authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-10-05 19:53:51-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-05 19:53:51-07:00
log6bfc8baeb8f280e57ef6d0d514680b0bce8538a2
tree39dd52130322cdcb0cf70304dededf2238cb24f1
parent41d2535a90fdfb49b2b9a7a7c8fb4bce09c4170f
signature Signed by PGP key B5690EEEBB952194

perf(relay): delete expired tunnels four at a time within a time budget (#13566)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

3 files changed, 186 insertions(+), 38 deletions(-)

docs/operations/release.md+6-2
......@@ -210,8 +210,12 @@ A deleted legacy tunnel keeps its allocation, so its hostname is kept. When the
2102103. Run the legacy steps of the disposable-host canary below.
2112114. Before enabling, confirm the web and mobile builds that show the "update T3 Code on that computer"
212212 message are live. Without them, a user whose older host lost its tunnel only sees it as offline.
2135. Set the legacy mode to `enabled`. One sweep deletes at most 100 tunnels, so a backlog of
214 20,000 takes about 17 hours. Watch `deletedLegacy`, `failed`, and `truncated`.
2135. Set the legacy mode to `enabled`. One sweep deletes at most 100 tunnels, four at a time, and
214 stops starting new deletions after 90 seconds. A backlog of 20,000 takes about 17 hours if each
215 sweep finishes its 100. Watch `deletedLegacy`, `attempted`, `failed`, and `truncated`. An
216 `attempted` well under 100 with `truncated` set means the sweep stopped early: either the time
217 budget ran out or Cloudflare rate-limited a deletion. The counters don't say which; the relay
218 logs a warning with the Cloudflare error for each failed deletion.
215219
216220### Disposable-host canary
217221
infra/relay/src/environments/ManagedEndpointReaper.test.ts+101-1
......@@ -75,6 +75,10 @@ function harness(input?: {
7575 readonly cleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode;
7676 readonly legacyCleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode;
7777 readonly legacyTunnelGraceMinutes?: number;
78 /** Simulated time each release takes, advanced on the test clock. */
79 readonly releaseDelayMs?: number;
80 /** Simulated time each Cloudflare list request takes. */
81 readonly listDelayMs?: number;
7882}) {
7983 const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = [];
8084 const deleted: string[] = [];
......@@ -119,7 +123,10 @@ function harness(input?: {
119123 return Effect.succeed(found);
120124 }),
121125 list: (request) =>
122 Effect.sync(() => {
126 Effect.gen(function* () {
127 if (input?.listDelayMs !== undefined) {
128 yield* TestClock.adjust(input.listDelayMs);
129 }
123130 listRequests.push(request);
124131 const matching = remaining.filter((entry) => entry.status === request.status);
125132 const start = ((request.page ?? 1) - 1) * (request.perPage ?? 100);
......@@ -191,6 +198,9 @@ function harness(input?: {
191198 release: (request) =>
192199 Effect.gen(function* () {
193200 releases.push(request);
201 if (input?.releaseDelayMs !== undefined) {
202 yield* TestClock.adjust(input.releaseDelayMs);
203 }
194204 if (request.expectedTunnelId === input?.skipTunnelId) {
195205 return false;
196206 }
......@@ -689,6 +699,96 @@ describe("ManagedEndpointReaper", () => {
689699 }).pipe(Effect.provide(state.layer));
690700 });
691701
702 it.effect("starts no new deletion after a rate limit, even with deletions in flight", () => {
703 // Enough candidates to fill every concurrent slot several times over.
704 const entries = Array.from({ length: 12 }, (_, index) =>
705 tunnel({
706 id: index === 0 ? "limited" : `ok-${index}`,
707 suffix: index.toString(16).padStart(16, "0"),
708 status: "down",
709 timestamp: "2026-08-25T11:00:00.000Z",
710 }),
711 );
712 const state = harness({
713 tunnels: entries,
714 allocations: recoverableOwners(entries),
715 rateLimitedTunnelId: "limited",
716 });
717
718 return Effect.gen(function* () {
719 yield* TestClock.setTime(NOW_MILLIS);
720 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
721 const result = yield* reaper.sweep;
722 expect(result.truncated).toBe(true);
723 expect(result.failed).toBe(1);
724 // Releases already running may finish, but none start afterwards.
725 expect(result.attempted).toBeLessThanOrEqual(
726 ManagedEndpointReaper.MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY,
727 );
728 expect(state.releases.length).toBe(result.attempted);
729 }).pipe(Effect.provide(state.layer));
730 });
731
732 it.effect("stops starting deletions when the sweep's time budget runs out", () => {
733 const entries = Array.from({ length: 40 }, (_, index) =>
734 tunnel({
735 id: `slow-${index}`,
736 suffix: index.toString(16).padStart(16, "0"),
737 status: "down",
738 timestamp: "2026-08-25T11:00:00.000Z",
739 }),
740 );
741 const state = harness({
742 tunnels: entries,
743 allocations: recoverableOwners(entries),
744 // Ten seconds each: the 90-second budget allows about 9 rounds.
745 releaseDelayMs: 10_000,
746 });
747
748 return Effect.gen(function* () {
749 yield* TestClock.setTime(NOW_MILLIS);
750 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
751 const result = yield* reaper.sweep;
752 expect(result.truncated).toBe(true);
753 expect(result.attempted).toBeGreaterThan(0);
754 expect(result.attempted).toBeLessThan(entries.length);
755 expect(result.deleted).toBe(result.attempted);
756 }).pipe(Effect.provide(state.layer));
757 });
758
759 it.effect("counts listing time against the deletion budget", () => {
760 const entries = Array.from({ length: 40 }, (_, index) =>
761 tunnel({
762 id: `slow-${index}`,
763 suffix: index.toString(16).padStart(16, "0"),
764 status: "down",
765 timestamp: "2026-08-25T11:00:00.000Z",
766 }),
767 );
768 const sweepWith = (listDelayMs: number) =>
769 Effect.gen(function* () {
770 yield* TestClock.setTime(NOW_MILLIS);
771 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
772 return (yield* reaper.sweep).attempted;
773 }).pipe(
774 Effect.provide(
775 harness({
776 tunnels: entries,
777 allocations: recoverableOwners(entries),
778 releaseDelayMs: 10_000,
779 listDelayMs,
780 }).layer,
781 ),
782 );
783
784 return Effect.gen(function* () {
785 const fastListing = yield* sweepWith(0);
786 // Two list requests of 20 seconds each use 40 of the 90-second budget.
787 const slowListing = yield* sweepWith(20_000);
788 expect(slowListing).toBeLessThan(fastListing);
789 });
790 });
791
692792 it.effect("continues past a page of older hosts to find recoverable tunnels", () => {
693793 const entries = Array.from({ length: 101 }, (_, index) =>
694794 tunnel({
infra/relay/src/environments/ManagedEndpointReaper.ts+79-35
......@@ -1,3 +1,4 @@
1import * as Clock from "effect/Clock";
12import * as Context from "effect/Context";
23import * as DateTime from "effect/DateTime";
34import * as Effect from "effect/Effect";
......@@ -27,6 +28,16 @@ const MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS = [7, 30, 90] as const;
2728// returning host that has updated recovers the tunnel at the same hostname;
2829// one that has not sees the client's "update T3 Code" message instead.
2930const MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS = 7;
31// Deletions run a few at a time: each one is a row-locked database
32// transaction plus two Cloudflare calls, so one at a time cannot finish a
33// full attempt budget inside the cron's timeout. Each holds a Hyperdrive
34// connection while it runs; keep this well under the 20-connection origin
35// limit that request handlers share.
36export const MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY = 4;
37// Stop starting deletions this long after the sweep starts, leaving room under
38// the cron's two-minute timeout to finish in-flight ones and record the
39// counters. Counted from sweep start so slow listing eats into it.
40const MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS = 90_000;
3041
3142export interface ManagedEndpointSweepResult {
3243 readonly mode: RelayConfiguration.ManagedEndpointCleanupMode;
......@@ -180,6 +191,7 @@ export const make = Effect.gen(function* () {
180191 if ((mode === "off" && legacyMode === "off") || !namespace) {
181192 return emptyResult(mode, legacyMode);
182193 }
194 const sweepStartedAtMillis = yield* Clock.currentTimeMillis;
183195 // The override exists for the disposable canary stage; prod always
184196 // waits the full grace period.
185197 const legacyGraceMinutes =
......@@ -275,6 +287,16 @@ export const make = Effect.gen(function* () {
275287 let attempted = 0;
276288 let deleted = 0;
277289 let wouldDelete = 0;
290 const candidates: Array<{
291 readonly owner: ManagedEndpointAllocations.ManagedEndpointTunnelAllocation;
292 readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & {
293 readonly id: string;
294 readonly name: string;
295 };
296 readonly status: "down" | "inactive";
297 readonly legacy: boolean;
298 readonly inactiveBefore: string;
299 }> = [];
278300 let wouldDeleteLegacy = 0;
279301 let deletedLegacy = 0;
280302 let skippedLegacy = 0;
......@@ -330,46 +352,68 @@ export const make = Effect.gen(function* () {
330352 wouldDelete += 1;
331353 if (mode === "dry-run") continue;
332354 }
333 if (attempted >= MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) {
355 if (candidates.length >= MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) {
334356 truncated = true;
335357 break;
336358 }
337 attempted += 1;
338 // The release re-reads the tunnel and deletes only if it is still in
339 // this status and inactive since before this tunnel's cutoff.
340 const result = yield* provider
341 .release({
342 userId: owner.userId,
343 environmentId: owner.environmentId,
344 expectedTunnelId: tunnel.id,
345 expectedInactiveBefore: DateTime.formatIso(legacy ? legacyCutoff : cutoff),
346 expectedStatus: status,
347 })
348 .pipe(Effect.result);
349 if (result._tag === "Failure") {
350 failed += 1;
351 yield* Effect.logWarning("Failed to delete an inactive managed tunnel", {
352 tunnelId: tunnel.id,
353 tunnelName: tunnel.name,
354 legacy,
355 cause: result.failure,
356 });
357 if (isRateLimited(result.failure)) {
358 truncated = true;
359 break;
360 }
361 } else if (result.success) {
362 deleted += 1;
363 if (legacy) deletedLegacy += 1;
364 yield* Effect.logInfo("Deleted an inactive managed tunnel", {
365 tunnelId: tunnel.id,
366 tunnelName: tunnel.name,
367 status,
368 legacy,
369 });
370 }
359 candidates.push({
360 owner,
361 tunnel,
362 status,
363 legacy,
364 // The release re-reads the tunnel and deletes only if it is still in
365 // this status and inactive since before this cutoff.
366 inactiveBefore: DateTime.formatIso(legacy ? legacyCutoff : cutoff),
367 });
371368 }
372369
370 const deleteDeadline = sweepStartedAtMillis + MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS;
371 let stopDeleting = false;
372 yield* Effect.forEach(
373 candidates,
374 (candidate) =>
375 Effect.gen(function* () {
376 if (stopDeleting || (yield* Clock.currentTimeMillis) >= deleteDeadline) {
377 stopDeleting = true;
378 truncated = true;
379 return;
380 }
381 attempted += 1;
382 const result = yield* provider
383 .release({
384 userId: candidate.owner.userId,
385 environmentId: candidate.owner.environmentId,
386 expectedTunnelId: candidate.tunnel.id,
387 expectedInactiveBefore: candidate.inactiveBefore,
388 expectedStatus: candidate.status,
389 })
390 .pipe(Effect.result);
391 if (result._tag === "Failure") {
392 failed += 1;
393 yield* Effect.logWarning("Failed to delete an inactive managed tunnel", {
394 tunnelId: candidate.tunnel.id,
395 tunnelName: candidate.tunnel.name,
396 legacy: candidate.legacy,
397 cause: result.failure,
398 });
399 if (isRateLimited(result.failure)) {
400 stopDeleting = true;
401 truncated = true;
402 }
403 } else if (result.success) {
404 deleted += 1;
405 if (candidate.legacy) deletedLegacy += 1;
406 yield* Effect.logInfo("Deleted an inactive managed tunnel", {
407 tunnelId: candidate.tunnel.id,
408 tunnelName: candidate.tunnel.name,
409 status: candidate.status,
410 legacy: candidate.legacy,
411 });
412 }
413 }),
414 { concurrency: MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY, discard: true },
415 );
416
373417 return {
374418 mode,
375419 legacyMode,