authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-10-05 19:53:51-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-05 19:53:51-07:00
log41d2535a90fdfb49b2b9a7a7c8fb4bce09c4170f
tree2e0f4ff3534b991480733ee3c47456abe0c59ce6
parent7b32b2ca39892bc59e1d8cfff7aea2a8fbefa73a
signature Signed by PGP key B5690EEEBB952194

feat(relay): clean up tunnels of hosts that never registered recovery (#13565)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

8 files changed, 340 insertions(+), 24 deletions(-)

.github/workflows/deploy-relay.yml+1
......@@ -40,6 +40,7 @@ jobs:
4040 RELAY_API_ZONE_NAME: ${{ vars.RELAY_API_ZONE_NAME }}
4141 RELAY_TUNNEL_ZONE_NAME: ${{ vars.RELAY_TUNNEL_ZONE_NAME }}
4242 RELAY_TUNNEL_CLEANUP_MODE: ${{ vars.RELAY_TUNNEL_CLEANUP_MODE }}
43 RELAY_LEGACY_TUNNEL_CLEANUP_MODE: ${{ vars.RELAY_LEGACY_TUNNEL_CLEANUP_MODE }}
4344 CLERK_PUBLISHABLE_KEY: ${{ vars.CLERK_PUBLISHABLE_KEY }}
4445 CLERK_JWT_AUDIENCE: ${{ vars.CLERK_JWT_AUDIENCE }}
4546 APNS_ENVIRONMENT: ${{ vars.APNS_ENVIRONMENT }}
docs/operations/release.md+42-1
......@@ -134,6 +134,8 @@ Optional `production` environment variables:
134134- `RELAY_DOMAIN` when overriding the derived `relay.<RELAY_API_ZONE_NAME>` domain
135135- `RELAY_TUNNEL_CLEANUP_MODE` with `off`, `dry-run`, or `enabled`. Missing and blank values use
136136 `off`.
137- `RELAY_LEGACY_TUNNEL_CLEANUP_MODE` with the same values, for tunnels whose host never registered
138 recovery. Missing and blank values use `off`.
137139
138140Required `production` environment secrets:
139141
......@@ -168,7 +170,7 @@ because those builds register recovery and replace a deleted tunnel after wake.
168170
1691711. Deploy the relay and migration with cleanup `off`.
1701722. Release the server build and confirm current hosts register recovery. Older hosts stay marked
171 legacy and are never candidates.
173 legacy and are only candidates under the legacy switch below.
1721743. Set `dry-run`, run a relay deploy, and read the sweep counters (`scanned`, `wouldDelete`,
173175 `skippedLegacy`, `skippedOrphan`, `failed`, `truncated`) across several sweeps. Each sweep records
174176 them, and the active `mode`, as `relay.managed_endpoint_reaper.*` attributes on its
......@@ -185,6 +187,32 @@ deploy without force. Confirm the new `mode` on the next sweep span.
185187To roll back, set cleanup to `off` and run a relay deploy before downgrading any host. Keep the
186188recovery endpoints deployed while current server builds are in use. The nullable columns can stay.
187189
190### Legacy tunnel cleanup
191
192A legacy tunnel belongs to a host that never registered recovery, usually one that went offline
193before the recovery build shipped. `RELAY_LEGACY_TUNNEL_CLEANUP_MODE` deletes these once Cloudflare
194reports them down, or never connected, for more than 7 days. It is independent of
195`RELAY_TUNNEL_CLEANUP_MODE`, and every other check still applies.
196
197A deleted legacy tunnel keeps its allocation, so its hostname is kept. When the host comes back:
198
199- On a build with recovery, the connector is rejected and the host requests a replacement tunnel at
200 the same hostname.
201- On an older build with a CLI link, startup provisions a new tunnel.
202- On an older build linked from web or mobile, the host stays offline until T3 Code on that computer
203 is updated.
204
2051. Run `vp run --filter t3code-relay tunnels:census` with a read-only Cloudflare token. It counts
206 tunnels in every relay stage. The reaper only sees its own stage's tunnels, so clean up the rest
207 by hand.
2082. Set the legacy mode to `dry-run`, deploy, and read `wouldDeleteLegacy`, `legacyOver30Days`,
209 `totalDown`, and `totalInactive` on the sweep spans for a day.
2103. Run the legacy steps of the disposable-host canary below.
2114. Before enabling, confirm the web and mobile builds that show the "update T3 Code on that computer"
212 message are live. Without them, a user whose older host lost its tunnel only sees it as offline.
2135. Set the legacy mode to `enabled`. One sweep deletes at most 100 tunnels, so a backlog of
214 20,000 takes about 17 hours. Watch `deletedLegacy`, `failed`, and `truncated`.
215
188216### Disposable-host canary
189217
190218This test has not been run against a real Cloudflare account. Run it against a disposable relay
......@@ -210,6 +238,19 @@ stage, test Cloudflare account, disposable host, and disposable T3 home. Keep pr
2102388. Resume the legacy child with `kill -CONT <legacy-pid>` and confirm its tunnel reconnects.
2112399. Repeat with a physical sleep and wake cycle on a disposable laptop before broad rollout.
212240
241Legacy cleanup, on the same disposable stage:
242
24310. Set `RELAY_LEGACY_TUNNEL_GRACE_MINUTES=10` and the legacy mode to `dry-run`, then deploy. The
244 override shortens the 7-day grace period and is ignored on `prod`. Pause the legacy child again
245 and wait until Cloudflare reports it down for over ten minutes.
24611. Confirm the sweep counts it in `wouldDeleteLegacy`, then set the legacy mode to `enabled` and
247 deploy. Confirm the legacy tunnel is deleted and its allocation row remains.
24812. With the legacy host still on its old build, resume the child. A CLI-linked host provisions a
249 new tunnel on its next restart; a web- or mobile-linked host stays offline.
25013. Update that host to the current build and start it. Confirm it requests recovery and is
251 reachable at the same hostname.
25214. Remove `RELAY_LEGACY_TUNNEL_GRACE_MINUTES` from the disposable stage.
253
213254## Marketing site deployment
214255
215256On nightly releases, the release workflow builds the same commit as a staged
infra/relay/.env.example+6
......@@ -8,6 +8,12 @@ RELAY_TUNNEL_ZONE_NAME=tunnels.example.com
88# Optional: inactive tunnel cleanup. Start with dry-run, verify the cleanup
99# logs, then set enabled. Unset and off both disable cleanup.
1010# RELAY_TUNNEL_CLEANUP_MODE=off
11# Optional: cleanup for tunnels whose host never registered recovery, once they
12# have been down for 30 days. Same values, independent of the mode above.
13# RELAY_LEGACY_TUNNEL_CLEANUP_MODE=off
14# Canary stages only: shorten the 30-day legacy grace period, in minutes.
15# Ignored on prod.
16# RELAY_LEGACY_TUNNEL_GRACE_MINUTES=10
1117
1218# Optional: Relay domain override
1319# Set this only when the derived relay hostname should not be used.
infra/relay/src/Config.test.ts+44-1
......@@ -1,8 +1,13 @@
11import { expect, it } from "@effect/vitest";
22import * as ConfigProvider from "effect/ConfigProvider";
33import * as Effect from "effect/Effect";
4import * as Option from "effect/Option";
45
5import { managedEndpointCleanupModeConfig } from "./Config.ts";
6import {
7 legacyManagedEndpointCleanupModeConfig,
8 legacyTunnelGraceMinutesConfig,
9 managedEndpointCleanupModeConfig,
10} from "./Config.ts";
611
712it.effect.each([
813 { name: "missing", env: {}, expected: "off" },
......@@ -33,3 +38,41 @@ it.effect("rejects an invalid cleanup mode", () =>
3338 expect(error.message).toContain('Expected "off" | "dry-run" | "enabled"');
3439 }),
3540);
41
42it.effect("reads the legacy cleanup mode independently of the main one", () =>
43 Effect.gen(function* () {
44 const provider = ConfigProvider.fromEnv({
45 env: { RELAY_TUNNEL_CLEANUP_MODE: "enabled", RELAY_LEGACY_TUNNEL_CLEANUP_MODE: "dry-run" },
46 });
47 expect(yield* managedEndpointCleanupModeConfig.parse(provider)).toBe("enabled");
48 expect(yield* legacyManagedEndpointCleanupModeConfig.parse(provider)).toBe("dry-run");
49 expect(
50 yield* legacyManagedEndpointCleanupModeConfig.parse(ConfigProvider.fromEnv({ env: {} })),
51 ).toBe("off");
52 }),
53);
54
55it.effect.each([
56 { name: "missing", env: {}, expected: Option.none() },
57 {
58 name: "positive",
59 env: { RELAY_LEGACY_TUNNEL_GRACE_MINUTES: "10" },
60 expected: Option.some(10),
61 },
62] as const)("loads a $name legacy grace override", ({ env, expected }) =>
63 Effect.gen(function* () {
64 const minutes = yield* legacyTunnelGraceMinutesConfig.parse(ConfigProvider.fromEnv({ env }));
65 expect(minutes).toEqual(expected);
66 }),
67);
68
69it.effect.each(["0", "-10"])("rejects a grace override of %s minutes", (value) =>
70 Effect.gen(function* () {
71 const error = yield* Effect.flip(
72 legacyTunnelGraceMinutesConfig.parse(
73 ConfigProvider.fromEnv({ env: { RELAY_LEGACY_TUNNEL_GRACE_MINUTES: value } }),
74 ),
75 );
76 expect(error._tag).toBe("ConfigError");
77 }),
78);
infra/relay/src/Config.ts+34-6
......@@ -12,13 +12,38 @@ export const ManagedEndpointCleanupMode = Schema.Literals(["off", "dry-run", "en
1212export type ManagedEndpointCleanupMode = typeof ManagedEndpointCleanupMode.Type;
1313const decodeManagedEndpointCleanupMode = Schema.decodeUnknownEffect(ManagedEndpointCleanupMode);
1414
15export const managedEndpointCleanupModeConfig = Config.String("RELAY_TUNNEL_CLEANUP_MODE").pipe(
16 Config.withDefault("off"),
17 Config.map((value) => value.trim() || "off"),
18 Config.mapEffect((value) =>
19 decodeManagedEndpointCleanupMode(value).pipe(
20 Effect.mapError((error) => new Config.ConfigError(error)),
15const RELAY_TUNNEL_CLEANUP_MODE = "RELAY_TUNNEL_CLEANUP_MODE";
16/** Separate switch for tunnels whose host never registered recovery. */
17const RELAY_LEGACY_TUNNEL_CLEANUP_MODE = "RELAY_LEGACY_TUNNEL_CLEANUP_MODE";
18
19const cleanupModeConfig = (name: string) =>
20 Config.String(name).pipe(
21 Config.withDefault("off"),
22 Config.map((value) => value.trim() || "off"),
23 Config.mapEffect((value) =>
24 decodeManagedEndpointCleanupMode(value).pipe(
25 Effect.mapError((error) => new Config.ConfigError(error)),
26 ),
2127 ),
28 );
29
30export const managedEndpointCleanupModeConfig = cleanupModeConfig(RELAY_TUNNEL_CLEANUP_MODE);
31export const legacyManagedEndpointCleanupModeConfig = cleanupModeConfig(
32 RELAY_LEGACY_TUNNEL_CLEANUP_MODE,
33);
34
35/**
36 * Overrides the 7-day legacy grace period, in minutes, so the disposable
37 * canary stage can exercise legacy cleanup. Ignored on the prod stage.
38 */
39const RELAY_LEGACY_TUNNEL_GRACE_MINUTES = "RELAY_LEGACY_TUNNEL_GRACE_MINUTES";
40
41// A zero or negative override would be ignored at runtime, silently leaving
42// the canary on the 7-day grace period, so reject it when the deploy reads it.
43export const legacyTunnelGraceMinutesConfig = Config.option(
44 Config.schema(
45 Schema.NumberFromString.pipe(Schema.check(Schema.isInt(), Schema.isGreaterThan(0))),
46 RELAY_LEGACY_TUNNEL_GRACE_MINUTES,
2247 ),
2348);
2449
......@@ -45,6 +70,9 @@ export class RelayConfiguration extends Context.Service<
4570 readonly managedEndpointBaseDomain: string | undefined;
4671 readonly managedEndpointNamespace: string | undefined;
4772 readonly managedEndpointCleanupMode?: ManagedEndpointCleanupMode;
73 readonly legacyManagedEndpointCleanupMode?: ManagedEndpointCleanupMode;
74 /** Canary-only override of the legacy grace period; ignored on prod. */
75 readonly legacyTunnelGraceMinutes?: number;
4876 }
4977>()("t3code-relay/Config/RelayConfiguration") {}
5078
infra/relay/src/environments/ManagedEndpointReaper.test.ts+144
......@@ -73,6 +73,8 @@ function harness(input?: {
7373 readonly refreshedTunnels?: ReadonlyMap<string, ManagedEndpointProvider.ManagedEndpointTunnel>;
7474 readonly skipTunnelId?: string;
7575 readonly cleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode;
76 readonly legacyCleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode;
77 readonly legacyTunnelGraceMinutes?: number;
7678}) {
7779 const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = [];
7880 const deleted: string[] = [];
......@@ -228,6 +230,10 @@ function harness(input?: {
228230 managedEndpointBaseDomain: "example.test",
229231 managedEndpointNamespace: input?.namespace ?? "prod",
230232 managedEndpointCleanupMode: input?.cleanupMode ?? "enabled",
233 legacyManagedEndpointCleanupMode: input?.legacyCleanupMode ?? "off",
234 ...(input?.legacyTunnelGraceMinutes === undefined
235 ? {}
236 : { legacyTunnelGraceMinutes: input.legacyTunnelGraceMinutes }),
231237 });
232238
233239 return {
......@@ -490,6 +496,141 @@ describe("ManagedEndpointReaper", () => {
490496 }).pipe(Effect.provide(state.layer));
491497 });
492498
499 describe("legacy cleanup", () => {
500 const legacyTunnels = [
501 // Down for 5 days: inside the legacy grace period.
502 tunnel({
503 id: "legacy-recent",
504 suffix: "1111111111111111",
505 status: "down",
506 timestamp: "2026-08-20T12:00:00.000Z",
507 }),
508 // Down for 40 days: past it.
509 tunnel({
510 id: "legacy-old",
511 suffix: "2222222222222222",
512 status: "down",
513 timestamp: "2026-07-16T12:00:00.000Z",
514 }),
515 // Never connected, created 40 days ago.
516 tunnel({
517 id: "legacy-never",
518 suffix: "3333333333333333",
519 status: "inactive",
520 timestamp: "2026-07-16T12:00:00.000Z",
521 }),
522 ];
523 const legacyOwners = legacyTunnels.map((entry) =>
524 allocation({ tunnelId: entry.id!, recoveryEnabled: false }),
525 );
526
527 it.effect("leaves legacy tunnels alone while legacy cleanup is off", () => {
528 const state = harness({ tunnels: legacyTunnels, allocations: legacyOwners });
529 return Effect.gen(function* () {
530 yield* TestClock.setTime(NOW_MILLIS);
531 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
532 expect(yield* reaper.sweep).toMatchObject({
533 skippedLegacy: 3,
534 wouldDeleteLegacy: 0,
535 deletedLegacy: 0,
536 });
537 expect(state.deleted).toEqual([]);
538 }).pipe(Effect.provide(state.layer));
539 });
540
541 it.effect("counts legacy tunnels past the grace period in dry-run", () => {
542 const state = harness({
543 tunnels: legacyTunnels,
544 allocations: legacyOwners,
545 legacyCleanupMode: "dry-run",
546 });
547 return Effect.gen(function* () {
548 yield* TestClock.setTime(NOW_MILLIS);
549 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
550 expect(yield* reaper.sweep).toMatchObject({
551 legacyMode: "dry-run",
552 wouldDeleteLegacy: 2,
553 deletedLegacy: 0,
554 attempted: 0,
555 });
556 expect(state.deleted).toEqual([]);
557 }).pipe(Effect.provide(state.layer));
558 });
559
560 it.effect("deletes only legacy tunnels past the grace period", () => {
561 const state = harness({
562 tunnels: legacyTunnels,
563 allocations: legacyOwners,
564 legacyCleanupMode: "enabled",
565 });
566 return Effect.gen(function* () {
567 yield* TestClock.setTime(NOW_MILLIS);
568 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
569 expect(yield* reaper.sweep).toMatchObject({
570 wouldDeleteLegacy: 2,
571 deletedLegacy: 2,
572 deleted: 2,
573 });
574 expect([...state.deleted].sort()).toEqual(["legacy-never", "legacy-old"]);
575 // The release re-checks each tunnel against the 7-day cutoff, not
576 // the 5-minute cutoff used for recoverable tunnels.
577 for (const release of state.releases) {
578 expect(release.expectedInactiveBefore).toBe("2026-08-18T12:00:00.000Z");
579 }
580 }).pipe(Effect.provide(state.layer));
581 });
582
583 it.effect.each([
584 { stage: "canary", expected: 3 },
585 { stage: "prod", expected: 2 },
586 ] as const)(
587 "applies the grace-period override only off prod ($stage)",
588 ({ stage, expected }) => {
589 const stageLegacyTunnels = legacyTunnels.map((entry) => ({
590 ...entry,
591 name: entry.name!.replace(PREFIX, `t3coderelay-managedendpoint-${stage}-`),
592 }));
593 const state = harness({
594 namespace: stage,
595 tunnels: stageLegacyTunnels,
596 allocations: legacyOwners,
597 legacyCleanupMode: "dry-run",
598 // Ten minutes: the 10-day-old tunnel qualifies only if this applies.
599 legacyTunnelGraceMinutes: 10,
600 });
601 return Effect.gen(function* () {
602 yield* TestClock.setTime(NOW_MILLIS);
603 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
604 expect((yield* reaper.sweep).wouldDeleteLegacy).toBe(expected);
605 }).pipe(Effect.provide(state.layer));
606 },
607 );
608
609 it.effect("runs legacy cleanup while recoverable cleanup is off", () => {
610 const recoverable = tunnel({
611 id: "recoverable",
612 suffix: "4444444444444444",
613 status: "down",
614 timestamp: "2026-08-25T11:00:00.000Z",
615 });
616 const state = harness({
617 tunnels: [...legacyTunnels, recoverable],
618 allocations: [
619 ...legacyOwners,
620 allocation({ tunnelId: "recoverable", recoveryEnabled: true }),
621 ],
622 cleanupMode: "off",
623 legacyCleanupMode: "enabled",
624 });
625 return Effect.gen(function* () {
626 yield* TestClock.setTime(NOW_MILLIS);
627 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
628 expect(yield* reaper.sweep).toMatchObject({ wouldDelete: 0, deletedLegacy: 2 });
629 expect(state.deleted).not.toContain("recoverable");
630 }).pipe(Effect.provide(state.layer));
631 });
632 });
633
493634 it.effect("does not count a tunnel that was replaced before its release", () => {
494635 const state = harness({
495636 tunnels: [
......@@ -697,11 +838,14 @@ describe("ManagedEndpointReaper", () => {
697838 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
698839 expect(yield* reaper.sweep).toEqual({
699840 mode: "off",
841 legacyMode: "off",
700842 listRequests: 0,
701843 scanned: 0,
702844 attempted: 0,
703845 deleted: 0,
704846 wouldDelete: 0,
847 wouldDeleteLegacy: 0,
848 deletedLegacy: 0,
705849 skippedLegacy: 0,
706850 legacyOver7Days: 0,
707851 legacyOver30Days: 0,
infra/relay/src/environments/ManagedEndpointReaper.ts+62-16
......@@ -5,7 +5,10 @@ import * as Layer from "effect/Layer";
55import * as Option from "effect/Option";
66
77import * as RelayConfiguration from "../Config.ts";
8import { managedEndpointTunnelNamePrefix } from "../deploymentConfig.ts";
8import {
9 MANAGED_ENDPOINT_ZONE_OWNER_STAGE,
10 managedEndpointTunnelNamePrefix,
11} from "../deploymentConfig.ts";
912import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";
1013import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts";
1114
......@@ -19,14 +22,23 @@ export const MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT = 10;
1922// Age buckets for legacy candidates, in days since the tunnel went down (or
2023// was created, for one that never connected).
2124const MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS = [7, 30, 90] as const;
25// A host that never registered recovery cannot replace a deleted tunnel on
26// its own build. Only delete its tunnel after it has been gone this long. A
27// returning host that has updated recovers the tunnel at the same hostname;
28// one that has not sees the client's "update T3 Code" message instead.
29const MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS = 7;
2230
2331export interface ManagedEndpointSweepResult {
2432 readonly mode: RelayConfiguration.ManagedEndpointCleanupMode;
33 readonly legacyMode: RelayConfiguration.ManagedEndpointCleanupMode;
2534 readonly listRequests: number;
2635 readonly scanned: number;
2736 readonly attempted: number;
2837 readonly deleted: number;
2938 readonly wouldDelete: number;
39 /** Legacy tunnels past the legacy grace period, and how many were deleted. */
40 readonly wouldDeleteLegacy: number;
41 readonly deletedLegacy: number;
3042 readonly skippedLegacy: number;
3143 /** Legacy candidates, by days since they went down: over 7, 30, and 90. */
3244 readonly legacyOver7Days: number;
......@@ -131,13 +143,17 @@ function rotatedPages(input: {
131143
132144const emptyResult = (
133145 mode: RelayConfiguration.ManagedEndpointCleanupMode,
146 legacyMode: RelayConfiguration.ManagedEndpointCleanupMode,
134147): ManagedEndpointSweepResult => ({
135148 mode,
149 legacyMode,
136150 listRequests: 0,
137151 scanned: 0,
138152 attempted: 0,
139153 deleted: 0,
140154 wouldDelete: 0,
155 wouldDeleteLegacy: 0,
156 deletedLegacy: 0,
141157 skippedLegacy: 0,
142158 legacyOver7Days: 0,
143159 legacyOver30Days: 0,
......@@ -159,8 +175,19 @@ export const make = Effect.gen(function* () {
159175
160176 const sweep = Effect.gen(function* () {
161177 const mode = config.managedEndpointCleanupMode ?? "off";
178 const legacyMode = config.legacyManagedEndpointCleanupMode ?? "off";
162179 const namespace = config.managedEndpointNamespace;
163 if (mode === "off" || !namespace) return emptyResult(mode);
180 if ((mode === "off" && legacyMode === "off") || !namespace) {
181 return emptyResult(mode, legacyMode);
182 }
183 // The override exists for the disposable canary stage; prod always
184 // waits the full grace period.
185 const legacyGraceMinutes =
186 namespace !== MANAGED_ENDPOINT_ZONE_OWNER_STAGE &&
187 config.legacyTunnelGraceMinutes !== undefined
188 ? config.legacyTunnelGraceMinutes
189 : MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS * 24 * 60;
190 const legacyCutoff = DateTime.subtract(yield* DateTime.now, { minutes: legacyGraceMinutes });
164191
165192 const now = yield* DateTime.now;
166193 const cutoffFor = (status: "down" | "inactive") =>
......@@ -248,6 +275,8 @@ export const make = Effect.gen(function* () {
248275 let attempted = 0;
249276 let deleted = 0;
250277 let wouldDelete = 0;
278 let wouldDeleteLegacy = 0;
279 let deletedLegacy = 0;
251280 let skippedLegacy = 0;
252281 const legacyOverDays = new Map<number, number>(
253282 MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS.map((days) => [days, 0]),
......@@ -258,7 +287,6 @@ export const make = Effect.gen(function* () {
258287 let failed = 0;
259288
260289 for (const { tunnel, status, cutoff } of uniqueExpired) {
261 const cutoffIso = DateTime.formatIso(cutoff);
262290 const allocation = recordedByTunnelName.get(tunnel.name);
263291 if (
264292 allocation !== undefined &&
......@@ -269,16 +297,6 @@ export const make = Effect.gen(function* () {
269297 continue;
270298 }
271299 const owner = allocation?.tunnelId === tunnel.id ? allocation : undefined;
272 if (owner !== undefined && !owner.recoveryEnabled) {
273 skippedLegacy += 1;
274 const inactiveDays = inactiveDaysAt(tunnel, status, now);
275 for (const days of MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS) {
276 if (inactiveDays !== null && inactiveDays > days) {
277 legacyOverDays.set(days, (legacyOverDays.get(days) ?? 0) + 1);
278 }
279 }
280 continue;
281 }
282300 if (allocation !== undefined && owner === undefined) {
283301 skippedUnrecorded += 1;
284302 continue;
......@@ -290,19 +308,41 @@ export const make = Effect.gen(function* () {
290308 skippedOrphan += 1;
291309 continue;
292310 }
293 wouldDelete += 1;
294 if (mode === "dry-run") continue;
311 const legacy = !owner.recoveryEnabled;
312 if (legacy) {
313 skippedLegacy += 1;
314 const inactiveDays = inactiveDaysAt(tunnel, status, now);
315 for (const days of MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS) {
316 if (inactiveDays !== null && inactiveDays > days) {
317 legacyOverDays.set(days, (legacyOverDays.get(days) ?? 0) + 1);
318 }
319 }
320 if (
321 legacyMode === "off" ||
322 !isExpiredManagedTunnel({ tunnel, status, prefix, cutoff: legacyCutoff })
323 ) {
324 continue;
325 }
326 wouldDeleteLegacy += 1;
327 if (legacyMode === "dry-run") continue;
328 } else {
329 if (mode === "off") continue;
330 wouldDelete += 1;
331 if (mode === "dry-run") continue;
332 }
295333 if (attempted >= MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) {
296334 truncated = true;
297335 break;
298336 }
299337 attempted += 1;
338 // The release re-reads the tunnel and deletes only if it is still in
339 // this status and inactive since before this tunnel's cutoff.
300340 const result = yield* provider
301341 .release({
302342 userId: owner.userId,
303343 environmentId: owner.environmentId,
304344 expectedTunnelId: tunnel.id,
305 expectedInactiveBefore: cutoffIso,
345 expectedInactiveBefore: DateTime.formatIso(legacy ? legacyCutoff : cutoff),
306346 expectedStatus: status,
307347 })
308348 .pipe(Effect.result);
......@@ -311,6 +351,7 @@ export const make = Effect.gen(function* () {
311351 yield* Effect.logWarning("Failed to delete an inactive managed tunnel", {
312352 tunnelId: tunnel.id,
313353 tunnelName: tunnel.name,
354 legacy,
314355 cause: result.failure,
315356 });
316357 if (isRateLimited(result.failure)) {
......@@ -319,21 +360,26 @@ export const make = Effect.gen(function* () {
319360 }
320361 } else if (result.success) {
321362 deleted += 1;
363 if (legacy) deletedLegacy += 1;
322364 yield* Effect.logInfo("Deleted an inactive managed tunnel", {
323365 tunnelId: tunnel.id,
324366 tunnelName: tunnel.name,
325367 status,
368 legacy,
326369 });
327370 }
328371 }
329372
330373 return {
331374 mode,
375 legacyMode,
332376 listRequests,
333377 scanned: uniqueExpired.length,
334378 attempted,
335379 deleted,
336380 wouldDelete,
381 wouldDeleteLegacy,
382 deletedLegacy,
337383 skippedLegacy,
338384 legacyOver7Days: legacyOverDays.get(7) ?? 0,
339385 legacyOver30Days: legacyOverDays.get(30) ?? 0,
infra/relay/src/worker.ts+7
......@@ -184,6 +184,11 @@ export const layer = Api.make(
184184 const managedEndpointDnsBinding = yield* Cloudflare.DNS.ReadWriteDns(managedEndpointZone);
185185 const managedEndpointZoneName = yield* managedEndpointZone.name;
186186 const managedEndpointCleanupMode = yield* RelayConfiguration.managedEndpointCleanupModeConfig;
187 const legacyManagedEndpointCleanupMode =
188 yield* RelayConfiguration.legacyManagedEndpointCleanupModeConfig;
189 const legacyTunnelGraceMinutes = Option.getOrUndefined(
190 yield* RelayConfiguration.legacyTunnelGraceMinutesConfig,
191 );
187192 // Keys are endpoint keys or hashes over them, which already differ per
188193 // stage, so stages sharing an account cannot collide in these namespaces.
189194 const hookRateLimit = yield* Cloudflare.RateLimit("HOOK_RATE_LIMIT", {
......@@ -221,6 +226,8 @@ export const layer = Api.make(
221226 managedEndpointBaseDomain: yield* managedEndpointZoneName,
222227 managedEndpointNamespace: stage,
223228 managedEndpointCleanupMode,
229 legacyManagedEndpointCleanupMode,
230 ...(legacyTunnelGraceMinutes === undefined ? {} : { legacyTunnelGraceMinutes }),
224231 });
225232 });
226233