authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-10-05 19:53:51-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-05 19:53:51-07:00
log41d2535a90fdfb49b2b9a7a7c8fb4bce09c4170f
tree2e0f4ff3534b991480733ee3c47456abe0c59ce6
parent7b32b2ca39892bc59e1d8cfff7aea2a8fbefa73a
signature Signed by PGP key B5690EEEBB952194

feat(relay): clean up tunnels of hosts that never registered recovery (#13565)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

8 files changed, 340 insertions(+), 24 deletions(-)

.github/workflows/deploy-relay.yml+1
...@@ -40,6 +40,7 @@ jobs:...@@ -40,6 +40,7 @@ jobs:
40 RELAY_API_ZONE_NAME: ${{ vars.RELAY_API_ZONE_NAME }}40 RELAY_API_ZONE_NAME: ${{ vars.RELAY_API_ZONE_NAME }}
41 RELAY_TUNNEL_ZONE_NAME: ${{ vars.RELAY_TUNNEL_ZONE_NAME }}41 RELAY_TUNNEL_ZONE_NAME: ${{ vars.RELAY_TUNNEL_ZONE_NAME }}
42 RELAY_TUNNEL_CLEANUP_MODE: ${{ vars.RELAY_TUNNEL_CLEANUP_MODE }}42 RELAY_TUNNEL_CLEANUP_MODE: ${{ vars.RELAY_TUNNEL_CLEANUP_MODE }}
43 RELAY_LEGACY_TUNNEL_CLEANUP_MODE: ${{ vars.RELAY_LEGACY_TUNNEL_CLEANUP_MODE }}
43 CLERK_PUBLISHABLE_KEY: ${{ vars.CLERK_PUBLISHABLE_KEY }}44 CLERK_PUBLISHABLE_KEY: ${{ vars.CLERK_PUBLISHABLE_KEY }}
44 CLERK_JWT_AUDIENCE: ${{ vars.CLERK_JWT_AUDIENCE }}45 CLERK_JWT_AUDIENCE: ${{ vars.CLERK_JWT_AUDIENCE }}
45 APNS_ENVIRONMENT: ${{ vars.APNS_ENVIRONMENT }}46 APNS_ENVIRONMENT: ${{ vars.APNS_ENVIRONMENT }}
docs/operations/release.md+42-1
...@@ -134,6 +134,8 @@ Optional `production` environment variables:...@@ -134,6 +134,8 @@ Optional `production` environment variables:
134- `RELAY_DOMAIN` when overriding the derived `relay.<RELAY_API_ZONE_NAME>` domain134- `RELAY_DOMAIN` when overriding the derived `relay.<RELAY_API_ZONE_NAME>` domain
135- `RELAY_TUNNEL_CLEANUP_MODE` with `off`, `dry-run`, or `enabled`. Missing and blank values use135- `RELAY_TUNNEL_CLEANUP_MODE` with `off`, `dry-run`, or `enabled`. Missing and blank values use
136 `off`.136 `off`.
137- `RELAY_LEGACY_TUNNEL_CLEANUP_MODE` with the same values, for tunnels whose host never registered
138 recovery. Missing and blank values use `off`.
137139
138Required `production` environment secrets:140Required `production` environment secrets:
139141
...@@ -168,7 +170,7 @@ because those builds register recovery and replace a deleted tunnel after wake....@@ -168,7 +170,7 @@ because those builds register recovery and replace a deleted tunnel after wake.
168170
1691. Deploy the relay and migration with cleanup `off`.1711. Deploy the relay and migration with cleanup `off`.
1702. Release the server build and confirm current hosts register recovery. Older hosts stay marked1722. Release the server build and confirm current hosts register recovery. Older hosts stay marked
171 legacy and are never candidates.173 legacy and are only candidates under the legacy switch below.
1723. Set `dry-run`, run a relay deploy, and read the sweep counters (`scanned`, `wouldDelete`,1743. Set `dry-run`, run a relay deploy, and read the sweep counters (`scanned`, `wouldDelete`,
173 `skippedLegacy`, `skippedOrphan`, `failed`, `truncated`) across several sweeps. Each sweep records175 `skippedLegacy`, `skippedOrphan`, `failed`, `truncated`) across several sweeps. Each sweep records
174 them, and the active `mode`, as `relay.managed_endpoint_reaper.*` attributes on its176 them, and the active `mode`, as `relay.managed_endpoint_reaper.*` attributes on its
...@@ -185,6 +187,32 @@ deploy without force. Confirm the new `mode` on the next sweep span....@@ -185,6 +187,32 @@ deploy without force. Confirm the new `mode` on the next sweep span.
185To roll back, set cleanup to `off` and run a relay deploy before downgrading any host. Keep the187To roll back, set cleanup to `off` and run a relay deploy before downgrading any host. Keep the
186recovery endpoints deployed while current server builds are in use. The nullable columns can stay.188recovery endpoints deployed while current server builds are in use. The nullable columns can stay.
187189
190### Legacy tunnel cleanup
191
192A legacy tunnel belongs to a host that never registered recovery, usually one that went offline
193before the recovery build shipped. `RELAY_LEGACY_TUNNEL_CLEANUP_MODE` deletes these once Cloudflare
194reports them down, or never connected, for more than 7 days. It is independent of
195`RELAY_TUNNEL_CLEANUP_MODE`, and every other check still applies.
196
197A deleted legacy tunnel keeps its allocation, so its hostname is kept. When the host comes back:
198
199- On a build with recovery, the connector is rejected and the host requests a replacement tunnel at
200 the same hostname.
201- On an older build with a CLI link, startup provisions a new tunnel.
202- On an older build linked from web or mobile, the host stays offline until T3 Code on that computer
203 is updated.
204
2051. Run `vp run --filter t3code-relay tunnels:census` with a read-only Cloudflare token. It counts
206 tunnels in every relay stage. The reaper only sees its own stage's tunnels, so clean up the rest
207 by hand.
2082. Set the legacy mode to `dry-run`, deploy, and read `wouldDeleteLegacy`, `legacyOver30Days`,
209 `totalDown`, and `totalInactive` on the sweep spans for a day.
2103. Run the legacy steps of the disposable-host canary below.
2114. Before enabling, confirm the web and mobile builds that show the "update T3 Code on that computer"
212 message are live. Without them, a user whose older host lost its tunnel only sees it as offline.
2135. Set the legacy mode to `enabled`. One sweep deletes at most 100 tunnels, so a backlog of
214 20,000 takes about 17 hours. Watch `deletedLegacy`, `failed`, and `truncated`.
215
188### Disposable-host canary216### Disposable-host canary
189217
190This test has not been run against a real Cloudflare account. Run it against a disposable relay218This test has not been run against a real Cloudflare account. Run it against a disposable relay
...@@ -210,6 +238,19 @@ stage, test Cloudflare account, disposable host, and disposable T3 home. Keep pr...@@ -210,6 +238,19 @@ stage, test Cloudflare account, disposable host, and disposable T3 home. Keep pr
2108. Resume the legacy child with `kill -CONT <legacy-pid>` and confirm its tunnel reconnects.2388. Resume the legacy child with `kill -CONT <legacy-pid>` and confirm its tunnel reconnects.
2119. Repeat with a physical sleep and wake cycle on a disposable laptop before broad rollout.2399. Repeat with a physical sleep and wake cycle on a disposable laptop before broad rollout.
212240
241Legacy cleanup, on the same disposable stage:
242
24310. Set `RELAY_LEGACY_TUNNEL_GRACE_MINUTES=10` and the legacy mode to `dry-run`, then deploy. The
244 override shortens the 7-day grace period and is ignored on `prod`. Pause the legacy child again
245 and wait until Cloudflare reports it down for over ten minutes.
24611. Confirm the sweep counts it in `wouldDeleteLegacy`, then set the legacy mode to `enabled` and
247 deploy. Confirm the legacy tunnel is deleted and its allocation row remains.
24812. With the legacy host still on its old build, resume the child. A CLI-linked host provisions a
249 new tunnel on its next restart; a web- or mobile-linked host stays offline.
25013. Update that host to the current build and start it. Confirm it requests recovery and is
251 reachable at the same hostname.
25214. Remove `RELAY_LEGACY_TUNNEL_GRACE_MINUTES` from the disposable stage.
253
213## Marketing site deployment254## Marketing site deployment
214255
215On nightly releases, the release workflow builds the same commit as a staged256On nightly releases, the release workflow builds the same commit as a staged
infra/relay/.env.example+6
...@@ -8,6 +8,12 @@ RELAY_TUNNEL_ZONE_NAME=tunnels.example.com...@@ -8,6 +8,12 @@ RELAY_TUNNEL_ZONE_NAME=tunnels.example.com
8# Optional: inactive tunnel cleanup. Start with dry-run, verify the cleanup8# Optional: inactive tunnel cleanup. Start with dry-run, verify the cleanup
9# logs, then set enabled. Unset and off both disable cleanup.9# logs, then set enabled. Unset and off both disable cleanup.
10# RELAY_TUNNEL_CLEANUP_MODE=off10# RELAY_TUNNEL_CLEANUP_MODE=off
11# Optional: cleanup for tunnels whose host never registered recovery, once they
12# have been down for 30 days. Same values, independent of the mode above.
13# RELAY_LEGACY_TUNNEL_CLEANUP_MODE=off
14# Canary stages only: shorten the 30-day legacy grace period, in minutes.
15# Ignored on prod.
16# RELAY_LEGACY_TUNNEL_GRACE_MINUTES=10
1117
12# Optional: Relay domain override18# Optional: Relay domain override
13# Set this only when the derived relay hostname should not be used.19# Set this only when the derived relay hostname should not be used.
infra/relay/src/Config.test.ts+44-1
...@@ -1,8 +1,13 @@...@@ -1,8 +1,13 @@
1import { expect, it } from "@effect/vitest";1import { expect, it } from "@effect/vitest";
2import * as ConfigProvider from "effect/ConfigProvider";2import * as ConfigProvider from "effect/ConfigProvider";
3import * as Effect from "effect/Effect";3import * as Effect from "effect/Effect";
4import * as Option from "effect/Option";
45
5import { managedEndpointCleanupModeConfig } from "./Config.ts";6import {
7 legacyManagedEndpointCleanupModeConfig,
8 legacyTunnelGraceMinutesConfig,
9 managedEndpointCleanupModeConfig,
10} from "./Config.ts";
611
7it.effect.each([12it.effect.each([
8 { name: "missing", env: {}, expected: "off" },13 { name: "missing", env: {}, expected: "off" },
...@@ -33,3 +38,41 @@ it.effect("rejects an invalid cleanup mode", () =>...@@ -33,3 +38,41 @@ it.effect("rejects an invalid cleanup mode", () =>
33 expect(error.message).toContain('Expected "off" | "dry-run" | "enabled"');38 expect(error.message).toContain('Expected "off" | "dry-run" | "enabled"');
34 }),39 }),
35);40);
41
42it.effect("reads the legacy cleanup mode independently of the main one", () =>
43 Effect.gen(function* () {
44 const provider = ConfigProvider.fromEnv({
45 env: { RELAY_TUNNEL_CLEANUP_MODE: "enabled", RELAY_LEGACY_TUNNEL_CLEANUP_MODE: "dry-run" },
46 });
47 expect(yield* managedEndpointCleanupModeConfig.parse(provider)).toBe("enabled");
48 expect(yield* legacyManagedEndpointCleanupModeConfig.parse(provider)).toBe("dry-run");
49 expect(
50 yield* legacyManagedEndpointCleanupModeConfig.parse(ConfigProvider.fromEnv({ env: {} })),
51 ).toBe("off");
52 }),
53);
54
55it.effect.each([
56 { name: "missing", env: {}, expected: Option.none() },
57 {
58 name: "positive",
59 env: { RELAY_LEGACY_TUNNEL_GRACE_MINUTES: "10" },
60 expected: Option.some(10),
61 },
62] as const)("loads a $name legacy grace override", ({ env, expected }) =>
63 Effect.gen(function* () {
64 const minutes = yield* legacyTunnelGraceMinutesConfig.parse(ConfigProvider.fromEnv({ env }));
65 expect(minutes).toEqual(expected);
66 }),
67);
68
69it.effect.each(["0", "-10"])("rejects a grace override of %s minutes", (value) =>
70 Effect.gen(function* () {
71 const error = yield* Effect.flip(
72 legacyTunnelGraceMinutesConfig.parse(
73 ConfigProvider.fromEnv({ env: { RELAY_LEGACY_TUNNEL_GRACE_MINUTES: value } }),
74 ),
75 );
76 expect(error._tag).toBe("ConfigError");
77 }),
78);
infra/relay/src/Config.ts+34-6
...@@ -12,13 +12,38 @@ export const ManagedEndpointCleanupMode = Schema.Literals(["off", "dry-run", "en...@@ -12,13 +12,38 @@ export const ManagedEndpointCleanupMode = Schema.Literals(["off", "dry-run", "en
12export type ManagedEndpointCleanupMode = typeof ManagedEndpointCleanupMode.Type;12export type ManagedEndpointCleanupMode = typeof ManagedEndpointCleanupMode.Type;
13const decodeManagedEndpointCleanupMode = Schema.decodeUnknownEffect(ManagedEndpointCleanupMode);13const decodeManagedEndpointCleanupMode = Schema.decodeUnknownEffect(ManagedEndpointCleanupMode);
1414
15export const managedEndpointCleanupModeConfig = Config.String("RELAY_TUNNEL_CLEANUP_MODE").pipe(15const RELAY_TUNNEL_CLEANUP_MODE = "RELAY_TUNNEL_CLEANUP_MODE";
16 Config.withDefault("off"),16/** Separate switch for tunnels whose host never registered recovery. */
17 Config.map((value) => value.trim() || "off"),17const RELAY_LEGACY_TUNNEL_CLEANUP_MODE = "RELAY_LEGACY_TUNNEL_CLEANUP_MODE";
18 Config.mapEffect((value) =>18
19 decodeManagedEndpointCleanupMode(value).pipe(19const cleanupModeConfig = (name: string) =>
20 Effect.mapError((error) => new Config.ConfigError(error)),20 Config.String(name).pipe(
21 Config.withDefault("off"),
22 Config.map((value) => value.trim() || "off"),
23 Config.mapEffect((value) =>
24 decodeManagedEndpointCleanupMode(value).pipe(
25 Effect.mapError((error) => new Config.ConfigError(error)),
26 ),
21 ),27 ),
28 );
29
30export const managedEndpointCleanupModeConfig = cleanupModeConfig(RELAY_TUNNEL_CLEANUP_MODE);
31export const legacyManagedEndpointCleanupModeConfig = cleanupModeConfig(
32 RELAY_LEGACY_TUNNEL_CLEANUP_MODE,
33);
34
35/**
36 * Overrides the 7-day legacy grace period, in minutes, so the disposable
37 * canary stage can exercise legacy cleanup. Ignored on the prod stage.
38 */
39const RELAY_LEGACY_TUNNEL_GRACE_MINUTES = "RELAY_LEGACY_TUNNEL_GRACE_MINUTES";
40
41// A zero or negative override would be ignored at runtime, silently leaving
42// the canary on the 7-day grace period, so reject it when the deploy reads it.
43export const legacyTunnelGraceMinutesConfig = Config.option(
44 Config.schema(
45 Schema.NumberFromString.pipe(Schema.check(Schema.isInt(), Schema.isGreaterThan(0))),
46 RELAY_LEGACY_TUNNEL_GRACE_MINUTES,
22 ),47 ),
23);48);
2449
...@@ -45,6 +70,9 @@ export class RelayConfiguration extends Context.Service<...@@ -45,6 +70,9 @@ export class RelayConfiguration extends Context.Service<
45 readonly managedEndpointBaseDomain: string | undefined;70 readonly managedEndpointBaseDomain: string | undefined;
46 readonly managedEndpointNamespace: string | undefined;71 readonly managedEndpointNamespace: string | undefined;
47 readonly managedEndpointCleanupMode?: ManagedEndpointCleanupMode;72 readonly managedEndpointCleanupMode?: ManagedEndpointCleanupMode;
73 readonly legacyManagedEndpointCleanupMode?: ManagedEndpointCleanupMode;
74 /** Canary-only override of the legacy grace period; ignored on prod. */
75 readonly legacyTunnelGraceMinutes?: number;
48 }76 }
49>()("t3code-relay/Config/RelayConfiguration") {}77>()("t3code-relay/Config/RelayConfiguration") {}
5078
infra/relay/src/environments/ManagedEndpointReaper.test.ts+144
...@@ -73,6 +73,8 @@ function harness(input?: {...@@ -73,6 +73,8 @@ function harness(input?: {
73 readonly refreshedTunnels?: ReadonlyMap<string, ManagedEndpointProvider.ManagedEndpointTunnel>;73 readonly refreshedTunnels?: ReadonlyMap<string, ManagedEndpointProvider.ManagedEndpointTunnel>;
74 readonly skipTunnelId?: string;74 readonly skipTunnelId?: string;
75 readonly cleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode;75 readonly cleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode;
76 readonly legacyCleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode;
77 readonly legacyTunnelGraceMinutes?: number;
76}) {78}) {
77 const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = [];79 const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = [];
78 const deleted: string[] = [];80 const deleted: string[] = [];
...@@ -228,6 +230,10 @@ function harness(input?: {...@@ -228,6 +230,10 @@ function harness(input?: {
228 managedEndpointBaseDomain: "example.test",230 managedEndpointBaseDomain: "example.test",
229 managedEndpointNamespace: input?.namespace ?? "prod",231 managedEndpointNamespace: input?.namespace ?? "prod",
230 managedEndpointCleanupMode: input?.cleanupMode ?? "enabled",232 managedEndpointCleanupMode: input?.cleanupMode ?? "enabled",
233 legacyManagedEndpointCleanupMode: input?.legacyCleanupMode ?? "off",
234 ...(input?.legacyTunnelGraceMinutes === undefined
235 ? {}
236 : { legacyTunnelGraceMinutes: input.legacyTunnelGraceMinutes }),
231 });237 });
232238
233 return {239 return {
...@@ -490,6 +496,141 @@ describe("ManagedEndpointReaper", () => {...@@ -490,6 +496,141 @@ describe("ManagedEndpointReaper", () => {
490 }).pipe(Effect.provide(state.layer));496 }).pipe(Effect.provide(state.layer));
491 });497 });
492498
499 describe("legacy cleanup", () => {
500 const legacyTunnels = [
501 // Down for 5 days: inside the legacy grace period.
502 tunnel({
503 id: "legacy-recent",
504 suffix: "1111111111111111",
505 status: "down",
506 timestamp: "2026-08-20T12:00:00.000Z",
507 }),
508 // Down for 40 days: past it.
509 tunnel({
510 id: "legacy-old",
511 suffix: "2222222222222222",
512 status: "down",
513 timestamp: "2026-07-16T12:00:00.000Z",
514 }),
515 // Never connected, created 40 days ago.
516 tunnel({
517 id: "legacy-never",
518 suffix: "3333333333333333",
519 status: "inactive",
520 timestamp: "2026-07-16T12:00:00.000Z",
521 }),
522 ];
523 const legacyOwners = legacyTunnels.map((entry) =>
524 allocation({ tunnelId: entry.id!, recoveryEnabled: false }),
525 );
526
527 it.effect("leaves legacy tunnels alone while legacy cleanup is off", () => {
528 const state = harness({ tunnels: legacyTunnels, allocations: legacyOwners });
529 return Effect.gen(function* () {
530 yield* TestClock.setTime(NOW_MILLIS);
531 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
532 expect(yield* reaper.sweep).toMatchObject({
533 skippedLegacy: 3,
534 wouldDeleteLegacy: 0,
535 deletedLegacy: 0,
536 });
537 expect(state.deleted).toEqual([]);
538 }).pipe(Effect.provide(state.layer));
539 });
540
541 it.effect("counts legacy tunnels past the grace period in dry-run", () => {
542 const state = harness({
543 tunnels: legacyTunnels,
544 allocations: legacyOwners,
545 legacyCleanupMode: "dry-run",
546 });
547 return Effect.gen(function* () {
548 yield* TestClock.setTime(NOW_MILLIS);
549 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
550 expect(yield* reaper.sweep).toMatchObject({
551 legacyMode: "dry-run",
552 wouldDeleteLegacy: 2,
553 deletedLegacy: 0,
554 attempted: 0,
555 });
556 expect(state.deleted).toEqual([]);
557 }).pipe(Effect.provide(state.layer));
558 });
559
560 it.effect("deletes only legacy tunnels past the grace period", () => {
561 const state = harness({
562 tunnels: legacyTunnels,
563 allocations: legacyOwners,
564 legacyCleanupMode: "enabled",
565 });
566 return Effect.gen(function* () {
567 yield* TestClock.setTime(NOW_MILLIS);
568 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
569 expect(yield* reaper.sweep).toMatchObject({
570 wouldDeleteLegacy: 2,
571 deletedLegacy: 2,
572 deleted: 2,
573 });
574 expect([...state.deleted].sort()).toEqual(["legacy-never", "legacy-old"]);
575 // The release re-checks each tunnel against the 7-day cutoff, not
576 // the 5-minute cutoff used for recoverable tunnels.
577 for (const release of state.releases) {
578 expect(release.expectedInactiveBefore).toBe("2026-08-18T12:00:00.000Z");
579 }
580 }).pipe(Effect.provide(state.layer));
581 });
582
583 it.effect.each([
584 { stage: "canary", expected: 3 },
585 { stage: "prod", expected: 2 },
586 ] as const)(
587 "applies the grace-period override only off prod ($stage)",
588 ({ stage, expected }) => {
589 const stageLegacyTunnels = legacyTunnels.map((entry) => ({
590 ...entry,
591 name: entry.name!.replace(PREFIX, `t3coderelay-managedendpoint-${stage}-`),
592 }));
593 const state = harness({
594 namespace: stage,
595 tunnels: stageLegacyTunnels,
596 allocations: legacyOwners,
597 legacyCleanupMode: "dry-run",
598 // Ten minutes: the 10-day-old tunnel qualifies only if this applies.
599 legacyTunnelGraceMinutes: 10,
600 });
601 return Effect.gen(function* () {
602 yield* TestClock.setTime(NOW_MILLIS);
603 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
604 expect((yield* reaper.sweep).wouldDeleteLegacy).toBe(expected);
605 }).pipe(Effect.provide(state.layer));
606 },
607 );
608
609 it.effect("runs legacy cleanup while recoverable cleanup is off", () => {
610 const recoverable = tunnel({
611 id: "recoverable",
612 suffix: "4444444444444444",
613 status: "down",
614 timestamp: "2026-08-25T11:00:00.000Z",
615 });
616 const state = harness({
617 tunnels: [...legacyTunnels, recoverable],
618 allocations: [
619 ...legacyOwners,
620 allocation({ tunnelId: "recoverable", recoveryEnabled: true }),
621 ],
622 cleanupMode: "off",
623 legacyCleanupMode: "enabled",
624 });
625 return Effect.gen(function* () {
626 yield* TestClock.setTime(NOW_MILLIS);
627 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
628 expect(yield* reaper.sweep).toMatchObject({ wouldDelete: 0, deletedLegacy: 2 });
629 expect(state.deleted).not.toContain("recoverable");
630 }).pipe(Effect.provide(state.layer));
631 });
632 });
633
493 it.effect("does not count a tunnel that was replaced before its release", () => {634 it.effect("does not count a tunnel that was replaced before its release", () => {
494 const state = harness({635 const state = harness({
495 tunnels: [636 tunnels: [
...@@ -697,11 +838,14 @@ describe("ManagedEndpointReaper", () => {...@@ -697,11 +838,14 @@ describe("ManagedEndpointReaper", () => {
697 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;838 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
698 expect(yield* reaper.sweep).toEqual({839 expect(yield* reaper.sweep).toEqual({
699 mode: "off",840 mode: "off",
841 legacyMode: "off",
700 listRequests: 0,842 listRequests: 0,
701 scanned: 0,843 scanned: 0,
702 attempted: 0,844 attempted: 0,
703 deleted: 0,845 deleted: 0,
704 wouldDelete: 0,846 wouldDelete: 0,
847 wouldDeleteLegacy: 0,
848 deletedLegacy: 0,
705 skippedLegacy: 0,849 skippedLegacy: 0,
706 legacyOver7Days: 0,850 legacyOver7Days: 0,
707 legacyOver30Days: 0,851 legacyOver30Days: 0,
infra/relay/src/environments/ManagedEndpointReaper.ts+62-16
...@@ -5,7 +5,10 @@ import * as Layer from "effect/Layer";...@@ -5,7 +5,10 @@ import * as Layer from "effect/Layer";
5import * as Option from "effect/Option";5import * as Option from "effect/Option";
66
7import * as RelayConfiguration from "../Config.ts";7import * as RelayConfiguration from "../Config.ts";
8import { managedEndpointTunnelNamePrefix } from "../deploymentConfig.ts";8import {
9 MANAGED_ENDPOINT_ZONE_OWNER_STAGE,
10 managedEndpointTunnelNamePrefix,
11} from "../deploymentConfig.ts";
9import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";12import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";
10import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts";13import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts";
1114
...@@ -19,14 +22,23 @@ export const MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT = 10;...@@ -19,14 +22,23 @@ export const MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT = 10;
19// Age buckets for legacy candidates, in days since the tunnel went down (or22// Age buckets for legacy candidates, in days since the tunnel went down (or
20// was created, for one that never connected).23// was created, for one that never connected).
21const MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS = [7, 30, 90] as const;24const MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS = [7, 30, 90] as const;
25// A host that never registered recovery cannot replace a deleted tunnel on
26// its own build. Only delete its tunnel after it has been gone this long. A
27// returning host that has updated recovers the tunnel at the same hostname;
28// one that has not sees the client's "update T3 Code" message instead.
29const MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS = 7;
2230
23export interface ManagedEndpointSweepResult {31export interface ManagedEndpointSweepResult {
24 readonly mode: RelayConfiguration.ManagedEndpointCleanupMode;32 readonly mode: RelayConfiguration.ManagedEndpointCleanupMode;
33 readonly legacyMode: RelayConfiguration.ManagedEndpointCleanupMode;
25 readonly listRequests: number;34 readonly listRequests: number;
26 readonly scanned: number;35 readonly scanned: number;
27 readonly attempted: number;36 readonly attempted: number;
28 readonly deleted: number;37 readonly deleted: number;
29 readonly wouldDelete: number;38 readonly wouldDelete: number;
39 /** Legacy tunnels past the legacy grace period, and how many were deleted. */
40 readonly wouldDeleteLegacy: number;
41 readonly deletedLegacy: number;
30 readonly skippedLegacy: number;42 readonly skippedLegacy: number;
31 /** Legacy candidates, by days since they went down: over 7, 30, and 90. */43 /** Legacy candidates, by days since they went down: over 7, 30, and 90. */
32 readonly legacyOver7Days: number;44 readonly legacyOver7Days: number;
...@@ -131,13 +143,17 @@ function rotatedPages(input: {...@@ -131,13 +143,17 @@ function rotatedPages(input: {
131143
132const emptyResult = (144const emptyResult = (
133 mode: RelayConfiguration.ManagedEndpointCleanupMode,145 mode: RelayConfiguration.ManagedEndpointCleanupMode,
146 legacyMode: RelayConfiguration.ManagedEndpointCleanupMode,
134): ManagedEndpointSweepResult => ({147): ManagedEndpointSweepResult => ({
135 mode,148 mode,
149 legacyMode,
136 listRequests: 0,150 listRequests: 0,
137 scanned: 0,151 scanned: 0,
138 attempted: 0,152 attempted: 0,
139 deleted: 0,153 deleted: 0,
140 wouldDelete: 0,154 wouldDelete: 0,
155 wouldDeleteLegacy: 0,
156 deletedLegacy: 0,
141 skippedLegacy: 0,157 skippedLegacy: 0,
142 legacyOver7Days: 0,158 legacyOver7Days: 0,
143 legacyOver30Days: 0,159 legacyOver30Days: 0,
...@@ -159,8 +175,19 @@ export const make = Effect.gen(function* () {...@@ -159,8 +175,19 @@ export const make = Effect.gen(function* () {
159175
160 const sweep = Effect.gen(function* () {176 const sweep = Effect.gen(function* () {
161 const mode = config.managedEndpointCleanupMode ?? "off";177 const mode = config.managedEndpointCleanupMode ?? "off";
178 const legacyMode = config.legacyManagedEndpointCleanupMode ?? "off";
162 const namespace = config.managedEndpointNamespace;179 const namespace = config.managedEndpointNamespace;
163 if (mode === "off" || !namespace) return emptyResult(mode);180 if ((mode === "off" && legacyMode === "off") || !namespace) {
181 return emptyResult(mode, legacyMode);
182 }
183 // The override exists for the disposable canary stage; prod always
184 // waits the full grace period.
185 const legacyGraceMinutes =
186 namespace !== MANAGED_ENDPOINT_ZONE_OWNER_STAGE &&
187 config.legacyTunnelGraceMinutes !== undefined
188 ? config.legacyTunnelGraceMinutes
189 : MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS * 24 * 60;
190 const legacyCutoff = DateTime.subtract(yield* DateTime.now, { minutes: legacyGraceMinutes });
164191
165 const now = yield* DateTime.now;192 const now = yield* DateTime.now;
166 const cutoffFor = (status: "down" | "inactive") =>193 const cutoffFor = (status: "down" | "inactive") =>
...@@ -248,6 +275,8 @@ export const make = Effect.gen(function* () {...@@ -248,6 +275,8 @@ export const make = Effect.gen(function* () {
248 let attempted = 0;275 let attempted = 0;
249 let deleted = 0;276 let deleted = 0;
250 let wouldDelete = 0;277 let wouldDelete = 0;
278 let wouldDeleteLegacy = 0;
279 let deletedLegacy = 0;
251 let skippedLegacy = 0;280 let skippedLegacy = 0;
252 const legacyOverDays = new Map<number, number>(281 const legacyOverDays = new Map<number, number>(
253 MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS.map((days) => [days, 0]),282 MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS.map((days) => [days, 0]),
...@@ -258,7 +287,6 @@ export const make = Effect.gen(function* () {...@@ -258,7 +287,6 @@ export const make = Effect.gen(function* () {
258 let failed = 0;287 let failed = 0;
259288
260 for (const { tunnel, status, cutoff } of uniqueExpired) {289 for (const { tunnel, status, cutoff } of uniqueExpired) {
261 const cutoffIso = DateTime.formatIso(cutoff);
262 const allocation = recordedByTunnelName.get(tunnel.name);290 const allocation = recordedByTunnelName.get(tunnel.name);
263 if (291 if (
264 allocation !== undefined &&292 allocation !== undefined &&
...@@ -269,16 +297,6 @@ export const make = Effect.gen(function* () {...@@ -269,16 +297,6 @@ export const make = Effect.gen(function* () {
269 continue;297 continue;
270 }298 }
271 const owner = allocation?.tunnelId === tunnel.id ? allocation : undefined;299 const owner = allocation?.tunnelId === tunnel.id ? allocation : undefined;
272 if (owner !== undefined && !owner.recoveryEnabled) {
273 skippedLegacy += 1;
274 const inactiveDays = inactiveDaysAt(tunnel, status, now);
275 for (const days of MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS) {
276 if (inactiveDays !== null && inactiveDays > days) {
277 legacyOverDays.set(days, (legacyOverDays.get(days) ?? 0) + 1);
278 }
279 }
280 continue;
281 }
282 if (allocation !== undefined && owner === undefined) {300 if (allocation !== undefined && owner === undefined) {
283 skippedUnrecorded += 1;301 skippedUnrecorded += 1;
284 continue;302 continue;
...@@ -290,19 +308,41 @@ export const make = Effect.gen(function* () {...@@ -290,19 +308,41 @@ export const make = Effect.gen(function* () {
290 skippedOrphan += 1;308 skippedOrphan += 1;
291 continue;309 continue;
292 }310 }
293 wouldDelete += 1;311 const legacy = !owner.recoveryEnabled;
294 if (mode === "dry-run") continue;312 if (legacy) {
313 skippedLegacy += 1;
314 const inactiveDays = inactiveDaysAt(tunnel, status, now);
315 for (const days of MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS) {
316 if (inactiveDays !== null && inactiveDays > days) {
317 legacyOverDays.set(days, (legacyOverDays.get(days) ?? 0) + 1);
318 }
319 }
320 if (
321 legacyMode === "off" ||
322 !isExpiredManagedTunnel({ tunnel, status, prefix, cutoff: legacyCutoff })
323 ) {
324 continue;
325 }
326 wouldDeleteLegacy += 1;
327 if (legacyMode === "dry-run") continue;
328 } else {
329 if (mode === "off") continue;
330 wouldDelete += 1;
331 if (mode === "dry-run") continue;
332 }
295 if (attempted >= MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) {333 if (attempted >= MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) {
296 truncated = true;334 truncated = true;
297 break;335 break;
298 }336 }
299 attempted += 1;337 attempted += 1;
338 // The release re-reads the tunnel and deletes only if it is still in
339 // this status and inactive since before this tunnel's cutoff.
300 const result = yield* provider340 const result = yield* provider
301 .release({341 .release({
302 userId: owner.userId,342 userId: owner.userId,
303 environmentId: owner.environmentId,343 environmentId: owner.environmentId,
304 expectedTunnelId: tunnel.id,344 expectedTunnelId: tunnel.id,
305 expectedInactiveBefore: cutoffIso,345 expectedInactiveBefore: DateTime.formatIso(legacy ? legacyCutoff : cutoff),
306 expectedStatus: status,346 expectedStatus: status,
307 })347 })
308 .pipe(Effect.result);348 .pipe(Effect.result);
...@@ -311,6 +351,7 @@ export const make = Effect.gen(function* () {...@@ -311,6 +351,7 @@ export const make = Effect.gen(function* () {
311 yield* Effect.logWarning("Failed to delete an inactive managed tunnel", {351 yield* Effect.logWarning("Failed to delete an inactive managed tunnel", {
312 tunnelId: tunnel.id,352 tunnelId: tunnel.id,
313 tunnelName: tunnel.name,353 tunnelName: tunnel.name,
354 legacy,
314 cause: result.failure,355 cause: result.failure,
315 });356 });
316 if (isRateLimited(result.failure)) {357 if (isRateLimited(result.failure)) {
...@@ -319,21 +360,26 @@ export const make = Effect.gen(function* () {...@@ -319,21 +360,26 @@ export const make = Effect.gen(function* () {
319 }360 }
320 } else if (result.success) {361 } else if (result.success) {
321 deleted += 1;362 deleted += 1;
363 if (legacy) deletedLegacy += 1;
322 yield* Effect.logInfo("Deleted an inactive managed tunnel", {364 yield* Effect.logInfo("Deleted an inactive managed tunnel", {
323 tunnelId: tunnel.id,365 tunnelId: tunnel.id,
324 tunnelName: tunnel.name,366 tunnelName: tunnel.name,
325 status,367 status,
368 legacy,
326 });369 });
327 }370 }
328 }371 }
329372
330 return {373 return {
331 mode,374 mode,
375 legacyMode,
332 listRequests,376 listRequests,
333 scanned: uniqueExpired.length,377 scanned: uniqueExpired.length,
334 attempted,378 attempted,
335 deleted,379 deleted,
336 wouldDelete,380 wouldDelete,
381 wouldDeleteLegacy,
382 deletedLegacy,
337 skippedLegacy,383 skippedLegacy,
338 legacyOver7Days: legacyOverDays.get(7) ?? 0,384 legacyOver7Days: legacyOverDays.get(7) ?? 0,
339 legacyOver30Days: legacyOverDays.get(30) ?? 0,385 legacyOver30Days: legacyOverDays.get(30) ?? 0,
infra/relay/src/worker.ts+7
...@@ -184,6 +184,11 @@ export const layer = Api.make(...@@ -184,6 +184,11 @@ export const layer = Api.make(
184 const managedEndpointDnsBinding = yield* Cloudflare.DNS.ReadWriteDns(managedEndpointZone);184 const managedEndpointDnsBinding = yield* Cloudflare.DNS.ReadWriteDns(managedEndpointZone);
185 const managedEndpointZoneName = yield* managedEndpointZone.name;185 const managedEndpointZoneName = yield* managedEndpointZone.name;
186 const managedEndpointCleanupMode = yield* RelayConfiguration.managedEndpointCleanupModeConfig;186 const managedEndpointCleanupMode = yield* RelayConfiguration.managedEndpointCleanupModeConfig;
187 const legacyManagedEndpointCleanupMode =
188 yield* RelayConfiguration.legacyManagedEndpointCleanupModeConfig;
189 const legacyTunnelGraceMinutes = Option.getOrUndefined(
190 yield* RelayConfiguration.legacyTunnelGraceMinutesConfig,
191 );
187 // Keys are endpoint keys or hashes over them, which already differ per192 // Keys are endpoint keys or hashes over them, which already differ per
188 // stage, so stages sharing an account cannot collide in these namespaces.193 // stage, so stages sharing an account cannot collide in these namespaces.
189 const hookRateLimit = yield* Cloudflare.RateLimit("HOOK_RATE_LIMIT", {194 const hookRateLimit = yield* Cloudflare.RateLimit("HOOK_RATE_LIMIT", {
...@@ -221,6 +226,8 @@ export const layer = Api.make(...@@ -221,6 +226,8 @@ export const layer = Api.make(
221 managedEndpointBaseDomain: yield* managedEndpointZoneName,226 managedEndpointBaseDomain: yield* managedEndpointZoneName,
222 managedEndpointNamespace: stage,227 managedEndpointNamespace: stage,
223 managedEndpointCleanupMode,228 managedEndpointCleanupMode,
229 legacyManagedEndpointCleanupMode,
230 ...(legacyTunnelGraceMinutes === undefined ? {} : { legacyTunnelGraceMinutes }),
224 });231 });
225 });232 });
226233