authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-08-06 02:29:30-07:00
committergravatar for me@theo.liTheo Browne <me@theo.li> 2026-08-06 08:09:51-04:00
logf9e8236899f25653ce95d688fac798cd070afbc1
treeb7a4ad34cb9ea4e8aaa8339a46097fd577e588a6
parent8f341f20c91e708de1dab949dbd723e86f2ff328

fix(server): release the tunnel when an explicit stop interrupts an update

A pending update in the launcher state is not proof a replacement server is coming: `t3 service uninstall` or `systemctl stop` during the pending window also tears the server down, permanently. The launcher now writes a stop marker before signalling its child on an explicit stop and clears it on the next start; the shutdown tunnel release keeps the tunnel only for pending updates without the marker. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

5 files changed, 100 insertions(+), 5 deletions(-)

apps/server/src/cloud/http.test.ts+31
...@@ -1,6 +1,7 @@...@@ -1,6 +1,7 @@
1import * as NodeServices from "@effect/platform-node/NodeServices";1import * as NodeServices from "@effect/platform-node/NodeServices";
2import { describe, expect, it } from "@effect/vitest";2import { describe, expect, it } from "@effect/vitest";
3import * as Effect from "effect/Effect";3import * as Effect from "effect/Effect";
4import * as FileSystem from "effect/FileSystem";
4import * as Layer from "effect/Layer";5import * as Layer from "effect/Layer";
5import * as Option from "effect/Option";6import * as Option from "effect/Option";
6import * as Path from "effect/Path";7import * as Path from "effect/Path";
...@@ -22,6 +23,7 @@ import { writeServiceState } from "../serviceLauncher.ts";...@@ -22,6 +23,7 @@ import { writeServiceState } from "../serviceLauncher.ts";
22import {23import {
23 SERVICE_LAUNCHER_PROTOCOL,24 SERVICE_LAUNCHER_PROTOCOL,
24 SERVICE_STATE_FILE,25 SERVICE_STATE_FILE,
26 SERVICE_STOP_MARKER_FILE,
25 type ServiceUpdateRecord,27 type ServiceUpdateRecord,
26} from "./serviceProtocol.ts";28} from "./serviceProtocol.ts";
27import * as ServerEnvironment from "../environment/ServerEnvironment.ts";29import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
...@@ -451,6 +453,35 @@ describe("releaseManagedTunnelOnShutdown", () => {...@@ -451,6 +453,35 @@ describe("releaseManagedTunnelOnShutdown", () => {
451 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));453 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
452 });454 });
453455
456 it.effect("still releases a pending update when the launcher is stopping", () => {
457 // `t3 service uninstall` or `systemctl stop` during the pending window:
458 // the launcher writes its stop marker before signalling the child, so no
459 // replacement server is coming and the tunnel must not be kept.
460 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
461 const applyConfigCalls: Array<unknown> = [];
462 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
463
464 return Effect.gen(function* () {
465 yield* writeLauncherState({
466 id: "update-1",
467 fromVersion: "0.0.30",
468 targetVersion: "0.0.31",
469 dbPath: "/tmp/state.sqlite",
470 status: "pending",
471 });
472 const fs = yield* FileSystem.FileSystem;
473 const path = yield* Path.Path;
474 const config = yield* ServerConfigModule.ServerConfig;
475 yield* fs.writeFileString(path.join(config.baseDir, "runtime", SERVICE_STOP_MARKER_FILE), "");
476
477 const released = yield* releaseManagedTunnelOnShutdown();
478
479 expect(released).toBe(true);
480 expect(requests).toHaveLength(1);
481 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false);
482 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
483 });
484
454 it.effect("still releases when the recorded update already settled", () => {485 it.effect("still releases when the recorded update already settled", () => {
455 const { store, values } = makeMemorySecretStore(managedLinkSecrets);486 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
456 const applyConfigCalls: Array<unknown> = [];487 const applyConfigCalls: Array<unknown> = [];
apps/server/src/cloud/http.ts+22-5
...@@ -61,7 +61,11 @@ import { requireEnvironmentScope } from "../auth/http.ts";...@@ -61,7 +61,11 @@ import { requireEnvironmentScope } from "../auth/http.ts";
61import * as ServerConfig from "../config.ts";61import * as ServerConfig from "../config.ts";
62import * as ServerEnvironment from "../environment/ServerEnvironment.ts";62import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
63import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";63import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
64import { SERVICE_STATE_FILE, serviceStateHasPendingUpdate } from "./serviceProtocol.ts";64import {
65 SERVICE_STATE_FILE,
66 SERVICE_STOP_MARKER_FILE,
67 serviceStateHasPendingUpdate,
68} from "./serviceProtocol.ts";
65import {69import {
66 CLOUD_ENDPOINT_RUNTIME_CONFIG,70 CLOUD_ENDPOINT_RUNTIME_CONFIG,
67 CLOUD_LINKED_USER_ID,71 CLOUD_LINKED_USER_ID,
...@@ -635,14 +639,27 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD...@@ -635,14 +639,27 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD
635// straight from disk: the launcher owns that file, and this runs while the639// straight from disk: the launcher owns that file, and this runs while the
636// server is tearing down. Unreadable or unparseable state means no pending640// server is tearing down. Unreadable or unparseable state means no pending
637// update — the release then proceeds as before.641// update — the release then proceeds as before.
638const pendingServiceUpdateExists = Effect.gen(function* () {642//
643// A pending update alone is not proof a replacement server is coming: an
644// explicit launcher stop (`t3 service uninstall`, `systemctl stop`) during
645// the pending window also tears this server down, with nothing starting
646// afterwards. The launcher marks that case with a stop-marker file just
647// before it signals the child, so pending + no marker is the update handoff.
648const pendingUpdateHandoffExists = Effect.gen(function* () {
639 const config = yield* ServerConfig.ServerConfig;649 const config = yield* ServerConfig.ServerConfig;
640 const fs = yield* FileSystem.FileSystem;650 const fs = yield* FileSystem.FileSystem;
641 const path = yield* Path.Path;651 const path = yield* Path.Path;
652 const runtimeDir = path.join(config.baseDir, "runtime");
642 const stateText = yield* fs653 const stateText = yield* fs
643 .readFileString(path.join(config.baseDir, "runtime", SERVICE_STATE_FILE))654 .readFileString(path.join(runtimeDir, SERVICE_STATE_FILE))
644 .pipe(Effect.option);655 .pipe(Effect.option);
645 return Option.isSome(stateText) && serviceStateHasPendingUpdate(stateText.value);656 if (Option.isNone(stateText) || !serviceStateHasPendingUpdate(stateText.value)) {
657 return false;
658 }
659 const stopping = yield* fs
660 .exists(path.join(runtimeDir, SERVICE_STOP_MARKER_FILE))
661 .pipe(Effect.orElseSucceed(() => false));
662 return !stopping;
646});663});
647664
648// Cloudflare bills per provisioned tunnel, so an environment that goes offline665// Cloudflare bills per provisioned tunnel, so an environment that goes offline
...@@ -676,7 +693,7 @@ export const releaseManagedTunnelOnShutdown = Effect.fn(...@@ -676,7 +693,7 @@ export const releaseManagedTunnelOnShutdown = Effect.fn(
676 // boot respawns the connector from the stored config and is reachable as693 // boot respawns the connector from the stored config and is reachable as
677 // soon as it connects, and the reconcile confirms the still-live tunnel694 // soon as it connects, and the reconcile confirms the still-live tunnel
678 // without replacing it.695 // without replacing it.
679 if (yield* pendingServiceUpdateExists) {696 if (yield* pendingUpdateHandoffExists) {
680 yield* Effect.logInfo("Keeping the managed tunnel across the update restart");697 yield* Effect.logInfo("Keeping the managed tunnel across the update restart");
681 return false;698 return false;
682 }699 }
apps/server/src/cloud/serviceProtocol.ts+4
...@@ -5,6 +5,10 @@ export const SERVICE_LAUNCHER_PROTOCOL = 2 as const;...@@ -5,6 +5,10 @@ export const SERVICE_LAUNCHER_PROTOCOL = 2 as const;
5export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT";5export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT";
6export const SERVICE_LAUNCHER_FILE = "service-launcher.mjs";6export const SERVICE_LAUNCHER_FILE = "service-launcher.mjs";
7export const SERVICE_STATE_FILE = "service-state.json";7export const SERVICE_STATE_FILE = "service-state.json";
8/** Written by the launcher just before an explicit stop kills its child, so
9 the child can tell "the service is going away" from "the launcher is about
10 to start my replacement" while a pending update is recorded. */
11export const SERVICE_STOP_MARKER_FILE = ".service-stopping";
812
9export interface PendingServiceUpdate {13export interface PendingServiceUpdate {
10 readonly id: string;14 readonly id: string;
apps/server/src/serviceLauncher.test.ts+26
...@@ -1,3 +1,6 @@...@@ -1,3 +1,6 @@
1// @effect-diagnostics-next-line nodeBuiltinImport:off - sync marker poll outside the test clock.
2import * as NodeFS from "node:fs";
3
1import * as NodeServices from "@effect/platform-node/NodeServices";4import * as NodeServices from "@effect/platform-node/NodeServices";
2import { assert, it } from "@effect/vitest";5import { assert, it } from "@effect/vitest";
3import * as Effect from "effect/Effect";6import * as Effect from "effect/Effect";
...@@ -10,6 +13,7 @@ import {...@@ -10,6 +13,7 @@ import {
10 decodeServiceState,13 decodeServiceState,
11 isExactServiceVersion,14 isExactServiceVersion,
12 SERVICE_LAUNCHER_PROTOCOL,15 SERVICE_LAUNCHER_PROTOCOL,
16 SERVICE_STOP_MARKER_FILE,
13} from "./cloud/serviceProtocol.ts";17} from "./cloud/serviceProtocol.ts";
1418
15it("accepts only exact semantic versions", () => {19it("accepts only exact semantic versions", () => {
...@@ -113,6 +117,28 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {...@@ -113,6 +117,28 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {
113 const running = launcher.run();117 const running = launcher.run();
114 yield* Effect.promise(() => launcher.stop("SIGTERM"));118 yield* Effect.promise(() => launcher.stop("SIGTERM"));
115 yield* Effect.promise(() => running);119 yield* Effect.promise(() => running);
120 // An explicit stop leaves the marker that tells a child shutting down
121 // mid-update that no replacement server is coming.
122 assert.isTrue(yield* fs.exists(path.join(root, "runtime", SERVICE_STOP_MARKER_FILE)));
123
124 // The next launcher start clears the stale marker so a later update
125 // handoff keeps its tunnel again. Recovery runs before any stop
126 // transition, so polling until the marker disappears observes it; the
127 // poll uses real timers because it.effect runs under the test clock.
128 const restarted = new Launcher(
129 root,
130 yield* Effect.promise(() => readServiceState(statePath)),
131 );
132 const runningAgain = restarted.run();
133 const markerPath = path.join(root, "runtime", SERVICE_STOP_MARKER_FILE);
134 yield* Effect.promise(async () => {
135 while (NodeFS.existsSync(markerPath)) {
136 // @effect-diagnostics-next-line globalTimers:off - real delay under the test clock.
137 await new Promise((resolve) => setTimeout(resolve, 25));
138 }
139 });
140 yield* Effect.promise(() => restarted.stop("SIGTERM"));
141 yield* Effect.promise(() => runningAgain);
116 }),142 }),
117 );143 );
118144
apps/server/src/serviceLauncher.ts+17
...@@ -24,6 +24,7 @@ import {...@@ -24,6 +24,7 @@ import {
24 SERVICE_LAUNCHER_CONTEXT_ENV,24 SERVICE_LAUNCHER_CONTEXT_ENV,
25 SERVICE_LAUNCHER_PROTOCOL,25 SERVICE_LAUNCHER_PROTOCOL,
26 SERVICE_STATE_FILE,26 SERVICE_STATE_FILE,
27 SERVICE_STOP_MARKER_FILE,
27} from "./cloud/serviceProtocol.ts";28} from "./cloud/serviceProtocol.ts";
2829
29const HANDOFF_DELAY_MS = 2_000;30const HANDOFF_DELAY_MS = 2_000;
...@@ -257,6 +258,9 @@ async function terminateChild(...@@ -257,6 +258,9 @@ async function terminateChild(
257 }258 }
258}259}
259260
261const stopMarkerPath = (baseDir: string) =>
262 NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE);
263
260export class Launcher {264export class Launcher {
261 readonly #baseDir: string;265 readonly #baseDir: string;
262 readonly #statePath: string;266 readonly #statePath: string;
...@@ -315,6 +319,15 @@ export class Launcher {...@@ -315,6 +319,15 @@ export class Launcher {
315 this.#stopping = true;319 this.#stopping = true;
316 this.#clearTimer();320 this.#clearTimer();
317 this.#enqueue(async () => {321 this.#enqueue(async () => {
322 // An explicit stop is the service going away, not an update handoff.
323 // The marker lets the child's shutdown release the managed tunnel even
324 // while an update is recorded as pending; the next launcher start
325 // clears it. Written before the child is signalled so the child sees
326 // it during teardown. Best-effort: if the write fails the child errs
327 // toward keeping the tunnel, and the next link or unlink reconciles it.
328 await NodeFSP.writeFile(stopMarkerPath(this.#baseDir), "", { mode: 0o600 }).catch(
329 () => undefined,
330 );
318 const child = this.#child?.process;331 const child = this.#child?.process;
319 this.#child = null;332 this.#child = null;
320 if (child !== undefined) await terminateChild(child, signal);333 if (child !== undefined) await terminateChild(child, signal);
...@@ -330,6 +343,10 @@ export class Launcher {...@@ -330,6 +343,10 @@ export class Launcher {
330 }343 }
331344
332 async #recover(): Promise<void> {345 async #recover(): Promise<void> {
346 // A fresh launcher means servers are running again: any stop marker from
347 // a previous explicit stop is stale and must not make a future update
348 // handoff release its tunnel.
349 await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined);
333 const update = this.#state.update;350 const update = this.#state.update;
334 if (update?.status !== "pending") {351 if (update?.status !== "pending") {
335 if (update !== undefined) {352 if (update !== undefined) {