authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-08-06 02:29:30-07:00
committergravatar for me@theo.liTheo Browne <me@theo.li> 2026-08-06 08:09:51-04:00
logf9e8236899f25653ce95d688fac798cd070afbc1
treeb7a4ad34cb9ea4e8aaa8339a46097fd577e588a6
parent8f341f20c91e708de1dab949dbd723e86f2ff328

fix(server): release the tunnel when an explicit stop interrupts an update

A pending update in the launcher state is not proof a replacement server is coming: `t3 service uninstall` or `systemctl stop` during the pending window also tears the server down, permanently. The launcher now writes a stop marker before signalling its child on an explicit stop and clears it on the next start; the shutdown tunnel release keeps the tunnel only for pending updates without the marker. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

5 files changed, 100 insertions(+), 5 deletions(-)

apps/server/src/cloud/http.test.ts+31
......@@ -1,6 +1,7 @@
11import * as NodeServices from "@effect/platform-node/NodeServices";
22import { describe, expect, it } from "@effect/vitest";
33import * as Effect from "effect/Effect";
4import * as FileSystem from "effect/FileSystem";
45import * as Layer from "effect/Layer";
56import * as Option from "effect/Option";
67import * as Path from "effect/Path";
......@@ -22,6 +23,7 @@ import { writeServiceState } from "../serviceLauncher.ts";
2223import {
2324 SERVICE_LAUNCHER_PROTOCOL,
2425 SERVICE_STATE_FILE,
26 SERVICE_STOP_MARKER_FILE,
2527 type ServiceUpdateRecord,
2628} from "./serviceProtocol.ts";
2729import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
......@@ -451,6 +453,35 @@ describe("releaseManagedTunnelOnShutdown", () => {
451453 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
452454 });
453455
456 it.effect("still releases a pending update when the launcher is stopping", () => {
457 // `t3 service uninstall` or `systemctl stop` during the pending window:
458 // the launcher writes its stop marker before signalling the child, so no
459 // replacement server is coming and the tunnel must not be kept.
460 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
461 const applyConfigCalls: Array<unknown> = [];
462 const requests: Array<HttpClientRequest.HttpClientRequest> = [];
463
464 return Effect.gen(function* () {
465 yield* writeLauncherState({
466 id: "update-1",
467 fromVersion: "0.0.30",
468 targetVersion: "0.0.31",
469 dbPath: "/tmp/state.sqlite",
470 status: "pending",
471 });
472 const fs = yield* FileSystem.FileSystem;
473 const path = yield* Path.Path;
474 const config = yield* ServerConfigModule.ServerConfig;
475 yield* fs.writeFileString(path.join(config.baseDir, "runtime", SERVICE_STOP_MARKER_FILE), "");
476
477 const released = yield* releaseManagedTunnelOnShutdown();
478
479 expect(released).toBe(true);
480 expect(requests).toHaveLength(1);
481 expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false);
482 }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
483 });
484
454485 it.effect("still releases when the recorded update already settled", () => {
455486 const { store, values } = makeMemorySecretStore(managedLinkSecrets);
456487 const applyConfigCalls: Array<unknown> = [];
apps/server/src/cloud/http.ts+22-5
......@@ -61,7 +61,11 @@ import { requireEnvironmentScope } from "../auth/http.ts";
6161import * as ServerConfig from "../config.ts";
6262import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
6363import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
64import { SERVICE_STATE_FILE, serviceStateHasPendingUpdate } from "./serviceProtocol.ts";
64import {
65 SERVICE_STATE_FILE,
66 SERVICE_STOP_MARKER_FILE,
67 serviceStateHasPendingUpdate,
68} from "./serviceProtocol.ts";
6569import {
6670 CLOUD_ENDPOINT_RUNTIME_CONFIG,
6771 CLOUD_LINKED_USER_ID,
......@@ -635,14 +639,27 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD
635639// straight from disk: the launcher owns that file, and this runs while the
636640// server is tearing down. Unreadable or unparseable state means no pending
637641// update — the release then proceeds as before.
638const pendingServiceUpdateExists = Effect.gen(function* () {
642//
643// A pending update alone is not proof a replacement server is coming: an
644// explicit launcher stop (`t3 service uninstall`, `systemctl stop`) during
645// the pending window also tears this server down, with nothing starting
646// afterwards. The launcher marks that case with a stop-marker file just
647// before it signals the child, so pending + no marker is the update handoff.
648const pendingUpdateHandoffExists = Effect.gen(function* () {
639649 const config = yield* ServerConfig.ServerConfig;
640650 const fs = yield* FileSystem.FileSystem;
641651 const path = yield* Path.Path;
652 const runtimeDir = path.join(config.baseDir, "runtime");
642653 const stateText = yield* fs
643 .readFileString(path.join(config.baseDir, "runtime", SERVICE_STATE_FILE))
654 .readFileString(path.join(runtimeDir, SERVICE_STATE_FILE))
644655 .pipe(Effect.option);
645 return Option.isSome(stateText) && serviceStateHasPendingUpdate(stateText.value);
656 if (Option.isNone(stateText) || !serviceStateHasPendingUpdate(stateText.value)) {
657 return false;
658 }
659 const stopping = yield* fs
660 .exists(path.join(runtimeDir, SERVICE_STOP_MARKER_FILE))
661 .pipe(Effect.orElseSucceed(() => false));
662 return !stopping;
646663});
647664
648665// Cloudflare bills per provisioned tunnel, so an environment that goes offline
......@@ -676,7 +693,7 @@ export const releaseManagedTunnelOnShutdown = Effect.fn(
676693 // boot respawns the connector from the stored config and is reachable as
677694 // soon as it connects, and the reconcile confirms the still-live tunnel
678695 // without replacing it.
679 if (yield* pendingServiceUpdateExists) {
696 if (yield* pendingUpdateHandoffExists) {
680697 yield* Effect.logInfo("Keeping the managed tunnel across the update restart");
681698 return false;
682699 }
apps/server/src/cloud/serviceProtocol.ts+4
......@@ -5,6 +5,10 @@ export const SERVICE_LAUNCHER_PROTOCOL = 2 as const;
55export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT";
66export const SERVICE_LAUNCHER_FILE = "service-launcher.mjs";
77export const SERVICE_STATE_FILE = "service-state.json";
8/** Written by the launcher just before an explicit stop kills its child, so
9 the child can tell "the service is going away" from "the launcher is about
10 to start my replacement" while a pending update is recorded. */
11export const SERVICE_STOP_MARKER_FILE = ".service-stopping";
812
913export interface PendingServiceUpdate {
1014 readonly id: string;
apps/server/src/serviceLauncher.test.ts+26
......@@ -1,3 +1,6 @@
1// @effect-diagnostics-next-line nodeBuiltinImport:off - sync marker poll outside the test clock.
2import * as NodeFS from "node:fs";
3
14import * as NodeServices from "@effect/platform-node/NodeServices";
25import { assert, it } from "@effect/vitest";
36import * as Effect from "effect/Effect";
......@@ -10,6 +13,7 @@ import {
1013 decodeServiceState,
1114 isExactServiceVersion,
1215 SERVICE_LAUNCHER_PROTOCOL,
16 SERVICE_STOP_MARKER_FILE,
1317} from "./cloud/serviceProtocol.ts";
1418
1519it("accepts only exact semantic versions", () => {
......@@ -113,6 +117,28 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {
113117 const running = launcher.run();
114118 yield* Effect.promise(() => launcher.stop("SIGTERM"));
115119 yield* Effect.promise(() => running);
120 // An explicit stop leaves the marker that tells a child shutting down
121 // mid-update that no replacement server is coming.
122 assert.isTrue(yield* fs.exists(path.join(root, "runtime", SERVICE_STOP_MARKER_FILE)));
123
124 // The next launcher start clears the stale marker so a later update
125 // handoff keeps its tunnel again. Recovery runs before any stop
126 // transition, so polling until the marker disappears observes it; the
127 // poll uses real timers because it.effect runs under the test clock.
128 const restarted = new Launcher(
129 root,
130 yield* Effect.promise(() => readServiceState(statePath)),
131 );
132 const runningAgain = restarted.run();
133 const markerPath = path.join(root, "runtime", SERVICE_STOP_MARKER_FILE);
134 yield* Effect.promise(async () => {
135 while (NodeFS.existsSync(markerPath)) {
136 // @effect-diagnostics-next-line globalTimers:off - real delay under the test clock.
137 await new Promise((resolve) => setTimeout(resolve, 25));
138 }
139 });
140 yield* Effect.promise(() => restarted.stop("SIGTERM"));
141 yield* Effect.promise(() => runningAgain);
116142 }),
117143 );
118144
apps/server/src/serviceLauncher.ts+17
......@@ -24,6 +24,7 @@ import {
2424 SERVICE_LAUNCHER_CONTEXT_ENV,
2525 SERVICE_LAUNCHER_PROTOCOL,
2626 SERVICE_STATE_FILE,
27 SERVICE_STOP_MARKER_FILE,
2728} from "./cloud/serviceProtocol.ts";
2829
2930const HANDOFF_DELAY_MS = 2_000;
......@@ -257,6 +258,9 @@ async function terminateChild(
257258 }
258259}
259260
261const stopMarkerPath = (baseDir: string) =>
262 NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE);
263
260264export class Launcher {
261265 readonly #baseDir: string;
262266 readonly #statePath: string;
......@@ -315,6 +319,15 @@ export class Launcher {
315319 this.#stopping = true;
316320 this.#clearTimer();
317321 this.#enqueue(async () => {
322 // An explicit stop is the service going away, not an update handoff.
323 // The marker lets the child's shutdown release the managed tunnel even
324 // while an update is recorded as pending; the next launcher start
325 // clears it. Written before the child is signalled so the child sees
326 // it during teardown. Best-effort: if the write fails the child errs
327 // toward keeping the tunnel, and the next link or unlink reconciles it.
328 await NodeFSP.writeFile(stopMarkerPath(this.#baseDir), "", { mode: 0o600 }).catch(
329 () => undefined,
330 );
318331 const child = this.#child?.process;
319332 this.#child = null;
320333 if (child !== undefined) await terminateChild(child, signal);
......@@ -330,6 +343,10 @@ export class Launcher {
330343 }
331344
332345 async #recover(): Promise<void> {
346 // A fresh launcher means servers are running again: any stop marker from
347 // a previous explicit stop is stale and must not make a future update
348 // handoff release its tunnel.
349 await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined);
333350 const update = this.#state.update;
334351 if (update?.status !== "pending") {
335352 if (update !== undefined) {