From f9e8236899f25653ce95d688fac798cd070afbc1 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Thu, 6 Aug 2026 02:29:30 -0700 Subject: [PATCH] fix(server): release the tunnel when an explicit stop interrupts an update A pending update in the launcher state is not proof a replacement server is coming: `t3 service uninstall` or `systemctl stop` during the pending window also tears the server down, permanently. The launcher now writes a stop marker before signalling its child on an explicit stop and clears it on the next start; the shutdown tunnel release keeps the tunnel only for pending updates without the marker. Co-Authored-By: Claude Fable 5 --- apps/server/src/cloud/http.test.ts | 31 ++++++++++++++++++++++++ apps/server/src/cloud/http.ts | 27 +++++++++++++++++---- apps/server/src/cloud/serviceProtocol.ts | 4 +++ apps/server/src/serviceLauncher.test.ts | 26 ++++++++++++++++++++ apps/server/src/serviceLauncher.ts | 17 +++++++++++++ 5 files changed, 100 insertions(+), 5 deletions(-) diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index 8aa800a33106ad734fcae255cc9571fe058c014c..d3cd9c8a47d4cf820407955630eedf42a872c30b 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -1,6 +1,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { describe, expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; @@ -22,6 +23,7 @@ import { writeServiceState } from "../serviceLauncher.ts"; import { SERVICE_LAUNCHER_PROTOCOL, SERVICE_STATE_FILE, + SERVICE_STOP_MARKER_FILE, type ServiceUpdateRecord, } from "./serviceProtocol.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; @@ -451,6 +453,35 @@ describe("releaseManagedTunnelOnShutdown", () => { }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); }); + it.effect("still releases a pending update when the launcher is stopping", () => { + // `t3 service uninstall` or `systemctl stop` during the pending window: + // the launcher writes its stop marker before signalling the child, so no + // replacement server is coming and the tunnel must not be kept. + const { store, values } = makeMemorySecretStore(managedLinkSecrets); + const applyConfigCalls: Array = []; + const requests: Array = []; + + return Effect.gen(function* () { + yield* writeLauncherState({ + id: "update-1", + fromVersion: "0.0.30", + targetVersion: "0.0.31", + dbPath: "/tmp/state.sqlite", + status: "pending", + }); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const config = yield* ServerConfigModule.ServerConfig; + yield* fs.writeFileString(path.join(config.baseDir, "runtime", SERVICE_STOP_MARKER_FILE), ""); + + const released = yield* releaseManagedTunnelOnShutdown(); + + expect(released).toBe(true); + expect(requests).toHaveLength(1); + expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false); + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); + }); + it.effect("still releases when the recorded update already settled", () => { const { store, values } = makeMemorySecretStore(managedLinkSecrets); const applyConfigCalls: Array = []; diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index e51ec6f3c88bf6af75ba40e2850431e80e0ae3a6..3dc12f41c5504bda6ffc421ed4aa13a862221cca 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -61,7 +61,11 @@ import { requireEnvironmentScope } from "../auth/http.ts"; import * as ServerConfig from "../config.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; -import { SERVICE_STATE_FILE, serviceStateHasPendingUpdate } from "./serviceProtocol.ts"; +import { + SERVICE_STATE_FILE, + SERVICE_STOP_MARKER_FILE, + serviceStateHasPendingUpdate, +} from "./serviceProtocol.ts"; import { CLOUD_ENDPOINT_RUNTIME_CONFIG, CLOUD_LINKED_USER_ID, @@ -635,14 +639,27 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD // straight from disk: the launcher owns that file, and this runs while the // server is tearing down. Unreadable or unparseable state means no pending // update — the release then proceeds as before. -const pendingServiceUpdateExists = Effect.gen(function* () { +// +// A pending update alone is not proof a replacement server is coming: an +// explicit launcher stop (`t3 service uninstall`, `systemctl stop`) during +// the pending window also tears this server down, with nothing starting +// afterwards. The launcher marks that case with a stop-marker file just +// before it signals the child, so pending + no marker is the update handoff. +const pendingUpdateHandoffExists = Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; + const runtimeDir = path.join(config.baseDir, "runtime"); const stateText = yield* fs - .readFileString(path.join(config.baseDir, "runtime", SERVICE_STATE_FILE)) + .readFileString(path.join(runtimeDir, SERVICE_STATE_FILE)) .pipe(Effect.option); - return Option.isSome(stateText) && serviceStateHasPendingUpdate(stateText.value); + if (Option.isNone(stateText) || !serviceStateHasPendingUpdate(stateText.value)) { + return false; + } + const stopping = yield* fs + .exists(path.join(runtimeDir, SERVICE_STOP_MARKER_FILE)) + .pipe(Effect.orElseSucceed(() => false)); + return !stopping; }); // Cloudflare bills per provisioned tunnel, so an environment that goes offline @@ -676,7 +693,7 @@ export const releaseManagedTunnelOnShutdown = Effect.fn( // boot respawns the connector from the stored config and is reachable as // soon as it connects, and the reconcile confirms the still-live tunnel // without replacing it. - if (yield* pendingServiceUpdateExists) { + if (yield* pendingUpdateHandoffExists) { yield* Effect.logInfo("Keeping the managed tunnel across the update restart"); return false; } diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index ebc5d15d54f00f7178af74849b471a2bbca91ae1..0faf88948377158ce399ff89bf558cc1be3fe0db 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -5,6 +5,10 @@ export const SERVICE_LAUNCHER_PROTOCOL = 2 as const; export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT"; export const SERVICE_LAUNCHER_FILE = "service-launcher.mjs"; export const SERVICE_STATE_FILE = "service-state.json"; +/** Written by the launcher just before an explicit stop kills its child, so + the child can tell "the service is going away" from "the launcher is about + to start my replacement" while a pending update is recorded. */ +export const SERVICE_STOP_MARKER_FILE = ".service-stopping"; export interface PendingServiceUpdate { readonly id: string; diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 4562c6a6de8ec75a03d38c532540ca76a79d84db..90c76b85b3196e42f5442879e1dc02c960c4f76f 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -1,3 +1,6 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off - sync marker poll outside the test clock. +import * as NodeFS from "node:fs"; + import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; @@ -10,6 +13,7 @@ import { decodeServiceState, isExactServiceVersion, SERVICE_LAUNCHER_PROTOCOL, + SERVICE_STOP_MARKER_FILE, } from "./cloud/serviceProtocol.ts"; it("accepts only exact semantic versions", () => { @@ -113,6 +117,28 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { const running = launcher.run(); yield* Effect.promise(() => launcher.stop("SIGTERM")); yield* Effect.promise(() => running); + // An explicit stop leaves the marker that tells a child shutting down + // mid-update that no replacement server is coming. + assert.isTrue(yield* fs.exists(path.join(root, "runtime", SERVICE_STOP_MARKER_FILE))); + + // The next launcher start clears the stale marker so a later update + // handoff keeps its tunnel again. Recovery runs before any stop + // transition, so polling until the marker disappears observes it; the + // poll uses real timers because it.effect runs under the test clock. + const restarted = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + ); + const runningAgain = restarted.run(); + const markerPath = path.join(root, "runtime", SERVICE_STOP_MARKER_FILE); + yield* Effect.promise(async () => { + while (NodeFS.existsSync(markerPath)) { + // @effect-diagnostics-next-line globalTimers:off - real delay under the test clock. + await new Promise((resolve) => setTimeout(resolve, 25)); + } + }); + yield* Effect.promise(() => restarted.stop("SIGTERM")); + yield* Effect.promise(() => runningAgain); }), ); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 211c0138bc4b6e176aab4899690b53f9c30d2388..b704dc4c0320bf7b28bcad6d89486ff4c9072b15 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -24,6 +24,7 @@ import { SERVICE_LAUNCHER_CONTEXT_ENV, SERVICE_LAUNCHER_PROTOCOL, SERVICE_STATE_FILE, + SERVICE_STOP_MARKER_FILE, } from "./cloud/serviceProtocol.ts"; const HANDOFF_DELAY_MS = 2_000; @@ -257,6 +258,9 @@ async function terminateChild( } } +const stopMarkerPath = (baseDir: string) => + NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE); + export class Launcher { readonly #baseDir: string; readonly #statePath: string; @@ -315,6 +319,15 @@ export class Launcher { this.#stopping = true; this.#clearTimer(); this.#enqueue(async () => { + // An explicit stop is the service going away, not an update handoff. + // The marker lets the child's shutdown release the managed tunnel even + // while an update is recorded as pending; the next launcher start + // clears it. Written before the child is signalled so the child sees + // it during teardown. Best-effort: if the write fails the child errs + // toward keeping the tunnel, and the next link or unlink reconciles it. + await NodeFSP.writeFile(stopMarkerPath(this.#baseDir), "", { mode: 0o600 }).catch( + () => undefined, + ); const child = this.#child?.process; this.#child = null; if (child !== undefined) await terminateChild(child, signal); @@ -330,6 +343,10 @@ export class Launcher { } async #recover(): Promise { + // A fresh launcher means servers are running again: any stop marker from + // a previous explicit stop is stale and must not make a future update + // handoff release its tunnel. + await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined); const update = this.#state.update; if (update?.status !== "pending") { if (update !== undefined) { -- 2.54.0