From 570542662a3f9f3fb44f88778cb24753072f78aa Mon Sep 17 00:00:00 2001 From: maria Date: Wed, 14 Oct 2026 07:55:25 +0000 Subject: [PATCH] fix(server): enforce one database owner and guard update rollback (#16102) Co-authored-by: maria-rcks <254055478+maria-rcks@users.noreply.github.com> --- .../src/backend/DesktopBackendManager.test.ts | 36 +++++ .../src/backend/DesktopBackendManager.ts | 11 +- .../src/backend/DesktopBackendPool.test.ts | 2 + .../desktop/src/backend/DesktopBackendPool.ts | 9 ++ .../desktop/src/wsl/DesktopWslBackend.test.ts | 54 +++++-- apps/desktop/src/wsl/DesktopWslBackend.ts | 22 ++- apps/server/src/cli/project.ts | 10 +- apps/server/src/server.ts | 57 +++---- apps/server/src/serverOwnership.ts | 147 ++++++++++++++++++ apps/server/src/serverRuntimeState.test.ts | 99 ++++++++++++ apps/server/src/serverRuntimeState.ts | 2 + apps/server/src/serviceLauncher.test.ts | 42 +++++ apps/server/src/serviceLauncher.ts | 33 ++++ docs/user/updating.md | 4 + packages/contracts/src/desktopBootstrap.ts | 3 + 15 files changed, 486 insertions(+), 45 deletions(-) create mode 100644 apps/server/src/serverOwnership.ts diff --git a/apps/desktop/src/backend/DesktopBackendManager.test.ts b/apps/desktop/src/backend/DesktopBackendManager.test.ts index e3e9ba62f317c461d22cbd1d8db74e885979afd6..760e5a24f897163ee5d685d66a9251da7cb3c4e4 100644 --- a/apps/desktop/src/backend/DesktopBackendManager.test.ts +++ b/apps/desktop/src/backend/DesktopBackendManager.test.ts @@ -1,4 +1,5 @@ import { + SERVER_EXIT_CODE_STATE_DIR_OWNED, DesktopBackendBootstrap, type DesktopBackendBootstrap as DesktopBackendBootstrapValue, DesktopTelemetryControlMessage, @@ -124,6 +125,7 @@ interface MakeInstanceInput { readonly backendOutputLog?: Partial; readonly onReady?: Effect.Effect; readonly onShutdown?: Effect.Effect; + readonly onStateDirOwned?: Effect.Effect; readonly onPreflightFailed?: ( failure: DesktopBackendManager.PreflightFailure, ) => Effect.Effect; @@ -186,6 +188,7 @@ function makeTestInstance(input: MakeInstanceInput) { configResolve: input.configResolve ?? Effect.succeed(input.config ?? baseConfig), ...(input.onReady ? { onReady: () => input.onReady! } : {}), ...(input.onShutdown ? { onShutdown: () => input.onShutdown! } : {}), + ...(input.onStateDirOwned ? { onStateDirOwned: () => input.onStateDirOwned! } : {}), ...(input.onPreflightFailed ? { onPreflightFailed: input.onPreflightFailed } : {}), }); @@ -1206,6 +1209,39 @@ describe("DesktopBackendManager", () => { ), ); + it.effect("reports state-directory ownership once without scheduling another backend", () => + Effect.scoped( + Effect.gen(function* () { + let startCount = 0; + const refused = yield* Deferred.make(); + const instance = yield* makeTestInstance({ + spawnerLayer: Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.sync(() => { + startCount += 1; + return makeProcess({ + exitCode: Effect.succeed( + ChildProcessSpawner.ExitCode(SERVER_EXIT_CODE_STATE_DIR_OWNED), + ), + }); + }), + ), + ), + httpClientLayer: layerHttpClient(() => Effect.never), + onStateDirOwned: Deferred.succeed(refused, undefined).pipe(Effect.asVoid), + }); + yield* instance.start; + yield* Deferred.await(refused); + const snapshot = yield* instance.snapshot; + assert.isFalse(snapshot.desiredRunning); + assert.isFalse(snapshot.restartScheduled); + yield* TestClock.adjust(Duration.minutes(1)); + assert.equal(startCount, 1); + }).pipe(Effect.provide(TestClock.layer())), + ), + ); + it.effect("does not notify shutdown when a scheduled restart starts from non-ready state", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/desktop/src/backend/DesktopBackendManager.ts b/apps/desktop/src/backend/DesktopBackendManager.ts index 313e8ba5ef37b698ef72c3cb6bde5c8b6caad488..c758b27dd282ce5f2f4f40b8f97056e58d746ba7 100644 --- a/apps/desktop/src/backend/DesktopBackendManager.ts +++ b/apps/desktop/src/backend/DesktopBackendManager.ts @@ -42,6 +42,7 @@ import { HttpClient } from "effect/http"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { + SERVER_EXIT_CODE_STATE_DIR_OWNED, DesktopBackendBootstrap, type DesktopBackendBootstrap as DesktopBackendBootstrapValue, PRIMARY_LOCAL_ENVIRONMENT_ID, @@ -306,6 +307,7 @@ export interface BackendInstanceSpec { // retries. Returns true when the callback changed configuration and the // manager should resolve once more; false stops the failed instance. readonly onPreflightFailed?: (failure: PreflightFailure) => Effect.Effect; + readonly onStateDirOwned?: () => Effect.Effect; } interface ActiveBackendRun { @@ -869,7 +871,9 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( const finalizeRun = Effect.fn("desktop.backendInstance.finalizeRun")(function* ( reason: string, + exitCode?: number, ) { + const stateDirOwned = exitCode === SERVER_EXIT_CODE_STATE_DIR_OWNED; yield* mutex.withPermits(1)( Effect.gen(function* () { const { isCurrentRun, nextState, pid, exitObserved, stopRequested, wasReady } = @@ -939,6 +943,11 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( } if (isCurrentRun && nextState.desiredRunning) { + if (stateDirOwned && !stopRequested) { + yield* Ref.update(state, (latest) => ({ ...latest, desiredRunning: false })); + yield* (spec.onStateDirOwned?.() ?? Effect.void).pipe(Effect.ignore); + return; + } yield* scheduleRestart(reason); } }), @@ -1015,7 +1024,7 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* ( Scope.provide(runScope), Effect.matchEffect({ onFailure: (error) => finalizeRun(error.message), - onSuccess: (exit) => finalizeRun(exit.reason), + onSuccess: (exit) => finalizeRun(exit.reason, Option.getOrUndefined(exit.code)), }), Effect.ensuring(Scope.close(runScope, Exit.void).pipe(Effect.ignore)), ); diff --git a/apps/desktop/src/backend/DesktopBackendPool.test.ts b/apps/desktop/src/backend/DesktopBackendPool.test.ts index d8d7813e79721d981ae119d5128f769a484c2232..c5f31fc47a09bd31184b0876fe553015721fa953 100644 --- a/apps/desktop/src/backend/DesktopBackendPool.test.ts +++ b/apps/desktop/src/backend/DesktopBackendPool.test.ts @@ -14,6 +14,7 @@ import * as DesktopAppSettings from "../settings/DesktopAppSettings.ts"; import * as DesktopTelemetryPublisher from "../telemetry/DesktopTelemetryPublisher.ts"; import * as DesktopBrowserHost from "../preview/DesktopBrowserHost.ts"; import * as ElectronDialog from "../electron/ElectronDialog.ts"; +import * as ElectronApp from "../electron/ElectronApp.ts"; import * as DesktopWindow from "../window/DesktopWindow.ts"; import * as DesktopWslEnvironment from "../wsl/DesktopWslEnvironment.ts"; import * as DesktopBackendConfiguration from "./DesktopBackendConfiguration.ts"; @@ -86,6 +87,7 @@ function layerPool(labelRef: Ref.Ref): Layer.Layer desktopWindow.handleBackendNotReady, onPreflightFailed: handlePrimaryPreflightFailure, + onStateDirOwned: () => + electronDialog + .showErrorBox( + "This T3 home is unavailable", + "Another T3 Code server, such as the background service, is using this data directory. Stop that server and reopen T3 Code, or set a separate T3CODE_HOME and pair with the running server.", + ) + .pipe(Effect.andThen(electronApp.quit)), }); const instancesRef = yield* SynchronizedRef.make< diff --git a/apps/desktop/src/wsl/DesktopWslBackend.test.ts b/apps/desktop/src/wsl/DesktopWslBackend.test.ts index 8afc1bdd948b8c5bf0b808706d271d0b3ae23df8..e6dcb7bd7133ee4607a4335fcb40d6b18cecacdd 100644 --- a/apps/desktop/src/wsl/DesktopWslBackend.test.ts +++ b/apps/desktop/src/wsl/DesktopWslBackend.test.ts @@ -109,16 +109,13 @@ describe("DesktopWslBackend", () => { ); it.effect("clears the stored preflight error when a registered WSL backend becomes ready", () => { let registeredSpec: DesktopBackendPool.BackendInstanceSpec | undefined; + let registeredWsl: DesktopBackendPool.DesktopBackendInstance | undefined; + let startCount = 0; const primary = makeStubInstance({ id: DesktopBackendPool.PRIMARY_INSTANCE_ID, label: "Windows", snapshot: primarySnapshot, }); - const wsl = makeStubInstance({ - id: DesktopBackendPool.BackendInstanceId("wsl:Ubuntu"), - label: "WSL (Ubuntu)", - snapshot: primarySnapshot, - }); const layerPool = Layer.succeed(DesktopBackendPool.DesktopBackendPool, { get: (id) => Effect.succeed( @@ -126,14 +123,25 @@ describe("DesktopWslBackend", () => { ? Option.some(primary) : Option.none(), ), - list: Effect.succeed([primary]), + list: Effect.sync(() => (registeredWsl === undefined ? [primary] : [primary, registeredWsl])), primary: Effect.succeed(primary), register: (spec) => Effect.sync(() => { registeredSpec = spec; - return wsl; + registeredWsl = makeStubInstance({ + id: spec.id, + label: "WSL", + snapshot: idleSnapshot, + start: Effect.sync(() => { + startCount += 1; + }), + }); + return registeredWsl; + }), + unregister: () => + Effect.sync(() => { + registeredWsl = undefined; }), - unregister: () => Effect.die("unexpected unregister"), } satisfies DesktopBackendPool.DesktopBackendPool["Service"]); return Effect.gen(function* () { @@ -146,18 +154,46 @@ describe("DesktopWslBackend", () => { throw new Error("Expected WSL backend registration"); } const recordFailure = spec.onPreflightFailed; + const recordOwnershipFailure = spec.onStateDirOwned; const clearFailure = spec.onReady; assert.isDefined(recordFailure); + assert.isDefined(recordOwnershipFailure); assert.isDefined(clearFailure); - if (recordFailure === undefined || clearFailure === undefined) { + if ( + recordFailure === undefined || + recordOwnershipFailure === undefined || + clearFailure === undefined + ) { throw new Error("Expected WSL backend callbacks"); } assert.isFalse(yield* recordFailure({ reason: "Node.js not found", fatal: true })); assert.deepEqual(yield* backend.lastPreflightError, Option.some("Node.js not found")); + yield* recordOwnershipFailure(); + const refusal = yield* backend.lastPreflightError; + assert.include(Option.getOrThrow(refusal), "Another T3 Code server"); + yield* backend.reconcile; + assert.equal(startCount, 1); + assert.deepEqual(yield* backend.lastPreflightError, refusal); + yield* clearFailure(new URL("http://127.0.0.1:41773")); assert.deepEqual(yield* backend.lastPreflightError, Option.none()); + yield* recordOwnershipFailure(); + const settings = yield* DesktopAppSettings.DesktopAppSettings; + yield* settings.setWslDistro("Debian"); + yield* backend.reconcile; + assert.equal(startCount, 2); + assert.deepEqual(yield* backend.lastPreflightError, Option.none()); + const nextSpec = registeredSpec; + if (nextSpec?.onStateDirOwned === undefined) throw new Error("Expected ownership callback"); + yield* nextSpec.onStateDirOwned(); + yield* settings.setWslBackendEnabled(false); + yield* backend.reconcile; + yield* settings.setWslBackendEnabled(true); + yield* backend.reconcile; + assert.equal(startCount, 3); + assert.deepEqual(yield* backend.lastPreflightError, Option.none()); }).pipe( Effect.provide( DesktopWslBackend.layer.pipe( diff --git a/apps/desktop/src/wsl/DesktopWslBackend.ts b/apps/desktop/src/wsl/DesktopWslBackend.ts index 72bad30c06cc9a3e216a976ae4bea7a7439bdf8d..fe131ca4f9c01cb5a6d81dc2f321735d35ca14f3 100644 --- a/apps/desktop/src/wsl/DesktopWslBackend.ts +++ b/apps/desktop/src/wsl/DesktopWslBackend.ts @@ -113,6 +113,9 @@ export const layer = Layer.effect( // Windows, so we record the reason rather than interrupting. Cleared on any // reconcile state change so it reflects the current attempt. const preflightErrorRef = yield* Ref.make(Option.none()); + const ownershipRefusedRef = yield* Ref.make( + Option.none(), + ); const findExistingWslInstance = pool.list.pipe( Effect.map((instances) => instances.find((instance) => isWslInstanceId(instance.id))), @@ -168,7 +171,21 @@ export const layer = Layer.effect( // fallback — Windows is the primary and keeps working. onPreflightFailed: (failure) => Ref.set(preflightErrorRef, Option.some(failure.reason)).pipe(Effect.as(false)), - onReady: () => Ref.set(preflightErrorRef, Option.none()), + onStateDirOwned: () => + Ref.set(ownershipRefusedRef, Option.some(targetId)).pipe( + Effect.andThen( + Ref.set( + preflightErrorRef, + Option.some( + "Another T3 Code server is using this distro's T3 home. Stop it, or pair with it instead.", + ), + ), + ), + ), + onReady: () => + Ref.set(ownershipRefusedRef, Option.none()).pipe( + Effect.andThen(Ref.set(preflightErrorRef, Option.none())), + ), }) .pipe( Effect.asSome, @@ -217,6 +234,8 @@ export const layer = Layer.effect( const isIdle = !snapshot.ready && Option.isNone(snapshot.activePid) && !snapshot.restartScheduled; if (isIdle) { + const refused = yield* Ref.get(ownershipRefusedRef); + if (Option.isSome(refused) && refused.value === existingInstance.id) return; yield* logWslBackendInfo("retrying idle WSL backend", { id: existingInstance.id }); yield* Ref.set(preflightErrorRef, Option.none()); yield* existingInstance.start; @@ -228,6 +247,7 @@ export const layer = Layer.effect( // any stale secondary preflight error so it reflects this fresh attempt; // onPreflightFailed re-sets it only if the new secondary exhausts retries. yield* Ref.set(preflightErrorRef, Option.none()); + yield* Ref.set(ownershipRefusedRef, Option.none()); if (Option.isSome(existingId)) { yield* logWslBackendInfo("tearing down WSL backend", { id: existingId.value }); diff --git a/apps/server/src/cli/project.ts b/apps/server/src/cli/project.ts index c3fa9e98c5230c67cb84f9f51f5cd02022a02aa7..43253040a942a1b7d3661e9b0b6fcd98220ce617 100644 --- a/apps/server/src/cli/project.ts +++ b/apps/server/src/cli/project.ts @@ -26,6 +26,8 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as ServerConfig from "../config.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ProcessRunner from "../processRunner.ts"; +import * as ServerOwnership from "../serverOwnership.ts"; import * as RuntimeLayer from "../orchestration-v2/runtimeLayer.ts"; import * as ProjectEnrichmentService from "../project/ProjectEnrichmentService.ts"; import * as ProjectFaviconResolver from "../project/ProjectFaviconResolver.ts"; @@ -373,7 +375,7 @@ const tryResolveLiveProjectExecutionMode = Effect.fn("tryResolveLiveProjectExecu origin: runtimeState.value.origin, cause: attempted.failure, }); - yield* clearPersistedServerRuntimeState(config.serverRuntimeStatePath); + // The offline path decides whether that server still owns the home. return Option.none<{ readonly origin: string }>(); }, ); @@ -424,6 +426,11 @@ const runProjectMutation = Effect.fn("runProjectMutation")(function* ( Layer.provide(Layer.succeed(References.MinimumLogLevel, minimumLogLevel)), ); + // An unreachable server may still be running; never write behind it. + yield* ServerOwnership.acquireServerOwnership(config.serverRuntimeStatePath).pipe( + Effect.provide(ProcessRunner.layer), + ); + yield* clearPersistedServerRuntimeState(config.serverRuntimeStatePath); return yield* Effect.gen(function* () { const snapshot = yield* getOfflineSnapshot(); const projects = yield* ProjectService.ProjectService; @@ -435,6 +442,7 @@ const runProjectMutation = Effect.fn("runProjectMutation")(function* ( yield* Console.log(output); }).pipe(Effect.provide(layerOfflineRuntime)); }).pipe( + Effect.scoped, Effect.provide( Layer.mergeAll(EnvironmentAuth.layerRuntime, WorkspacePaths.layer).pipe( Layer.provideMerge(FetchHttpClient.layer), diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 3c1539a70985a91ed5bb31b6a9b3e024bd513109..8148479f8318447e1153174c06862f0dbf793ac0 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -76,6 +76,7 @@ import * as GitManager from "./git/GitManager.ts"; import * as EnvironmentTheme from "./environmentTheme.ts"; import * as Keybindings from "./keybindings.ts"; import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts"; +import * as ServerOwnership from "./serverOwnership.ts"; import * as AgentAwarenessRelay from "./relay/AgentAwarenessRelay.ts"; import { hasCloudPublicConfig } from "./cloud/publicConfig.ts"; import * as ServerSettings from "./serverSettings.ts"; @@ -170,11 +171,7 @@ import * as ThreadSettlementService from "./orchestration-v2/ThreadSettlementSer import * as ThreadPullRequestService from "./orchestration-v2/ThreadPullRequestService.ts"; import * as RunFinalizationService from "./orchestration-v2/RunFinalizationService.ts"; import * as ProjectionStoreV2 from "./orchestration-v2/ProjectionStore.ts"; -import { - clearPersistedServerRuntimeState, - makePersistedServerRuntimeState, - persistServerRuntimeState, -} from "./serverRuntimeState.ts"; +import { makePersistedServerRuntimeState } from "./serverRuntimeState.ts"; import * as OrchestrationHttp from "./orchestration-v2/http.ts"; import * as ProjectHttp from "./project/http.ts"; import * as NetService from "@t3tools/shared/Net"; @@ -746,6 +743,8 @@ const layerMakeRoutes = Layer.mergeAll( const layerMakeServer = Layer.unwrap( Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; + // Before anything opens the database: one server per T3 home. + const ownership = yield* ServerOwnership.acquireServerOwnership(config.serverRuntimeStatePath); const activation = yield* Deferred.make(); const awaitActivation = Deferred.await(activation); const layerActivation = Layer.succeed(ServerActivation.ServerActivation, awaitActivation); @@ -765,38 +764,30 @@ const layerMakeServer = Layer.unwrap( }), ); const layerRuntimeState = Layer.effectDiscard( - Effect.acquireRelease( - Effect.gen(function* () { - yield* Deferred.succeed(runtimeStateParked, undefined).pipe(Effect.orDie); - yield* awaitActivation; - const server = yield* HttpServer.HttpServer; - const address = server.address; - if (typeof address === "string" || !("port" in address)) { - return; - } + Effect.gen(function* () { + yield* Deferred.succeed(runtimeStateParked, undefined).pipe(Effect.orDie); + yield* awaitActivation; + const server = yield* HttpServer.HttpServer; + const address = server.address; + if (typeof address === "string" || !("port" in address)) { + return; + } - const launcher = yield* ServiceLauncherClient.ServiceLauncherClient; - const state = yield* makePersistedServerRuntimeState({ - config, - port: address.port, - serviceManaged: launcher.managed, - }); - yield* persistServerRuntimeState({ - path: config.serverRuntimeStatePath, - state, - }).pipe( + const launcher = yield* ServiceLauncherClient.ServiceLauncherClient; + const state = yield* makePersistedServerRuntimeState({ + config, + port: address.port, + serviceManaged: launcher.managed, + }); + // Ownership release removes this record. + yield* ownership + .publish(state) + .pipe( Effect.catchCause((cause) => Effect.logWarning("Failed to persist server runtime state", { cause }), ), ); - }), - () => - clearPersistedServerRuntimeState(config.serverRuntimeStatePath).pipe( - Effect.catchCause((cause) => - Effect.logWarning("Failed to clear server runtime state", { cause }), - ), - ), - ), + }), ); const layerTailscaleServe = config.tailscaleServeEnabled ? Layer.effectDiscard( @@ -1119,7 +1110,7 @@ const layerMakeServer = Layer.unwrap( Layer.provideMerge(layerPlatformServices), ); }), -); +).pipe(Layer.provide(ProcessRunner.layer)); // The CLI supplies configuration. export const runServer = Layer.launch(layerMakeServer); diff --git a/apps/server/src/serverOwnership.ts b/apps/server/src/serverOwnership.ts new file mode 100644 index 0000000000000000000000000000000000000000..818c4cce29d04b475054cf9daca21c78de8edde7 --- /dev/null +++ b/apps/server/src/serverOwnership.ts @@ -0,0 +1,147 @@ +// @effect-diagnostics nodeBuiltinImport:off - The ownership lock is a native SQLite handle held for the server's lifetime. +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; +import * as NodeSqlite from "node:sqlite"; + +import { SERVER_EXIT_CODE_STATE_DIR_OWNED } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Crypto from "effect/Crypto"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Runtime from "effect/Runtime"; +import * as Schema from "effect/Schema"; + +import * as ProcessRunner from "./processRunner.ts"; +import { + clearPersistedServerRuntimeState, + isProcessAlive, + persistServerRuntimeState, + readPersistedServerRuntimeState, + type PersistedServerRuntimeState, +} from "./serverRuntimeState.ts"; + +export class ServerAlreadyRunningError extends Schema.TaggedError()( + "ServerAlreadyRunningError", + { stateDir: Schema.String }, +) { + // Distinct process exit code so a supervisor can tell "owned by another + // server" apart from a crash and stop restarting. + override readonly [Runtime.errorExitCode] = SERVER_EXIT_CODE_STATE_DIR_OWNED; + + override get message(): string { + return `A T3 Code server already owns ${this.stateDir}. Stop that server, or use a separate T3 home and pair with it, then retry. No server was stopped.`; + } +} + +export class ServerOwnershipError extends Schema.TaggedError()( + "ServerOwnershipError", + { stateDir: Schema.String, cause: Schema.Defect() }, +) { + override get message(): string { + return `Could not acquire server ownership of ${this.stateDir}.`; + } +} + +const isSqliteBusy = (cause: unknown) => + cause instanceof Error && + (("errcode" in cause && cause.errcode === 5) || + ("code" in cause && cause.code === "SQLITE_BUSY")); + +/** + * Servers from before this lock only publish `server-runtime.json`. Treat + * their record as stale only when the process start time proves PID reuse. + */ +const legacyOwnerIsLive = Effect.fn("legacyOwnerIsLive")(function* ( + state: PersistedServerRuntimeState, +) { + if (!isProcessAlive(state.pid)) return false; + const recordedAt = Date.parse(state.startedAt); + if (!Number.isFinite(recordedAt)) return true; + const windows = (yield* HostProcess.Platform) === "win32"; + const runner = yield* ProcessRunner.ProcessRunner; + const result = yield* runner + .run({ + command: windows ? "powershell.exe" : "ps", + args: windows + ? [ + "-NoProfile", + "-NonInteractive", + "-Command", + `(Get-Process -Id ${state.pid} -ErrorAction Stop).StartTime.ToUniversalTime().ToString('o')`, + ] + : ["-p", String(state.pid), "-o", "lstart="], + env: { LC_ALL: "C", TZ: "UTC" }, + timeout: Duration.seconds(2), + maxOutputBytes: 16_384, + }) + .pipe(Effect.option); + if (Option.isNone(result) || result.value.code !== 0) return true; + const output = result.value.stdout.trim(); + const startedAt = Date.parse(windows ? output : `${output} UTC`); + // ps reports whole seconds. Unknown identity stays conservative. + return !Number.isFinite(startedAt) || startedAt <= recordedAt + 1_000; +}); + +/** + * Hold an exclusive lock on the state directory until the scope closes. The + * lock is an exclusive transaction on `server-owner.sqlite`, which never holds + * data and must never be unlinked: the OS releases it when the process exits, + * even on SIGKILL, so a crashed owner never blocks the next start. + */ +export const acquireServerOwnership = Effect.fn("acquireServerOwnership")(function* ( + statePath: string, +) { + const stateDir = NodePath.dirname(statePath); + const crypto = yield* Crypto.Crypto; + const ownerId = yield* crypto.randomUUIDv4.pipe( + Effect.mapError((cause) => new ServerOwnershipError({ stateDir, cause })), + ); + yield* Effect.acquireRelease( + Effect.try({ + try: () => { + NodeFS.mkdirSync(stateDir, { recursive: true }); + const db = new NodeSqlite.DatabaseSync(NodePath.join(stateDir, "server-owner.sqlite")); + try { + db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;"); + } catch (cause) { + db.close(); + throw cause; + } + return db; + }, + catch: (cause) => + isSqliteBusy(cause) + ? new ServerAlreadyRunningError({ stateDir }) + : new ServerOwnershipError({ stateDir, cause }), + }), + (db) => + readPersistedServerRuntimeState(statePath).pipe( + // Only remove discovery this owner published. + Effect.flatMap((state) => + Option.isSome(state) && state.value.ownerId === ownerId + ? clearPersistedServerRuntimeState(statePath) + : Effect.void, + ), + Effect.ignore({ log: true }), + Effect.ensuring(Effect.sync(() => db.close())), + ), + ); + + // An unreadable record cannot name a live owner; this server replaces it. + const previous = yield* readPersistedServerRuntimeState(statePath).pipe( + Effect.orElseSucceed(() => Option.none()), + ); + if ( + Option.isSome(previous) && + previous.value.ownerId === undefined && + (yield* legacyOwnerIsLive(previous.value)) + ) { + return yield* new ServerAlreadyRunningError({ stateDir }); + } + + return { + publish: (state: PersistedServerRuntimeState) => + persistServerRuntimeState({ path: statePath, state: { ...state, ownerId } }), + }; +}); diff --git a/apps/server/src/serverRuntimeState.test.ts b/apps/server/src/serverRuntimeState.test.ts index 6fbdab817b63ca632bb9740ad18df0e331cb6591..dee6afb00c39d3fa972d297bc8033d5c4ba70695 100644 --- a/apps/server/src/serverRuntimeState.test.ts +++ b/apps/server/src/serverRuntimeState.test.ts @@ -9,6 +9,8 @@ import * as Path from "effect/Path"; import * as References from "effect/References"; import * as Schema from "effect/Schema"; +import * as ProcessRunner from "./processRunner.ts"; +import * as ServerOwnership from "./serverOwnership.ts"; import * as ServerRuntimeState from "./serverRuntimeState.ts"; const isServerRuntimeStateError = Schema.is(ServerRuntimeState.ServerRuntimeStateError); @@ -18,6 +20,103 @@ interface CapturedLog { readonly annotations: Readonly>; } +describe("server ownership", () => { + const layerOwnership = ProcessRunner.layer.pipe(Layer.provideMerge(NodeServices.layer)); + const makeState = ServerRuntimeState.makePersistedServerRuntimeState({ + config: { host: undefined, devUrl: undefined }, + port: 3773, + }); + + it.effect("lets only one of two racing starts own a state directory", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-owner-race-" }); + const statePath = path.join(root, "server-runtime.json"); + const attempts = yield* Effect.all( + [ + ServerOwnership.acquireServerOwnership(statePath).pipe(Effect.result), + ServerOwnership.acquireServerOwnership(statePath).pipe(Effect.result), + ], + { concurrency: "unbounded" }, + ); + assert.equal(attempts.filter((result) => result._tag === "Success").length, 1); + const refused = attempts.find((result) => result._tag === "Failure"); + assert.equal(refused?.failure._tag, "ServerAlreadyRunningError"); + }).pipe(Effect.provide(layerOwnership)), + ); + + it.effect("refuses a second owner without touching its record, then allows a restart", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-owner-" }); + const statePath = path.join(root, "server-runtime.json"); + const state = yield* makeState; + yield* Effect.scoped( + Effect.gen(function* () { + const owner = yield* ServerOwnership.acquireServerOwnership(statePath); + yield* owner.publish(state); + const before = yield* fs.readFileString(statePath); + const refused = yield* ServerOwnership.acquireServerOwnership(statePath).pipe( + Effect.flip, + ); + assert.equal(refused._tag, "ServerAlreadyRunningError"); + assert.equal(yield* fs.readFileString(statePath), before); + }), + ); + assert.isFalse(yield* fs.exists(statePath)); + yield* Effect.scoped(ServerOwnership.acquireServerOwnership(statePath)); + }).pipe(Effect.provide(layerOwnership)), + ); + + it.effect("refuses beside a live pre-lock server and ignores a stale record", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-owner-legacy-" }); + const statePath = path.join(root, "server-runtime.json"); + const state = yield* makeState; + yield* ServerRuntimeState.persistServerRuntimeState({ + path: statePath, + // @effect-diagnostics-next-line globalDateInEffect:off - Legacy identity uses the real process start time, not TestClock. + state: { ...state, startedAt: new Date().toISOString() }, + }); + const refused = yield* Effect.scoped(ServerOwnership.acquireServerOwnership(statePath)).pipe( + Effect.flip, + ); + assert.equal(refused._tag, "ServerAlreadyRunningError"); + yield* ServerRuntimeState.persistServerRuntimeState({ + path: statePath, + state: { ...state, pid: 0 }, + }); + yield* Effect.scoped(ServerOwnership.acquireServerOwnership(statePath)); + }).pipe(Effect.provide(layerOwnership)), + ); + + it.effect("never removes a runtime record another server published", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-owner-cleanup-" }); + const statePath = path.join(root, "server-runtime.json"); + const state = yield* makeState; + yield* Effect.scoped( + Effect.gen(function* () { + const owner = yield* ServerOwnership.acquireServerOwnership(statePath); + yield* owner.publish(state); + yield* ServerRuntimeState.persistServerRuntimeState({ + path: statePath, + state: { ...state, ownerId: "other-owner" }, + }); + }), + ); + const remaining = yield* ServerRuntimeState.readPersistedServerRuntimeState(statePath); + assert.equal(Option.getOrThrow(remaining).ownerId, "other-owner"); + }).pipe(Effect.provide(layerOwnership)), + ); +}); + describe("serverRuntimeState", () => { it.effect("persists and reads the runtime state", () => Effect.gen(function* () { diff --git a/apps/server/src/serverRuntimeState.ts b/apps/server/src/serverRuntimeState.ts index 8e565b46e26b7e1c32b90aeae9811551dc645e57..862b3174b7949a704f2ec008eaef529ab06c98a1 100644 --- a/apps/server/src/serverRuntimeState.ts +++ b/apps/server/src/serverRuntimeState.ts @@ -11,6 +11,7 @@ import { formatHostForUrl, isWildcardHost } from "./startupAccess.ts"; export const PersistedServerRuntimeState = Schema.Struct({ version: Schema.Literal(1), pid: Schema.Int, + ownerId: Schema.optional(Schema.String), host: Schema.optional(Schema.String), port: Schema.Int, origin: Schema.String, @@ -119,6 +120,7 @@ export const clearPersistedServerRuntimeState = (path: string) => * alive. */ export const isProcessAlive = (pid: number): boolean => { + if (!Number.isInteger(pid) || pid <= 0) return false; try { process.kill(pid, 0); return true; diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index df2ab9e9cd0618df9ffa46666474d1c8de5a182f..a9f91c2c87bbeba82fd2a6cdc748c6a9a570abfe 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -1,5 +1,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; +import { SERVER_EXIT_CODE_STATE_DIR_OWNED } from "@t3tools/contracts"; +import { vi } from "vite-plus/test"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; @@ -158,6 +160,46 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { }), ); + it.effect("stays idle instead of exiting when another server owns the T3 home", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-refused-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", "1.0.0"), + `process.exit(${SERVER_EXIT_CODE_STATE_DIR_OWNED});\n`, + ); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + const refused = Promise.withResolvers(); + yield* Effect.acquireRelease( + Effect.sync(() => + vi.spyOn(process.stderr, "write").mockImplementation((chunk) => { + if (String(chunk).includes("owns this T3 home")) refused.resolve(); + return true; + }), + ), + (spy) => Effect.sync(() => spy.mockRestore()), + ); + const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); + let completed = false; + const running = launcher.run().finally(() => { + completed = true; + }); + yield* Effect.promise(() => refused.promise); + assert.isFalse(completed); + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); + }), + ); + it.effect("serializes shutdown with launcher recovery", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index cfdd3d75b57ee7cb7ebca52dd3ae126460664dc9..b3b005abfaa74e51f588003329c80fca4d6947f8 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -33,6 +33,9 @@ import { const HANDOFF_DELAY_MS = 2_000; const PREPARED_TIMEOUT_MS = 120_000; const TERMINATE_GRACE_MS = 5_000; +// Matches SERVER_EXIT_CODE_STATE_DIR_OWNED in contracts. The standalone +// launcher stays independent of the Effect-backed contracts package. +const STATE_DIR_OWNED_EXIT_CODE = 78; type TerminalStatus = "committed" | "rolled-back" | "failed"; type ChildRole = "active" | "trial"; @@ -579,6 +582,10 @@ export class Launcher { ): Promise { if (this.#child !== child || this.#stopping) return; this.#child = null; + if (code === STATE_DIR_OWNED_EXIT_CODE) { + await this.#idleWhileStateDirOwned(); + return; + } if (child.role === "trial") { this.#clearTimer(); const pending = this.#state.update; @@ -602,6 +609,32 @@ export class Launcher { throw new Error(`Active child exited unexpectedly (${String(code ?? signal ?? "unknown")}).`); } + /** + * Another server owns the T3 home. Exiting would make systemd or launchd + * start another refused server, so stay idle until stopped or restarted. + * A pending update fails without restoring its backup: the other owner may + * have written to the database since. + */ + async #idleWhileStateDirOwned(): Promise { + this.#clearTimer(); + const pending = this.#state.update; + if (pending?.status === "pending") { + const next: ServiceState = { + ...this.#state, + activeVersion: pending.fromVersion, + update: terminalUpdate({ pending, status: "failed", reason: "state-dir-owned" }), + }; + await writeServiceState(this.#statePath, next); + this.#state = next; + await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined); + } + // Signal listeners alone do not keep Node alive. + this.#timer = setInterval(() => {}, 2_147_483_647); + process.stderr.write( + "[service-launcher] Another T3 Code server owns this T3 home. Stop it, then restart this service.\n", + ); + } + async #returnToPrevious( pending: PendingServiceUpdate, status: "rolled-back" | "failed", diff --git a/docs/user/updating.md b/docs/user/updating.md index 07c0942a2b618b338f78e8f11cda81524672bca3..1c7ac45a8b16d126ee09b68442b5a824683f76e3 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -99,6 +99,10 @@ update can roll back to the previous version. If the update still fails: 2. Check that you updated the server's machine, not only the device you are using. 3. For a command-line server, stop it and relaunch the exact version shown in the notice. +Only one server can use a T3 home directory at a time. If startup reports that +another server owns it, stop that server before retrying, or use a separate home +directory and pair with the running server. + ## Update providers **Settings → Providers** shows provider updates for the selected environment. diff --git a/packages/contracts/src/desktopBootstrap.ts b/packages/contracts/src/desktopBootstrap.ts index a61379a3906e9884924156da393134224abef3ff..8a3f379df329013248fd08a397f0f6974d0b6141 100644 --- a/packages/contracts/src/desktopBootstrap.ts +++ b/packages/contracts/src/desktopBootstrap.ts @@ -2,6 +2,9 @@ import * as Schema from "effect/Schema"; import { PortSchema, PositiveInt, TrimmedNonEmptyString } from "./baseSchemas.ts"; +// A supervisor must not restart a server refused by the state-directory owner. +export const SERVER_EXIT_CODE_STATE_DIR_OWNED = 78; + export const DesktopBackendBootstrap = Schema.Struct({ mode: Schema.Literal("desktop"), noBrowser: Schema.Boolean, -- 2.54.0