authorgravatar for maria@kuuro.netmaria <maria@kuuro.net> 2026-10-14 07:55:25+00:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-14 00:55:25-07:00
log570542662a3f9f3fb44f88778cb24753072f78aa
treed97bb4248c69f9274f5ec28c298b26bf5bebdc4e
parenteeaac49a9d20d94970b26fe7775fc8ddb4672f4f
signature Signed by PGP key B5690EEEBB952194

fix(server): enforce one database owner and guard update rollback (#16102)

Co-authored-by: maria-rcks <254055478+maria-rcks@users.noreply.github.com>

15 files changed, 486 insertions(+), 45 deletions(-)

apps/desktop/src/backend/DesktopBackendManager.test.ts+36
......@@ -1,4 +1,5 @@
11import {
2 SERVER_EXIT_CODE_STATE_DIR_OWNED,
23 DesktopBackendBootstrap,
34 type DesktopBackendBootstrap as DesktopBackendBootstrapValue,
45 DesktopTelemetryControlMessage,
......@@ -124,6 +125,7 @@ interface MakeInstanceInput {
124125 readonly backendOutputLog?: Partial<DesktopObservability.DesktopBackendOutputLogShape>;
125126 readonly onReady?: Effect.Effect<void>;
126127 readonly onShutdown?: Effect.Effect<void>;
128 readonly onStateDirOwned?: Effect.Effect<void>;
127129 readonly onPreflightFailed?: (
128130 failure: DesktopBackendManager.PreflightFailure,
129131 ) => Effect.Effect<boolean>;
......@@ -186,6 +188,7 @@ function makeTestInstance(input: MakeInstanceInput) {
186188 configResolve: input.configResolve ?? Effect.succeed(input.config ?? baseConfig),
187189 ...(input.onReady ? { onReady: () => input.onReady! } : {}),
188190 ...(input.onShutdown ? { onShutdown: () => input.onShutdown! } : {}),
191 ...(input.onStateDirOwned ? { onStateDirOwned: () => input.onStateDirOwned! } : {}),
189192 ...(input.onPreflightFailed ? { onPreflightFailed: input.onPreflightFailed } : {}),
190193 });
191194
......@@ -1206,6 +1209,39 @@ describe("DesktopBackendManager", () => {
12061209 ),
12071210 );
12081211
1212 it.effect("reports state-directory ownership once without scheduling another backend", () =>
1213 Effect.scoped(
1214 Effect.gen(function* () {
1215 let startCount = 0;
1216 const refused = yield* Deferred.make<void>();
1217 const instance = yield* makeTestInstance({
1218 spawnerLayer: Layer.succeed(
1219 ChildProcessSpawner.ChildProcessSpawner,
1220 ChildProcessSpawner.make(() =>
1221 Effect.sync(() => {
1222 startCount += 1;
1223 return makeProcess({
1224 exitCode: Effect.succeed(
1225 ChildProcessSpawner.ExitCode(SERVER_EXIT_CODE_STATE_DIR_OWNED),
1226 ),
1227 });
1228 }),
1229 ),
1230 ),
1231 httpClientLayer: layerHttpClient(() => Effect.never),
1232 onStateDirOwned: Deferred.succeed(refused, undefined).pipe(Effect.asVoid),
1233 });
1234 yield* instance.start;
1235 yield* Deferred.await(refused);
1236 const snapshot = yield* instance.snapshot;
1237 assert.isFalse(snapshot.desiredRunning);
1238 assert.isFalse(snapshot.restartScheduled);
1239 yield* TestClock.adjust(Duration.minutes(1));
1240 assert.equal(startCount, 1);
1241 }).pipe(Effect.provide(TestClock.layer())),
1242 ),
1243 );
1244
12091245 it.effect("does not notify shutdown when a scheduled restart starts from non-ready state", () =>
12101246 Effect.scoped(
12111247 Effect.gen(function* () {
apps/desktop/src/backend/DesktopBackendManager.ts+10-1
......@@ -42,6 +42,7 @@ import { HttpClient } from "effect/http";
4242import { ChildProcess, ChildProcessSpawner } from "effect/process";
4343
4444import {
45 SERVER_EXIT_CODE_STATE_DIR_OWNED,
4546 DesktopBackendBootstrap,
4647 type DesktopBackendBootstrap as DesktopBackendBootstrapValue,
4748 PRIMARY_LOCAL_ENVIRONMENT_ID,
......@@ -306,6 +307,7 @@ export interface BackendInstanceSpec {
306307 // retries. Returns true when the callback changed configuration and the
307308 // manager should resolve once more; false stops the failed instance.
308309 readonly onPreflightFailed?: (failure: PreflightFailure) => Effect.Effect<boolean>;
310 readonly onStateDirOwned?: () => Effect.Effect<void>;
309311}
310312
311313interface ActiveBackendRun {
......@@ -869,7 +871,9 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (
869871
870872 const finalizeRun = Effect.fn("desktop.backendInstance.finalizeRun")(function* (
871873 reason: string,
874 exitCode?: number,
872875 ) {
876 const stateDirOwned = exitCode === SERVER_EXIT_CODE_STATE_DIR_OWNED;
873877 yield* mutex.withPermits(1)(
874878 Effect.gen(function* () {
875879 const { isCurrentRun, nextState, pid, exitObserved, stopRequested, wasReady } =
......@@ -939,6 +943,11 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (
939943 }
940944
941945 if (isCurrentRun && nextState.desiredRunning) {
946 if (stateDirOwned && !stopRequested) {
947 yield* Ref.update(state, (latest) => ({ ...latest, desiredRunning: false }));
948 yield* (spec.onStateDirOwned?.() ?? Effect.void).pipe(Effect.ignore);
949 return;
950 }
942951 yield* scheduleRestart(reason);
943952 }
944953 }),
......@@ -1015,7 +1024,7 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (
10151024 Scope.provide(runScope),
10161025 Effect.matchEffect({
10171026 onFailure: (error) => finalizeRun(error.message),
1018 onSuccess: (exit) => finalizeRun(exit.reason),
1027 onSuccess: (exit) => finalizeRun(exit.reason, Option.getOrUndefined(exit.code)),
10191028 }),
10201029 Effect.ensuring(Scope.close(runScope, Exit.void).pipe(Effect.ignore)),
10211030 );
apps/desktop/src/backend/DesktopBackendPool.test.ts+2
......@@ -14,6 +14,7 @@ import * as DesktopAppSettings from "../settings/DesktopAppSettings.ts";
1414import * as DesktopTelemetryPublisher from "../telemetry/DesktopTelemetryPublisher.ts";
1515import * as DesktopBrowserHost from "../preview/DesktopBrowserHost.ts";
1616import * as ElectronDialog from "../electron/ElectronDialog.ts";
17import * as ElectronApp from "../electron/ElectronApp.ts";
1718import * as DesktopWindow from "../window/DesktopWindow.ts";
1819import * as DesktopWslEnvironment from "../wsl/DesktopWslEnvironment.ts";
1920import * as DesktopBackendConfiguration from "./DesktopBackendConfiguration.ts";
......@@ -86,6 +87,7 @@ function layerPool(labelRef: Ref.Ref<string>): Layer.Layer<DesktopBackendPool.De
8687 DesktopAppSettings.layerTest(),
8788 DesktopWslEnvironment.layerTest(),
8889 ElectronDialog.layer,
90 ElectronApp.layer,
8991 Layer.succeed(DesktopWindow.DesktopWindow, {
9092 createMain: Effect.die("unexpected window create"),
9193 ensureMain: Effect.die("unexpected window ensure"),
apps/desktop/src/backend/DesktopBackendPool.ts+9
......@@ -102,6 +102,7 @@ import * as DesktopBrowserHost from "../preview/DesktopBrowserHost.ts";
102102import * as DesktopWindow from "../window/DesktopWindow.ts";
103103import * as DesktopWslEnvironment from "../wsl/DesktopWslEnvironment.ts";
104104import * as ElectronDialog from "../electron/ElectronDialog.ts";
105import * as ElectronApp from "../electron/ElectronApp.ts";
105106
106107const { logWarning: logBackendPoolWarning } =
107108 DesktopObservability.makeComponentLogger("desktop-backend-pool");
......@@ -215,6 +216,7 @@ export const layer = Layer.effect(
215216 const configuration = yield* DesktopBackendConfiguration.DesktopBackendConfiguration;
216217 const desktopWindow = yield* DesktopWindow.DesktopWindow;
217218 const electronDialog = yield* ElectronDialog.ElectronDialog;
219 const electronApp = yield* ElectronApp.ElectronApp;
218220 const appSettings = yield* DesktopAppSettings.DesktopAppSettings;
219221 // Anchor the pool's lifetime to its layer scope so registered
220222 // instance scopes can be forked off it. Without this, instance
......@@ -303,6 +305,13 @@ export const layer = Layer.effect(
303305 ),
304306 onShutdown: () => desktopWindow.handleBackendNotReady,
305307 onPreflightFailed: handlePrimaryPreflightFailure,
308 onStateDirOwned: () =>
309 electronDialog
310 .showErrorBox(
311 "This T3 home is unavailable",
312 "Another T3 Code server, such as the background service, is using this data directory. Stop that server and reopen T3 Code, or set a separate T3CODE_HOME and pair with the running server.",
313 )
314 .pipe(Effect.andThen(electronApp.quit)),
306315 });
307316
308317 const instancesRef = yield* SynchronizedRef.make<
apps/desktop/src/wsl/DesktopWslBackend.test.ts+45-9
......@@ -109,16 +109,13 @@ describe("DesktopWslBackend", () => {
109109 );
110110 it.effect("clears the stored preflight error when a registered WSL backend becomes ready", () => {
111111 let registeredSpec: DesktopBackendPool.BackendInstanceSpec | undefined;
112 let registeredWsl: DesktopBackendPool.DesktopBackendInstance | undefined;
113 let startCount = 0;
112114 const primary = makeStubInstance({
113115 id: DesktopBackendPool.PRIMARY_INSTANCE_ID,
114116 label: "Windows",
115117 snapshot: primarySnapshot,
116118 });
117 const wsl = makeStubInstance({
118 id: DesktopBackendPool.BackendInstanceId("wsl:Ubuntu"),
119 label: "WSL (Ubuntu)",
120 snapshot: primarySnapshot,
121 });
122119 const layerPool = Layer.succeed(DesktopBackendPool.DesktopBackendPool, {
123120 get: (id) =>
124121 Effect.succeed(
......@@ -126,14 +123,25 @@ describe("DesktopWslBackend", () => {
126123 ? Option.some(primary)
127124 : Option.none<DesktopBackendPool.DesktopBackendInstance>(),
128125 ),
129 list: Effect.succeed([primary]),
126 list: Effect.sync(() => (registeredWsl === undefined ? [primary] : [primary, registeredWsl])),
130127 primary: Effect.succeed(primary),
131128 register: (spec) =>
132129 Effect.sync(() => {
133130 registeredSpec = spec;
134 return wsl;
131 registeredWsl = makeStubInstance({
132 id: spec.id,
133 label: "WSL",
134 snapshot: idleSnapshot,
135 start: Effect.sync(() => {
136 startCount += 1;
137 }),
138 });
139 return registeredWsl;
140 }),
141 unregister: () =>
142 Effect.sync(() => {
143 registeredWsl = undefined;
135144 }),
136 unregister: () => Effect.die("unexpected unregister"),
137145 } satisfies DesktopBackendPool.DesktopBackendPool["Service"]);
138146
139147 return Effect.gen(function* () {
......@@ -146,18 +154,46 @@ describe("DesktopWslBackend", () => {
146154 throw new Error("Expected WSL backend registration");
147155 }
148156 const recordFailure = spec.onPreflightFailed;
157 const recordOwnershipFailure = spec.onStateDirOwned;
149158 const clearFailure = spec.onReady;
150159 assert.isDefined(recordFailure);
160 assert.isDefined(recordOwnershipFailure);
151161 assert.isDefined(clearFailure);
152 if (recordFailure === undefined || clearFailure === undefined) {
162 if (
163 recordFailure === undefined ||
164 recordOwnershipFailure === undefined ||
165 clearFailure === undefined
166 ) {
153167 throw new Error("Expected WSL backend callbacks");
154168 }
155169
156170 assert.isFalse(yield* recordFailure({ reason: "Node.js not found", fatal: true }));
157171 assert.deepEqual(yield* backend.lastPreflightError, Option.some("Node.js not found"));
158172
173 yield* recordOwnershipFailure();
174 const refusal = yield* backend.lastPreflightError;
175 assert.include(Option.getOrThrow(refusal), "Another T3 Code server");
176 yield* backend.reconcile;
177 assert.equal(startCount, 1);
178 assert.deepEqual(yield* backend.lastPreflightError, refusal);
179
159180 yield* clearFailure(new URL("http://127.0.0.1:41773"));
160181 assert.deepEqual(yield* backend.lastPreflightError, Option.none());
182 yield* recordOwnershipFailure();
183 const settings = yield* DesktopAppSettings.DesktopAppSettings;
184 yield* settings.setWslDistro("Debian");
185 yield* backend.reconcile;
186 assert.equal(startCount, 2);
187 assert.deepEqual(yield* backend.lastPreflightError, Option.none());
188 const nextSpec = registeredSpec;
189 if (nextSpec?.onStateDirOwned === undefined) throw new Error("Expected ownership callback");
190 yield* nextSpec.onStateDirOwned();
191 yield* settings.setWslBackendEnabled(false);
192 yield* backend.reconcile;
193 yield* settings.setWslBackendEnabled(true);
194 yield* backend.reconcile;
195 assert.equal(startCount, 3);
196 assert.deepEqual(yield* backend.lastPreflightError, Option.none());
161197 }).pipe(
162198 Effect.provide(
163199 DesktopWslBackend.layer.pipe(
apps/desktop/src/wsl/DesktopWslBackend.ts+21-1
......@@ -113,6 +113,9 @@ export const layer = Layer.effect(
113113 // Windows, so we record the reason rather than interrupting. Cleared on any
114114 // reconcile state change so it reflects the current attempt.
115115 const preflightErrorRef = yield* Ref.make(Option.none<string>());
116 const ownershipRefusedRef = yield* Ref.make(
117 Option.none<DesktopBackendPool.BackendInstanceId>(),
118 );
116119
117120 const findExistingWslInstance = pool.list.pipe(
118121 Effect.map((instances) => instances.find((instance) => isWslInstanceId(instance.id))),
......@@ -168,7 +171,21 @@ export const layer = Layer.effect(
168171 // fallback — Windows is the primary and keeps working.
169172 onPreflightFailed: (failure) =>
170173 Ref.set(preflightErrorRef, Option.some(failure.reason)).pipe(Effect.as(false)),
171 onReady: () => Ref.set(preflightErrorRef, Option.none()),
174 onStateDirOwned: () =>
175 Ref.set(ownershipRefusedRef, Option.some(targetId)).pipe(
176 Effect.andThen(
177 Ref.set(
178 preflightErrorRef,
179 Option.some(
180 "Another T3 Code server is using this distro's T3 home. Stop it, or pair with it instead.",
181 ),
182 ),
183 ),
184 ),
185 onReady: () =>
186 Ref.set(ownershipRefusedRef, Option.none()).pipe(
187 Effect.andThen(Ref.set(preflightErrorRef, Option.none())),
188 ),
172189 })
173190 .pipe(
174191 Effect.asSome,
......@@ -217,6 +234,8 @@ export const layer = Layer.effect(
217234 const isIdle =
218235 !snapshot.ready && Option.isNone(snapshot.activePid) && !snapshot.restartScheduled;
219236 if (isIdle) {
237 const refused = yield* Ref.get(ownershipRefusedRef);
238 if (Option.isSome(refused) && refused.value === existingInstance.id) return;
220239 yield* logWslBackendInfo("retrying idle WSL backend", { id: existingInstance.id });
221240 yield* Ref.set(preflightErrorRef, Option.none());
222241 yield* existingInstance.start;
......@@ -228,6 +247,7 @@ export const layer = Layer.effect(
228247 // any stale secondary preflight error so it reflects this fresh attempt;
229248 // onPreflightFailed re-sets it only if the new secondary exhausts retries.
230249 yield* Ref.set(preflightErrorRef, Option.none());
250 yield* Ref.set(ownershipRefusedRef, Option.none());
231251
232252 if (Option.isSome(existingId)) {
233253 yield* logWslBackendInfo("tearing down WSL backend", { id: existingId.value });
apps/server/src/cli/project.ts+9-1
......@@ -26,6 +26,8 @@ import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
2626
2727import * as ServerConfig from "../config.ts";
2828import * as SqlitePersistence from "../persistence/Sqlite.ts";
29import * as ProcessRunner from "../processRunner.ts";
30import * as ServerOwnership from "../serverOwnership.ts";
2931import * as RuntimeLayer from "../orchestration-v2/runtimeLayer.ts";
3032import * as ProjectEnrichmentService from "../project/ProjectEnrichmentService.ts";
3133import * as ProjectFaviconResolver from "../project/ProjectFaviconResolver.ts";
......@@ -373,7 +375,7 @@ const tryResolveLiveProjectExecutionMode = Effect.fn("tryResolveLiveProjectExecu
373375 origin: runtimeState.value.origin,
374376 cause: attempted.failure,
375377 });
376 yield* clearPersistedServerRuntimeState(config.serverRuntimeStatePath);
378 // The offline path decides whether that server still owns the home.
377379 return Option.none<{ readonly origin: string }>();
378380 },
379381);
......@@ -424,6 +426,11 @@ const runProjectMutation = Effect.fn("runProjectMutation")(function* (
424426 Layer.provide(Layer.succeed(References.MinimumLogLevel, minimumLogLevel)),
425427 );
426428
429 // An unreachable server may still be running; never write behind it.
430 yield* ServerOwnership.acquireServerOwnership(config.serverRuntimeStatePath).pipe(
431 Effect.provide(ProcessRunner.layer),
432 );
433 yield* clearPersistedServerRuntimeState(config.serverRuntimeStatePath);
427434 return yield* Effect.gen(function* () {
428435 const snapshot = yield* getOfflineSnapshot();
429436 const projects = yield* ProjectService.ProjectService;
......@@ -435,6 +442,7 @@ const runProjectMutation = Effect.fn("runProjectMutation")(function* (
435442 yield* Console.log(output);
436443 }).pipe(Effect.provide(layerOfflineRuntime));
437444 }).pipe(
445 Effect.scoped,
438446 Effect.provide(
439447 Layer.mergeAll(EnvironmentAuth.layerRuntime, WorkspacePaths.layer).pipe(
440448 Layer.provideMerge(FetchHttpClient.layer),
apps/server/src/server.ts+24-33
......@@ -76,6 +76,7 @@ import * as GitManager from "./git/GitManager.ts";
7676import * as EnvironmentTheme from "./environmentTheme.ts";
7777import * as Keybindings from "./keybindings.ts";
7878import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts";
79import * as ServerOwnership from "./serverOwnership.ts";
7980import * as AgentAwarenessRelay from "./relay/AgentAwarenessRelay.ts";
8081import { hasCloudPublicConfig } from "./cloud/publicConfig.ts";
8182import * as ServerSettings from "./serverSettings.ts";
......@@ -170,11 +171,7 @@ import * as ThreadSettlementService from "./orchestration-v2/ThreadSettlementSer
170171import * as ThreadPullRequestService from "./orchestration-v2/ThreadPullRequestService.ts";
171172import * as RunFinalizationService from "./orchestration-v2/RunFinalizationService.ts";
172173import * as ProjectionStoreV2 from "./orchestration-v2/ProjectionStore.ts";
173import {
174 clearPersistedServerRuntimeState,
175 makePersistedServerRuntimeState,
176 persistServerRuntimeState,
177} from "./serverRuntimeState.ts";
174import { makePersistedServerRuntimeState } from "./serverRuntimeState.ts";
178175import * as OrchestrationHttp from "./orchestration-v2/http.ts";
179176import * as ProjectHttp from "./project/http.ts";
180177import * as NetService from "@t3tools/shared/Net";
......@@ -746,6 +743,8 @@ const layerMakeRoutes = Layer.mergeAll(
746743const layerMakeServer = Layer.unwrap(
747744 Effect.gen(function* () {
748745 const config = yield* ServerConfig.ServerConfig;
746 // Before anything opens the database: one server per T3 home.
747 const ownership = yield* ServerOwnership.acquireServerOwnership(config.serverRuntimeStatePath);
749748 const activation = yield* Deferred.make<void>();
750749 const awaitActivation = Deferred.await(activation);
751750 const layerActivation = Layer.succeed(ServerActivation.ServerActivation, awaitActivation);
......@@ -765,38 +764,30 @@ const layerMakeServer = Layer.unwrap(
765764 }),
766765 );
767766 const layerRuntimeState = Layer.effectDiscard(
768 Effect.acquireRelease(
769 Effect.gen(function* () {
770 yield* Deferred.succeed(runtimeStateParked, undefined).pipe(Effect.orDie);
771 yield* awaitActivation;
772 const server = yield* HttpServer.HttpServer;
773 const address = server.address;
774 if (typeof address === "string" || !("port" in address)) {
775 return;
776 }
767 Effect.gen(function* () {
768 yield* Deferred.succeed(runtimeStateParked, undefined).pipe(Effect.orDie);
769 yield* awaitActivation;
770 const server = yield* HttpServer.HttpServer;
771 const address = server.address;
772 if (typeof address === "string" || !("port" in address)) {
773 return;
774 }
777775
778 const launcher = yield* ServiceLauncherClient.ServiceLauncherClient;
779 const state = yield* makePersistedServerRuntimeState({
780 config,
781 port: address.port,
782 serviceManaged: launcher.managed,
783 });
784 yield* persistServerRuntimeState({
785 path: config.serverRuntimeStatePath,
786 state,
787 }).pipe(
776 const launcher = yield* ServiceLauncherClient.ServiceLauncherClient;
777 const state = yield* makePersistedServerRuntimeState({
778 config,
779 port: address.port,
780 serviceManaged: launcher.managed,
781 });
782 // Ownership release removes this record.
783 yield* ownership
784 .publish(state)
785 .pipe(
788786 Effect.catchCause((cause) =>
789787 Effect.logWarning("Failed to persist server runtime state", { cause }),
790788 ),
791789 );
792 }),
793 () =>
794 clearPersistedServerRuntimeState(config.serverRuntimeStatePath).pipe(
795 Effect.catchCause((cause) =>
796 Effect.logWarning("Failed to clear server runtime state", { cause }),
797 ),
798 ),
799 ),
790 }),
800791 );
801792 const layerTailscaleServe = config.tailscaleServeEnabled
802793 ? Layer.effectDiscard(
......@@ -1119,7 +1110,7 @@ const layerMakeServer = Layer.unwrap(
11191110 Layer.provideMerge(layerPlatformServices),
11201111 );
11211112 }),
1122);
1113).pipe(Layer.provide(ProcessRunner.layer));
11231114
11241115// The CLI supplies configuration.
11251116export const runServer = Layer.launch(layerMakeServer);
apps/server/src/serverOwnership.ts created+147
......@@ -0,0 +1,147 @@
1// @effect-diagnostics nodeBuiltinImport:off - The ownership lock is a native SQLite handle held for the server's lifetime.
2import * as NodeFS from "node:fs";
3import * as NodePath from "node:path";
4import * as NodeSqlite from "node:sqlite";
5
6import { SERVER_EXIT_CODE_STATE_DIR_OWNED } from "@t3tools/contracts";
7import * as HostProcess from "@t3tools/shared/HostProcess";
8import * as Crypto from "effect/Crypto";
9import * as Duration from "effect/Duration";
10import * as Effect from "effect/Effect";
11import * as Option from "effect/Option";
12import * as Runtime from "effect/Runtime";
13import * as Schema from "effect/Schema";
14
15import * as ProcessRunner from "./processRunner.ts";
16import {
17 clearPersistedServerRuntimeState,
18 isProcessAlive,
19 persistServerRuntimeState,
20 readPersistedServerRuntimeState,
21 type PersistedServerRuntimeState,
22} from "./serverRuntimeState.ts";
23
24export class ServerAlreadyRunningError extends Schema.TaggedError<ServerAlreadyRunningError>()(
25 "ServerAlreadyRunningError",
26 { stateDir: Schema.String },
27) {
28 // Distinct process exit code so a supervisor can tell "owned by another
29 // server" apart from a crash and stop restarting.
30 override readonly [Runtime.errorExitCode] = SERVER_EXIT_CODE_STATE_DIR_OWNED;
31
32 override get message(): string {
33 return `A T3 Code server already owns ${this.stateDir}. Stop that server, or use a separate T3 home and pair with it, then retry. No server was stopped.`;
34 }
35}
36
37export class ServerOwnershipError extends Schema.TaggedError<ServerOwnershipError>()(
38 "ServerOwnershipError",
39 { stateDir: Schema.String, cause: Schema.Defect() },
40) {
41 override get message(): string {
42 return `Could not acquire server ownership of ${this.stateDir}.`;
43 }
44}
45
46const isSqliteBusy = (cause: unknown) =>
47 cause instanceof Error &&
48 (("errcode" in cause && cause.errcode === 5) ||
49 ("code" in cause && cause.code === "SQLITE_BUSY"));
50
51/**
52 * Servers from before this lock only publish `server-runtime.json`. Treat
53 * their record as stale only when the process start time proves PID reuse.
54 */
55const legacyOwnerIsLive = Effect.fn("legacyOwnerIsLive")(function* (
56 state: PersistedServerRuntimeState,
57) {
58 if (!isProcessAlive(state.pid)) return false;
59 const recordedAt = Date.parse(state.startedAt);
60 if (!Number.isFinite(recordedAt)) return true;
61 const windows = (yield* HostProcess.Platform) === "win32";
62 const runner = yield* ProcessRunner.ProcessRunner;
63 const result = yield* runner
64 .run({
65 command: windows ? "powershell.exe" : "ps",
66 args: windows
67 ? [
68 "-NoProfile",
69 "-NonInteractive",
70 "-Command",
71 `(Get-Process -Id ${state.pid} -ErrorAction Stop).StartTime.ToUniversalTime().ToString('o')`,
72 ]
73 : ["-p", String(state.pid), "-o", "lstart="],
74 env: { LC_ALL: "C", TZ: "UTC" },
75 timeout: Duration.seconds(2),
76 maxOutputBytes: 16_384,
77 })
78 .pipe(Effect.option);
79 if (Option.isNone(result) || result.value.code !== 0) return true;
80 const output = result.value.stdout.trim();
81 const startedAt = Date.parse(windows ? output : `${output} UTC`);
82 // ps reports whole seconds. Unknown identity stays conservative.
83 return !Number.isFinite(startedAt) || startedAt <= recordedAt + 1_000;
84});
85
86/**
87 * Hold an exclusive lock on the state directory until the scope closes. The
88 * lock is an exclusive transaction on `server-owner.sqlite`, which never holds
89 * data and must never be unlinked: the OS releases it when the process exits,
90 * even on SIGKILL, so a crashed owner never blocks the next start.
91 */
92export const acquireServerOwnership = Effect.fn("acquireServerOwnership")(function* (
93 statePath: string,
94) {
95 const stateDir = NodePath.dirname(statePath);
96 const crypto = yield* Crypto.Crypto;
97 const ownerId = yield* crypto.randomUUIDv4.pipe(
98 Effect.mapError((cause) => new ServerOwnershipError({ stateDir, cause })),
99 );
100 yield* Effect.acquireRelease(
101 Effect.try({
102 try: () => {
103 NodeFS.mkdirSync(stateDir, { recursive: true });
104 const db = new NodeSqlite.DatabaseSync(NodePath.join(stateDir, "server-owner.sqlite"));
105 try {
106 db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;");
107 } catch (cause) {
108 db.close();
109 throw cause;
110 }
111 return db;
112 },
113 catch: (cause) =>
114 isSqliteBusy(cause)
115 ? new ServerAlreadyRunningError({ stateDir })
116 : new ServerOwnershipError({ stateDir, cause }),
117 }),
118 (db) =>
119 readPersistedServerRuntimeState(statePath).pipe(
120 // Only remove discovery this owner published.
121 Effect.flatMap((state) =>
122 Option.isSome(state) && state.value.ownerId === ownerId
123 ? clearPersistedServerRuntimeState(statePath)
124 : Effect.void,
125 ),
126 Effect.ignore({ log: true }),
127 Effect.ensuring(Effect.sync(() => db.close())),
128 ),
129 );
130
131 // An unreadable record cannot name a live owner; this server replaces it.
132 const previous = yield* readPersistedServerRuntimeState(statePath).pipe(
133 Effect.orElseSucceed(() => Option.none<PersistedServerRuntimeState>()),
134 );
135 if (
136 Option.isSome(previous) &&
137 previous.value.ownerId === undefined &&
138 (yield* legacyOwnerIsLive(previous.value))
139 ) {
140 return yield* new ServerAlreadyRunningError({ stateDir });
141 }
142
143 return {
144 publish: (state: PersistedServerRuntimeState) =>
145 persistServerRuntimeState({ path: statePath, state: { ...state, ownerId } }),
146 };
147});
apps/server/src/serverRuntimeState.test.ts+99
......@@ -9,6 +9,8 @@ import * as Path from "effect/Path";
99import * as References from "effect/References";
1010import * as Schema from "effect/Schema";
1111
12import * as ProcessRunner from "./processRunner.ts";
13import * as ServerOwnership from "./serverOwnership.ts";
1214import * as ServerRuntimeState from "./serverRuntimeState.ts";
1315
1416const isServerRuntimeStateError = Schema.is(ServerRuntimeState.ServerRuntimeStateError);
......@@ -18,6 +20,103 @@ interface CapturedLog {
1820 readonly annotations: Readonly<Record<string, unknown>>;
1921}
2022
23describe("server ownership", () => {
24 const layerOwnership = ProcessRunner.layer.pipe(Layer.provideMerge(NodeServices.layer));
25 const makeState = ServerRuntimeState.makePersistedServerRuntimeState({
26 config: { host: undefined, devUrl: undefined },
27 port: 3773,
28 });
29
30 it.effect("lets only one of two racing starts own a state directory", () =>
31 Effect.gen(function* () {
32 const fs = yield* FileSystem.FileSystem;
33 const path = yield* Path.Path;
34 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-owner-race-" });
35 const statePath = path.join(root, "server-runtime.json");
36 const attempts = yield* Effect.all(
37 [
38 ServerOwnership.acquireServerOwnership(statePath).pipe(Effect.result),
39 ServerOwnership.acquireServerOwnership(statePath).pipe(Effect.result),
40 ],
41 { concurrency: "unbounded" },
42 );
43 assert.equal(attempts.filter((result) => result._tag === "Success").length, 1);
44 const refused = attempts.find((result) => result._tag === "Failure");
45 assert.equal(refused?.failure._tag, "ServerAlreadyRunningError");
46 }).pipe(Effect.provide(layerOwnership)),
47 );
48
49 it.effect("refuses a second owner without touching its record, then allows a restart", () =>
50 Effect.gen(function* () {
51 const fs = yield* FileSystem.FileSystem;
52 const path = yield* Path.Path;
53 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-owner-" });
54 const statePath = path.join(root, "server-runtime.json");
55 const state = yield* makeState;
56 yield* Effect.scoped(
57 Effect.gen(function* () {
58 const owner = yield* ServerOwnership.acquireServerOwnership(statePath);
59 yield* owner.publish(state);
60 const before = yield* fs.readFileString(statePath);
61 const refused = yield* ServerOwnership.acquireServerOwnership(statePath).pipe(
62 Effect.flip,
63 );
64 assert.equal(refused._tag, "ServerAlreadyRunningError");
65 assert.equal(yield* fs.readFileString(statePath), before);
66 }),
67 );
68 assert.isFalse(yield* fs.exists(statePath));
69 yield* Effect.scoped(ServerOwnership.acquireServerOwnership(statePath));
70 }).pipe(Effect.provide(layerOwnership)),
71 );
72
73 it.effect("refuses beside a live pre-lock server and ignores a stale record", () =>
74 Effect.gen(function* () {
75 const fs = yield* FileSystem.FileSystem;
76 const path = yield* Path.Path;
77 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-owner-legacy-" });
78 const statePath = path.join(root, "server-runtime.json");
79 const state = yield* makeState;
80 yield* ServerRuntimeState.persistServerRuntimeState({
81 path: statePath,
82 // @effect-diagnostics-next-line globalDateInEffect:off - Legacy identity uses the real process start time, not TestClock.
83 state: { ...state, startedAt: new Date().toISOString() },
84 });
85 const refused = yield* Effect.scoped(ServerOwnership.acquireServerOwnership(statePath)).pipe(
86 Effect.flip,
87 );
88 assert.equal(refused._tag, "ServerAlreadyRunningError");
89 yield* ServerRuntimeState.persistServerRuntimeState({
90 path: statePath,
91 state: { ...state, pid: 0 },
92 });
93 yield* Effect.scoped(ServerOwnership.acquireServerOwnership(statePath));
94 }).pipe(Effect.provide(layerOwnership)),
95 );
96
97 it.effect("never removes a runtime record another server published", () =>
98 Effect.gen(function* () {
99 const fs = yield* FileSystem.FileSystem;
100 const path = yield* Path.Path;
101 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-owner-cleanup-" });
102 const statePath = path.join(root, "server-runtime.json");
103 const state = yield* makeState;
104 yield* Effect.scoped(
105 Effect.gen(function* () {
106 const owner = yield* ServerOwnership.acquireServerOwnership(statePath);
107 yield* owner.publish(state);
108 yield* ServerRuntimeState.persistServerRuntimeState({
109 path: statePath,
110 state: { ...state, ownerId: "other-owner" },
111 });
112 }),
113 );
114 const remaining = yield* ServerRuntimeState.readPersistedServerRuntimeState(statePath);
115 assert.equal(Option.getOrThrow(remaining).ownerId, "other-owner");
116 }).pipe(Effect.provide(layerOwnership)),
117 );
118});
119
21120describe("serverRuntimeState", () => {
22121 it.effect("persists and reads the runtime state", () =>
23122 Effect.gen(function* () {
apps/server/src/serverRuntimeState.ts+2
......@@ -11,6 +11,7 @@ import { formatHostForUrl, isWildcardHost } from "./startupAccess.ts";
1111export const PersistedServerRuntimeState = Schema.Struct({
1212 version: Schema.Literal(1),
1313 pid: Schema.Int,
14 ownerId: Schema.optional(Schema.String),
1415 host: Schema.optional(Schema.String),
1516 port: Schema.Int,
1617 origin: Schema.String,
......@@ -119,6 +120,7 @@ export const clearPersistedServerRuntimeState = (path: string) =>
119120 * alive.
120121 */
121122export const isProcessAlive = (pid: number): boolean => {
123 if (!Number.isInteger(pid) || pid <= 0) return false;
122124 try {
123125 process.kill(pid, 0);
124126 return true;
apps/server/src/serviceLauncher.test.ts+42
......@@ -1,5 +1,7 @@
11import * as NodeServices from "@effect/platform-node/NodeServices";
22import { assert, it } from "@effect/vitest";
3import { SERVER_EXIT_CODE_STATE_DIR_OWNED } from "@t3tools/contracts";
4import { vi } from "vite-plus/test";
35import * as Effect from "effect/Effect";
46import * as FileSystem from "effect/FileSystem";
57import * as Path from "effect/Path";
......@@ -158,6 +160,46 @@ it.layer(NodeServices.layer)("service state persistence", (it) => {
158160 }),
159161 );
160162
163 it.effect("stays idle instead of exiting when another server owns the T3 home", () =>
164 Effect.gen(function* () {
165 const fs = yield* FileSystem.FileSystem;
166 const path = yield* Path.Path;
167 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-refused-" });
168 const statePath = path.join(root, "runtime", "service-state.json");
169 yield* writeFakeRuntime(
170 fs,
171 path,
172 path.join(root, "runtime", "versions", "1.0.0"),
173 `process.exit(${SERVER_EXIT_CODE_STATE_DIR_OWNED});\n`,
174 );
175 yield* Effect.promise(() =>
176 writeServiceState(statePath, {
177 protocol: SERVICE_LAUNCHER_PROTOCOL,
178 activeVersion: "1.0.0",
179 }),
180 );
181 const refused = Promise.withResolvers<void>();
182 yield* Effect.acquireRelease(
183 Effect.sync(() =>
184 vi.spyOn(process.stderr, "write").mockImplementation((chunk) => {
185 if (String(chunk).includes("owns this T3 home")) refused.resolve();
186 return true;
187 }),
188 ),
189 (spy) => Effect.sync(() => spy.mockRestore()),
190 );
191 const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)));
192 let completed = false;
193 const running = launcher.run().finally(() => {
194 completed = true;
195 });
196 yield* Effect.promise(() => refused.promise);
197 assert.isFalse(completed);
198 yield* Effect.promise(() => launcher.stop("SIGTERM"));
199 yield* Effect.promise(() => running);
200 }),
201 );
202
161203 it.effect("serializes shutdown with launcher recovery", () =>
162204 Effect.gen(function* () {
163205 const fs = yield* FileSystem.FileSystem;
apps/server/src/serviceLauncher.ts+33
......@@ -33,6 +33,9 @@ import {
3333const HANDOFF_DELAY_MS = 2_000;
3434const PREPARED_TIMEOUT_MS = 120_000;
3535const TERMINATE_GRACE_MS = 5_000;
36// Matches SERVER_EXIT_CODE_STATE_DIR_OWNED in contracts. The standalone
37// launcher stays independent of the Effect-backed contracts package.
38const STATE_DIR_OWNED_EXIT_CODE = 78;
3639
3740type TerminalStatus = "committed" | "rolled-back" | "failed";
3841type ChildRole = "active" | "trial";
......@@ -579,6 +582,10 @@ export class Launcher {
579582 ): Promise<void> {
580583 if (this.#child !== child || this.#stopping) return;
581584 this.#child = null;
585 if (code === STATE_DIR_OWNED_EXIT_CODE) {
586 await this.#idleWhileStateDirOwned();
587 return;
588 }
582589 if (child.role === "trial") {
583590 this.#clearTimer();
584591 const pending = this.#state.update;
......@@ -602,6 +609,32 @@ export class Launcher {
602609 throw new Error(`Active child exited unexpectedly (${String(code ?? signal ?? "unknown")}).`);
603610 }
604611
612 /**
613 * Another server owns the T3 home. Exiting would make systemd or launchd
614 * start another refused server, so stay idle until stopped or restarted.
615 * A pending update fails without restoring its backup: the other owner may
616 * have written to the database since.
617 */
618 async #idleWhileStateDirOwned(): Promise<void> {
619 this.#clearTimer();
620 const pending = this.#state.update;
621 if (pending?.status === "pending") {
622 const next: ServiceState = {
623 ...this.#state,
624 activeVersion: pending.fromVersion,
625 update: terminalUpdate({ pending, status: "failed", reason: "state-dir-owned" }),
626 };
627 await writeServiceState(this.#statePath, next);
628 this.#state = next;
629 await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined);
630 }
631 // Signal listeners alone do not keep Node alive.
632 this.#timer = setInterval(() => {}, 2_147_483_647);
633 process.stderr.write(
634 "[service-launcher] Another T3 Code server owns this T3 home. Stop it, then restart this service.\n",
635 );
636 }
637
605638 async #returnToPrevious(
606639 pending: PendingServiceUpdate,
607640 status: "rolled-back" | "failed",
docs/user/updating.md+4
......@@ -99,6 +99,10 @@ update can roll back to the previous version. If the update still fails:
99992. Check that you updated the server's machine, not only the device you are using.
1001003. For a command-line server, stop it and relaunch the exact version shown in the notice.
101101
102Only one server can use a T3 home directory at a time. If startup reports that
103another server owns it, stop that server before retrying, or use a separate home
104directory and pair with the running server.
105
102106## Update providers
103107
104108**Settings → Providers** shows provider updates for the selected environment.
packages/contracts/src/desktopBootstrap.ts+3
......@@ -2,6 +2,9 @@ import * as Schema from "effect/Schema";
22
33import { PortSchema, PositiveInt, TrimmedNonEmptyString } from "./baseSchemas.ts";
44
5// A supervisor must not restart a server refused by the state-directory owner.
6export const SERVER_EXIT_CODE_STATE_DIR_OWNED = 78;
7
58export const DesktopBackendBootstrap = Schema.Struct({
69 mode: Schema.Literal("desktop"),
710 noBrowser: Schema.Boolean,