authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-10-14 02:46:53-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-14 02:46:53-07:00
log3417bc3856a25fa772f969faeaf8428a58010d47
tree0ff8361d89503d84cd9d53ee53e07f439c33cdeb
parentbc95bbd790580ff22bf1351edfa84231493c4ecc
signature Signed by PGP key B5690EEEBB952194

fix(server): service updates no longer fail on full disks or large databases (#19311)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: SAPHID <saphid@users.noreply.github.com>

4 files changed, 604 insertions(+), 31 deletions(-)

apps/server/src/serviceLauncher.test.ts+379-19
...@@ -1,12 +1,22 @@...@@ -1,12 +1,22 @@
1// @effect-diagnostics-next-line nodeBuiltinImport:off - observes the children the launcher spawns with Node built-ins.
2import type * as NodeChildProcess from "node:child_process";
3
1import * as NodeServices from "@effect/platform-node/NodeServices";4import * as NodeServices from "@effect/platform-node/NodeServices";
2import { assert, it } from "@effect/vitest";5import { assert, it } from "@effect/vitest";
3import { SERVER_EXIT_CODE_STATE_DIR_OWNED } from "@t3tools/contracts";6import { SERVER_EXIT_CODE_STATE_DIR_OWNED } from "@t3tools/contracts";
4import { vi } from "vite-plus/test";
5import * as Effect from "effect/Effect";7import * as Effect from "effect/Effect";
6import * as FileSystem from "effect/FileSystem";8import * as FileSystem from "effect/FileSystem";
7import * as Path from "effect/Path";9import * as Path from "effect/Path";
10import { vi } from "vite-plus/test";
811
9import { Launcher, readServiceState, writeServiceState } from "./serviceLauncher.ts";12import {
13 databaseBackupBlocker,
14 Launcher,
15 PREPARED_TIMEOUT_MS,
16 type BackupSpaceProbe,
17 readServiceState,
18 writeServiceState,
19} from "./serviceLauncher.ts";
10import {20import {
11 compareExactServiceVersions,21 compareExactServiceVersions,
12 decodeServiceState,22 decodeServiceState,
...@@ -16,6 +26,25 @@ import {...@@ -16,6 +26,25 @@ import {
16 SERVICE_STOP_MARKER_FILE,26 SERVICE_STOP_MARKER_FILE,
17} from "./cloud/serviceProtocol.ts";27} from "./cloud/serviceProtocol.ts";
1828
29const spawnedChildren = vi.hoisted((): Array<NodeChildProcess.ChildProcess> => []);
30
31vi.mock("node:child_process", async (importOriginal) => {
32 const actual = await importOriginal<typeof NodeChildProcess>();
33 const spawn = (...args: Parameters<typeof actual.spawn>) => {
34 const child = actual.spawn(...args);
35 spawnedChildren.push(child);
36 return child;
37 };
38 return { ...actual, spawn: spawn as typeof actual.spawn };
39});
40
41const writeSparseFile = (filePath: string, size: number) =>
42 Effect.gen(function* () {
43 const fileSystem = yield* FileSystem.FileSystem;
44 yield* fileSystem.writeFileString(filePath, "");
45 yield* fileSystem.truncate(filePath, size);
46 });
47
19it("accepts only exact semantic versions", () => {48it("accepts only exact semantic versions", () => {
20 for (const version of ["0.0.0", "1.2.3", "1.2.3-alpha.1", "1.2.3-0", "1.2.3+001"]) {49 for (const version of ["0.0.0", "1.2.3", "1.2.3-alpha.1", "1.2.3-0", "1.2.3+001"]) {
21 assert.isTrue(isExactServiceVersion(version), version);50 assert.isTrue(isExactServiceVersion(version), version);
...@@ -338,27 +367,198 @@ if (context.update?.status === "pending") {...@@ -338,27 +367,198 @@ if (context.update?.status === "pending") {
338 }),367 }),
339 );368 );
340369
341 it.effect("restores the database when a migrating trial exits", () =>370 // Migrations on a large database can keep a trial busy for minutes before it
371 // reports prepared. Only a trial that stays stuck past the window rolls back.
372 it.effect.each([
373 { prepares: true, status: "committed", activeVersion: "1.1.0" },
374 { prepares: false, status: "rolled-back", activeVersion: "1.0.0" },
375 ] as const)(
376 "gives a trial the full prepared window (prepares: $prepares)",
377 ({ prepares, status, activeVersion }) =>
378 Effect.gen(function* () {
379 const fs = yield* FileSystem.FileSystem;
380 const path = yield* Path.Path;
381 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-slow-" });
382 const statePath = path.join(root, "runtime", "service-state.json");
383 const databasePath = path.join(root, "userdata", "state.sqlite");
384 yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
385 yield* fs.writeFileString(databasePath, "before trial");
386 // The trial stands in for slow migrations: it reports prepared only
387 // when the test tells it to.
388 const childSource = `
389const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
390if (context.update?.status !== "pending") process.exit(0);
391process.on("message", (message) => {
392 if (message.type === "test-prepare") process.send({ type: "prepared", updateId: context.update.id });
393 if (message.type === "committed") process.exit(0);
394});
395setInterval(() => {}, 1_000);
396`;
397 for (const version of ["1.0.0", "1.1.0"]) {
398 yield* writeFakeRuntime(
399 fs,
400 path,
401 path.join(root, "runtime", "versions", version),
402 childSource,
403 );
404 }
405 yield* Effect.promise(() =>
406 writeServiceState(statePath, {
407 protocol: SERVICE_LAUNCHER_PROTOCOL,
408 activeVersion: "1.0.0",
409 update: {
410 id: "update-1",
411 fromVersion: "1.0.0",
412 targetVersion: "1.1.0",
413 dbPath: databasePath,
414 status: "pending",
415 },
416 }),
417 );
418
419 // Effect schedules on setImmediate, so faking only timeouts leaves it running.
420 vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
421 yield* Effect.addFinalizer(() => Effect.sync(() => vi.useRealTimers()));
422 const launcher = new Launcher(
423 root,
424 yield* Effect.promise(() => readServiceState(statePath)),
425 );
426 // Each run ends when the final active child exits on its own.
427 const running = launcher.run().then(
428 () => Promise.reject(new Error("launcher unexpectedly completed")),
429 () => Promise.resolve(),
430 );
431 // The prepared timer is armed once the trial has spawned.
432 yield* Effect.promise(() =>
433 vi.waitFor(() => assert.equal(vi.getTimerCount(), 1), { timeout: 10_000 }),
434 );
435 yield* Effect.promise(() => vi.advanceTimersByTimeAsync(121_000));
436 if (prepares) spawnedChildren.at(-1)!.send({ type: "test-prepare" });
437 else yield* Effect.promise(() => vi.advanceTimersByTimeAsync(PREPARED_TIMEOUT_MS));
438 yield* Effect.promise(() => running);
439
440 const state = yield* Effect.promise(() => readServiceState(statePath));
441 assert.equal(state.activeVersion, activeVersion);
442 assert.equal(state.update?.status, status);
443 if (!prepares) {
444 assert.equal(
445 state.update?.status === "rolled-back" ? state.update.reason : undefined,
446 "prepared-timeout",
447 );
448 }
449 }),
450 );
451
452 it.effect.each(["file", "symlink", "dangling symlink"] as const)(
453 "restores the database when a migrating trial exits, with the database a %s",
454 (kind) =>
455 Effect.gen(function* () {
456 const fs = yield* FileSystem.FileSystem;
457 const path = yield* Path.Path;
458 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-db-" });
459 const statePath = path.join(root, "runtime", "service-state.json");
460 const databasePath = path.join(root, "userdata", "state.sqlite");
461 // A symlinked database keeps its link: rollback restores the file it points to,
462 // even one the trial left missing, as a crash during a macOS clone restore can.
463 // The dangling case reaches its file through a directory link and a relative
464 // `..` link, which only resolves correctly by following the directory link.
465 const databaseFile =
466 kind === "file" ? databasePath : path.join(root, "data", "original.sqlite");
467 const linkTarget = kind === "dangling symlink" ? "../original.sqlite" : databaseFile;
468 const original = "database before migration";
469 yield* fs.makeDirectory(path.dirname(databaseFile), { recursive: true });
470 if (kind === "dangling symlink") {
471 yield* fs.makeDirectory(path.join(root, "data", "sub"));
472 yield* fs.symlink(path.join(root, "data", "sub"), path.dirname(databasePath));
473 } else {
474 yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
475 }
476 yield* fs.writeFileString(databaseFile, original);
477 if (kind !== "file") yield* fs.symlink(linkTarget, databasePath);
478 const encodedDatabasePath = JSON.stringify(databasePath);
479 const childSource = `
480import { realpathSync, unlinkSync, writeFileSync } from "node:fs";
481const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
482if (context.update?.status === "pending") {
483 writeFileSync(context.update.dbPath, "database after migration");
484 writeFileSync(context.update.dbPath + "-wal", "trial wal");
485 writeFileSync(context.update.dbPath + "-shm", "trial shm");
486 ${kind === "dangling symlink" ? "unlinkSync(realpathSync(context.update.dbPath));" : ""}
487 process.exit(1);
488} else if (context.update === undefined) {
489 process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} });
490 setInterval(() => {}, 1_000);
491} else {
492 process.exit(0);
493}
494`;
495 for (const version of ["1.0.0", "1.1.0"]) {
496 yield* writeFakeRuntime(
497 fs,
498 path,
499 path.join(root, "runtime", "versions", version),
500 childSource,
501 );
502 }
503 yield* Effect.promise(() =>
504 writeServiceState(statePath, {
505 protocol: SERVICE_LAUNCHER_PROTOCOL,
506 activeVersion: "1.0.0",
507 }),
508 );
509
510 const launcher = new Launcher(
511 root,
512 yield* Effect.promise(() => readServiceState(statePath)),
513 );
514 yield* Effect.promise(() =>
515 launcher.run().then(
516 () => Promise.reject(new Error("launcher unexpectedly completed")),
517 () => Promise.resolve(),
518 ),
519 );
520
521 const state = yield* Effect.promise(() => readServiceState(statePath));
522 assert.equal(state.activeVersion, "1.0.0");
523 assert.equal(state.update?.status, "rolled-back");
524 assert.equal(yield* fs.readFileString(databaseFile), original);
525 if (kind !== "file") assert.equal(yield* fs.readLink(databasePath), linkTarget);
526 assert.isFalse(yield* fs.exists(path.join(root, "original.sqlite")));
527 assert.isFalse(yield* fs.exists(`${databasePath}-wal`));
528 assert.isFalse(yield* fs.exists(`${databasePath}-shm`));
529 const updateId = state.update?.id;
530 assert.isDefined(updateId);
531 assert.isFalse(yield* fs.exists(path.join(root, "runtime", "db-backup", updateId)));
532 }),
533 );
534
535 const requestRejectedUpdate = (backupSpaceProbe: Partial<BackupSpaceProbe>) =>
342 Effect.gen(function* () {536 Effect.gen(function* () {
343 const fs = yield* FileSystem.FileSystem;537 const fs = yield* FileSystem.FileSystem;
344 const path = yield* Path.Path;538 const path = yield* Path.Path;
345 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-db-" });539 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-space-" });
346 const statePath = path.join(root, "runtime", "service-state.json");540 const statePath = path.join(root, "runtime", "service-state.json");
347 const databasePath = path.join(root, "userdata", "state.sqlite");541 const databasePath = path.join(root, "userdata", "state.sqlite");
348 const original = "database before migration";542 const replyPath = path.join(root, "rejection.txt");
543 const trialMarker = path.join(root, "trial-started");
349 yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });544 yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
350 yield* fs.writeFileString(databasePath, original);545 yield* fs.writeFileString(databasePath, "database");
351 const encodedDatabasePath = JSON.stringify(databasePath);546 const encoded = JSON.stringify({ databasePath, replyPath, trialMarker });
352 const childSource = `547 const childSource = `
353import { writeFileSync } from "node:fs";548import { writeFileSync } from "node:fs";
549const paths = ${encoded};
354const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);550const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT);
355if (context.update?.status === "pending") {551if (context.update?.status === "pending") {
356 writeFileSync(context.update.dbPath, "database after migration");552 writeFileSync(paths.trialMarker, "started");
357 writeFileSync(context.update.dbPath + "-wal", "trial wal");
358 writeFileSync(context.update.dbPath + "-shm", "trial shm");
359 process.exit(1);553 process.exit(1);
360} else if (context.update === undefined) {554} else if (context.update === undefined) {
361 process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} });555 process.on("message", (message) => {
556 if (message.type === "update-rejected") {
557 writeFileSync(paths.replyPath, message.reason);
558 process.exit(0);
559 }
560 });
561 process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: paths.databasePath });
362 setInterval(() => {}, 1_000);562 setInterval(() => {}, 1_000);
363} else {563} else {
364 process.exit(0);564 process.exit(0);
...@@ -379,7 +579,11 @@ if (context.update?.status === "pending") {...@@ -379,7 +579,11 @@ if (context.update?.status === "pending") {
379 }),579 }),
380 );580 );
381581
382 const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)));582 const launcher = new Launcher(
583 root,
584 yield* Effect.promise(() => readServiceState(statePath)),
585 { backupSpaceProbe },
586 );
383 yield* Effect.promise(() =>587 yield* Effect.promise(() =>
384 launcher.run().then(588 launcher.run().then(
385 () => Promise.reject(new Error("launcher unexpectedly completed")),589 () => Promise.reject(new Error("launcher unexpectedly completed")),
...@@ -387,15 +591,171 @@ if (context.update?.status === "pending") {...@@ -387,15 +591,171 @@ if (context.update?.status === "pending") {
387 ),591 ),
388 );592 );
389593
594 return {
595 reason: yield* fs.readFileString(replyPath),
596 trialStarted: yield* fs.exists(trialMarker),
597 state: yield* Effect.promise(() => readServiceState(statePath)),
598 };
599 });
600
601 it.effect("refuses a backup that cannot fit and leaves the server running", () =>
602 Effect.gen(function* () {
603 const { reason, trialStarted, state } = yield* requestRejectedUpdate({
604 readFreeBytes: () => Promise.resolve(0),
605 canClone: () => Promise.resolve(true),
606 });
607
608 assert.include(reason, "Not enough free disk space");
609 assert.include(reason, "The server was not stopped");
610 assert.isFalse(trialStarted);
611 assert.equal(state.activeVersion, "1.0.0");
612 assert.isUndefined(state.update);
613 }),
614 );
615
616 it.effect("refuses the update when free space cannot be checked", () =>
617 Effect.gen(function* () {
618 const { reason, trialStarted, state } = yield* requestRejectedUpdate({
619 readFreeBytes: () => Promise.reject(new Error("statfs unavailable")),
620 });
621
622 assert.include(
623 reason,
624 "Could not check that the database backup fits before updating: statfs unavailable",
625 );
626 assert.isFalse(trialStarted);
627 assert.equal(state.activeVersion, "1.0.0");
628 assert.isUndefined(state.update);
629 }),
630 );
631
632 it.effect("records no update when the server exits during the backup check", () =>
633 Effect.gen(function* () {
634 const fs = yield* FileSystem.FileSystem;
635 const path = yield* Path.Path;
636 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-exit-" });
637 const statePath = path.join(root, "runtime", "service-state.json");
638 const databasePath = path.join(root, "userdata", "state.sqlite");
639 yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
640 yield* fs.writeFileString(databasePath, "database");
641 const encoded = JSON.stringify({ databasePath });
642 const childSource = `
643const paths = ${encoded};
644process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: paths.databasePath }, () =>
645 process.exit(3),
646);
647setInterval(() => {}, 1_000);
648`;
649 for (const version of ["1.0.0", "1.1.0"]) {
650 yield* writeFakeRuntime(
651 fs,
652 path,
653 path.join(root, "runtime", "versions", version),
654 childSource,
655 );
656 }
657 yield* Effect.promise(() =>
658 writeServiceState(statePath, {
659 protocol: SERVICE_LAUNCHER_PROTOCOL,
660 activeVersion: "1.0.0",
661 }),
662 );
663 const launcher = new Launcher(
664 root,
665 yield* Effect.promise(() => readServiceState(statePath)),
666 {
667 backupSpaceProbe: {
668 // Answer only once the server that asked has disconnected.
669 readFreeBytes: () => {
670 const child = spawnedChildren.at(-1)!;
671 return new Promise((resolve) => {
672 if (!child.connected) resolve(Number.MAX_SAFE_INTEGER);
673 else child.once("disconnect", () => resolve(Number.MAX_SAFE_INTEGER));
674 });
675 },
676 },
677 },
678 );
679 const failure = yield* Effect.promise(() =>
680 launcher.run().then(
681 () => new Error("launcher unexpectedly completed"),
682 (cause: unknown) => cause as Error,
683 ),
684 );
685
686 assert.include(failure.message, "Active child exited unexpectedly (3)");
390 const state = yield* Effect.promise(() => readServiceState(statePath));687 const state = yield* Effect.promise(() => readServiceState(statePath));
391 assert.equal(state.activeVersion, "1.0.0");688 assert.equal(state.activeVersion, "1.0.0");
392 assert.equal(state.update?.status, "rolled-back");689 assert.isUndefined(state.update);
393 assert.equal(yield* fs.readFileString(databasePath), original);690 }),
394 assert.isFalse(yield* fs.exists(`${databasePath}-wal`));691 );
395 assert.isFalse(yield* fs.exists(`${databasePath}-shm`));692
396 const updateId = state.update?.id;693 it.effect("checks database backup space against clone or full-copy needs", () =>
397 assert.isDefined(updateId);694 Effect.gen(function* () {
398 assert.isFalse(yield* fs.exists(path.join(root, "runtime", "db-backup", updateId)));695 const fs = yield* FileSystem.FileSystem;
696 const path = yield* Path.Path;
697 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-blocker-" });
698 const databasePath = path.join(root, "state.sqlite");
699 yield* writeSparseFile(databasePath, 10 * 1024 ** 3);
700 const check = (free: number, cloneable: boolean) =>
701 Effect.promise(() =>
702 databaseBackupBlocker(root, databasePath, {
703 readFreeBytes: () => Promise.resolve(free),
704 canClone: () => Promise.resolve(cloneable),
705 }),
706 );
707
708 // A full copy fits: no clone needed.
709 assert.isUndefined(yield* check(11 * 1024 ** 3, false));
710 // A clone needs 10% headroom for the trial's writes, at least 1 GiB.
711 assert.isUndefined(yield* check(1.5 * 1024 ** 3, true));
712 assert.include((yield* check(0.5 * 1024 ** 3, true)) ?? "", "1.0 GB needed, 0.5 GB free");
713 // A filesystem that cannot clone needs room for the full copy.
714 const blocker = yield* check(8 * 1024 ** 3, false);
715 assert.include(blocker ?? "", "Not enough free disk space");
716 assert.include(blocker ?? "", "10.5 GB needed, 8.0 GB free");
717 assert.include(blocker ?? "", "The server was not stopped");
718 }),
719 );
720
721 it.effect("fails the backup check when the database cannot be inspected", () =>
722 Effect.gen(function* () {
723 const fs = yield* FileSystem.FileSystem;
724 const path = yield* Path.Path;
725 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-blocker-" });
726 const error = yield* Effect.promise(() =>
727 databaseBackupBlocker(root, path.join(root, "missing.sqlite"), {
728 readFreeBytes: () => Promise.resolve(0),
729 }).then(
730 () => undefined,
731 (cause: unknown) => cause,
732 ),
733 );
734 assert.equal((error as NodeJS.ErrnoException | undefined)?.code, "ENOENT");
735 }),
736 );
737
738 it.effect("recognizes a clone into the backup directory on this filesystem", () =>
739 Effect.gen(function* () {
740 const fs = yield* FileSystem.FileSystem;
741 const path = yield* Path.Path;
742 const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-clone-" });
743 const databasePath = path.join(root, "userdata", "state.sqlite");
744 yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true });
745 yield* fs.writeFileString(databasePath, "database");
746 // With no free space, the result shows which requirement applied.
747 const blocker = yield* Effect.promise(() =>
748 databaseBackupBlocker(root, databasePath, { readFreeBytes: () => Promise.resolve(0) }),
749 );
750 // A tiny database: 1.0 GB is the clone minimum, 0.0 GB the full copy. The
751 // macOS temp directory is on APFS, so the clone requirement applies there.
752 assert.include(
753 blocker ?? "",
754 // oxlint-disable-next-line t3code/no-global-process-runtime -- mirrors the standalone launcher's own platform check.
755 process.platform === "darwin" ? "1.0 GB needed, 0.0 GB free" : "GB needed, 0.0 GB free",
756 );
757 // Linux probes leave nothing behind in the backup directory.
758 assert.deepEqual(yield* fs.readDirectory(path.join(root, "runtime", "db-backup")), []);
399 }),759 }),
400 );760 );
401});761});
apps/server/src/serviceLauncher.ts+212-7
...@@ -31,7 +31,15 @@ import {...@@ -31,7 +31,15 @@ import {
31} from "./cloud/serviceProtocol.ts";31} from "./cloud/serviceProtocol.ts";
3232
33const HANDOFF_DELAY_MS = 2_000;33const HANDOFF_DELAY_MS = 2_000;
34const PREPARED_TIMEOUT_MS = 120_000;34/**
35 * How long a trial has to report prepared. Migrations and the open-time
36 * `PRAGMA optimize` count toward it and scale with the database: tens of GB on
37 * a cold page cache take minutes. They run synchronously on the child's main
38 * thread, so it cannot report progress meanwhile. A missed window rolls back
39 * the database, so the next attempt pays the same cost. Crashes are caught at
40 * exit; this only bounds a trial that is alive but stuck.
41 */
42export const PREPARED_TIMEOUT_MS = 30 * 60_000;
35const TERMINATE_GRACE_MS = 5_000;43const TERMINATE_GRACE_MS = 5_000;
36// Matches SERVER_EXIT_CODE_STATE_DIR_OWNED in contracts. The standalone44// Matches SERVER_EXIT_CODE_STATE_DIR_OWNED in contracts. The standalone
37// launcher stays independent of the Effect-backed contracts package.45// launcher stays independent of the Effect-backed contracts package.
...@@ -109,6 +117,147 @@ async function syncDirectory(directory: string): Promise<void> {...@@ -109,6 +117,147 @@ async function syncDirectory(directory: string): Promise<void> {
109 }117 }
110}118}
111119
120function runCp(args: ReadonlyArray<string>): Promise<boolean> {
121 return new Promise((resolve) => {
122 NodeChildProcess.execFile("cp", args, (error) => resolve(error === null));
123 });
124}
125
126/**
127 * Copies a database file as a copy-on-write clone where the filesystem has
128 * them (APFS via `cp -c`, btrfs/XFS reflinks via `cp --reflink=auto`), which
129 * takes no extra space and finishes instantly for a database of tens of GB.
130 * Node's own clone flag is not used: on macOS `fs.copyFile` falls back to a
131 * full copy. Anywhere cloning is unavailable, or `cp` fails, this is a
132 * regular copy.
133 */
134async function copyDatabaseFile(source: string, destination: string): Promise<void> {
135 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
136 const platform = process.platform;
137 // `cp -c` replaces the destination with a new file, which would cut a
138 // hard-linked database off from its other links; copy into it instead.
139 const hardLinked =
140 platform === "darwin" &&
141 ((await NodeFSP.stat(destination).catch(() => undefined))?.nlink ?? 1) > 1;
142 const args =
143 platform === "darwin" && !hardLinked
144 ? ["-c", source, destination]
145 : platform === "linux"
146 ? ["--reflink=auto", source, destination]
147 : undefined;
148 if (args === undefined || !(await runCp(args))) await NodeFSP.copyFile(source, destination);
149}
150
151const GIB = 1024 ** 3;
152/**
153 * Clones share blocks, but the trial server diverges from them as it writes,
154 * migrations included, so a cloned backup still needs room for that churn.
155 */
156const CLONED_BACKUP_MIN_HEADROOM_BYTES = GIB;
157const CLONED_BACKUP_HEADROOM_FRACTION = 0.1;
158
159/**
160 * Stats of the database and its existing sidecars, following symlinks. A main
161 * file that cannot be read is an error.
162 */
163async function statDatabaseFiles(dbPath: string): Promise<ReadonlyArray<NodeFS.Stats>> {
164 const stats: Array<NodeFS.Stats> = [];
165 for (const suffix of DB_FILE_SUFFIXES) {
166 const stat = await NodeFSP.stat(`${dbPath}${suffix}`).catch((cause: unknown) => {
167 if (suffix !== "" && (cause as NodeJS.ErrnoException).code === "ENOENT") return undefined;
168 throw cause;
169 });
170 if (stat !== undefined) stats.push(stat);
171 }
172 return stats;
173}
174
175/**
176 * Whether `copyDatabaseFile` will clone the database into the backup
177 * directory rather than copy it.
178 * - macOS: `cp -c` clones within one APFS volume and silently copies
179 * anywhere else, so every existing database file must share the backup
180 * directory's device, whose filesystem type must match the system volume's,
181 * which is always APFS.
182 * - Linux: clone each existing database file with `cp --reflink=always`,
183 * which fails instead of copying when a file cannot be reflinked (another
184 * filesystem, or a btrfs file marked No_COW), then discard the clones.
185 * - Elsewhere: never.
186 */
187async function backupCanClone(dbPath: string, backupRoot: string): Promise<boolean> {
188 // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
189 const platform = process.platform;
190 if (platform === "darwin") {
191 const [backupDirectory, backupFs, systemFs] = await Promise.all([
192 NodeFSP.stat(backupRoot),
193 NodeFSP.statfs(backupRoot),
194 NodeFSP.statfs("/"),
195 ]);
196 if (backupFs.type !== systemFs.type) return false;
197 // A file linked from another volume is caught because `stat` follows
198 // symlinks. A hard-linked file is restored by a full copy, so it needs the
199 // full space.
200 return (await statDatabaseFiles(dbPath)).every(
201 (source) => source.dev === backupDirectory.dev && source.nlink === 1,
202 );
203 }
204 if (platform !== "linux") return false;
205 const probe = NodePath.join(backupRoot, `.clone-probe-${NodeCrypto.randomUUID()}`);
206 try {
207 for (const suffix of DB_FILE_SUFFIXES) {
208 const source = `${dbPath}${suffix}`;
209 if (suffix !== "" && !(await pathExists(source))) continue;
210 if (!(await runCp(["--reflink=always", source, `${probe}${suffix}`]))) return false;
211 }
212 return true;
213 } finally {
214 for (const suffix of DB_FILE_SUFFIXES) {
215 await NodeFSP.rm(`${probe}${suffix}`, { force: true }).catch(() => undefined);
216 }
217 }
218}
219
220export interface BackupSpaceProbe {
221 readonly readFreeBytes: (directory: string) => Promise<number>;
222 readonly canClone: (dbPath: string, backupRoot: string) => Promise<boolean>;
223}
224
225const filesystemBackupSpaceProbe: BackupSpaceProbe = {
226 readFreeBytes: async (directory) => {
227 const stats = await NodeFSP.statfs(directory);
228 return stats.bavail * stats.bsize;
229 },
230 canClone: backupCanClone,
231};
232
233/**
234 * Returns a client-visible reason when the pre-update database backup cannot
235 * fit, or undefined when it can. Runs before the server is stopped so a full
236 * disk refuses the update instead of causing an outage and a cold restart.
237 * Throws when the database or the filesystem cannot be inspected.
238 */
239export async function databaseBackupBlocker(
240 baseDir: string,
241 dbPath: string,
242 probe: Partial<BackupSpaceProbe> = {},
243): Promise<string | undefined> {
244 const { readFreeBytes, canClone } = { ...filesystemBackupSpaceProbe, ...probe };
245 const backupRoot = NodePath.join(baseDir, "runtime", "db-backup");
246 await NodeFSP.mkdir(backupRoot, { recursive: true, mode: 0o700 });
247 const [files, free] = await Promise.all([statDatabaseFiles(dbPath), readFreeBytes(backupRoot)]);
248 const size = files.reduce((total, file) => total + file.size, 0);
249 const fullCopyBytes = size * 1.05;
250 if (free >= fullCopyBytes) return undefined;
251 const clonedBytes = Math.max(
252 CLONED_BACKUP_MIN_HEADROOM_BYTES,
253 size * CLONED_BACKUP_HEADROOM_FRACTION,
254 );
255 const needed = (await canClone(dbPath, backupRoot)) ? clonedBytes : fullCopyBytes;
256 if (free >= needed) return undefined;
257 const gb = (bytes: number) => (bytes / GIB).toFixed(1);
258 return `Not enough free disk space to back up the database before updating: ${gb(needed)} GB needed, ${gb(free)} GB free. Free up disk space and try again. The server was not stopped.`;
259}
260
112/**261/**
113 * Snapshots the database once per update before the first trial. A completed262 * Snapshots the database once per update before the first trial. A completed
114 * backup is never overwritten because a restarted launcher may be looking at263 * backup is never overwritten because a restarted launcher may be looking at
...@@ -126,7 +275,7 @@ async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate...@@ -126,7 +275,7 @@ async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate
126 const source = `${pending.dbPath}${suffix}`;275 const source = `${pending.dbPath}${suffix}`;
127 if (suffix !== "" && !(await pathExists(source))) continue;276 if (suffix !== "" && !(await pathExists(source))) continue;
128 const destination = databaseBackupFile(stagingDir, suffix);277 const destination = databaseBackupFile(stagingDir, suffix);
129 await NodeFSP.copyFile(source, destination);278 await copyDatabaseFile(source, destination);
130 await syncFile(destination);279 await syncFile(destination);
131 }280 }
132 await NodeFSP.rename(stagingDir, backupDir);281 await NodeFSP.rename(stagingDir, backupDir);
...@@ -157,6 +306,34 @@ async function markDatabaseRestorePending(backupDir: string): Promise<void> {...@@ -157,6 +306,34 @@ async function markDatabaseRestorePending(backupDir: string): Promise<void> {
157 }306 }
158}307}
159308
309/**
310 * The file a restore writes for `path`: where its symlinks lead, even when the
311 * last target is missing, as after a crash between `cp -c` removing a file
312 * and cloning over it.
313 */
314async function restoreTarget(path: string): Promise<string> {
315 let current = path;
316 for (let hops = 0; hops < 40; hops++) {
317 try {
318 return await NodeFSP.realpath(current);
319 } catch (cause) {
320 if ((cause as NodeJS.ErrnoException).code !== "ENOENT") throw cause;
321 }
322 const link = await NodeFSP.readlink(current).catch((cause: unknown) => {
323 const code = (cause as NodeJS.ErrnoException).code;
324 if (code === "ENOENT" || code === "EINVAL") return undefined;
325 throw cause;
326 });
327 if (link === undefined) return current;
328 // Joined without normalizing: a `..` in the link must follow directory links
329 // the way the filesystem does, not cancel a path segment.
330 current = NodePath.isAbsolute(link)
331 ? link
332 : `${await NodeFSP.realpath(NodePath.dirname(current))}${NodePath.sep}${link}`;
333 }
334 throw new Error(`Too many symbolic links resolving ${path}.`);
335}
336
160/** Restore is retryable after any process crash while the backup directory remains. */337/** Restore is retryable after any process crash while the backup directory remains. */
161async function restoreDatabaseBackup(338async function restoreDatabaseBackup(
162 baseDir: string,339 baseDir: string,
...@@ -166,17 +343,22 @@ async function restoreDatabaseBackup(...@@ -166,17 +343,22 @@ async function restoreDatabaseBackup(
166 if (!(await pathExists(backupDir))) return;343 if (!(await pathExists(backupDir))) return;
167344
168 await markDatabaseRestorePending(backupDir);345 await markDatabaseRestorePending(backupDir);
346 const directories = new Set([NodePath.dirname(pending.dbPath)]);
169 for (const suffix of DB_FILE_SUFFIXES) {347 for (const suffix of DB_FILE_SUFFIXES) {
170 const target = `${pending.dbPath}${suffix}`;348 const livePath = `${pending.dbPath}${suffix}`;
171 const source = databaseBackupFile(backupDir, suffix);349 const source = databaseBackupFile(backupDir, suffix);
172 if (await pathExists(source)) {350 if (await pathExists(source)) {
173 await NodeFSP.copyFile(source, target);351 // A clone replaces its destination, so restore into the file a symlinked
352 // database resolves to and keep the link itself.
353 const target = await restoreTarget(livePath);
354 await copyDatabaseFile(source, target);
174 await syncFile(target);355 await syncFile(target);
356 directories.add(NodePath.dirname(target));
175 } else {357 } else {
176 await NodeFSP.rm(target, { force: true });358 await NodeFSP.rm(livePath, { force: true });
177 }359 }
178 }360 }
179 await syncDirectory(NodePath.dirname(pending.dbPath));361 for (const directory of directories) await syncDirectory(directory);
180}362}
181363
182async function discardDatabaseBackup(baseDir: string, updateId: string): Promise<void> {364async function discardDatabaseBackup(baseDir: string, updateId: string): Promise<void> {
...@@ -291,9 +473,15 @@ export class Launcher {...@@ -291,9 +473,15 @@ export class Launcher {
291 #stopping = false;473 #stopping = false;
292 #done = false;474 #done = false;
293 readonly #completion = Promise.withResolvers<void>();475 readonly #completion = Promise.withResolvers<void>();
476 readonly #backupSpaceProbe: Partial<BackupSpaceProbe> | undefined;
294477
295 constructor(baseDir: string, state: ServiceState) {478 constructor(
479 baseDir: string,
480 state: ServiceState,
481 options: { readonly backupSpaceProbe?: Partial<BackupSpaceProbe> } = {},
482 ) {
296 this.#baseDir = baseDir;483 this.#baseDir = baseDir;
484 this.#backupSpaceProbe = options.backupSpaceProbe;
297 this.#statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE);485 this.#statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE);
298 this.#state = state;486 this.#state = state;
299 }487 }
...@@ -512,6 +700,23 @@ export class Launcher {...@@ -512,6 +700,23 @@ export class Launcher {
512 await reject("The requested target runtime is missing or incomplete.");700 await reject("The requested target runtime is missing or incomplete.");
513 return;701 return;
514 }702 }
703 // A check that cannot run refuses the update too: the alternative is
704 // stopping the server without knowing whether the backup fits.
705 const backupBlocker = await databaseBackupBlocker(
706 this.#baseDir,
707 message.dbPath,
708 this.#backupSpaceProbe,
709 ).catch(
710 (error: unknown) =>
711 `Could not check that the database backup fits before updating: ${error instanceof Error ? error.message : String(error)}. The server was not stopped.`,
712 );
713 // The child may have exited during the check. Its queued exit transition
714 // reports that; answering it or recording a pending update would not.
715 if (!child.process.connected) return;
716 if (backupBlocker !== undefined) {
717 await reject(backupBlocker);
718 return;
719 }
515720
516 const pending: PendingServiceUpdate = {721 const pending: PendingServiceUpdate = {
517 id: NodeCrypto.randomUUID(),722 id: NodeCrypto.randomUUID(),
docs/internals/server-updates.md+10-2
...@@ -30,10 +30,18 @@ ready to commit....@@ -30,10 +30,18 @@ ready to commit.
30A failed or timed-out trial returns to the old version. After commit, the target30A failed or timed-out trial returns to the old version. After commit, the target
31is authoritative and the service manager's ordinary restart policy applies.31is authoritative and the service manager's ordinary restart policy applies.
3232
33The prepared deadline is long because migrations and SQLite's open-time optimize
34scale with the database and block the trial's main thread, so it cannot report
35progress. A missed deadline restores the snapshot, and the retry pays the same
36cost again. A crashed trial is caught when it exits; the deadline bounds only a
37stuck one. Client update waits must outlast the deadline, or a slow update shows
38as failed.
39
33## Database rollback40## Database rollback
3441
35After the old child exits, the launcher snapshots SQLite's main file, WAL, and42Before stopping the old child, the launcher refuses an update whose snapshot will
36shared-memory file. This makes trial migrations reversible without down43not fit on disk, so a full disk does not cause an outage. After the old child
44exits, the launcher snapshots SQLite's main file, WAL, and shared-memory file. This makes trial migrations reversible without down
37migrations. The snapshot is made once per update and survives launcher restarts;45migrations. The snapshot is made once per update and survives launcher restarts;
38replacing it during a retry could capture changes from the failed trial.46replacing it during a retry could capture changes from the failed trial.
3947
packages/client-runtime/src/state/server.ts+3-3
...@@ -127,9 +127,9 @@ export class ServerUpdateTerminalError extends Schema.TaggedError<ServerUpdateTe...@@ -127,9 +127,9 @@ export class ServerUpdateTerminalError extends Schema.TaggedError<ServerUpdateTe
127 }127 }
128}128}
129129
130// Covers the 120-second trial deadline and a final restart of the previous130// Covers the service launcher's 30-minute trial deadline (PREPARED_TIMEOUT_MS)
131// version when the trial rolls back.131// and a final restart of the previous version when the trial rolls back.
132const SERVER_UPDATE_RESUME_TIMEOUT = Duration.minutes(4);132const SERVER_UPDATE_RESUME_TIMEOUT = Duration.minutes(32);
133133
134export function matchesServerUpdateReadyEvent(134export function matchesServerUpdateReadyEvent(
135 result: ServerSelfUpdateResult,135 result: ServerSelfUpdateResult,