diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index a9f91c2c87bbeba82fd2a6cdc748c6a9a570abfe..9f86b5d71238efc18ed068368d95635df2c78eca 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -1,12 +1,22 @@ +// @effect-diagnostics-next-line nodeBuiltinImport:off - observes the children the launcher spawns with Node built-ins. +import type * as NodeChildProcess from "node:child_process"; + import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import { SERVER_EXIT_CODE_STATE_DIR_OWNED } from "@t3tools/contracts"; -import { vi } from "vite-plus/test"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; +import { vi } from "vite-plus/test"; -import { Launcher, readServiceState, writeServiceState } from "./serviceLauncher.ts"; +import { + databaseBackupBlocker, + Launcher, + PREPARED_TIMEOUT_MS, + type BackupSpaceProbe, + readServiceState, + writeServiceState, +} from "./serviceLauncher.ts"; import { compareExactServiceVersions, decodeServiceState, @@ -16,6 +26,25 @@ import { SERVICE_STOP_MARKER_FILE, } from "./cloud/serviceProtocol.ts"; +const spawnedChildren = vi.hoisted((): Array => []); + +vi.mock("node:child_process", async (importOriginal) => { + const actual = await importOriginal(); + const spawn = (...args: Parameters) => { + const child = actual.spawn(...args); + spawnedChildren.push(child); + return child; + }; + return { ...actual, spawn: spawn as typeof actual.spawn }; +}); + +const writeSparseFile = (filePath: string, size: number) => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + yield* fileSystem.writeFileString(filePath, ""); + yield* fileSystem.truncate(filePath, size); + }); + it("accepts only exact semantic versions", () => { for (const version of ["0.0.0", "1.2.3", "1.2.3-alpha.1", "1.2.3-0", "1.2.3+001"]) { assert.isTrue(isExactServiceVersion(version), version); @@ -338,24 +367,123 @@ if (context.update?.status === "pending") { }), ); - it.effect("restores the database when a migrating trial exits", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-db-" }); - const statePath = path.join(root, "runtime", "service-state.json"); - const databasePath = path.join(root, "userdata", "state.sqlite"); - const original = "database before migration"; - yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); - yield* fs.writeFileString(databasePath, original); - const encodedDatabasePath = JSON.stringify(databasePath); - const childSource = ` -import { writeFileSync } from "node:fs"; + // Migrations on a large database can keep a trial busy for minutes before it + // reports prepared. Only a trial that stays stuck past the window rolls back. + it.effect.each([ + { prepares: true, status: "committed", activeVersion: "1.1.0" }, + { prepares: false, status: "rolled-back", activeVersion: "1.0.0" }, + ] as const)( + "gives a trial the full prepared window (prepares: $prepares)", + ({ prepares, status, activeVersion }) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-slow-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + yield* fs.writeFileString(databasePath, "before trial"); + // The trial stands in for slow migrations: it reports prepared only + // when the test tells it to. + const childSource = ` +const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); +if (context.update?.status !== "pending") process.exit(0); +process.on("message", (message) => { + if (message.type === "test-prepare") process.send({ type: "prepared", updateId: context.update.id }); + if (message.type === "committed") process.exit(0); +}); +setInterval(() => {}, 1_000); +`; + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + childSource, + ); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + update: { + id: "update-1", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + dbPath: databasePath, + status: "pending", + }, + }), + ); + + // Effect schedules on setImmediate, so faking only timeouts leaves it running. + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + yield* Effect.addFinalizer(() => Effect.sync(() => vi.useRealTimers())); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + ); + // Each run ends when the final active child exits on its own. + const running = launcher.run().then( + () => Promise.reject(new Error("launcher unexpectedly completed")), + () => Promise.resolve(), + ); + // The prepared timer is armed once the trial has spawned. + yield* Effect.promise(() => + vi.waitFor(() => assert.equal(vi.getTimerCount(), 1), { timeout: 10_000 }), + ); + yield* Effect.promise(() => vi.advanceTimersByTimeAsync(121_000)); + if (prepares) spawnedChildren.at(-1)!.send({ type: "test-prepare" }); + else yield* Effect.promise(() => vi.advanceTimersByTimeAsync(PREPARED_TIMEOUT_MS)); + yield* Effect.promise(() => running); + + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(state.activeVersion, activeVersion); + assert.equal(state.update?.status, status); + if (!prepares) { + assert.equal( + state.update?.status === "rolled-back" ? state.update.reason : undefined, + "prepared-timeout", + ); + } + }), + ); + + it.effect.each(["file", "symlink", "dangling symlink"] as const)( + "restores the database when a migrating trial exits, with the database a %s", + (kind) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-db-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + // A symlinked database keeps its link: rollback restores the file it points to, + // even one the trial left missing, as a crash during a macOS clone restore can. + // The dangling case reaches its file through a directory link and a relative + // `..` link, which only resolves correctly by following the directory link. + const databaseFile = + kind === "file" ? databasePath : path.join(root, "data", "original.sqlite"); + const linkTarget = kind === "dangling symlink" ? "../original.sqlite" : databaseFile; + const original = "database before migration"; + yield* fs.makeDirectory(path.dirname(databaseFile), { recursive: true }); + if (kind === "dangling symlink") { + yield* fs.makeDirectory(path.join(root, "data", "sub")); + yield* fs.symlink(path.join(root, "data", "sub"), path.dirname(databasePath)); + } else { + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + } + yield* fs.writeFileString(databaseFile, original); + if (kind !== "file") yield* fs.symlink(linkTarget, databasePath); + const encodedDatabasePath = JSON.stringify(databasePath); + const childSource = ` +import { realpathSync, unlinkSync, writeFileSync } from "node:fs"; const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); if (context.update?.status === "pending") { writeFileSync(context.update.dbPath, "database after migration"); writeFileSync(context.update.dbPath + "-wal", "trial wal"); writeFileSync(context.update.dbPath + "-shm", "trial shm"); + ${kind === "dangling symlink" ? "unlinkSync(realpathSync(context.update.dbPath));" : ""} process.exit(1); } else if (context.update === undefined) { process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} }); @@ -363,6 +491,78 @@ if (context.update?.status === "pending") { } else { process.exit(0); } +`; + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + childSource, + ); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + ); + yield* Effect.promise(() => + launcher.run().then( + () => Promise.reject(new Error("launcher unexpectedly completed")), + () => Promise.resolve(), + ), + ); + + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(state.activeVersion, "1.0.0"); + assert.equal(state.update?.status, "rolled-back"); + assert.equal(yield* fs.readFileString(databaseFile), original); + if (kind !== "file") assert.equal(yield* fs.readLink(databasePath), linkTarget); + assert.isFalse(yield* fs.exists(path.join(root, "original.sqlite"))); + assert.isFalse(yield* fs.exists(`${databasePath}-wal`)); + assert.isFalse(yield* fs.exists(`${databasePath}-shm`)); + const updateId = state.update?.id; + assert.isDefined(updateId); + assert.isFalse(yield* fs.exists(path.join(root, "runtime", "db-backup", updateId))); + }), + ); + + const requestRejectedUpdate = (backupSpaceProbe: Partial) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-space-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + const replyPath = path.join(root, "rejection.txt"); + const trialMarker = path.join(root, "trial-started"); + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + yield* fs.writeFileString(databasePath, "database"); + const encoded = JSON.stringify({ databasePath, replyPath, trialMarker }); + const childSource = ` +import { writeFileSync } from "node:fs"; +const paths = ${encoded}; +const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); +if (context.update?.status === "pending") { + writeFileSync(paths.trialMarker, "started"); + process.exit(1); +} else if (context.update === undefined) { + process.on("message", (message) => { + if (message.type === "update-rejected") { + writeFileSync(paths.replyPath, message.reason); + process.exit(0); + } + }); + process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: paths.databasePath }); + setInterval(() => {}, 1_000); +} else { + process.exit(0); +} `; for (const version of ["1.0.0", "1.1.0"]) { yield* writeFakeRuntime( @@ -379,7 +579,11 @@ if (context.update?.status === "pending") { }), ); - const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { backupSpaceProbe }, + ); yield* Effect.promise(() => launcher.run().then( () => Promise.reject(new Error("launcher unexpectedly completed")), @@ -387,15 +591,171 @@ if (context.update?.status === "pending") { ), ); + return { + reason: yield* fs.readFileString(replyPath), + trialStarted: yield* fs.exists(trialMarker), + state: yield* Effect.promise(() => readServiceState(statePath)), + }; + }); + + it.effect("refuses a backup that cannot fit and leaves the server running", () => + Effect.gen(function* () { + const { reason, trialStarted, state } = yield* requestRejectedUpdate({ + readFreeBytes: () => Promise.resolve(0), + canClone: () => Promise.resolve(true), + }); + + assert.include(reason, "Not enough free disk space"); + assert.include(reason, "The server was not stopped"); + assert.isFalse(trialStarted); + assert.equal(state.activeVersion, "1.0.0"); + assert.isUndefined(state.update); + }), + ); + + it.effect("refuses the update when free space cannot be checked", () => + Effect.gen(function* () { + const { reason, trialStarted, state } = yield* requestRejectedUpdate({ + readFreeBytes: () => Promise.reject(new Error("statfs unavailable")), + }); + + assert.include( + reason, + "Could not check that the database backup fits before updating: statfs unavailable", + ); + assert.isFalse(trialStarted); + assert.equal(state.activeVersion, "1.0.0"); + assert.isUndefined(state.update); + }), + ); + + it.effect("records no update when the server exits during the backup check", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-exit-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + yield* fs.writeFileString(databasePath, "database"); + const encoded = JSON.stringify({ databasePath }); + const childSource = ` +const paths = ${encoded}; +process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: paths.databasePath }, () => + process.exit(3), +); +setInterval(() => {}, 1_000); +`; + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + childSource, + ); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { + backupSpaceProbe: { + // Answer only once the server that asked has disconnected. + readFreeBytes: () => { + const child = spawnedChildren.at(-1)!; + return new Promise((resolve) => { + if (!child.connected) resolve(Number.MAX_SAFE_INTEGER); + else child.once("disconnect", () => resolve(Number.MAX_SAFE_INTEGER)); + }); + }, + }, + }, + ); + const failure = yield* Effect.promise(() => + launcher.run().then( + () => new Error("launcher unexpectedly completed"), + (cause: unknown) => cause as Error, + ), + ); + + assert.include(failure.message, "Active child exited unexpectedly (3)"); const state = yield* Effect.promise(() => readServiceState(statePath)); assert.equal(state.activeVersion, "1.0.0"); - assert.equal(state.update?.status, "rolled-back"); - assert.equal(yield* fs.readFileString(databasePath), original); - assert.isFalse(yield* fs.exists(`${databasePath}-wal`)); - assert.isFalse(yield* fs.exists(`${databasePath}-shm`)); - const updateId = state.update?.id; - assert.isDefined(updateId); - assert.isFalse(yield* fs.exists(path.join(root, "runtime", "db-backup", updateId))); + assert.isUndefined(state.update); + }), + ); + + it.effect("checks database backup space against clone or full-copy needs", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-blocker-" }); + const databasePath = path.join(root, "state.sqlite"); + yield* writeSparseFile(databasePath, 10 * 1024 ** 3); + const check = (free: number, cloneable: boolean) => + Effect.promise(() => + databaseBackupBlocker(root, databasePath, { + readFreeBytes: () => Promise.resolve(free), + canClone: () => Promise.resolve(cloneable), + }), + ); + + // A full copy fits: no clone needed. + assert.isUndefined(yield* check(11 * 1024 ** 3, false)); + // A clone needs 10% headroom for the trial's writes, at least 1 GiB. + assert.isUndefined(yield* check(1.5 * 1024 ** 3, true)); + assert.include((yield* check(0.5 * 1024 ** 3, true)) ?? "", "1.0 GB needed, 0.5 GB free"); + // A filesystem that cannot clone needs room for the full copy. + const blocker = yield* check(8 * 1024 ** 3, false); + assert.include(blocker ?? "", "Not enough free disk space"); + assert.include(blocker ?? "", "10.5 GB needed, 8.0 GB free"); + assert.include(blocker ?? "", "The server was not stopped"); + }), + ); + + it.effect("fails the backup check when the database cannot be inspected", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-blocker-" }); + const error = yield* Effect.promise(() => + databaseBackupBlocker(root, path.join(root, "missing.sqlite"), { + readFreeBytes: () => Promise.resolve(0), + }).then( + () => undefined, + (cause: unknown) => cause, + ), + ); + assert.equal((error as NodeJS.ErrnoException | undefined)?.code, "ENOENT"); + }), + ); + + it.effect("recognizes a clone into the backup directory on this filesystem", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-clone-" }); + const databasePath = path.join(root, "userdata", "state.sqlite"); + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + yield* fs.writeFileString(databasePath, "database"); + // With no free space, the result shows which requirement applied. + const blocker = yield* Effect.promise(() => + databaseBackupBlocker(root, databasePath, { readFreeBytes: () => Promise.resolve(0) }), + ); + // A tiny database: 1.0 GB is the clone minimum, 0.0 GB the full copy. The + // macOS temp directory is on APFS, so the clone requirement applies there. + assert.include( + blocker ?? "", + // oxlint-disable-next-line t3code/no-global-process-runtime -- mirrors the standalone launcher's own platform check. + process.platform === "darwin" ? "1.0 GB needed, 0.0 GB free" : "GB needed, 0.0 GB free", + ); + // Linux probes leave nothing behind in the backup directory. + assert.deepEqual(yield* fs.readDirectory(path.join(root, "runtime", "db-backup")), []); }), ); }); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index b3b005abfaa74e51f588003329c80fca4d6947f8..313a25421f88e2ac47e57087a0291da48877ddd7 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -31,7 +31,15 @@ import { } from "./cloud/serviceProtocol.ts"; const HANDOFF_DELAY_MS = 2_000; -const PREPARED_TIMEOUT_MS = 120_000; +/** + * How long a trial has to report prepared. Migrations and the open-time + * `PRAGMA optimize` count toward it and scale with the database: tens of GB on + * a cold page cache take minutes. They run synchronously on the child's main + * thread, so it cannot report progress meanwhile. A missed window rolls back + * the database, so the next attempt pays the same cost. Crashes are caught at + * exit; this only bounds a trial that is alive but stuck. + */ +export const PREPARED_TIMEOUT_MS = 30 * 60_000; const TERMINATE_GRACE_MS = 5_000; // Matches SERVER_EXIT_CODE_STATE_DIR_OWNED in contracts. The standalone // launcher stays independent of the Effect-backed contracts package. @@ -109,6 +117,147 @@ async function syncDirectory(directory: string): Promise { } } +function runCp(args: ReadonlyArray): Promise { + return new Promise((resolve) => { + NodeChildProcess.execFile("cp", args, (error) => resolve(error === null)); + }); +} + +/** + * Copies a database file as a copy-on-write clone where the filesystem has + * them (APFS via `cp -c`, btrfs/XFS reflinks via `cp --reflink=auto`), which + * takes no extra space and finishes instantly for a database of tens of GB. + * Node's own clone flag is not used: on macOS `fs.copyFile` falls back to a + * full copy. Anywhere cloning is unavailable, or `cp` fails, this is a + * regular copy. + */ +async function copyDatabaseFile(source: string, destination: string): Promise { + // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime. + const platform = process.platform; + // `cp -c` replaces the destination with a new file, which would cut a + // hard-linked database off from its other links; copy into it instead. + const hardLinked = + platform === "darwin" && + ((await NodeFSP.stat(destination).catch(() => undefined))?.nlink ?? 1) > 1; + const args = + platform === "darwin" && !hardLinked + ? ["-c", source, destination] + : platform === "linux" + ? ["--reflink=auto", source, destination] + : undefined; + if (args === undefined || !(await runCp(args))) await NodeFSP.copyFile(source, destination); +} + +const GIB = 1024 ** 3; +/** + * Clones share blocks, but the trial server diverges from them as it writes, + * migrations included, so a cloned backup still needs room for that churn. + */ +const CLONED_BACKUP_MIN_HEADROOM_BYTES = GIB; +const CLONED_BACKUP_HEADROOM_FRACTION = 0.1; + +/** + * Stats of the database and its existing sidecars, following symlinks. A main + * file that cannot be read is an error. + */ +async function statDatabaseFiles(dbPath: string): Promise> { + const stats: Array = []; + for (const suffix of DB_FILE_SUFFIXES) { + const stat = await NodeFSP.stat(`${dbPath}${suffix}`).catch((cause: unknown) => { + if (suffix !== "" && (cause as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw cause; + }); + if (stat !== undefined) stats.push(stat); + } + return stats; +} + +/** + * Whether `copyDatabaseFile` will clone the database into the backup + * directory rather than copy it. + * - macOS: `cp -c` clones within one APFS volume and silently copies + * anywhere else, so every existing database file must share the backup + * directory's device, whose filesystem type must match the system volume's, + * which is always APFS. + * - Linux: clone each existing database file with `cp --reflink=always`, + * which fails instead of copying when a file cannot be reflinked (another + * filesystem, or a btrfs file marked No_COW), then discard the clones. + * - Elsewhere: never. + */ +async function backupCanClone(dbPath: string, backupRoot: string): Promise { + // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime. + const platform = process.platform; + if (platform === "darwin") { + const [backupDirectory, backupFs, systemFs] = await Promise.all([ + NodeFSP.stat(backupRoot), + NodeFSP.statfs(backupRoot), + NodeFSP.statfs("/"), + ]); + if (backupFs.type !== systemFs.type) return false; + // A file linked from another volume is caught because `stat` follows + // symlinks. A hard-linked file is restored by a full copy, so it needs the + // full space. + return (await statDatabaseFiles(dbPath)).every( + (source) => source.dev === backupDirectory.dev && source.nlink === 1, + ); + } + if (platform !== "linux") return false; + const probe = NodePath.join(backupRoot, `.clone-probe-${NodeCrypto.randomUUID()}`); + try { + for (const suffix of DB_FILE_SUFFIXES) { + const source = `${dbPath}${suffix}`; + if (suffix !== "" && !(await pathExists(source))) continue; + if (!(await runCp(["--reflink=always", source, `${probe}${suffix}`]))) return false; + } + return true; + } finally { + for (const suffix of DB_FILE_SUFFIXES) { + await NodeFSP.rm(`${probe}${suffix}`, { force: true }).catch(() => undefined); + } + } +} + +export interface BackupSpaceProbe { + readonly readFreeBytes: (directory: string) => Promise; + readonly canClone: (dbPath: string, backupRoot: string) => Promise; +} + +const filesystemBackupSpaceProbe: BackupSpaceProbe = { + readFreeBytes: async (directory) => { + const stats = await NodeFSP.statfs(directory); + return stats.bavail * stats.bsize; + }, + canClone: backupCanClone, +}; + +/** + * Returns a client-visible reason when the pre-update database backup cannot + * fit, or undefined when it can. Runs before the server is stopped so a full + * disk refuses the update instead of causing an outage and a cold restart. + * Throws when the database or the filesystem cannot be inspected. + */ +export async function databaseBackupBlocker( + baseDir: string, + dbPath: string, + probe: Partial = {}, +): Promise { + const { readFreeBytes, canClone } = { ...filesystemBackupSpaceProbe, ...probe }; + const backupRoot = NodePath.join(baseDir, "runtime", "db-backup"); + await NodeFSP.mkdir(backupRoot, { recursive: true, mode: 0o700 }); + const [files, free] = await Promise.all([statDatabaseFiles(dbPath), readFreeBytes(backupRoot)]); + const size = files.reduce((total, file) => total + file.size, 0); + const fullCopyBytes = size * 1.05; + if (free >= fullCopyBytes) return undefined; + const clonedBytes = Math.max( + CLONED_BACKUP_MIN_HEADROOM_BYTES, + size * CLONED_BACKUP_HEADROOM_FRACTION, + ); + const needed = (await canClone(dbPath, backupRoot)) ? clonedBytes : fullCopyBytes; + if (free >= needed) return undefined; + const gb = (bytes: number) => (bytes / GIB).toFixed(1); + return `Not enough free disk space to back up the database before updating: ${gb(needed)} GB needed, ${gb(free)} GB free. Free up disk space and try again. The server was not stopped.`; +} + /** * Snapshots the database once per update before the first trial. A completed * backup is never overwritten because a restarted launcher may be looking at @@ -126,7 +275,7 @@ async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate const source = `${pending.dbPath}${suffix}`; if (suffix !== "" && !(await pathExists(source))) continue; const destination = databaseBackupFile(stagingDir, suffix); - await NodeFSP.copyFile(source, destination); + await copyDatabaseFile(source, destination); await syncFile(destination); } await NodeFSP.rename(stagingDir, backupDir); @@ -157,6 +306,34 @@ async function markDatabaseRestorePending(backupDir: string): Promise { } } +/** + * The file a restore writes for `path`: where its symlinks lead, even when the + * last target is missing, as after a crash between `cp -c` removing a file + * and cloning over it. + */ +async function restoreTarget(path: string): Promise { + let current = path; + for (let hops = 0; hops < 40; hops++) { + try { + return await NodeFSP.realpath(current); + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code !== "ENOENT") throw cause; + } + const link = await NodeFSP.readlink(current).catch((cause: unknown) => { + const code = (cause as NodeJS.ErrnoException).code; + if (code === "ENOENT" || code === "EINVAL") return undefined; + throw cause; + }); + if (link === undefined) return current; + // Joined without normalizing: a `..` in the link must follow directory links + // the way the filesystem does, not cancel a path segment. + current = NodePath.isAbsolute(link) + ? link + : `${await NodeFSP.realpath(NodePath.dirname(current))}${NodePath.sep}${link}`; + } + throw new Error(`Too many symbolic links resolving ${path}.`); +} + /** Restore is retryable after any process crash while the backup directory remains. */ async function restoreDatabaseBackup( baseDir: string, @@ -166,17 +343,22 @@ async function restoreDatabaseBackup( if (!(await pathExists(backupDir))) return; await markDatabaseRestorePending(backupDir); + const directories = new Set([NodePath.dirname(pending.dbPath)]); for (const suffix of DB_FILE_SUFFIXES) { - const target = `${pending.dbPath}${suffix}`; + const livePath = `${pending.dbPath}${suffix}`; const source = databaseBackupFile(backupDir, suffix); if (await pathExists(source)) { - await NodeFSP.copyFile(source, target); + // A clone replaces its destination, so restore into the file a symlinked + // database resolves to and keep the link itself. + const target = await restoreTarget(livePath); + await copyDatabaseFile(source, target); await syncFile(target); + directories.add(NodePath.dirname(target)); } else { - await NodeFSP.rm(target, { force: true }); + await NodeFSP.rm(livePath, { force: true }); } } - await syncDirectory(NodePath.dirname(pending.dbPath)); + for (const directory of directories) await syncDirectory(directory); } async function discardDatabaseBackup(baseDir: string, updateId: string): Promise { @@ -291,9 +473,15 @@ export class Launcher { #stopping = false; #done = false; readonly #completion = Promise.withResolvers(); + readonly #backupSpaceProbe: Partial | undefined; - constructor(baseDir: string, state: ServiceState) { + constructor( + baseDir: string, + state: ServiceState, + options: { readonly backupSpaceProbe?: Partial } = {}, + ) { this.#baseDir = baseDir; + this.#backupSpaceProbe = options.backupSpaceProbe; this.#statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE); this.#state = state; } @@ -512,6 +700,23 @@ export class Launcher { await reject("The requested target runtime is missing or incomplete."); return; } + // A check that cannot run refuses the update too: the alternative is + // stopping the server without knowing whether the backup fits. + const backupBlocker = await databaseBackupBlocker( + this.#baseDir, + message.dbPath, + this.#backupSpaceProbe, + ).catch( + (error: unknown) => + `Could not check that the database backup fits before updating: ${error instanceof Error ? error.message : String(error)}. The server was not stopped.`, + ); + // The child may have exited during the check. Its queued exit transition + // reports that; answering it or recording a pending update would not. + if (!child.process.connected) return; + if (backupBlocker !== undefined) { + await reject(backupBlocker); + return; + } const pending: PendingServiceUpdate = { id: NodeCrypto.randomUUID(), diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index 59d06b0abe3de5978cab6dbf1658040439f5ea7a..3a8774f4d530c6b7bcd26ebc74ff7b7143bfd2a4 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -30,10 +30,18 @@ ready to commit. A failed or timed-out trial returns to the old version. After commit, the target is authoritative and the service manager's ordinary restart policy applies. +The prepared deadline is long because migrations and SQLite's open-time optimize +scale with the database and block the trial's main thread, so it cannot report +progress. A missed deadline restores the snapshot, and the retry pays the same +cost again. A crashed trial is caught when it exits; the deadline bounds only a +stuck one. Client update waits must outlast the deadline, or a slow update shows +as failed. + ## Database rollback -After the old child exits, the launcher snapshots SQLite's main file, WAL, and -shared-memory file. This makes trial migrations reversible without down +Before stopping the old child, the launcher refuses an update whose snapshot will +not fit on disk, so a full disk does not cause an outage. After the old child +exits, the launcher snapshots SQLite's main file, WAL, and shared-memory file. This makes trial migrations reversible without down migrations. The snapshot is made once per update and survives launcher restarts; replacing it during a retry could capture changes from the failed trial. diff --git a/packages/client-runtime/src/state/server.ts b/packages/client-runtime/src/state/server.ts index 31aadd0025bb7c9d6facbb7fa1e9d8f693ebbb9c..cc2f21b7271d9960db24102f29d4bf0fdfbd38eb 100644 --- a/packages/client-runtime/src/state/server.ts +++ b/packages/client-runtime/src/state/server.ts @@ -127,9 +127,9 @@ export class ServerUpdateTerminalError extends Schema.TaggedError