diff --git a/apps/server/src/observability/HeapSnapshot.test.ts b/apps/server/src/observability/HeapSnapshot.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..0beb466e50194ee968cc3391be7f74cd77627167 --- /dev/null +++ b/apps/server/src/observability/HeapSnapshot.test.ts @@ -0,0 +1,36 @@ +// @effect-diagnostics nodeBuiltinImport:off - tests fake a failed write at the native v8 boundary. +import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; +import * as NodeV8 from "node:v8"; +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import { vi } from "vite-plus/test"; + +import { writeHeapSnapshot } from "./HeapSnapshot.ts"; + +vi.mock("node:v8", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, writeHeapSnapshot: vi.fn(actual.writeHeapSnapshot) }; +}); + +it.layer(NodeServices.layer)("writeHeapSnapshot", (it) => { + it.effect("removes the partial file when the write fails", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const logsDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-heap-snapshot-test-" }); + let partialPath: string | undefined; + vi.mocked(NodeV8.writeHeapSnapshot).mockImplementationOnce((path) => { + partialPath = path; + if (path) NodeFS.writeFileSync(path, "partial"); + throw new Error("ENOSPC: no space left on device"); + }); + + yield* writeHeapSnapshot(logsDir); + + assert.strictEqual(NodePath.dirname(partialPath ?? ""), logsDir); + assert.deepEqual(yield* fs.readDirectory(logsDir), []); + }), + ); +}); diff --git a/apps/server/src/observability/HeapSnapshot.ts b/apps/server/src/observability/HeapSnapshot.ts new file mode 100644 index 0000000000000000000000000000000000000000..827dfab2c6d6911677347a1fe5016d59cbea5588 --- /dev/null +++ b/apps/server/src/observability/HeapSnapshot.ts @@ -0,0 +1,52 @@ +// @effect-diagnostics nodeBuiltinImport:off - v8.writeHeapSnapshot has no Effect equivalent. +import * as NodePath from "node:path"; +import * as NodeV8 from "node:v8"; + +import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; + +import * as ServerConfig from "../config.ts"; + +/** + * Writes one V8 heap snapshot into `logsDir` and logs its path. A failed write + * logs a warning and removes any partial file, because that file can hold + * secrets and the failure is often a full disk. + */ +export const writeHeapSnapshot = Effect.fn("server.heapSnapshot", { root: true })( + function* (logsDir: string) { + const fs = yield* FileSystem.FileSystem; + const timestamp = DateTime.formatIso(yield* DateTime.now).replaceAll(":", "-"); + const path = NodePath.join(logsDir, `server-${process.pid}-${timestamp}.heapsnapshot`); + yield* Effect.annotateCurrentSpan({ path }); + yield* Effect.try(() => NodeV8.writeHeapSnapshot(path)).pipe( + Effect.tapError(() => fs.remove(path, { force: true }).pipe(Effect.ignore)), + ); + yield* Effect.logInfo("Wrote heap snapshot.", { path }); + }, + Effect.catch((cause) => Effect.logWarning("Failed to write heap snapshot.", { cause })), +); + +/** + * Writes a heap snapshot when the process gets SIGUSR2 (`kill -USR2 `), + * so a maintainer can see what a long-running server holds. See "Heap + * Snapshots" in docs/operations/observability.md. + * + * The write blocks the event loop, so two snapshots never overlap: a signal + * sent during a write waits until it finishes. Windows has no SIGUSR2, so the + * layer does nothing there. + */ +export const layer = Layer.effectDiscard( + Effect.gen(function* () { + if ((yield* HostProcessPlatform) === "win32") return; + const { logsDir } = yield* ServerConfig.ServerConfig; + const runFork = Effect.runForkWith(yield* Effect.context()); + const onSignal = () => void runFork(writeHeapSnapshot(logsDir)); + yield* Effect.acquireRelease( + Effect.sync(() => process.on("SIGUSR2", onSignal)), + () => Effect.sync(() => process.off("SIGUSR2", onSignal)), + ); + }), +); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 4f264ae1cb0d50037e4c5d1b8b6b80488b96520a..e64b0702eb83e4fda6794f43f42b23b679739820 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -119,6 +119,7 @@ import * as SourceControlRepositoryService from "./sourceControl/SourceControlRe import * as ProjectSetupScriptRunner from "./project/ProjectSetupScriptRunner.ts"; import * as WorktreeSetupTracker from "./project/WorktreeSetupTracker.ts"; import { ObservabilityLive } from "./observability/Layers/Observability.ts"; +import * as HeapSnapshot from "./observability/HeapSnapshot.ts"; import * as ServerEnvironment from "./environment/ServerEnvironment.ts"; import * as RemoteOpenTargets from "./environment/RemoteOpenTargets.ts"; import { authHttpApiLayer, environmentAuthenticatedAuthLayer } from "./auth/http.ts"; @@ -957,6 +958,7 @@ const makeServerLayer = Layer.unwrap( runtimeStateLayer.pipe(Layer.provide(launcherLayer)), tailscaleServeLayer, cloudDesiredLinkReconcileLayer, + HeapSnapshot.layer, ); return serverApplicationLayer.pipe( diff --git a/docs/operations/observability.md b/docs/operations/observability.md index da020ac051530d43c815ae18badedf4499f2f4f9..d9dac58e2c3b0abbc42f90be061a8fc47daef03d 100644 --- a/docs/operations/observability.md +++ b/docs/operations/observability.md @@ -618,3 +618,40 @@ Current high-value span and metric boundaries include: - logs outside spans are not persisted in the trace file; SSH-managed launch stdout/stderr is still captured in its launcher log - metrics are not snapshotted locally + +## Heap Snapshots + +To see what a long-running server holds in memory, send it `SIGUSR2`. The server writes a V8 heap +snapshot to its logs dir and logs the path. This works for desktop, `npx t3`, and service installs +on macOS and Linux. Windows has no `SIGUSR2`. + +Send the signal to the server pid in `server-runtime.json`, which sits in the server's state dir +next to the `logs` dir. For a dev server or a `--home-dir` launch, use that server's state dir from +[Traces](#traces). Do not send it to the desktop app or the service launcher: a process without the +handler exits on `SIGUSR2`. After a crash the file can keep a stale pid that now belongs to a +different process, so check the pid first. + +```bash +pid="$(jq .pid "${T3CODE_HOME:-$HOME/.t3}/userdata/server-runtime.json")" +ps -p "$pid" -o command= +``` + +If `ps` shows the T3 Code server, send the signal: + +```bash +kill -USR2 "$pid" +``` + +The file is `/server--.heapsnapshot`, next to `server.trace.ndjson`. To +open it, use the Memory tab in Chrome DevTools and select Load. + +Before you take one: + +- The server stops while it writes the file. For a large heap this can take a minute or more. + Connected clients can reconnect during the pause, and an event loop monitor, if the server has + one, records the pause as a stall. Send the signal once. A second signal sent during a write + takes another snapshot after the first one finishes. +- The write needs about as much free memory as the heap uses. On a machine that is already + swapping, it can make the problem worse or crash the server. +- The file contains everything in server memory, including tokens, secrets, and thread content. Do + not share it publicly. Delete it when you are done, because storage cleanup does not remove it.