authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-09-25 20:20:21-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-25 20:20:21-07:00
log1dc8cbe6d14a7b4e131a2806f43bde8b60d6aaa7
tree715cacb9edf4305b14cd2beac718e4fa58703f40
parent6530de0339d2ca49957d0039133c49e3a08557f7
signature Signed by PGP key B5690EEEBB952194

feat(observability): write a server heap snapshot on SIGUSR2 (#13694)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

4 files changed, 127 insertions(+), 0 deletions(-)

apps/server/src/observability/HeapSnapshot.test.ts created+36
...@@ -0,0 +1,36 @@
1// @effect-diagnostics nodeBuiltinImport:off - tests fake a failed write at the native v8 boundary.
2import * as NodeServices from "@effect/platform-node/NodeServices";
3import * as NodeFS from "node:fs";
4import * as NodePath from "node:path";
5import * as NodeV8 from "node:v8";
6import { assert, it } from "@effect/vitest";
7import * as Effect from "effect/Effect";
8import * as FileSystem from "effect/FileSystem";
9import { vi } from "vite-plus/test";
10
11import { writeHeapSnapshot } from "./HeapSnapshot.ts";
12
13vi.mock("node:v8", async (importOriginal) => {
14 const actual = await importOriginal<typeof NodeV8>();
15 return { ...actual, writeHeapSnapshot: vi.fn(actual.writeHeapSnapshot) };
16});
17
18it.layer(NodeServices.layer)("writeHeapSnapshot", (it) => {
19 it.effect("removes the partial file when the write fails", () =>
20 Effect.gen(function* () {
21 const fs = yield* FileSystem.FileSystem;
22 const logsDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-heap-snapshot-test-" });
23 let partialPath: string | undefined;
24 vi.mocked(NodeV8.writeHeapSnapshot).mockImplementationOnce((path) => {
25 partialPath = path;
26 if (path) NodeFS.writeFileSync(path, "partial");
27 throw new Error("ENOSPC: no space left on device");
28 });
29
30 yield* writeHeapSnapshot(logsDir);
31
32 assert.strictEqual(NodePath.dirname(partialPath ?? ""), logsDir);
33 assert.deepEqual(yield* fs.readDirectory(logsDir), []);
34 }),
35 );
36});
apps/server/src/observability/HeapSnapshot.ts created+52
...@@ -0,0 +1,52 @@
1// @effect-diagnostics nodeBuiltinImport:off - v8.writeHeapSnapshot has no Effect equivalent.
2import * as NodePath from "node:path";
3import * as NodeV8 from "node:v8";
4
5import { HostProcessPlatform } from "@t3tools/shared/hostProcess";
6import * as DateTime from "effect/DateTime";
7import * as Effect from "effect/Effect";
8import * as FileSystem from "effect/FileSystem";
9import * as Layer from "effect/Layer";
10
11import * as ServerConfig from "../config.ts";
12
13/**
14 * Writes one V8 heap snapshot into `logsDir` and logs its path. A failed write
15 * logs a warning and removes any partial file, because that file can hold
16 * secrets and the failure is often a full disk.
17 */
18export const writeHeapSnapshot = Effect.fn("server.heapSnapshot", { root: true })(
19 function* (logsDir: string) {
20 const fs = yield* FileSystem.FileSystem;
21 const timestamp = DateTime.formatIso(yield* DateTime.now).replaceAll(":", "-");
22 const path = NodePath.join(logsDir, `server-${process.pid}-${timestamp}.heapsnapshot`);
23 yield* Effect.annotateCurrentSpan({ path });
24 yield* Effect.try(() => NodeV8.writeHeapSnapshot(path)).pipe(
25 Effect.tapError(() => fs.remove(path, { force: true }).pipe(Effect.ignore)),
26 );
27 yield* Effect.logInfo("Wrote heap snapshot.", { path });
28 },
29 Effect.catch((cause) => Effect.logWarning("Failed to write heap snapshot.", { cause })),
30);
31
32/**
33 * Writes a heap snapshot when the process gets SIGUSR2 (`kill -USR2 <pid>`),
34 * so a maintainer can see what a long-running server holds. See "Heap
35 * Snapshots" in docs/operations/observability.md.
36 *
37 * The write blocks the event loop, so two snapshots never overlap: a signal
38 * sent during a write waits until it finishes. Windows has no SIGUSR2, so the
39 * layer does nothing there.
40 */
41export const layer = Layer.effectDiscard(
42 Effect.gen(function* () {
43 if ((yield* HostProcessPlatform) === "win32") return;
44 const { logsDir } = yield* ServerConfig.ServerConfig;
45 const runFork = Effect.runForkWith(yield* Effect.context<FileSystem.FileSystem>());
46 const onSignal = () => void runFork(writeHeapSnapshot(logsDir));
47 yield* Effect.acquireRelease(
48 Effect.sync(() => process.on("SIGUSR2", onSignal)),
49 () => Effect.sync(() => process.off("SIGUSR2", onSignal)),
50 );
51 }),
52);
apps/server/src/server.ts+2
...@@ -119,6 +119,7 @@ import * as SourceControlRepositoryService from "./sourceControl/SourceControlRe...@@ -119,6 +119,7 @@ import * as SourceControlRepositoryService from "./sourceControl/SourceControlRe
119import * as ProjectSetupScriptRunner from "./project/ProjectSetupScriptRunner.ts";119import * as ProjectSetupScriptRunner from "./project/ProjectSetupScriptRunner.ts";
120import * as WorktreeSetupTracker from "./project/WorktreeSetupTracker.ts";120import * as WorktreeSetupTracker from "./project/WorktreeSetupTracker.ts";
121import { ObservabilityLive } from "./observability/Layers/Observability.ts";121import { ObservabilityLive } from "./observability/Layers/Observability.ts";
122import * as HeapSnapshot from "./observability/HeapSnapshot.ts";
122import * as ServerEnvironment from "./environment/ServerEnvironment.ts";123import * as ServerEnvironment from "./environment/ServerEnvironment.ts";
123import * as RemoteOpenTargets from "./environment/RemoteOpenTargets.ts";124import * as RemoteOpenTargets from "./environment/RemoteOpenTargets.ts";
124import { authHttpApiLayer, environmentAuthenticatedAuthLayer } from "./auth/http.ts";125import { authHttpApiLayer, environmentAuthenticatedAuthLayer } from "./auth/http.ts";
...@@ -957,6 +958,7 @@ const makeServerLayer = Layer.unwrap(...@@ -957,6 +958,7 @@ const makeServerLayer = Layer.unwrap(
957 runtimeStateLayer.pipe(Layer.provide(launcherLayer)),958 runtimeStateLayer.pipe(Layer.provide(launcherLayer)),
958 tailscaleServeLayer,959 tailscaleServeLayer,
959 cloudDesiredLinkReconcileLayer,960 cloudDesiredLinkReconcileLayer,
961 HeapSnapshot.layer,
960 );962 );
961963
962 return serverApplicationLayer.pipe(964 return serverApplicationLayer.pipe(
docs/operations/observability.md+37
...@@ -618,3 +618,40 @@ Current high-value span and metric boundaries include:...@@ -618,3 +618,40 @@ Current high-value span and metric boundaries include:
618- logs outside spans are not persisted in the trace file; SSH-managed launch stdout/stderr is still618- logs outside spans are not persisted in the trace file; SSH-managed launch stdout/stderr is still
619 captured in its launcher log619 captured in its launcher log
620- metrics are not snapshotted locally620- metrics are not snapshotted locally
621
622## Heap Snapshots
623
624To see what a long-running server holds in memory, send it `SIGUSR2`. The server writes a V8 heap
625snapshot to its logs dir and logs the path. This works for desktop, `npx t3`, and service installs
626on macOS and Linux. Windows has no `SIGUSR2`.
627
628Send the signal to the server pid in `server-runtime.json`, which sits in the server's state dir
629next to the `logs` dir. For a dev server or a `--home-dir` launch, use that server's state dir from
630[Traces](#traces). Do not send it to the desktop app or the service launcher: a process without the
631handler exits on `SIGUSR2`. After a crash the file can keep a stale pid that now belongs to a
632different process, so check the pid first.
633
634```bash
635pid="$(jq .pid "${T3CODE_HOME:-$HOME/.t3}/userdata/server-runtime.json")"
636ps -p "$pid" -o command=
637```
638
639If `ps` shows the T3 Code server, send the signal:
640
641```bash
642kill -USR2 "$pid"
643```
644
645The file is `<logsDir>/server-<pid>-<timestamp>.heapsnapshot`, next to `server.trace.ndjson`. To
646open it, use the Memory tab in Chrome DevTools and select Load.
647
648Before you take one:
649
650- The server stops while it writes the file. For a large heap this can take a minute or more.
651 Connected clients can reconnect during the pause, and an event loop monitor, if the server has
652 one, records the pause as a stall. Send the signal once. A second signal sent during a write
653 takes another snapshot after the first one finishes.
654- The write needs about as much free memory as the heap uses. On a machine that is already
655 swapping, it can make the problem worse or crash the server.
656- The file contains everything in server memory, including tokens, secrets, and thread content. Do
657 not share it publicly. Delete it when you are done, because storage cleanup does not remove it.