authorgravatar for me@t3.ggTheo Browne <me@t3.gg> 2026-10-14 04:55:28-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-14 04:55:28-07:00
logcdba9a038e41c6c807a2b58bd43e83dcf98d74b7
tree246ef475668942b87e9ce9fcce61e68f4fbcbf8b
parent8e018ff91f64d6caedc029e498eca446ff0fad20
signature Signed by PGP key B5690EEEBB952194

perf(server): idle servers no longer scan every process every 5 seconds (#19399)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

4 files changed, 130 insertions(+), 21 deletions(-)

apps/server/src/resourceTelemetry/NativeTelemetryClient.test.ts+108-9
...@@ -1,12 +1,30 @@...@@ -1,12 +1,30 @@
1import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
2import * as NodeServices from "@effect/platform-node/NodeServices";
1import type { HostPowerSnapshot } from "@t3tools/contracts";3import type { HostPowerSnapshot } from "@t3tools/contracts";
4import {
5 RESOURCE_MONITOR_PROTOCOL_VERSION,
6 ResourceMonitorCommand,
7 type ResourceMonitorHelloEvent,
8} from "@t3tools/contracts";
2import { describe, expect, it } from "@effect/vitest";9import { describe, expect, it } from "@effect/vitest";
3import * as DateTime from "effect/DateTime";10import * as DateTime from "effect/DateTime";
4import * as Deferred from "effect/Deferred";11import * as Deferred from "effect/Deferred";
5import * as Effect from "effect/Effect";12import * as Effect from "effect/Effect";
13import * as Exit from "effect/Exit";
6import * as Fiber from "effect/Fiber";14import * as Fiber from "effect/Fiber";
15import * as Layer from "effect/Layer";
16import * as Queue from "effect/Queue";
7import * as Ref from "effect/Ref";17import * as Ref from "effect/Ref";
18import * as Schema from "effect/Schema";
19import * as Scope from "effect/Scope";
8import * as Semaphore from "effect/Semaphore";20import * as Semaphore from "effect/Semaphore";
21import * as Sink from "effect/Sink";
22import * as Stream from "effect/Stream";
23import { ChildProcessSpawner } from "effect/process";
924
25import * as ServerConfig from "../config.ts";
26import * as NativeTelemetryClient from "./NativeTelemetryClient.ts";
27import * as ResourceMonitorBinary from "./ResourceMonitorBinary.ts";
10import {28import {
11 canCommandNativeTelemetrySidecar,29 canCommandNativeTelemetrySidecar,
12 canRequestNativeTelemetryRetry,30 canRequestNativeTelemetryRetry,
...@@ -40,25 +58,106 @@ describe("resolveNativeSampleIntervalMs", () => {...@@ -40,25 +58,106 @@ describe("resolveNativeSampleIntervalMs", () => {
40 expect(resolveNativeSampleIntervalMs({ ...basePower, onBattery: "true" }, 1)).toBe(5_000);58 expect(resolveNativeSampleIntervalMs({ ...basePower, onBattery: "true" }, 1)).toBe(5_000);
41 });59 });
4260
43 it("slows background telemetry and serves live diagnostics at 1Hz", () => {61 it("stops timed sampling without a live subscriber and serves live diagnostics at 1Hz", () => {
44 const unknown: HostPowerSnapshot = {62 const unknown: HostPowerSnapshot = {
45 ...basePower,63 ...basePower,
46 source: "unknown",64 source: "unknown",
47 stale: true,65 stale: true,
48 };66 };
49 expect(resolveNativeSampleIntervalMs(unknown, 0)).toBe(5_000);67 expect(resolveNativeSampleIntervalMs(unknown, 0)).toBe(0);
50 expect(resolveNativeSampleIntervalMs(unknown, 1)).toBe(1_000);68 expect(resolveNativeSampleIntervalMs(unknown, 1)).toBe(1_000);
51 expect(69 expect(resolveNativeSampleIntervalMs({ ...basePower, locked: "true" }, 0)).toBe(0);
52 resolveNativeSampleIntervalMs(70 expect(resolveNativeSampleIntervalMs({ ...basePower, onBattery: "true" }, 0)).toBe(0);
53 { ...basePower, stale: true, locked: "true", suspended: true },71 expect(resolveNativeSampleIntervalMs(basePower, 0)).toBe(0);
54 0,
55 ),
56 ).toBe(5_000);
57 expect(resolveNativeSampleIntervalMs(basePower, 0)).toBe(5_000);
58 expect(resolveNativeSampleIntervalMs(basePower, 1)).toBe(1_000);72 expect(resolveNativeSampleIntervalMs(basePower, 1)).toBe(1_000);
59 });73 });
60});74});
6175
76const decodeCommandLine = Schema.decodeUnknownSync(Schema.fromJsonString(ResourceMonitorCommand));
77
78const hello: ResourceMonitorHelloEvent = {
79 version: RESOURCE_MONITOR_PROTOCOL_VERSION,
80 type: "hello",
81 sidecarVersion: "test",
82 sidecarPid: 4242,
83 platform: "linux",
84 arch: "x64",
85 capabilities: {
86 cumulativeCpuTime: true,
87 currentCpuPercent: true,
88 residentMemory: true,
89 virtualMemory: true,
90 ioBytes: true,
91 processStartTime: true,
92 processTree: true,
93 },
94};
95
96describe("NativeTelemetryClient", () => {
97 it.effect("asks the sidecar for timed samples only while a live subscriber exists", () =>
98 Effect.gen(function* () {
99 const commands = yield* Queue.unbounded<ResourceMonitorCommand>();
100 // A sidecar that says hello, stays alive, and records every command it receives.
101 const spawner = ChildProcessSpawner.make(() =>
102 Effect.succeed(
103 ChildProcessSpawner.makeHandle({
104 pid: ChildProcessSpawner.ProcessId(hello.sidecarPid),
105 exitCode: Effect.never,
106 isRunning: Effect.succeed(true),
107 kill: () => Effect.void,
108 unref: Effect.succeed(Effect.void),
109 stdin: Sink.forEach((chunk: Uint8Array) =>
110 Effect.forEach(new TextDecoder().decode(chunk).split("\n").filter(Boolean), (line) =>
111 Queue.offer(commands, decodeCommandLine(line)),
112 ),
113 ),
114 stdout: Stream.encodeText(Stream.make(`${JSON.stringify(hello)}\n`)).pipe(
115 Stream.concat(Stream.never),
116 ),
117 stderr: Stream.empty,
118 all: Stream.empty,
119 getInputFd: () => Sink.drain,
120 getOutputFd: () => Stream.empty,
121 }),
122 ),
123 );
124 const client = yield* NativeTelemetryClient.make().pipe(
125 Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
126 Effect.provideService(ResourceMonitorBinary.ResourceMonitorBinary, {
127 resolve: Effect.succeed("/test/t3-resource-monitor"),
128 }),
129 );
130
131 const configure = yield* Queue.take(commands);
132 expect(configure).toMatchObject({ type: "configure", sampleIntervalMs: 0 });
133 expect(yield* Queue.take(commands)).toMatchObject({ type: "setExternalProcesses" });
134
135 const live = yield* Scope.make();
136 yield* client.snapshots.pipe(Stream.runDrain, Effect.forkIn(live));
137 expect(yield* Queue.take(commands)).toMatchObject({
138 type: "setSampleInterval",
139 sampleIntervalMs: 1_000,
140 });
141 expect(yield* Queue.take(commands)).toMatchObject({ type: "setStreaming", enabled: true });
142
143 yield* Scope.close(live, Exit.void);
144 expect(yield* Queue.take(commands)).toMatchObject({
145 type: "setSampleInterval",
146 sampleIntervalMs: 0,
147 });
148 expect(yield* Queue.take(commands)).toMatchObject({ type: "setStreaming", enabled: false });
149 expect((yield* client.health).sampleIntervalMs).toBe(0);
150 }).pipe(
151 Effect.provide(
152 Layer.mergeAll(
153 ServerConfig.layerTest(process.cwd(), { prefix: "t3-native-telemetry-test-" }),
154 NodeCrypto.layer,
155 ).pipe(Layer.provideMerge(NodeServices.layer)),
156 ),
157 ),
158 );
159});
160
62describe("canRequestNativeTelemetryRetry", () => {161describe("canRequestNativeTelemetryRetry", () => {
63 it("only accepts retry while the supervisor is waiting without a live sidecar", () => {162 it("only accepts retry while the supervisor is waiting without a live sidecar", () => {
64 expect(canRequestNativeTelemetryRetry("degraded", false)).toBe(true);163 expect(canRequestNativeTelemetryRetry("degraded", false)).toBe(true);
apps/server/src/resourceTelemetry/NativeTelemetryClient.ts+9-7
...@@ -40,7 +40,8 @@ import * as ServerConfig from "../config.ts";...@@ -40,7 +40,8 @@ import * as ServerConfig from "../config.ts";
40import { subscribeBeforeSnapshotWithoutMutex } from "../utils/subscribeBeforeSnapshot.ts";40import { subscribeBeforeSnapshotWithoutMutex } from "../utils/subscribeBeforeSnapshot.ts";
4141
42const SAMPLE_INTERVAL_MS = 1_000;42const SAMPLE_INTERVAL_MS = 1_000;
43const UNKNOWN_BACKGROUND_SAMPLE_INTERVAL_MS = 5_000;43// The sidecar treats 0 as "sample only on request".
44const IDLE_SAMPLE_INTERVAL_MS = 0;
44const BATTERY_SAMPLE_INTERVAL_MS = 5_000;45const BATTERY_SAMPLE_INTERVAL_MS = 5_000;
45const CONSTRAINED_SAMPLE_INTERVAL_MS = 15_000;46const CONSTRAINED_SAMPLE_INTERVAL_MS = 15_000;
46const HANDSHAKE_TIMEOUT = Duration.seconds(5);47const HANDSHAKE_TIMEOUT = Duration.seconds(5);
...@@ -258,13 +259,14 @@ function isThermallyConstrained(snapshot: HostPowerSnapshot): boolean {...@@ -258,13 +259,14 @@ function isThermallyConstrained(snapshot: HostPowerSnapshot): boolean {
258 return snapshot.thermalState === "serious" || snapshot.thermalState === "critical";259 return snapshot.thermalState === "serious" || snapshot.thermalState === "critical";
259}260}
260261
262// Without a live subscriber the sidecar does not sample on a timer, so an idle server
263// never walks the process table. History then covers only time with a subscriber.
261export function resolveNativeSampleIntervalMs(264export function resolveNativeSampleIntervalMs(
262 snapshot: HostPowerSnapshot,265 snapshot: HostPowerSnapshot,
263 liveSubscriberCount: number,266 liveSubscriberCount: number,
264): number {267): number {
265 if (snapshot.stale || snapshot.source === "unknown") {268 if (liveSubscriberCount === 0) return IDLE_SAMPLE_INTERVAL_MS;
266 return liveSubscriberCount > 0 ? SAMPLE_INTERVAL_MS : UNKNOWN_BACKGROUND_SAMPLE_INTERVAL_MS;269 if (snapshot.stale || snapshot.source === "unknown") return SAMPLE_INTERVAL_MS;
267 }
268 if (270 if (
269 snapshot.suspended ||271 snapshot.suspended ||
270 snapshot.locked === "true" ||272 snapshot.locked === "true" ||
...@@ -274,7 +276,7 @@ export function resolveNativeSampleIntervalMs(...@@ -274,7 +276,7 @@ export function resolveNativeSampleIntervalMs(
274 return CONSTRAINED_SAMPLE_INTERVAL_MS;276 return CONSTRAINED_SAMPLE_INTERVAL_MS;
275 }277 }
276 if (snapshot.onBattery === "true") return BATTERY_SAMPLE_INTERVAL_MS;278 if (snapshot.onBattery === "true") return BATTERY_SAMPLE_INTERVAL_MS;
277 return liveSubscriberCount > 0 ? SAMPLE_INTERVAL_MS : UNKNOWN_BACKGROUND_SAMPLE_INTERVAL_MS;279 return SAMPLE_INTERVAL_MS;
278}280}
279281
280export function commitCollectionControlUpdate<E, R>(282export function commitCollectionControlUpdate<E, R>(
...@@ -382,7 +384,7 @@ export const make = Effect.fn("resourceTelemetry.nativeTelemetryClient.make")(fu...@@ -382,7 +384,7 @@ export const make = Effect.fn("resourceTelemetry.nativeTelemetryClient.make")(fu
382 updatedAt: initializedAt,384 updatedAt: initializedAt,
383 },385 },
384 liveSubscriberCount: 0,386 liveSubscriberCount: 0,
385 sampleIntervalMs: UNKNOWN_BACKGROUND_SAMPLE_INTERVAL_MS,387 sampleIntervalMs: IDLE_SAMPLE_INTERVAL_MS,
386 });388 });
387 const appliedCollectionControl = yield* Ref.make(yield* Ref.get(collectionControl));389 const appliedCollectionControl = yield* Ref.make(yield* Ref.get(collectionControl));
388 const externalProcesses = yield* Ref.make<ReadonlyArray<ResourceMonitorExternalProcess>>([]);390 const externalProcesses = yield* Ref.make<ReadonlyArray<ResourceMonitorExternalProcess>>([]);
...@@ -1071,7 +1073,7 @@ export const layerTest = (...@@ -1071,7 +1073,7 @@ export const layerTest = (
1071 lastSampleAt: Option.none<DateTime.Utc>(),1073 lastSampleAt: Option.none<DateTime.Utc>(),
1072 lastError: Option.some("Resource monitor test implementation is unavailable."),1074 lastError: Option.some("Resource monitor test implementation is unavailable."),
1073 restartCount: 0,1075 restartCount: 0,
1074 sampleIntervalMs: UNKNOWN_BACKGROUND_SAMPLE_INTERVAL_MS,1076 sampleIntervalMs: IDLE_SAMPLE_INTERVAL_MS,
1075 });1077 });
1076 return Layer.succeed(1078 return Layer.succeed(
1077 NativeTelemetryClient,1079 NativeTelemetryClient,
apps/web/src/components/settings/DiagnosticsSettings.tsx+8-2
...@@ -1038,11 +1038,17 @@ export function DiagnosticsSettingsPanel() {...@@ -1038,11 +1038,17 @@ export function DiagnosticsSettingsPanel() {
1038 <StatBlock1038 <StatBlock
1039 label="Samples"1039 label="Samples"
1040 value={resourceData ? formatCount(resourceData.retainedSampleCount) : "..."}1040 value={resourceData ? formatCount(resourceData.retainedSampleCount) : "..."}
1041 tooltip="In-memory process samples retained by the server. This resets when the server restarts."1041 tooltip="In-memory process samples retained by the server. The server samples only while diagnostics is open, and this resets when the server restarts."
1042 />1042 />
1043 <StatBlock1043 <StatBlock
1044 label="Interval"1044 label="Interval"
1045 value={resourceData ? formatDuration(resourceData.sampleIntervalMs) : "..."}1045 value={
1046 resourceData
1047 ? resourceData.sampleIntervalMs === 0
1048 ? "On demand"
1049 : formatDuration(resourceData.sampleIntervalMs)
1050 : "..."
1051 }
1046 />1052 />
1047 <StatBlock1053 <StatBlock
1048 label="Processes"1054 label="Processes"
docs/internals/resource-telemetry.md+5-3
...@@ -8,9 +8,11 @@ protocol. A missing or failed collector leaves the server running....@@ -8,9 +8,11 @@ protocol. A missing or failed collector leaves the server running.
88
9## Collection cost9## Collection cost
1010
11The native child owns sampling and bounded in-memory history. The server requests11The native child owns sampling and bounded in-memory history. It samples on a timer
12continuous snapshots only while diagnostics has live subscribers and fetches12only while diagnostics has live subscribers. With none, the server sends a sample
13history on demand. Consuming host power for background scheduling must not retain13interval of 0 and the child samples only on request, so an idle server never walks
14the process table. History therefore covers only time with a live subscriber, plus
15on-demand samples. Consuming host power for background scheduling must not retain
14live diagnostics. There is no telemetry database or recurring shell-probe fallback.16live diagnostics. There is no telemetry database or recurring shell-probe fallback.
1517
16History has independent bounds for age, snapshot count, process rows, and retained18History has independent bounds for age, snapshot count, process rows, and retained