authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-06-20 11:11:15-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-06-20 11:11:15-07:00
logd84ebe831922167d556e16cb40a238a904e0f14a
treed4bd46fbbb357f8e132ee6facbc0da90ed82098b
parent5a2c92e86d16ecba823fdc61e163b8c35969f668
signature Signed by PGP key B5690EEEBB952194

[codex] Structure process resource sampling failures (#3415)

Co-authored-by: codex <codex@users.noreply.github.com>

3 files changed, 90 insertions(+), 20 deletions(-)

apps/server/src/diagnostics/ProcessResourceMonitor.test.ts+30-3
...@@ -111,7 +111,7 @@ describe("ProcessResourceMonitor", () => {...@@ -111,7 +111,7 @@ describe("ProcessResourceMonitor", () => {
111 readAtMs: DateTime.toEpochMillis(secondAt),111 readAtMs: DateTime.toEpochMillis(secondAt),
112 windowMs: 60_000,112 windowMs: 60_000,
113 bucketMs: 10_000,113 bucketMs: 10_000,
114 lastError: null,114 lastFailure: null,
115 });115 });
116116
117 expect(Option.isNone(result.error)).toBe(true);117 expect(Option.isNone(result.error)).toBe(true);
...@@ -171,7 +171,7 @@ describe("ProcessResourceMonitor", () => {...@@ -171,7 +171,7 @@ describe("ProcessResourceMonitor", () => {
171 readAtMs: DateTime.toEpochMillis(secondAt),171 readAtMs: DateTime.toEpochMillis(secondAt),
172 windowMs: 60_000,172 windowMs: 60_000,
173 bucketMs: 10_000,173 bucketMs: 10_000,
174 lastError: null,174 lastFailure: null,
175 });175 });
176176
177 expect(result.topProcesses).toHaveLength(1);177 expect(result.topProcesses).toHaveLength(1);
...@@ -218,11 +218,38 @@ describe("ProcessResourceMonitor", () => {...@@ -218,11 +218,38 @@ describe("ProcessResourceMonitor", () => {
218 readAtMs: DateTime.toEpochMillis(sampledAt),218 readAtMs: DateTime.toEpochMillis(sampledAt),
219 windowMs: 60_000,219 windowMs: 60_000,
220 bucketMs: 10_000,220 bucketMs: 10_000,
221 lastError: null,221 lastFailure: null,
222 });222 });
223223
224 expect(result.topProcesses).toHaveLength(36);224 expect(result.topProcesses).toHaveLength(36);
225 expect(result.topProcesses.some((process) => process.command === "worker 34")).toBe(true);225 expect(result.topProcesses.some((process) => process.command === "worker 34")).toBe(true);
226 }),226 }),
227 );227 );
228
229 it.effect("exposes bounded failure diagnostics while retaining the exact cause", () =>
230 Effect.sync(() => {
231 const readAt = DateTime.makeUnsafe("2026-05-05T10:00:00.000Z");
232 const cause = new Error("stderr included credential=secret-value");
233 const failure = new ProcessResourceMonitor.ProcessResourceSamplingError({
234 failureTag: "ProcessDiagnosticsQueryFailedError",
235 cause,
236 });
237
238 const result = ProcessResourceMonitor.aggregateProcessResourceHistory({
239 samples: [],
240 readAt,
241 readAtMs: DateTime.toEpochMillis(readAt),
242 windowMs: 60_000,
243 bucketMs: 10_000,
244 lastFailure: failure,
245 });
246
247 expect(failure.cause).toBe(cause);
248 expect(Option.getOrThrow(result.error)).toEqual({
249 failureTag: "ProcessDiagnosticsQueryFailedError",
250 message: "Failed to sample process resources (ProcessDiagnosticsQueryFailedError).",
251 });
252 expect(Option.getOrThrow(result.error).message).not.toContain("secret-value");
253 }),
254 );
228});255});
apps/server/src/diagnostics/ProcessResourceMonitor.ts+49-17
...@@ -1,8 +1,10 @@...@@ -1,8 +1,10 @@
1import type {1import {
2 ServerProcessResourceHistoryBucket,2 ServerProcessResourceHistoryFailureTag,
3 ServerProcessResourceHistoryInput,3 type ServerProcessResourceHistoryBucket,
4 ServerProcessResourceHistoryResult,4 type ServerProcessResourceHistoryFailureTag as ServerProcessResourceHistoryFailureTagType,
5 ServerProcessResourceHistorySummary,5 type ServerProcessResourceHistoryInput,
6 type ServerProcessResourceHistoryResult,
7 type ServerProcessResourceHistorySummary,
6} from "@t3tools/contracts";8} from "@t3tools/contracts";
7import * as Context from "effect/Context";9import * as Context from "effect/Context";
8import * as DateTime from "effect/DateTime";10import * as DateTime from "effect/DateTime";
...@@ -10,6 +12,7 @@ import * as Effect from "effect/Effect";...@@ -10,6 +12,7 @@ import * as Effect from "effect/Effect";
10import * as Layer from "effect/Layer";12import * as Layer from "effect/Layer";
11import * as Option from "effect/Option";13import * as Option from "effect/Option";
12import * as Ref from "effect/Ref";14import * as Ref from "effect/Ref";
15import * as Schema from "effect/Schema";
13import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner";16import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner";
1417
15import * as ProcessDiagnostics from "./ProcessDiagnostics.ts";18import * as ProcessDiagnostics from "./ProcessDiagnostics.ts";
...@@ -31,9 +34,21 @@ export interface ProcessResourceSample {...@@ -31,9 +34,21 @@ export interface ProcessResourceSample {
31 readonly isServerRoot: boolean;34 readonly isServerRoot: boolean;
32}35}
3336
37export class ProcessResourceSamplingError extends Schema.TaggedErrorClass<ProcessResourceSamplingError>()(
38 "ProcessResourceSamplingError",
39 {
40 failureTag: ServerProcessResourceHistoryFailureTag,
41 cause: Schema.Defect(),
42 },
43) {
44 override get message(): string {
45 return `Failed to sample process resources (${this.failureTag}).`;
46 }
47}
48
34interface MonitorState {49interface MonitorState {
35 readonly samples: ReadonlyArray<ProcessResourceSample>;50 readonly samples: ReadonlyArray<ProcessResourceSample>;
36 readonly lastError: string | null;51 readonly lastFailure: ProcessResourceSamplingError | null;
37}52}
3853
39export class ProcessResourceMonitor extends Context.Service<54export class ProcessResourceMonitor extends Context.Service<
...@@ -218,7 +233,7 @@ export function aggregateProcessResourceHistory(input: {...@@ -218,7 +233,7 @@ export function aggregateProcessResourceHistory(input: {
218 readonly readAtMs: number;233 readonly readAtMs: number;
219 readonly windowMs: number;234 readonly windowMs: number;
220 readonly bucketMs: number;235 readonly bucketMs: number;
221 readonly lastError: string | null;236 readonly lastFailure: ProcessResourceSamplingError | null;
222}): ServerProcessResourceHistoryResult {237}): ServerProcessResourceHistoryResult {
223 const windowMs = Math.max(1_000, input.windowMs);238 const windowMs = Math.max(1_000, input.windowMs);
224 const bucketMs = Math.max(1_000, input.bucketMs);239 const bucketMs = Math.max(1_000, input.bucketMs);
...@@ -239,13 +254,29 @@ export function aggregateProcessResourceHistory(input: {...@@ -239,13 +254,29 @@ export function aggregateProcessResourceHistory(input: {
239 totalCpuSecondsApprox,254 totalCpuSecondsApprox,
240 buckets: buildBuckets({ samples, nowMs: input.readAtMs, windowMs, bucketMs }),255 buckets: buildBuckets({ samples, nowMs: input.readAtMs, windowMs, bucketMs }),
241 topProcesses,256 topProcesses,
242 error: input.lastError ? Option.some({ message: input.lastError }) : Option.none(),257 error: input.lastFailure
258 ? Option.some({
259 failureTag: input.lastFailure.failureTag,
260 message: input.lastFailure.message,
261 })
262 : Option.none(),
243 };263 };
244}264}
245265
246export const make = Effect.gen(function* () {266export const make = Effect.gen(function* () {
247 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;267 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
248 const state = yield* Ref.make<MonitorState>({ samples: [], lastError: null });268 const state = yield* Ref.make<MonitorState>({ samples: [], lastFailure: null });
269
270 const recordSamplingFailure = (cause: {
271 readonly _tag: ServerProcessResourceHistoryFailureTagType;
272 }) =>
273 Ref.update(state, (current) => ({
274 ...current,
275 lastFailure: new ProcessResourceSamplingError({
276 failureTag: cause._tag,
277 cause,
278 }),
279 }));
249280
250 const sampleOnce = Effect.gen(function* () {281 const sampleOnce = Effect.gen(function* () {
251 const sampledAt = yield* DateTime.now;282 const sampledAt = yield* DateTime.now;
...@@ -261,15 +292,16 @@ export const make = Effect.gen(function* () {...@@ -261,15 +292,16 @@ export const make = Effect.gen(function* () {
261 });292 });
262 yield* Ref.update(state, (current) => ({293 yield* Ref.update(state, (current) => ({
263 samples: trimSamples([...current.samples, ...samples], sampledAtMs),294 samples: trimSamples([...current.samples, ...samples], sampledAtMs),
264 lastError: null,295 lastFailure: null,
265 }));296 }));
266 }).pipe(297 }).pipe(
267 Effect.catch((error: unknown) =>298 Effect.catchTags({
268 Ref.update(state, (current) => ({299 ProcessDiagnosticsQueryTimeoutError: recordSamplingFailure,
269 ...current,300 ProcessDiagnosticsQueryFailedError: recordSamplingFailure,
270 lastError: error instanceof Error ? error.message : "Failed to sample process resources.",301 ProcessDiagnosticsServerProcessSignalError: recordSamplingFailure,
271 })),302 ProcessDiagnosticsNotDescendantError: recordSamplingFailure,
272 ),303 ProcessDiagnosticsSignalFailedError: recordSamplingFailure,
304 }),
273 );305 );
274306
275 yield* Effect.forever(sampleOnce.pipe(Effect.andThen(Effect.sleep(SAMPLE_INTERVAL_MS)))).pipe(307 yield* Effect.forever(sampleOnce.pipe(Effect.andThen(Effect.sleep(SAMPLE_INTERVAL_MS)))).pipe(
...@@ -287,7 +319,7 @@ export const make = Effect.gen(function* () {...@@ -287,7 +319,7 @@ export const make = Effect.gen(function* () {
287 readAtMs,319 readAtMs,
288 windowMs: input.windowMs,320 windowMs: input.windowMs,
289 bucketMs: input.bucketMs,321 bucketMs: input.bucketMs,
290 lastError: current.lastError,322 lastFailure: current.lastFailure,
291 });323 });
292 });324 });
293325
packages/contracts/src/server.ts+11
...@@ -364,6 +364,16 @@ export const ServerProcessResourceHistorySummary = Schema.Struct({...@@ -364,6 +364,16 @@ export const ServerProcessResourceHistorySummary = Schema.Struct({
364});364});
365export type ServerProcessResourceHistorySummary = typeof ServerProcessResourceHistorySummary.Type;365export type ServerProcessResourceHistorySummary = typeof ServerProcessResourceHistorySummary.Type;
366366
367export const ServerProcessResourceHistoryFailureTag = Schema.Literals([
368 "ProcessDiagnosticsQueryTimeoutError",
369 "ProcessDiagnosticsQueryFailedError",
370 "ProcessDiagnosticsServerProcessSignalError",
371 "ProcessDiagnosticsNotDescendantError",
372 "ProcessDiagnosticsSignalFailedError",
373]);
374export type ServerProcessResourceHistoryFailureTag =
375 typeof ServerProcessResourceHistoryFailureTag.Type;
376
367export const ServerProcessResourceHistoryResult = Schema.Struct({377export const ServerProcessResourceHistoryResult = Schema.Struct({
368 readAt: Schema.DateTimeUtc,378 readAt: Schema.DateTimeUtc,
369 windowMs: NonNegativeInt,379 windowMs: NonNegativeInt,
...@@ -375,6 +385,7 @@ export const ServerProcessResourceHistoryResult = Schema.Struct({...@@ -375,6 +385,7 @@ export const ServerProcessResourceHistoryResult = Schema.Struct({
375 topProcesses: Schema.Array(ServerProcessResourceHistorySummary),385 topProcesses: Schema.Array(ServerProcessResourceHistorySummary),
376 error: Schema.Option(386 error: Schema.Option(
377 Schema.Struct({387 Schema.Struct({
388 failureTag: ServerProcessResourceHistoryFailureTag,
378 message: TrimmedNonEmptyString,389 message: TrimmedNonEmptyString,
379 }),390 }),
380 ),391 ),