authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-06-20 11:11:15-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-06-20 11:11:15-07:00
logd84ebe831922167d556e16cb40a238a904e0f14a
treed4bd46fbbb357f8e132ee6facbc0da90ed82098b
parent5a2c92e86d16ecba823fdc61e163b8c35969f668
signature Signed by PGP key B5690EEEBB952194

[codex] Structure process resource sampling failures (#3415)

Co-authored-by: codex <codex@users.noreply.github.com>

3 files changed, 90 insertions(+), 20 deletions(-)

apps/server/src/diagnostics/ProcessResourceMonitor.test.ts+30-3
......@@ -111,7 +111,7 @@ describe("ProcessResourceMonitor", () => {
111111 readAtMs: DateTime.toEpochMillis(secondAt),
112112 windowMs: 60_000,
113113 bucketMs: 10_000,
114 lastError: null,
114 lastFailure: null,
115115 });
116116
117117 expect(Option.isNone(result.error)).toBe(true);
......@@ -171,7 +171,7 @@ describe("ProcessResourceMonitor", () => {
171171 readAtMs: DateTime.toEpochMillis(secondAt),
172172 windowMs: 60_000,
173173 bucketMs: 10_000,
174 lastError: null,
174 lastFailure: null,
175175 });
176176
177177 expect(result.topProcesses).toHaveLength(1);
......@@ -218,11 +218,38 @@ describe("ProcessResourceMonitor", () => {
218218 readAtMs: DateTime.toEpochMillis(sampledAt),
219219 windowMs: 60_000,
220220 bucketMs: 10_000,
221 lastError: null,
221 lastFailure: null,
222222 });
223223
224224 expect(result.topProcesses).toHaveLength(36);
225225 expect(result.topProcesses.some((process) => process.command === "worker 34")).toBe(true);
226226 }),
227227 );
228
229 it.effect("exposes bounded failure diagnostics while retaining the exact cause", () =>
230 Effect.sync(() => {
231 const readAt = DateTime.makeUnsafe("2026-05-05T10:00:00.000Z");
232 const cause = new Error("stderr included credential=secret-value");
233 const failure = new ProcessResourceMonitor.ProcessResourceSamplingError({
234 failureTag: "ProcessDiagnosticsQueryFailedError",
235 cause,
236 });
237
238 const result = ProcessResourceMonitor.aggregateProcessResourceHistory({
239 samples: [],
240 readAt,
241 readAtMs: DateTime.toEpochMillis(readAt),
242 windowMs: 60_000,
243 bucketMs: 10_000,
244 lastFailure: failure,
245 });
246
247 expect(failure.cause).toBe(cause);
248 expect(Option.getOrThrow(result.error)).toEqual({
249 failureTag: "ProcessDiagnosticsQueryFailedError",
250 message: "Failed to sample process resources (ProcessDiagnosticsQueryFailedError).",
251 });
252 expect(Option.getOrThrow(result.error).message).not.toContain("secret-value");
253 }),
254 );
228255});
apps/server/src/diagnostics/ProcessResourceMonitor.ts+49-17
......@@ -1,8 +1,10 @@
1import type {
2 ServerProcessResourceHistoryBucket,
3 ServerProcessResourceHistoryInput,
4 ServerProcessResourceHistoryResult,
5 ServerProcessResourceHistorySummary,
1import {
2 ServerProcessResourceHistoryFailureTag,
3 type ServerProcessResourceHistoryBucket,
4 type ServerProcessResourceHistoryFailureTag as ServerProcessResourceHistoryFailureTagType,
5 type ServerProcessResourceHistoryInput,
6 type ServerProcessResourceHistoryResult,
7 type ServerProcessResourceHistorySummary,
68} from "@t3tools/contracts";
79import * as Context from "effect/Context";
810import * as DateTime from "effect/DateTime";
......@@ -10,6 +12,7 @@ import * as Effect from "effect/Effect";
1012import * as Layer from "effect/Layer";
1113import * as Option from "effect/Option";
1214import * as Ref from "effect/Ref";
15import * as Schema from "effect/Schema";
1316import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner";
1417
1518import * as ProcessDiagnostics from "./ProcessDiagnostics.ts";
......@@ -31,9 +34,21 @@ export interface ProcessResourceSample {
3134 readonly isServerRoot: boolean;
3235}
3336
37export class ProcessResourceSamplingError extends Schema.TaggedErrorClass<ProcessResourceSamplingError>()(
38 "ProcessResourceSamplingError",
39 {
40 failureTag: ServerProcessResourceHistoryFailureTag,
41 cause: Schema.Defect(),
42 },
43) {
44 override get message(): string {
45 return `Failed to sample process resources (${this.failureTag}).`;
46 }
47}
48
3449interface MonitorState {
3550 readonly samples: ReadonlyArray<ProcessResourceSample>;
36 readonly lastError: string | null;
51 readonly lastFailure: ProcessResourceSamplingError | null;
3752}
3853
3954export class ProcessResourceMonitor extends Context.Service<
......@@ -218,7 +233,7 @@ export function aggregateProcessResourceHistory(input: {
218233 readonly readAtMs: number;
219234 readonly windowMs: number;
220235 readonly bucketMs: number;
221 readonly lastError: string | null;
236 readonly lastFailure: ProcessResourceSamplingError | null;
222237}): ServerProcessResourceHistoryResult {
223238 const windowMs = Math.max(1_000, input.windowMs);
224239 const bucketMs = Math.max(1_000, input.bucketMs);
......@@ -239,13 +254,29 @@ export function aggregateProcessResourceHistory(input: {
239254 totalCpuSecondsApprox,
240255 buckets: buildBuckets({ samples, nowMs: input.readAtMs, windowMs, bucketMs }),
241256 topProcesses,
242 error: input.lastError ? Option.some({ message: input.lastError }) : Option.none(),
257 error: input.lastFailure
258 ? Option.some({
259 failureTag: input.lastFailure.failureTag,
260 message: input.lastFailure.message,
261 })
262 : Option.none(),
243263 };
244264}
245265
246266export const make = Effect.gen(function* () {
247267 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
248 const state = yield* Ref.make<MonitorState>({ samples: [], lastError: null });
268 const state = yield* Ref.make<MonitorState>({ samples: [], lastFailure: null });
269
270 const recordSamplingFailure = (cause: {
271 readonly _tag: ServerProcessResourceHistoryFailureTagType;
272 }) =>
273 Ref.update(state, (current) => ({
274 ...current,
275 lastFailure: new ProcessResourceSamplingError({
276 failureTag: cause._tag,
277 cause,
278 }),
279 }));
249280
250281 const sampleOnce = Effect.gen(function* () {
251282 const sampledAt = yield* DateTime.now;
......@@ -261,15 +292,16 @@ export const make = Effect.gen(function* () {
261292 });
262293 yield* Ref.update(state, (current) => ({
263294 samples: trimSamples([...current.samples, ...samples], sampledAtMs),
264 lastError: null,
295 lastFailure: null,
265296 }));
266297 }).pipe(
267 Effect.catch((error: unknown) =>
268 Ref.update(state, (current) => ({
269 ...current,
270 lastError: error instanceof Error ? error.message : "Failed to sample process resources.",
271 })),
272 ),
298 Effect.catchTags({
299 ProcessDiagnosticsQueryTimeoutError: recordSamplingFailure,
300 ProcessDiagnosticsQueryFailedError: recordSamplingFailure,
301 ProcessDiagnosticsServerProcessSignalError: recordSamplingFailure,
302 ProcessDiagnosticsNotDescendantError: recordSamplingFailure,
303 ProcessDiagnosticsSignalFailedError: recordSamplingFailure,
304 }),
273305 );
274306
275307 yield* Effect.forever(sampleOnce.pipe(Effect.andThen(Effect.sleep(SAMPLE_INTERVAL_MS)))).pipe(
......@@ -287,7 +319,7 @@ export const make = Effect.gen(function* () {
287319 readAtMs,
288320 windowMs: input.windowMs,
289321 bucketMs: input.bucketMs,
290 lastError: current.lastError,
322 lastFailure: current.lastFailure,
291323 });
292324 });
293325
packages/contracts/src/server.ts+11
......@@ -364,6 +364,16 @@ export const ServerProcessResourceHistorySummary = Schema.Struct({
364364});
365365export type ServerProcessResourceHistorySummary = typeof ServerProcessResourceHistorySummary.Type;
366366
367export const ServerProcessResourceHistoryFailureTag = Schema.Literals([
368 "ProcessDiagnosticsQueryTimeoutError",
369 "ProcessDiagnosticsQueryFailedError",
370 "ProcessDiagnosticsServerProcessSignalError",
371 "ProcessDiagnosticsNotDescendantError",
372 "ProcessDiagnosticsSignalFailedError",
373]);
374export type ServerProcessResourceHistoryFailureTag =
375 typeof ServerProcessResourceHistoryFailureTag.Type;
376
367377export const ServerProcessResourceHistoryResult = Schema.Struct({
368378 readAt: Schema.DateTimeUtc,
369379 windowMs: NonNegativeInt,
......@@ -375,6 +385,7 @@ export const ServerProcessResourceHistoryResult = Schema.Struct({
375385 topProcesses: Schema.Array(ServerProcessResourceHistorySummary),
376386 error: Schema.Option(
377387 Schema.Struct({
388 failureTag: ServerProcessResourceHistoryFailureTag,
378389 message: TrimmedNonEmptyString,
379390 }),
380391 ),