authorgravatar for maria@kuuro.netmaria <maria@kuuro.net> 2026-10-09 22:50:44-03:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-09 22:50:44-03:00
logb87b13d41426418c4f42c6477ce62cfe2a43fbae
tree0cce475413c80c13f6e9c105bce42df83d8d74ad
parentb744bded016b15b9b5ade552a3e28270ac20176f
signature Signed by PGP key B5690EEEBB952194

fix(server): report incomplete transcript usage scans (#15661)

3 files changed, 56 insertions(+), 9 deletions(-)

apps/server/src/usage/UsageService.test.ts+28
...@@ -996,6 +996,34 @@ describe("UsageService", () => {...@@ -996,6 +996,34 @@ describe("UsageService", () => {
996 }).pipe(Effect.scoped),996 }).pipe(Effect.scoped),
997 );997 );
998998
999 it.live.skipIf(HostProcess.Platform.defaultValue() === "win32" || process.getuid?.() === 0)(
1000 "reports unreadable transcripts as partial and recovers once they can be read",
1001 () =>
1002 Effect.gen(function* () {
1003 const { transcript, settings, home } = yield* setup;
1004 const unreadable = NodePath.join(NodePath.dirname(transcript), "other.jsonl");
1005 yield* Effect.promise(async () => {
1006 await NodeFSP.writeFile(transcript, claudeLine(1, 5));
1007 await NodeFSP.writeFile(unreadable, claudeLine(2, 7));
1008 await NodeFSP.chmod(unreadable, 0);
1009 });
1010 yield* Effect.gen(function* () {
1011 const service = yield* UsageService.make;
1012 const partial = yield* service.readSummary(WINDOW);
1013 assert.strictEqual(totalOutputTokens(partial), 5);
1014 assert.strictEqual(partial.sources[0]?.status, "partial");
1015
1016 yield* Effect.promise(() => NodeFSP.chmod(unreadable, 0o600));
1017 const healthy = yield* service.readSummary(WINDOW);
1018 assert.strictEqual(totalOutputTokens(healthy), 12);
1019 assert.strictEqual(healthy.sources[0]?.status, "ok");
1020 assert.isNull(healthy.sources[0]?.message);
1021 }).pipe(
1022 Effect.provide(layerService({ prefix: "usage-service-unreadable", home, settings })),
1023 );
1024 }).pipe(Effect.scoped),
1025 );
1026
999 it.live("reprices unchanged transcripts when custom prices are added, edited, or removed", () =>1027 it.live("reprices unchanged transcripts when custom prices are added, edited, or removed", () =>
1000 Effect.gen(function* () {1028 Effect.gen(function* () {
1001 const { transcript, settings, home } = yield* setup;1029 const { transcript, settings, home } = yield* setup;
apps/server/src/usage/UsageService.ts+17-2
...@@ -519,6 +519,7 @@ export const make = Effect.gen(function* () {...@@ -519,6 +519,7 @@ export const make = Effect.gen(function* () {
519 format: TranscriptUsageFormat<unknown>,519 format: TranscriptUsageFormat<unknown>,
520 ): Effect.Effect<{520 ): Effect.Effect<{
521 readonly records: readonly UsageRecord[];521 readonly records: readonly UsageRecord[];
522 readonly failed?: true;
522 readonly update?: { readonly entry: CachedFile; readonly replaces: CachedFile | undefined };523 readonly update?: { readonly entry: CachedFile; readonly replaces: CachedFile | undefined };
523 }> =>524 }> =>
524 Effect.gen(function* () {525 Effect.gen(function* () {
...@@ -554,6 +555,7 @@ export const make = Effect.gen(function* () {...@@ -554,6 +555,7 @@ export const make = Effect.gen(function* () {
554 if (parsed === null)555 if (parsed === null)
555 return {556 return {
556 records: cached?.provider === provider ? [...cached.records, ...cached.tailRecords] : [],557 records: cached?.provider === provider ? [...cached.records, ...cached.tailRecords] : [],
558 failed: true,
557 };559 };
558560
559 // Stored already de-duplicated within the file, which is 99% of all561 // Stored already de-duplicated within the file, which is 99% of all
...@@ -600,7 +602,7 @@ export const make = Effect.gen(function* () {...@@ -600,7 +602,7 @@ export const make = Effect.gen(function* () {
600 .exists(dir)602 .exists(dir)
601 .pipe(Effect.catchCause(() => Effect.succeed(false)));603 .pipe(Effect.catchCause(() => Effect.succeed(false)));
602 if (!exists) return { provider, dir, volumeId, files: null } satisfies ScannedDir;604 if (!exists) return { provider, dir, volumeId, files: null } satisfies ScannedDir;
603 const files = yield* Effect.promise(() =>605 const { files, failedPaths } = yield* Effect.promise(() =>
604 listTranscriptFiles(dir, windowStartMs, fileName === undefined ? undefined : { fileName }),606 listTranscriptFiles(dir, windowStartMs, fileName === undefined ? undefined : { fileName }),
605 );607 );
606 // A cold parse waits on disk reads, so a few files in flight read608 // A cold parse waits on disk reads, so a few files in flight read
...@@ -629,7 +631,20 @@ export const make = Effect.gen(function* () {...@@ -629,7 +631,20 @@ export const make = Effect.gen(function* () {
629 }631 }
630 return { path, records };632 return { path, records };
631 });633 });
632 return { provider, dir, volumeId, files: parsedFiles } satisfies ScannedDir;634 // Unread files keep their cached usage, but the total may be short.
635 const unread = failedPaths + read.filter((file) => file.failed).length;
636 return {
637 provider,
638 dir,
639 volumeId,
640 files: parsedFiles,
641 ...(unread > 0
642 ? {
643 status: "partial",
644 message: `${unread} transcript path(s) could not be read; usage may be incomplete.`,
645 }
646 : {}),
647 } satisfies ScannedDir;
633 });648 });
634649
635 const collectDirs = Effect.fn("UsageService.collectDirs")(function* (650 const collectDirs = Effect.fn("UsageService.collectDirs")(function* (
apps/server/src/usage/usageTranscriptReader.ts+11-7
...@@ -109,9 +109,9 @@ function fnv1a(buffer: Buffer): number {...@@ -109,9 +109,9 @@ function fnv1a(buffer: Buffer): number {
109/**109/**
110 * Lists `.jsonl` transcripts under `root` last modified at or after `sinceMs`.110 * Lists `.jsonl` transcripts under `root` last modified at or after `sinceMs`.
111 *111 *
112 * Errors on individual entries are swallowed: session files rotate and get112 * Unreadable directories and files are counted in `failedPaths` rather than
113 * removed while the walk is in flight, and a partial listing is far better than113 * failing the page, so the source can report incomplete usage. Files that
114 * failing the page.114 * vanish between `readdir` and `stat` are ordinary rotation and not counted.
115 *115 *
116 * `fileName` restricts the walk to a single basename (Grok's `updates.jsonl`).116 * `fileName` restricts the walk to a single basename (Grok's `updates.jsonl`).
117 * Grok sessions also ship multi-megabyte `chat_history` and `events` logs that117 * Grok sessions also ship multi-megabyte `chat_history` and `events` logs that
...@@ -126,14 +126,16 @@ export async function listTranscriptFiles(...@@ -126,14 +126,16 @@ export async function listTranscriptFiles(
126 root: string,126 root: string,
127 sinceMs: number,127 sinceMs: number,
128 options?: { readonly fileName?: string },128 options?: { readonly fileName?: string },
129): Promise<readonly TranscriptFile[]> {129): Promise<{ readonly files: readonly TranscriptFile[]; readonly failedPaths: number }> {
130 const fileName = options?.fileName;130 const fileName = options?.fileName;
131 const candidates: string[] = [];131 const candidates: string[] = [];
132 let failedPaths = 0;
132 const walk = async (dir: string): Promise<void> => {133 const walk = async (dir: string): Promise<void> => {
133 let entries;134 let entries;
134 try {135 try {
135 entries = await NodeFSP.readdir(dir, { withFileTypes: true });136 entries = await NodeFSP.readdir(dir, { withFileTypes: true });
136 } catch {137 } catch {
138 failedPaths += 1;
137 return;139 return;
138 }140 }
139 for (const entry of entries) {141 for (const entry of entries) {
...@@ -156,15 +158,17 @@ export async function listTranscriptFiles(...@@ -156,15 +158,17 @@ export async function listTranscriptFiles(
156 if (stats.mtimeMs >= sinceMs) {158 if (stats.mtimeMs >= sinceMs) {
157 found[index] = { path, size: stats.size, mtimeMs: stats.mtimeMs };159 found[index] = { path, size: stats.size, mtimeMs: stats.mtimeMs };
158 }160 }
159 } catch {161 } catch (error) {
160 // Vanished between readdir and stat.162 if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) {
163 failedPaths += 1;
164 }
161 }165 }
162 }166 }
163 };167 };
164 await Promise.all(168 await Promise.all(
165 Array.from({ length: Math.min(STAT_CONCURRENCY, candidates.length) }, statQueued),169 Array.from({ length: Math.min(STAT_CONCURRENCY, candidates.length) }, statQueued),
166 );170 );
167 return found.filter((file) => file !== undefined);171 return { files: found.filter((file) => file !== undefined), failedPaths };
168}172}
169173
170/**174/**