authorgravatar for maria@kuuro.netmaria <maria@kuuro.net> 2026-09-17 17:51:37-03:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-17 17:51:37-03:00
log675869d2b0f9e059d5950b80390c0cad1cb1590e
treeca7b59d70d88563a3c1b9ec2e31c520d91f19294
parent592021f008010ee947c326df0452d093ead8aeb1
signature Signed by PGP key B5690EEEBB952194

fix(usage): preserve saved totals after transcript cleanup (#12304)

Co-authored-by: maria-rcks <254055478+maria-rcks@users.noreply.github.com>

4 files changed, 219 insertions(+), 158 deletions(-)

apps/server/src/usage/UsageService.test.ts+112-4
...@@ -7,7 +7,9 @@ import * as NodePath from "node:path";...@@ -7,7 +7,9 @@ import * as NodePath from "node:path";
7import { assert, describe, it } from "@effect/vitest";7import { assert, describe, it } from "@effect/vitest";
8import * as NodeServices from "@effect/platform-node/NodeServices";8import * as NodeServices from "@effect/platform-node/NodeServices";
9import { HostProcessEnvironment } from "@t3tools/shared/hostProcess";9import { HostProcessEnvironment } from "@t3tools/shared/hostProcess";
10import { mergeUsage } from "@t3tools/shared/usageMerge";
10import {11import {
12 EnvironmentId,
11 ProviderDriverKind,13 ProviderDriverKind,
12 ProviderInstanceId,14 ProviderInstanceId,
13 UsageDay,15 UsageDay,
...@@ -131,14 +133,15 @@ describe("UsageService", () => {...@@ -131,14 +133,15 @@ describe("UsageService", () => {
131 [133 [
132 { type: "session_meta", payload: { id: "codex-account-session" } },134 { type: "session_meta", payload: { id: "codex-account-session" } },
133 { type: "turn_context", payload: { model: "gpt-5.6-sol" } },135 { type: "turn_context", payload: { model: "gpt-5.6-sol" } },
134 {136 // A-B-A at one timestamp must preserve both equal A events.
137 ...[11, 12, 11].map((outputTokens) => ({
135 type: "event_msg",138 type: "event_msg",
136 timestamp: "2026-08-01T10:00:00Z",139 timestamp: "2026-08-01T10:00:00Z",
137 payload: {140 payload: {
138 type: "token_count",141 type: "token_count",
139 info: { last_token_usage: { input_tokens: 10, output_tokens: 11 } },142 info: { last_token_usage: { input_tokens: 10, output_tokens: outputTokens } },
140 },143 },
141 },144 })),
142 ]145 ]
143 .map((line) => encodeUnknownJsonString(line))146 .map((line) => encodeUnknownJsonString(line))
144 .join("\n") + "\n",147 .join("\n") + "\n",
...@@ -198,7 +201,21 @@ describe("UsageService", () => {...@@ -198,7 +201,21 @@ describe("UsageService", () => {
198 ),201 ),
199 );202 );
200 const summary = yield* service.readSummary(WINDOW);203 const summary = yield* service.readSummary(WINDOW);
201 assert.strictEqual(totalOutputTokens(summary), 36);204 assert.strictEqual(totalOutputTokens(summary), 59);
205 yield* Effect.promise(() =>
206 NodeFSP.rename(
207 NodePath.join(codexHome, "sessions", "rollout.jsonl"),
208 NodePath.join(codexHome, "sessions", "moved.jsonl"),
209 ),
210 );
211 const moved = yield* service.readSummary(WINDOW);
212 assert.deepStrictEqual(moved.buckets, summary.buckets);
213 yield* Effect.promise(() =>
214 NodeFSP.rm(NodePath.join(codexHome, "sessions"), { recursive: true }),
215 );
216 const removed = yield* service.readSummary(WINDOW);
217 assert.deepStrictEqual(removed.buckets, summary.buckets);
218
202 const sources = summary.sources.filter((source) => source.status === "ok");219 const sources = summary.sources.filter((source) => source.status === "ok");
203 assert.strictEqual(sources.length, 4);220 assert.strictEqual(sources.length, 4);
204 assert.strictEqual(221 assert.strictEqual(
...@@ -393,6 +410,97 @@ describe("UsageService", () => {...@@ -393,6 +410,97 @@ describe("UsageService", () => {
393 }).pipe(Effect.scoped),410 }).pipe(Effect.scoped),
394 );411 );
395412
413 it.live("preserves saved tokens, costs and sessions after transcript cleanup and restart", () =>
414 Effect.gen(function* () {
415 const { transcript, settings, home } = yield* setup;
416 const alias = NodePath.join(home, "claude-alias");
417 yield* Effect.promise(() =>
418 NodeFSP.symlink(NodePath.join(home, "claude"), alias, "junction"),
419 );
420 const content = claudeLine(1, 5);
421 yield* Effect.promise(() => NodeFSP.writeFile(transcript, content));
422 yield* Effect.gen(function* () {
423 const service = yield* UsageService.make;
424 const first = yield* service.readSummary(WINDOW);
425 assert.strictEqual(totalOutputTokens(first), 5);
426 assert.isAbove(first.buckets[0]?.costUsd ?? 0, 0);
427
428 yield* Effect.promise(() => NodeFSP.rm(transcript));
429 const deleted = yield* service.readSummary(WINDOW);
430 assert.deepStrictEqual(deleted.buckets, first.buckets);
431 assert.deepStrictEqual(deleted.sources, first.sources);
432
433 const restarted = yield* UsageService.make;
434 const restored = yield* restarted.readSummary(WINDOW);
435 assert.deepStrictEqual(restored.buckets, first.buckets);
436 assert.deepStrictEqual(restored.sources, first.sources);
437
438 // A moved transcript must not count the saved usage twice.
439 yield* Effect.promise(() => NodeFSP.writeFile(transcript + ".jsonl", content));
440 const moved = yield* restarted.readSummary(WINDOW);
441 assert.deepStrictEqual(moved.buckets, first.buckets);
442 assert.strictEqual(moved.sources[0]?.distinctSessions, 1);
443
444 const replacementProjects = NodePath.join(home, "replacement-projects");
445 yield* Effect.promise(() => NodeFSP.mkdir(replacementProjects));
446 yield* Effect.promise(() =>
447 NodeFSP.rm(NodePath.join(home, "claude", "projects"), { recursive: true }),
448 );
449 const afterRootCleanup = yield* UsageService.make;
450 const missingRoot = yield* afterRootCleanup.readSummary(WINDOW);
451 assert.deepStrictEqual(missingRoot.buckets, first.buckets);
452 assert.strictEqual(missingRoot.sources[0]?.distinctSessions, 1);
453 assert.strictEqual(missingRoot.sources[0]?.status, "ok");
454 assert.deepStrictEqual(missingRoot.sources[0]?.fingerprint, first.sources[0]?.fingerprint);
455 yield* Effect.promise(async () => {
456 const projects = NodePath.join(home, "claude", "projects");
457 await NodeFSP.rename(replacementProjects, projects);
458 await NodeFSP.writeFile(NodePath.join(projects, "new.jsonl"), claudeLine(2, 7));
459 });
460 const recreated = yield* afterRootCleanup.readSummary(WINDOW);
461 assert.strictEqual(totalOutputTokens(recreated), 12);
462 assert.deepStrictEqual(recreated.sources[0]?.fingerprint, first.sources[0]?.fingerprint);
463
464 const merged = mergeUsage(
465 [
466 {
467 environmentId: EnvironmentId.make("cleanup-test"),
468 label: "test",
469 summary: recreated,
470 },
471 {
472 environmentId: EnvironmentId.make("other-environment"),
473 label: "before cleanup",
474 summary: first,
475 },
476 ],
477 missingRoot.contractVersion,
478 );
479 assert.strictEqual(merged.outputTokens, 12);
480 assert.strictEqual(merged.sessions, 1);
481 assert.strictEqual(merged.costUsd, recreated.buckets[0]?.costUsd);
482
483 const outsideWindow = yield* restarted.readSummary({
484 ...WINDOW,
485 sinceDay: UsageDay.make("2026-08-02"),
486 });
487 assert.deepStrictEqual(outsideWindow.buckets, []);
488 assert.strictEqual(outsideWindow.sources[0]?.distinctSessions, 0);
489 }).pipe(
490 Effect.provide(
491 serviceLayers({
492 prefix: "usage-service-cleanup-test",
493 home,
494 settings: { providers: { ...settings.providers, claudeAgent: { homePath: alias } } },
495 ratesDocument: {
496 "claude-fable-5": { input_cost_per_token: 1e-5, output_cost_per_token: 5e-5 },
497 },
498 }),
499 ),
500 );
501 }).pipe(Effect.scoped),
502 );
503
396 it.live("does not share an in-flight scan after custom prices change", () =>504 it.live("does not share an in-flight scan after custom prices change", () =>
397 Effect.gen(function* () {505 Effect.gen(function* () {
398 const { transcript, settings, home } = yield* setup;506 const { transcript, settings, home } = yield* setup;
apps/server/src/usage/UsageService.ts+101-38
...@@ -101,6 +101,11 @@ const encodeRatesCache = Schema.encodeEffect(...@@ -101,6 +101,11 @@ const encodeRatesCache = Schema.encodeEffect(
101const ScanCacheJson = Schema.fromJsonString(Schema.Unknown as unknown as Schema.Codec<unknown>);101const ScanCacheJson = Schema.fromJsonString(Schema.Unknown as unknown as Schema.Codec<unknown>);
102const decodeScanCacheFile = Schema.decodeUnknownEffect(ScanCacheJson);102const decodeScanCacheFile = Schema.decodeUnknownEffect(ScanCacheJson);
103const encodeScanCacheFile = Schema.encodeEffect(ScanCacheJson);103const encodeScanCacheFile = Schema.encodeEffect(ScanCacheJson);
104const encodeUsageRecordKey = Schema.encodeSync(ScanCacheJson);
105const CachedSource = Schema.Struct({ dir: Schema.String, volumeId: Schema.String });
106const decodeCachedSources = Schema.decodeUnknownOption(
107 Schema.Struct({ sources: Schema.Record(Schema.String, CachedSource) }),
108);
104109
105export class UsageService extends Context.Service<110export class UsageService extends Context.Service<
106 UsageService,111 UsageService,
...@@ -147,7 +152,12 @@ export const make = Effect.gen(function* () {...@@ -147,7 +152,12 @@ export const make = Effect.gen(function* () {
147 const hostEnvironment = yield* HostProcessEnvironment;152 const hostEnvironment = yield* HostProcessEnvironment;
148153
149 const fileCache: ScanCache = new Map();154 const fileCache: ScanCache = new Map();
155 const sourceCache = new Map<string, typeof CachedSource.Type>();
150 let cacheDirty = false;156 let cacheDirty = false;
157 const isWithinDirectory = (filePath: string, dir: string) => {
158 const relative = path.relative(dir, filePath);
159 return relative !== ".." && !relative.startsWith(".." + path.sep) && !path.isAbsolute(relative);
160 };
151161
152 const ratesCachePath = path.join(config.stateDir, "usage-model-rates.json");162 const ratesCachePath = path.join(config.stateDir, "usage-model-rates.json");
153 const scanCachePath = path.join(config.stateDir, "usage-scan-cache.json");163 const scanCachePath = path.join(config.stateDir, "usage-scan-cache.json");
...@@ -241,8 +251,14 @@ export const make = Effect.gen(function* () {...@@ -241,8 +251,14 @@ export const make = Effect.gen(function* () {
241 /** Resolves the transcript directory for each provider. */251 /** Resolves the transcript directory for each provider. */
242 const resolveTranscriptDirs = Effect.fn("UsageService.resolveTranscriptDirs")(function* (252 const resolveTranscriptDirs = Effect.fn("UsageService.resolveTranscriptDirs")(function* (
243 settings: ServerSettingsValue,253 settings: ServerSettingsValue,
254 retentionCutoffMs: number,
244 ) {255 ) {
245 const dirs: Array<{ provider: UsageProviderKind; dir: string; fileName?: string }> = [];256 const dirs: Array<{
257 provider: UsageProviderKind;
258 dir: string;
259 volumeId: string;
260 fileName?: string;
261 }> = [];
246 const seen = new Set<string>();262 const seen = new Set<string>();
247 for (const driver of ["claudeAgent", "codex", "grok"] as const) {263 for (const driver of ["claudeAgent", "codex", "grok"] as const) {
248 // Disabled accounts still have history. Explicit default slots replace264 // Disabled accounts still have history. Explicit default slots replace
...@@ -280,14 +296,41 @@ export const make = Effect.gen(function* () {...@@ -280,14 +296,41 @@ export const make = Effect.gen(function* () {
280 );296 );
281 }297 }
282 const directory = path.resolve(home, provider === "claude" ? "projects" : "sessions");298 const directory = path.resolve(home, provider === "claude" ? "projects" : "sessions");
283 // Account aliases and Codex auth overlays can share the same history.299 const sourceKey = provider + "\0" + directory;
300 const previous = sourceCache.get(sourceKey);
301 // Keep canonical paths and source fingerprints stable after root cleanup,
302 // including aliases and clients merging pre-cleanup environment summaries.
284 const dir = yield* fileSystem303 const dir = yield* fileSystem
285 .realPath(directory)304 .realPath(directory)
286 .pipe(Effect.orElseSucceed(() => directory));305 .pipe(Effect.orElseSucceed(() => previous?.dir ?? directory));
306 const currentVolumeId = yield* Effect.promise(() => readDirectoryVolumeId(dir));
307 const hasRetainedHistory = fileCache
308 .entries()
309 .some(
310 ([filePath, entry]) =>
311 entry.provider === provider &&
312 entry.mtimeMs >= retentionCutoffMs &&
313 entry.records.length + entry.tailRecords.length > 0 &&
314 isWithinDirectory(filePath, dir),
315 );
316 // A recreated directory still reports the retained history under its old identity.
317 const volumeId =
318 previous?.dir === dir && (hasRetainedHistory || !currentVolumeId)
319 ? previous.volumeId || currentVolumeId
320 : currentVolumeId;
321 if (previous?.dir !== dir || previous.volumeId !== volumeId) {
322 sourceCache.set(sourceKey, { dir, volumeId });
323 cacheDirty = true;
324 }
287 const key = `${provider}\0${dir}`;325 const key = `${provider}\0${dir}`;
288 if (seen.has(key)) continue;326 if (seen.has(key)) continue;
289 seen.add(key);327 seen.add(key);
290 dirs.push({ provider, dir, ...(provider === "grok" ? { fileName: "updates.jsonl" } : {}) });328 dirs.push({
329 provider,
330 dir,
331 volumeId,
332 ...(provider === "grok" ? { fileName: "updates.jsonl" } : {}),
333 });
291 }334 }
292 }335 }
293 return dirs;336 return dirs;
...@@ -308,6 +351,11 @@ export const make = Effect.gen(function* () {...@@ -308,6 +351,11 @@ export const make = Effect.gen(function* () {
308 );351 );
309 if (document === null) return;352 if (document === null) return;
310 for (const [path, entry] of decodeScanCache(document)) fileCache.set(path, entry);353 for (const [path, entry] of decodeScanCache(document)) fileCache.set(path, entry);
354 const sources = decodeCachedSources(document);
355 if (Option.isSome(sources)) {
356 for (const [key, source] of Object.entries(sources.value.sources))
357 sourceCache.set(key, source);
358 }
311 }),359 }),
312 );360 );
313361
...@@ -315,7 +363,10 @@ export const make = Effect.gen(function* () {...@@ -315,7 +363,10 @@ export const make = Effect.gen(function* () {
315 if (!cacheDirty) return;363 if (!cacheDirty) return;
316 // Cleared only after the write lands, so a failed persist is retried on364 // Cleared only after the write lands, so a failed persist is retried on
317 // the next scan instead of leaving disk permanently stale.365 // the next scan instead of leaving disk permanently stale.
318 yield* encodeScanCacheFile(encodeScanCache(fileCache)).pipe(366 yield* encodeScanCacheFile({
367 ...encodeScanCache(fileCache),
368 sources: Object.fromEntries(sourceCache),
369 }).pipe(
319 Effect.flatMap((serialized) => fileSystem.writeFileString(scanCachePath, serialized)),370 Effect.flatMap((serialized) => fileSystem.writeFileString(scanCachePath, serialized)),
320 Effect.map(() => {371 Effect.map(() => {
321 cacheDirty = false;372 cacheDirty = false;
...@@ -366,7 +417,8 @@ export const make = Effect.gen(function* () {...@@ -366,7 +417,8 @@ export const make = Effect.gen(function* () {
366 );417 );
367 // A read failure is not an empty transcript: caching it under this418 // A read failure is not an empty transcript: caching it under this
368 // (size, mtime) would silently drop the file's usage until it changes.419 // (size, mtime) would silently drop the file's usage until it changes.
369 if (parsed === null) return [];420 if (parsed === null)
421 return cached?.provider === provider ? [...cached.records, ...cached.tailRecords] : [];
370422
371 // Stored already de-duplicated within the file, which is 99% of all423 // Stored already de-duplicated within the file, which is 99% of all
372 // duplicates. The aggregator still runs the cross-file dedupe pass. One424 // duplicates. The aggregator still runs the cross-file dedupe pass. One
...@@ -403,15 +455,15 @@ export const make = Effect.gen(function* () {...@@ -403,15 +455,15 @@ export const make = Effect.gen(function* () {
403 const collectDirs = Effect.fn("UsageService.collectDirs")(function* (455 const collectDirs = Effect.fn("UsageService.collectDirs")(function* (
404 windowStartMs: number,456 windowStartMs: number,
405 settings: ServerSettingsValue,457 settings: ServerSettingsValue,
458 retentionCutoffMs: number,
406 ) {459 ) {
407 // The home resolvers ask for `Path` themselves; satisfy them from the460 // The home resolvers ask for `Path` themselves; satisfy them from the
408 // instance we already hold so the scan stays context-free.461 // instance we already hold so the scan stays context-free.
409 const dirs = yield* resolveTranscriptDirs(settings).pipe(462 const dirs = yield* resolveTranscriptDirs(settings, retentionCutoffMs).pipe(
410 Effect.provideService(Path.Path, path),463 Effect.provideService(Path.Path, path),
411 );464 );
412 const scanned: ScannedDir[] = [];465 const scanned: ScannedDir[] = [];
413 for (const { provider, dir, fileName } of dirs) {466 for (const { provider, dir, volumeId, fileName } of dirs) {
414 const volumeId = yield* Effect.promise(() => readDirectoryVolumeId(dir));
415 const exists = yield* fileSystem467 const exists = yield* fileSystem
416 .exists(dir)468 .exists(dir)
417 .pipe(Effect.catchCause(() => Effect.succeed(false)));469 .pipe(Effect.catchCause(() => Effect.succeed(false)));
...@@ -481,11 +533,13 @@ export const make = Effect.gen(function* () {...@@ -481,11 +533,13 @@ export const make = Effect.gen(function* () {
481 const windowStartMs =533 const windowStartMs =
482 (hourlyWindow?.sinceTimeMs ?? DateTime.toEpochMillis(windowStart.value)) - MTIME_SLACK_MS;534 (hourlyWindow?.sinceTimeMs ?? DateTime.toEpochMillis(windowStart.value)) - MTIME_SLACK_MS;
483535
536 const retentionCutoffMs = startedAtMs - CACHE_RETENTION_DAYS * 24 * 60 * 60 * 1000;
537
484 // Pricing only matters once records are aggregated, so the rate table538 // Pricing only matters once records are aggregated, so the rate table
485 // loads while transcripts stream instead of gating them: a cold rates539 // loads while transcripts stream instead of gating them: a cold rates
486 // fetch on a slow network no longer delays the scan by its own timeout.540 // fetch on a slow network no longer delays the scan by its own timeout.
487 const [, scannedDirs] = yield* Effect.all(541 const [, scannedDirs] = yield* Effect.all(
488 [ensureRates(false), collectDirs(windowStartMs, settings)],542 [ensureRates(false), collectDirs(windowStartMs, settings, retentionCutoffMs)],
489 { concurrency: 2 },543 { concurrency: 2 },
490 );544 );
491545
...@@ -500,41 +554,54 @@ export const make = Effect.gen(function* () {...@@ -500,41 +554,54 @@ export const make = Effect.gen(function* () {
500 });554 });
501555
502 const sources: UsageSource[] = [];556 const sources: UsageSource[] = [];
503 const livePaths = new Set<string>();
504 const walkedRoots: string[] = [];
505557
506 for (const { provider, dir, volumeId, files } of scannedDirs) {558 for (const { provider, dir, volumeId, files } of scannedDirs) {
507 if (files === null) {559 const retainedFiles = [...(files ?? [])];
508 sources.push({560 const livePaths = new Set(retainedFiles.map((file) => file.path));
509 fingerprint: { hostId, provider, resolvedHomePath: dir, volumeId },561 // Cleanup may remove transcripts, but the usage we already saved still
510 status: "missing",562 // contributes to this source. Keep the normal aggregation and dedupe path.
511 scannedFiles: 0,563 for (const [filePath, entry] of fileCache) {
512 skippedFiles: 0,564 if (
513 malformedRecords: 0,565 entry.provider !== provider ||
514 distinctSessions: 0,566 entry.mtimeMs < retentionCutoffMs ||
515 message: "No transcript directory on this environment.",567 livePaths.has(filePath) ||
516 });568 !isWithinDirectory(filePath, dir)
517 continue;569 )
570 continue;
571 retainedFiles.push({ path: filePath, records: [...entry.records, ...entry.tailRecords] });
518 }572 }
519
520 walkedRoots.push(dir);
521 let scannedFiles = 0;573 let scannedFiles = 0;
522 let skippedFiles = 0;574 let skippedFiles = 0;
523 // Distinct per directory. Buckets carry per-cell session counts, but a575 // Distinct per directory. Buckets carry per-cell session counts, but a
524 // session spans days and models, so clients total this figure instead.576 // session spans days and models, so clients total this figure instead.
525 const sessionIds = new Set<string>();577 const sessionIds = new Set<string>();
526578
527 for (const file of files) {579 for (const file of retainedFiles) {
528 livePaths.add(file.path);
529 if (file.records.length === 0) {580 if (file.records.length === 0) {
530 skippedFiles += 1;581 skippedFiles += 1;
531 continue;582 continue;
532 }583 }
533 scannedFiles += 1;584 scannedFiles += 1;
585 const codexEventOccurrences = new Map<string, number>();
534 for (const record of file.records) {586 for (const record of file.records) {
535 // Only sessions that contributed in-window count: the mtime slack587 let usageRecord = record;
536 // admits boundary files whose records fall outside the range.588 if (record.provider === "codex" && record.sessionId.length > 0) {
537 if (aggregator.add(record) && record.sessionId.length > 0) {589 // Match moved rollout copies without collapsing repeated equal events
590 // within one rollout (timestamps can have only second precision).
591 const key = encodeUsageRecordKey([
592 record.provider,
593 record.sessionId,
594 record.timestampMs,
595 record.model,
596 record.totals,
597 ]);
598 const occurrence = (codexEventOccurrences.get(key) ?? 0) + 1;
599 codexEventOccurrences.set(key, occurrence);
600 usageRecord = { ...record, dedupeKey: key + ":" + occurrence };
601 }
602 // Only sessions contributing in-window count; the mtime slack can
603 // admit boundary files whose records fall outside the range.
604 if (aggregator.add(usageRecord) && record.sessionId.length > 0) {
538 sessionIds.add(record.sessionId);605 sessionIds.add(record.sessionId);
539 }606 }
540 }607 }
...@@ -542,21 +609,17 @@ export const make = Effect.gen(function* () {...@@ -542,21 +609,17 @@ export const make = Effect.gen(function* () {
542609
543 sources.push({610 sources.push({
544 fingerprint: { hostId, provider, resolvedHomePath: dir, volumeId },611 fingerprint: { hostId, provider, resolvedHomePath: dir, volumeId },
545 status: "ok",612 // Clients exclude missing sources, so saved records remain an available source.
613 status: files === null && scannedFiles === 0 ? "missing" : "ok",
546 scannedFiles,614 scannedFiles,
547 skippedFiles,615 skippedFiles,
548 malformedRecords: 0,616 malformedRecords: 0,
549 distinctSessions: sessionIds.size,617 distinctSessions: sessionIds.size,
550 message: null,618 message: files === null ? "No transcript directory on this environment." : null,
551 });619 });
552 }620 }
553621
554 const pruned = pruneScanCache(fileCache, {622 const pruned = pruneScanCache(fileCache, retentionCutoffMs);
555 livePaths,
556 walkedRoots,
557 windowStartMs,
558 retentionCutoffMs: startedAtMs - CACHE_RETENTION_DAYS * 24 * 60 * 60 * 1000,
559 });
560 if (pruned > 0) cacheDirty = true;623 if (pruned > 0) cacheDirty = true;
561 yield* persistScanCache();624 yield* persistScanCache();
562625
apps/server/src/usage/usageScanCache.test.ts+3-74
...@@ -194,88 +194,17 @@ describe("pruneScanCache", () => {...@@ -194,88 +194,17 @@ describe("pruneScanCache", () => {
194 it("drops entries older than retention", () => {194 it("drops entries older than retention", () => {
195 const cache = cacheWith([["/old.jsonl", 500, [record()]]]);195 const cache = cacheWith([["/old.jsonl", 500, [record()]]]);
196196
197 const removed = pruneScanCache(cache, {197 const removed = pruneScanCache(cache, retentionCutoffMs);
198 livePaths: new Set(),
199 walkedRoots: ["/"],
200 windowStartMs: 400,
201 retentionCutoffMs,
202 });
203198
204 expect(removed).toBe(1);199 expect(removed).toBe(1);
205 expect(cache.size).toBe(0);200 expect(cache.size).toBe(0);
206 });201 });
207202
208 it("drops in-window entries whose file has disappeared", () => {203 it("keeps entries whose file has disappeared", () => {
209 const cache = cacheWith([["/gone.jsonl", 5000, [record()]]]);204 const cache = cacheWith([["/gone.jsonl", 5000, [record()]]]);
210205
211 pruneScanCache(cache, {206 pruneScanCache(cache, retentionCutoffMs);
212 livePaths: new Set(),
213 walkedRoots: ["/"],
214 windowStartMs: 4000,
215 retentionCutoffMs,
216 });
217
218 expect(cache.size).toBe(0);
219 });
220
221 it("keeps entries outside the walked window that are still within retention", () => {
222 // Viewing 7 days must not evict the 30-day entries, which that walk never
223 // looked for and so cannot prove are gone.
224 const cache = cacheWith([["/older-but-valid.jsonl", 2000, [record()]]]);
225
226 const removed = pruneScanCache(cache, {
227 livePaths: new Set(),
228 walkedRoots: ["/"],
229 windowStartMs: 4000,
230 retentionCutoffMs,
231 });
232
233 expect(removed).toBe(0);
234 expect(cache.size).toBe(1);
235 });
236
237 it("keeps entries the walk saw", () => {
238 const cache = cacheWith([["/live.jsonl", 5000, [record()]]]);
239
240 pruneScanCache(cache, {
241 livePaths: new Set(["/live.jsonl"]),
242 walkedRoots: ["/"],
243 windowStartMs: 4000,
244 retentionCutoffMs,
245 });
246
247 expect(cache.size).toBe(1);
248 });
249});
250
251describe("pruneScanCache with an unwalked root", () => {
252 it("keeps in-window entries for a provider whose directory was not walked", () => {
253 // A missing provider root or failed settings read leaves livePaths without
254 // that provider's files. Its warm entries must survive the pass.
255 const cache = cacheWith([["/codex/sessions/a.jsonl", 5000, [record()]]]);
256
257 const removed = pruneScanCache(cache, {
258 livePaths: new Set(),
259 walkedRoots: ["/claude/projects"],
260 windowStartMs: 4000,
261 retentionCutoffMs: 1000,
262 });
263
264 expect(removed).toBe(0);
265 expect(cache.size).toBe(1);
266 });
267
268 it("keeps entries under a sibling path that only shares the walked root prefix", () => {
269 const cache = cacheWith([["/claude/projects-copy/a.jsonl", 5000, [record()]]]);
270
271 const removed = pruneScanCache(cache, {
272 livePaths: new Set(),
273 walkedRoots: ["/claude/projects"],
274 windowStartMs: 4000,
275 retentionCutoffMs: 1000,
276 });
277207
278 expect(removed).toBe(0);
279 expect(cache.size).toBe(1);208 expect(cache.size).toBe(1);
280 });209 });
281});210});
apps/server/src/usage/usageScanCache.ts+3-42
...@@ -14,9 +14,6 @@...@@ -14,9 +14,6 @@
14 *14 *
15 * @module usageScanCache15 * @module usageScanCache
16 */16 */
17// @effect-diagnostics nodeBuiltinImport:off
18import * as NodePath from "node:path";
19
20import type { UsageProviderKind } from "@t3tools/contracts";17import type { UsageProviderKind } from "@t3tools/contracts";
2118
22import { GUARD_LENGTH, type TranscriptParsePosition } from "./usageTranscriptReader.ts";19import { GUARD_LENGTH, type TranscriptParsePosition } from "./usageTranscriptReader.ts";
...@@ -295,47 +292,11 @@ function decodeCodexState(value: unknown): CodexScanState | null | undefined {...@@ -295,47 +292,11 @@ function decodeCodexState(value: unknown): CodexScanState | null | undefined {
295 };292 };
296}293}
297294
298export interface PruneOptions {295/** Keeps saved usage after transcript cleanup, until the reporting retention expires. */
299 /** Files the walk just saw. Only meaningful inside the walked window. */296export function pruneScanCache(cache: ScanCache, retentionCutoffMs: number): number {
300 readonly livePaths: ReadonlySet<string>;
301 /**
302 * Roots the walk actually completed. Absence from `livePaths` only proves a
303 * file is gone when its root was walked: a provider whose directory failed to
304 * resolve this pass must not have its warm entries purged.
305 */
306 readonly walkedRoots: readonly string[];
307 /** Start of the walked window; entries older than this were not looked for. */
308 readonly windowStartMs: number;
309 /** Entries older than this are dropped regardless. */
310 readonly retentionCutoffMs: number;
311}
312
313/**
314 * Drops aged-out entries, and entries for files that have disappeared.
315 *
316 * The walk only covers the requested window, so absence from `livePaths` only
317 * proves deletion for entries *inside* that window. Pruning everything the walk
318 * missed would evict the 30-day entries every time someone looked at 7 days.
319 *
320 * Replaces an earlier record cap that cleared the whole cache once exceeded,
321 * which meant a large enough window never warmed up at all.
322 */
323export function pruneScanCache(cache: ScanCache, options: PruneOptions): number {
324 let removed = 0;297 let removed = 0;
325 for (const [path, entry] of cache) {298 for (const [path, entry] of cache) {
326 const agedOut = entry.mtimeMs < options.retentionCutoffMs;299 if (entry.mtimeMs < retentionCutoffMs) {
327 const underWalkedRoot = options.walkedRoots.some((root) => {
328 const relative = NodePath.relative(root, path);
329 return (
330 relative === "" ||
331 (relative !== ".." &&
332 !relative.startsWith(`..${NodePath.sep}`) &&
333 !NodePath.isAbsolute(relative))
334 );
335 });
336 const deleted =
337 underWalkedRoot && entry.mtimeMs >= options.windowStartMs && !options.livePaths.has(path);
338 if (agedOut || deleted) {
339 cache.delete(path);300 cache.delete(path);
340 removed += 1;301 removed += 1;
341 }302 }