| author | |
| committer | |
| log | 675869d2b0f9e059d5950b80390c0cad1cb1590e |
| tree | ca7b59d70d88563a3c1b9ec2e31c520d91f19294 |
| parent | 592021f008010ee947c326df0452d093ead8aeb1 |
| signature | Signed by PGP key B5690EEEBB952194 |
Co-authored-by: maria-rcks <254055478+maria-rcks@users.noreply.github.com>4 files changed, 219 insertions(+), 158 deletions(-)
apps/server/src/usage/UsageService.test.ts+112-4| ... | ... | @@ -7,7 +7,9 @@ import * as NodePath from "node:path"; |
| 7 | 7 | import { assert, describe, it } from "@effect/vitest"; |
| 8 | 8 | import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 9 | 9 | import { HostProcessEnvironment } from "@t3tools/shared/hostProcess"; |
| 10 | import { mergeUsage } from "@t3tools/shared/usageMerge"; | |
| 10 | 11 | import { |
| 12 | EnvironmentId, | |
| 11 | 13 | ProviderDriverKind, |
| 12 | 14 | ProviderInstanceId, |
| 13 | 15 | UsageDay, |
| ... | ... | @@ -131,14 +133,15 @@ describe("UsageService", () => { |
| 131 | 133 | [ |
| 132 | 134 | { type: "session_meta", payload: { id: "codex-account-session" } }, |
| 133 | 135 | { type: "turn_context", payload: { model: "gpt-5.6-sol" } }, |
| 134 | { | |
| 136 | // A-B-A at one timestamp must preserve both equal A events. | |
| 137 | ...[11, 12, 11].map((outputTokens) => ({ | |
| 135 | 138 | type: "event_msg", |
| 136 | 139 | timestamp: "2026-08-01T10:00:00Z", |
| 137 | 140 | payload: { |
| 138 | 141 | type: "token_count", |
| 139 | info: { last_token_usage: { input_tokens: 10, output_tokens: 11 } }, | |
| 142 | info: { last_token_usage: { input_tokens: 10, output_tokens: outputTokens } }, | |
| 140 | 143 | }, |
| 141 | }, | |
| 144 | })), | |
| 142 | 145 | ] |
| 143 | 146 | .map((line) => encodeUnknownJsonString(line)) |
| 144 | 147 | .join("\n") + "\n", |
| ... | ... | @@ -198,7 +201,21 @@ describe("UsageService", () => { |
| 198 | 201 | ), |
| 199 | 202 | ); |
| 200 | 203 | const summary = yield* service.readSummary(WINDOW); |
| 201 | assert.strictEqual(totalOutputTokens(summary), 36); | |
| 204 | assert.strictEqual(totalOutputTokens(summary), 59); | |
| 205 | yield* Effect.promise(() => | |
| 206 | NodeFSP.rename( | |
| 207 | NodePath.join(codexHome, "sessions", "rollout.jsonl"), | |
| 208 | NodePath.join(codexHome, "sessions", "moved.jsonl"), | |
| 209 | ), | |
| 210 | ); | |
| 211 | const moved = yield* service.readSummary(WINDOW); | |
| 212 | assert.deepStrictEqual(moved.buckets, summary.buckets); | |
| 213 | yield* Effect.promise(() => | |
| 214 | NodeFSP.rm(NodePath.join(codexHome, "sessions"), { recursive: true }), | |
| 215 | ); | |
| 216 | const removed = yield* service.readSummary(WINDOW); | |
| 217 | assert.deepStrictEqual(removed.buckets, summary.buckets); | |
| 218 | ||
| 202 | 219 | const sources = summary.sources.filter((source) => source.status === "ok"); |
| 203 | 220 | assert.strictEqual(sources.length, 4); |
| 204 | 221 | assert.strictEqual( |
| ... | ... | @@ -393,6 +410,97 @@ describe("UsageService", () => { |
| 393 | 410 | }).pipe(Effect.scoped), |
| 394 | 411 | ); |
| 395 | 412 | |
| 413 | it.live("preserves saved tokens, costs and sessions after transcript cleanup and restart", () => | |
| 414 | Effect.gen(function* () { | |
| 415 | const { transcript, settings, home } = yield* setup; | |
| 416 | const alias = NodePath.join(home, "claude-alias"); | |
| 417 | yield* Effect.promise(() => | |
| 418 | NodeFSP.symlink(NodePath.join(home, "claude"), alias, "junction"), | |
| 419 | ); | |
| 420 | const content = claudeLine(1, 5); | |
| 421 | yield* Effect.promise(() => NodeFSP.writeFile(transcript, content)); | |
| 422 | yield* Effect.gen(function* () { | |
| 423 | const service = yield* UsageService.make; | |
| 424 | const first = yield* service.readSummary(WINDOW); | |
| 425 | assert.strictEqual(totalOutputTokens(first), 5); | |
| 426 | assert.isAbove(first.buckets[0]?.costUsd ?? 0, 0); | |
| 427 | ||
| 428 | yield* Effect.promise(() => NodeFSP.rm(transcript)); | |
| 429 | const deleted = yield* service.readSummary(WINDOW); | |
| 430 | assert.deepStrictEqual(deleted.buckets, first.buckets); | |
| 431 | assert.deepStrictEqual(deleted.sources, first.sources); | |
| 432 | ||
| 433 | const restarted = yield* UsageService.make; | |
| 434 | const restored = yield* restarted.readSummary(WINDOW); | |
| 435 | assert.deepStrictEqual(restored.buckets, first.buckets); | |
| 436 | assert.deepStrictEqual(restored.sources, first.sources); | |
| 437 | ||
| 438 | // A moved transcript must not count the saved usage twice. | |
| 439 | yield* Effect.promise(() => NodeFSP.writeFile(transcript + ".jsonl", content)); | |
| 440 | const moved = yield* restarted.readSummary(WINDOW); | |
| 441 | assert.deepStrictEqual(moved.buckets, first.buckets); | |
| 442 | assert.strictEqual(moved.sources[0]?.distinctSessions, 1); | |
| 443 | ||
| 444 | const replacementProjects = NodePath.join(home, "replacement-projects"); | |
| 445 | yield* Effect.promise(() => NodeFSP.mkdir(replacementProjects)); | |
| 446 | yield* Effect.promise(() => | |
| 447 | NodeFSP.rm(NodePath.join(home, "claude", "projects"), { recursive: true }), | |
| 448 | ); | |
| 449 | const afterRootCleanup = yield* UsageService.make; | |
| 450 | const missingRoot = yield* afterRootCleanup.readSummary(WINDOW); | |
| 451 | assert.deepStrictEqual(missingRoot.buckets, first.buckets); | |
| 452 | assert.strictEqual(missingRoot.sources[0]?.distinctSessions, 1); | |
| 453 | assert.strictEqual(missingRoot.sources[0]?.status, "ok"); | |
| 454 | assert.deepStrictEqual(missingRoot.sources[0]?.fingerprint, first.sources[0]?.fingerprint); | |
| 455 | yield* Effect.promise(async () => { | |
| 456 | const projects = NodePath.join(home, "claude", "projects"); | |
| 457 | await NodeFSP.rename(replacementProjects, projects); | |
| 458 | await NodeFSP.writeFile(NodePath.join(projects, "new.jsonl"), claudeLine(2, 7)); | |
| 459 | }); | |
| 460 | const recreated = yield* afterRootCleanup.readSummary(WINDOW); | |
| 461 | assert.strictEqual(totalOutputTokens(recreated), 12); | |
| 462 | assert.deepStrictEqual(recreated.sources[0]?.fingerprint, first.sources[0]?.fingerprint); | |
| 463 | ||
| 464 | const merged = mergeUsage( | |
| 465 | [ | |
| 466 | { | |
| 467 | environmentId: EnvironmentId.make("cleanup-test"), | |
| 468 | label: "test", | |
| 469 | summary: recreated, | |
| 470 | }, | |
| 471 | { | |
| 472 | environmentId: EnvironmentId.make("other-environment"), | |
| 473 | label: "before cleanup", | |
| 474 | summary: first, | |
| 475 | }, | |
| 476 | ], | |
| 477 | missingRoot.contractVersion, | |
| 478 | ); | |
| 479 | assert.strictEqual(merged.outputTokens, 12); | |
| 480 | assert.strictEqual(merged.sessions, 1); | |
| 481 | assert.strictEqual(merged.costUsd, recreated.buckets[0]?.costUsd); | |
| 482 | ||
| 483 | const outsideWindow = yield* restarted.readSummary({ | |
| 484 | ...WINDOW, | |
| 485 | sinceDay: UsageDay.make("2026-08-02"), | |
| 486 | }); | |
| 487 | assert.deepStrictEqual(outsideWindow.buckets, []); | |
| 488 | assert.strictEqual(outsideWindow.sources[0]?.distinctSessions, 0); | |
| 489 | }).pipe( | |
| 490 | Effect.provide( | |
| 491 | serviceLayers({ | |
| 492 | prefix: "usage-service-cleanup-test", | |
| 493 | home, | |
| 494 | settings: { providers: { ...settings.providers, claudeAgent: { homePath: alias } } }, | |
| 495 | ratesDocument: { | |
| 496 | "claude-fable-5": { input_cost_per_token: 1e-5, output_cost_per_token: 5e-5 }, | |
| 497 | }, | |
| 498 | }), | |
| 499 | ), | |
| 500 | ); | |
| 501 | }).pipe(Effect.scoped), | |
| 502 | ); | |
| 503 | ||
| 396 | 504 | it.live("does not share an in-flight scan after custom prices change", () => |
| 397 | 505 | Effect.gen(function* () { |
| 398 | 506 | const { transcript, settings, home } = yield* setup; |
apps/server/src/usage/UsageService.ts+101-38| ... | ... | @@ -101,6 +101,11 @@ const encodeRatesCache = Schema.encodeEffect( |
| 101 | 101 | const ScanCacheJson = Schema.fromJsonString(Schema.Unknown as unknown as Schema.Codec<unknown>); |
| 102 | 102 | const decodeScanCacheFile = Schema.decodeUnknownEffect(ScanCacheJson); |
| 103 | 103 | const encodeScanCacheFile = Schema.encodeEffect(ScanCacheJson); |
| 104 | const encodeUsageRecordKey = Schema.encodeSync(ScanCacheJson); | |
| 105 | const CachedSource = Schema.Struct({ dir: Schema.String, volumeId: Schema.String }); | |
| 106 | const decodeCachedSources = Schema.decodeUnknownOption( | |
| 107 | Schema.Struct({ sources: Schema.Record(Schema.String, CachedSource) }), | |
| 108 | ); | |
| 104 | 109 | |
| 105 | 110 | export class UsageService extends Context.Service< |
| 106 | 111 | UsageService, |
| ... | ... | @@ -147,7 +152,12 @@ export const make = Effect.gen(function* () { |
| 147 | 152 | const hostEnvironment = yield* HostProcessEnvironment; |
| 148 | 153 | |
| 149 | 154 | const fileCache: ScanCache = new Map(); |
| 155 | const sourceCache = new Map<string, typeof CachedSource.Type>(); | |
| 150 | 156 | let cacheDirty = false; |
| 157 | const isWithinDirectory = (filePath: string, dir: string) => { | |
| 158 | const relative = path.relative(dir, filePath); | |
| 159 | return relative !== ".." && !relative.startsWith(".." + path.sep) && !path.isAbsolute(relative); | |
| 160 | }; | |
| 151 | 161 | |
| 152 | 162 | const ratesCachePath = path.join(config.stateDir, "usage-model-rates.json"); |
| 153 | 163 | const scanCachePath = path.join(config.stateDir, "usage-scan-cache.json"); |
| ... | ... | @@ -241,8 +251,14 @@ export const make = Effect.gen(function* () { |
| 241 | 251 | /** Resolves the transcript directory for each provider. */ |
| 242 | 252 | const resolveTranscriptDirs = Effect.fn("UsageService.resolveTranscriptDirs")(function* ( |
| 243 | 253 | settings: ServerSettingsValue, |
| 254 | retentionCutoffMs: number, | |
| 244 | 255 | ) { |
| 245 | const dirs: Array<{ provider: UsageProviderKind; dir: string; fileName?: string }> = []; | |
| 256 | const dirs: Array<{ | |
| 257 | provider: UsageProviderKind; | |
| 258 | dir: string; | |
| 259 | volumeId: string; | |
| 260 | fileName?: string; | |
| 261 | }> = []; | |
| 246 | 262 | const seen = new Set<string>(); |
| 247 | 263 | for (const driver of ["claudeAgent", "codex", "grok"] as const) { |
| 248 | 264 | // Disabled accounts still have history. Explicit default slots replace |
| ... | ... | @@ -280,14 +296,41 @@ export const make = Effect.gen(function* () { |
| 280 | 296 | ); |
| 281 | 297 | } |
| 282 | 298 | const directory = path.resolve(home, provider === "claude" ? "projects" : "sessions"); |
| 283 | // Account aliases and Codex auth overlays can share the same history. | |
| 299 | const sourceKey = provider + "\0" + directory; | |
| 300 | const previous = sourceCache.get(sourceKey); | |
| 301 | // Keep canonical paths and source fingerprints stable after root cleanup, | |
| 302 | // including aliases and clients merging pre-cleanup environment summaries. | |
| 284 | 303 | const dir = yield* fileSystem |
| 285 | 304 | .realPath(directory) |
| 286 | .pipe(Effect.orElseSucceed(() => directory)); | |
| 305 | .pipe(Effect.orElseSucceed(() => previous?.dir ?? directory)); | |
| 306 | const currentVolumeId = yield* Effect.promise(() => readDirectoryVolumeId(dir)); | |
| 307 | const hasRetainedHistory = fileCache | |
| 308 | .entries() | |
| 309 | .some( | |
| 310 | ([filePath, entry]) => | |
| 311 | entry.provider === provider && | |
| 312 | entry.mtimeMs >= retentionCutoffMs && | |
| 313 | entry.records.length + entry.tailRecords.length > 0 && | |
| 314 | isWithinDirectory(filePath, dir), | |
| 315 | ); | |
| 316 | // A recreated directory still reports the retained history under its old identity. | |
| 317 | const volumeId = | |
| 318 | previous?.dir === dir && (hasRetainedHistory || !currentVolumeId) | |
| 319 | ? previous.volumeId || currentVolumeId | |
| 320 | : currentVolumeId; | |
| 321 | if (previous?.dir !== dir || previous.volumeId !== volumeId) { | |
| 322 | sourceCache.set(sourceKey, { dir, volumeId }); | |
| 323 | cacheDirty = true; | |
| 324 | } | |
| 287 | 325 | const key = `${provider}\0${dir}`; |
| 288 | 326 | if (seen.has(key)) continue; |
| 289 | 327 | seen.add(key); |
| 290 | dirs.push({ provider, dir, ...(provider === "grok" ? { fileName: "updates.jsonl" } : {}) }); | |
| 328 | dirs.push({ | |
| 329 | provider, | |
| 330 | dir, | |
| 331 | volumeId, | |
| 332 | ...(provider === "grok" ? { fileName: "updates.jsonl" } : {}), | |
| 333 | }); | |
| 291 | 334 | } |
| 292 | 335 | } |
| 293 | 336 | return dirs; |
| ... | ... | @@ -308,6 +351,11 @@ export const make = Effect.gen(function* () { |
| 308 | 351 | ); |
| 309 | 352 | if (document === null) return; |
| 310 | 353 | for (const [path, entry] of decodeScanCache(document)) fileCache.set(path, entry); |
| 354 | const sources = decodeCachedSources(document); | |
| 355 | if (Option.isSome(sources)) { | |
| 356 | for (const [key, source] of Object.entries(sources.value.sources)) | |
| 357 | sourceCache.set(key, source); | |
| 358 | } | |
| 311 | 359 | }), |
| 312 | 360 | ); |
| 313 | 361 | |
| ... | ... | @@ -315,7 +363,10 @@ export const make = Effect.gen(function* () { |
| 315 | 363 | if (!cacheDirty) return; |
| 316 | 364 | // Cleared only after the write lands, so a failed persist is retried on |
| 317 | 365 | // the next scan instead of leaving disk permanently stale. |
| 318 | yield* encodeScanCacheFile(encodeScanCache(fileCache)).pipe( | |
| 366 | yield* encodeScanCacheFile({ | |
| 367 | ...encodeScanCache(fileCache), | |
| 368 | sources: Object.fromEntries(sourceCache), | |
| 369 | }).pipe( | |
| 319 | 370 | Effect.flatMap((serialized) => fileSystem.writeFileString(scanCachePath, serialized)), |
| 320 | 371 | Effect.map(() => { |
| 321 | 372 | cacheDirty = false; |
| ... | ... | @@ -366,7 +417,8 @@ export const make = Effect.gen(function* () { |
| 366 | 417 | ); |
| 367 | 418 | // A read failure is not an empty transcript: caching it under this |
| 368 | 419 | // (size, mtime) would silently drop the file's usage until it changes. |
| 369 | if (parsed === null) return []; | |
| 420 | if (parsed === null) | |
| 421 | return cached?.provider === provider ? [...cached.records, ...cached.tailRecords] : []; | |
| 370 | 422 | |
| 371 | 423 | // Stored already de-duplicated within the file, which is 99% of all |
| 372 | 424 | // duplicates. The aggregator still runs the cross-file dedupe pass. One |
| ... | ... | @@ -403,15 +455,15 @@ export const make = Effect.gen(function* () { |
| 403 | 455 | const collectDirs = Effect.fn("UsageService.collectDirs")(function* ( |
| 404 | 456 | windowStartMs: number, |
| 405 | 457 | settings: ServerSettingsValue, |
| 458 | retentionCutoffMs: number, | |
| 406 | 459 | ) { |
| 407 | 460 | // The home resolvers ask for `Path` themselves; satisfy them from the |
| 408 | 461 | // instance we already hold so the scan stays context-free. |
| 409 | const dirs = yield* resolveTranscriptDirs(settings).pipe( | |
| 462 | const dirs = yield* resolveTranscriptDirs(settings, retentionCutoffMs).pipe( | |
| 410 | 463 | Effect.provideService(Path.Path, path), |
| 411 | 464 | ); |
| 412 | 465 | const scanned: ScannedDir[] = []; |
| 413 | for (const { provider, dir, fileName } of dirs) { | |
| 414 | const volumeId = yield* Effect.promise(() => readDirectoryVolumeId(dir)); | |
| 466 | for (const { provider, dir, volumeId, fileName } of dirs) { | |
| 415 | 467 | const exists = yield* fileSystem |
| 416 | 468 | .exists(dir) |
| 417 | 469 | .pipe(Effect.catchCause(() => Effect.succeed(false))); |
| ... | ... | @@ -481,11 +533,13 @@ export const make = Effect.gen(function* () { |
| 481 | 533 | const windowStartMs = |
| 482 | 534 | (hourlyWindow?.sinceTimeMs ?? DateTime.toEpochMillis(windowStart.value)) - MTIME_SLACK_MS; |
| 483 | 535 | |
| 536 | const retentionCutoffMs = startedAtMs - CACHE_RETENTION_DAYS * 24 * 60 * 60 * 1000; | |
| 537 | ||
| 484 | 538 | // Pricing only matters once records are aggregated, so the rate table |
| 485 | 539 | // loads while transcripts stream instead of gating them: a cold rates |
| 486 | 540 | // fetch on a slow network no longer delays the scan by its own timeout. |
| 487 | 541 | const [, scannedDirs] = yield* Effect.all( |
| 488 | [ensureRates(false), collectDirs(windowStartMs, settings)], | |
| 542 | [ensureRates(false), collectDirs(windowStartMs, settings, retentionCutoffMs)], | |
| 489 | 543 | { concurrency: 2 }, |
| 490 | 544 | ); |
| 491 | 545 | |
| ... | ... | @@ -500,41 +554,54 @@ export const make = Effect.gen(function* () { |
| 500 | 554 | }); |
| 501 | 555 | |
| 502 | 556 | const sources: UsageSource[] = []; |
| 503 | const livePaths = new Set<string>(); | |
| 504 | const walkedRoots: string[] = []; | |
| 505 | 557 | |
| 506 | 558 | for (const { provider, dir, volumeId, files } of scannedDirs) { |
| 507 | if (files === null) { | |
| 508 | sources.push({ | |
| 509 | fingerprint: { hostId, provider, resolvedHomePath: dir, volumeId }, | |
| 510 | status: "missing", | |
| 511 | scannedFiles: 0, | |
| 512 | skippedFiles: 0, | |
| 513 | malformedRecords: 0, | |
| 514 | distinctSessions: 0, | |
| 515 | message: "No transcript directory on this environment.", | |
| 516 | }); | |
| 517 | continue; | |
| 559 | const retainedFiles = [...(files ?? [])]; | |
| 560 | const livePaths = new Set(retainedFiles.map((file) => file.path)); | |
| 561 | // Cleanup may remove transcripts, but the usage we already saved still | |
| 562 | // contributes to this source. Keep the normal aggregation and dedupe path. | |
| 563 | for (const [filePath, entry] of fileCache) { | |
| 564 | if ( | |
| 565 | entry.provider !== provider || | |
| 566 | entry.mtimeMs < retentionCutoffMs || | |
| 567 | livePaths.has(filePath) || | |
| 568 | !isWithinDirectory(filePath, dir) | |
| 569 | ) | |
| 570 | continue; | |
| 571 | retainedFiles.push({ path: filePath, records: [...entry.records, ...entry.tailRecords] }); | |
| 518 | 572 | } |
| 519 | ||
| 520 | walkedRoots.push(dir); | |
| 521 | 573 | let scannedFiles = 0; |
| 522 | 574 | let skippedFiles = 0; |
| 523 | 575 | // Distinct per directory. Buckets carry per-cell session counts, but a |
| 524 | 576 | // session spans days and models, so clients total this figure instead. |
| 525 | 577 | const sessionIds = new Set<string>(); |
| 526 | 578 | |
| 527 | for (const file of files) { | |
| 528 | livePaths.add(file.path); | |
| 579 | for (const file of retainedFiles) { | |
| 529 | 580 | if (file.records.length === 0) { |
| 530 | 581 | skippedFiles += 1; |
| 531 | 582 | continue; |
| 532 | 583 | } |
| 533 | 584 | scannedFiles += 1; |
| 585 | const codexEventOccurrences = new Map<string, number>(); | |
| 534 | 586 | for (const record of file.records) { |
| 535 | // Only sessions that contributed in-window count: the mtime slack | |
| 536 | // admits boundary files whose records fall outside the range. | |
| 537 | if (aggregator.add(record) && record.sessionId.length > 0) { | |
| 587 | let usageRecord = record; | |
| 588 | if (record.provider === "codex" && record.sessionId.length > 0) { | |
| 589 | // Match moved rollout copies without collapsing repeated equal events | |
| 590 | // within one rollout (timestamps can have only second precision). | |
| 591 | const key = encodeUsageRecordKey([ | |
| 592 | record.provider, | |
| 593 | record.sessionId, | |
| 594 | record.timestampMs, | |
| 595 | record.model, | |
| 596 | record.totals, | |
| 597 | ]); | |
| 598 | const occurrence = (codexEventOccurrences.get(key) ?? 0) + 1; | |
| 599 | codexEventOccurrences.set(key, occurrence); | |
| 600 | usageRecord = { ...record, dedupeKey: key + ":" + occurrence }; | |
| 601 | } | |
| 602 | // Only sessions contributing in-window count; the mtime slack can | |
| 603 | // admit boundary files whose records fall outside the range. | |
| 604 | if (aggregator.add(usageRecord) && record.sessionId.length > 0) { | |
| 538 | 605 | sessionIds.add(record.sessionId); |
| 539 | 606 | } |
| 540 | 607 | } |
| ... | ... | @@ -542,21 +609,17 @@ export const make = Effect.gen(function* () { |
| 542 | 609 | |
| 543 | 610 | sources.push({ |
| 544 | 611 | fingerprint: { hostId, provider, resolvedHomePath: dir, volumeId }, |
| 545 | status: "ok", | |
| 612 | // Clients exclude missing sources, so saved records remain an available source. | |
| 613 | status: files === null && scannedFiles === 0 ? "missing" : "ok", | |
| 546 | 614 | scannedFiles, |
| 547 | 615 | skippedFiles, |
| 548 | 616 | malformedRecords: 0, |
| 549 | 617 | distinctSessions: sessionIds.size, |
| 550 | message: null, | |
| 618 | message: files === null ? "No transcript directory on this environment." : null, | |
| 551 | 619 | }); |
| 552 | 620 | } |
| 553 | 621 | |
| 554 | const pruned = pruneScanCache(fileCache, { | |
| 555 | livePaths, | |
| 556 | walkedRoots, | |
| 557 | windowStartMs, | |
| 558 | retentionCutoffMs: startedAtMs - CACHE_RETENTION_DAYS * 24 * 60 * 60 * 1000, | |
| 559 | }); | |
| 622 | const pruned = pruneScanCache(fileCache, retentionCutoffMs); | |
| 560 | 623 | if (pruned > 0) cacheDirty = true; |
| 561 | 624 | yield* persistScanCache(); |
| 562 | 625 |
apps/server/src/usage/usageScanCache.test.ts+3-74| ... | ... | @@ -194,88 +194,17 @@ describe("pruneScanCache", () => { |
| 194 | 194 | it("drops entries older than retention", () => { |
| 195 | 195 | const cache = cacheWith([["/old.jsonl", 500, [record()]]]); |
| 196 | 196 | |
| 197 | const removed = pruneScanCache(cache, { | |
| 198 | livePaths: new Set(), | |
| 199 | walkedRoots: ["/"], | |
| 200 | windowStartMs: 400, | |
| 201 | retentionCutoffMs, | |
| 202 | }); | |
| 197 | const removed = pruneScanCache(cache, retentionCutoffMs); | |
| 203 | 198 | |
| 204 | 199 | expect(removed).toBe(1); |
| 205 | 200 | expect(cache.size).toBe(0); |
| 206 | 201 | }); |
| 207 | 202 | |
| 208 | it("drops in-window entries whose file has disappeared", () => { | |
| 203 | it("keeps entries whose file has disappeared", () => { | |
| 209 | 204 | const cache = cacheWith([["/gone.jsonl", 5000, [record()]]]); |
| 210 | 205 | |
| 211 | pruneScanCache(cache, { | |
| 212 | livePaths: new Set(), | |
| 213 | walkedRoots: ["/"], | |
| 214 | windowStartMs: 4000, | |
| 215 | retentionCutoffMs, | |
| 216 | }); | |
| 217 | ||
| 218 | expect(cache.size).toBe(0); | |
| 219 | }); | |
| 220 | ||
| 221 | it("keeps entries outside the walked window that are still within retention", () => { | |
| 222 | // Viewing 7 days must not evict the 30-day entries, which that walk never | |
| 223 | // looked for and so cannot prove are gone. | |
| 224 | const cache = cacheWith([["/older-but-valid.jsonl", 2000, [record()]]]); | |
| 225 | ||
| 226 | const removed = pruneScanCache(cache, { | |
| 227 | livePaths: new Set(), | |
| 228 | walkedRoots: ["/"], | |
| 229 | windowStartMs: 4000, | |
| 230 | retentionCutoffMs, | |
| 231 | }); | |
| 232 | ||
| 233 | expect(removed).toBe(0); | |
| 234 | expect(cache.size).toBe(1); | |
| 235 | }); | |
| 236 | ||
| 237 | it("keeps entries the walk saw", () => { | |
| 238 | const cache = cacheWith([["/live.jsonl", 5000, [record()]]]); | |
| 239 | ||
| 240 | pruneScanCache(cache, { | |
| 241 | livePaths: new Set(["/live.jsonl"]), | |
| 242 | walkedRoots: ["/"], | |
| 243 | windowStartMs: 4000, | |
| 244 | retentionCutoffMs, | |
| 245 | }); | |
| 246 | ||
| 247 | expect(cache.size).toBe(1); | |
| 248 | }); | |
| 249 | }); | |
| 250 | ||
| 251 | describe("pruneScanCache with an unwalked root", () => { | |
| 252 | it("keeps in-window entries for a provider whose directory was not walked", () => { | |
| 253 | // A missing provider root or failed settings read leaves livePaths without | |
| 254 | // that provider's files. Its warm entries must survive the pass. | |
| 255 | const cache = cacheWith([["/codex/sessions/a.jsonl", 5000, [record()]]]); | |
| 256 | ||
| 257 | const removed = pruneScanCache(cache, { | |
| 258 | livePaths: new Set(), | |
| 259 | walkedRoots: ["/claude/projects"], | |
| 260 | windowStartMs: 4000, | |
| 261 | retentionCutoffMs: 1000, | |
| 262 | }); | |
| 263 | ||
| 264 | expect(removed).toBe(0); | |
| 265 | expect(cache.size).toBe(1); | |
| 266 | }); | |
| 267 | ||
| 268 | it("keeps entries under a sibling path that only shares the walked root prefix", () => { | |
| 269 | const cache = cacheWith([["/claude/projects-copy/a.jsonl", 5000, [record()]]]); | |
| 270 | ||
| 271 | const removed = pruneScanCache(cache, { | |
| 272 | livePaths: new Set(), | |
| 273 | walkedRoots: ["/claude/projects"], | |
| 274 | windowStartMs: 4000, | |
| 275 | retentionCutoffMs: 1000, | |
| 276 | }); | |
| 206 | pruneScanCache(cache, retentionCutoffMs); | |
| 277 | 207 | |
| 278 | expect(removed).toBe(0); | |
| 279 | 208 | expect(cache.size).toBe(1); |
| 280 | 209 | }); |
| 281 | 210 | }); |
apps/server/src/usage/usageScanCache.ts+3-42| ... | ... | @@ -14,9 +14,6 @@ |
| 14 | 14 | * |
| 15 | 15 | * @module usageScanCache |
| 16 | 16 | */ |
| 17 | // @effect-diagnostics nodeBuiltinImport:off | |
| 18 | import * as NodePath from "node:path"; | |
| 19 | ||
| 20 | 17 | import type { UsageProviderKind } from "@t3tools/contracts"; |
| 21 | 18 | |
| 22 | 19 | import { GUARD_LENGTH, type TranscriptParsePosition } from "./usageTranscriptReader.ts"; |
| ... | ... | @@ -295,47 +292,11 @@ function decodeCodexState(value: unknown): CodexScanState | null | undefined { |
| 295 | 292 | }; |
| 296 | 293 | } |
| 297 | 294 | |
| 298 | export interface PruneOptions { | |
| 299 | /** Files the walk just saw. Only meaningful inside the walked window. */ | |
| 300 | readonly livePaths: ReadonlySet<string>; | |
| 301 | /** | |
| 302 | * Roots the walk actually completed. Absence from `livePaths` only proves a | |
| 303 | * file is gone when its root was walked: a provider whose directory failed to | |
| 304 | * resolve this pass must not have its warm entries purged. | |
| 305 | */ | |
| 306 | readonly walkedRoots: readonly string[]; | |
| 307 | /** Start of the walked window; entries older than this were not looked for. */ | |
| 308 | readonly windowStartMs: number; | |
| 309 | /** Entries older than this are dropped regardless. */ | |
| 310 | readonly retentionCutoffMs: number; | |
| 311 | } | |
| 312 | ||
| 313 | /** | |
| 314 | * Drops aged-out entries, and entries for files that have disappeared. | |
| 315 | * | |
| 316 | * The walk only covers the requested window, so absence from `livePaths` only | |
| 317 | * proves deletion for entries *inside* that window. Pruning everything the walk | |
| 318 | * missed would evict the 30-day entries every time someone looked at 7 days. | |
| 319 | * | |
| 320 | * Replaces an earlier record cap that cleared the whole cache once exceeded, | |
| 321 | * which meant a large enough window never warmed up at all. | |
| 322 | */ | |
| 323 | export function pruneScanCache(cache: ScanCache, options: PruneOptions): number { | |
| 295 | /** Keeps saved usage after transcript cleanup, until the reporting retention expires. */ | |
| 296 | export function pruneScanCache(cache: ScanCache, retentionCutoffMs: number): number { | |
| 324 | 297 | let removed = 0; |
| 325 | 298 | for (const [path, entry] of cache) { |
| 326 | const agedOut = entry.mtimeMs < options.retentionCutoffMs; | |
| 327 | const underWalkedRoot = options.walkedRoots.some((root) => { | |
| 328 | const relative = NodePath.relative(root, path); | |
| 329 | return ( | |
| 330 | relative === "" || | |
| 331 | (relative !== ".." && | |
| 332 | !relative.startsWith(`..${NodePath.sep}`) && | |
| 333 | !NodePath.isAbsolute(relative)) | |
| 334 | ); | |
| 335 | }); | |
| 336 | const deleted = | |
| 337 | underWalkedRoot && entry.mtimeMs >= options.windowStartMs && !options.livePaths.has(path); | |
| 338 | if (agedOut || deleted) { | |
| 299 | if (entry.mtimeMs < retentionCutoffMs) { | |
| 339 | 300 | cache.delete(path); |
| 340 | 301 | removed += 1; |
| 341 | 302 | } |