From 4d1f24ba4123279708e169fcb196f78da8f11351 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Thu, 8 Oct 2026 02:18:01 -0700 Subject: [PATCH] fix(usage): repeat usage scans no longer decode unchanged Antigravity databases (#17139) Takeover of #13902. Co-authored-by: Derek Trimm <275381468+derektrimm@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- apps/server/src/usage/UsageService.ts | 9 +- .../src/usage/antigravityUsageReader.ts | 36 ++++++- .../src/usage/usageTranscriptReader.test.ts | 101 +++++++++++++++++- 3 files changed, 140 insertions(+), 6 deletions(-) diff --git a/apps/server/src/usage/UsageService.ts b/apps/server/src/usage/UsageService.ts index c2206a53d33cf81ad2ef9e052e1aa92a46865f44..059d4e176bca886324e64f95c779b80b7e122506 100644 --- a/apps/server/src/usage/UsageService.ts +++ b/apps/server/src/usage/UsageService.ts @@ -8,7 +8,9 @@ * `(size, mtime)`. A cold 30-day scan of ~1.4 GB lands around 2-3 seconds; warm * scans only reparse files that changed, and a file that merely grew resumes * from its cached parse position so only the appended bytes are read. - * SQLite readers query live databases each scan so WAL writes remain visible. + * OpenCode's SQLite reader queries the live database each scan so WAL writes + * remain visible. Antigravity databases are memoised in memory while the + * database and its WAL keep the same `(size, mtime, ctime)`. * * @module UsageService */ @@ -52,7 +54,7 @@ import { resolveCodexHomeLayout } from "../provider/Drivers/CodexHomeLayout.ts"; import { resolveAntigravityInstanceDirectories } from "../provider/antigravityAuthSupport.ts"; import { mergeProviderInstanceEnvironment } from "../provider/ProviderInstanceEnvironment.ts"; import { readOpenCodeUsage } from "./opencodeUsageReader.ts"; -import { readAntigravityUsage } from "./antigravityUsageReader.ts"; +import { makeAntigravityUsageCache, readAntigravityUsage } from "./antigravityUsageReader.ts"; import { readCursorAccountUsage } from "./cursorUsageReader.ts"; import { resolveModelAliases, UsageAggregator } from "./usageAggregation.ts"; import { createOverrideRateTable, parseRateTable, type RateTable } from "./usagePricing.ts"; @@ -195,6 +197,7 @@ export const make = Effect.gen(function* () { const platform = yield* HostProcessPlatform; const fileCache: ScanCache = new Map(); + const antigravityCache = makeAntigravityUsageCache(); const sourceCache = new Map(); let cacheDirty = false; const isWithinDirectory = (filePath: string, dir: string) => { @@ -661,7 +664,7 @@ export const make = Effect.gen(function* () { antigravityDirs.add(yield* fileSystem.realPath(dir).pipe(Effect.orElseSucceed(() => dir))); } const antigravity = yield* Effect.promise(() => - readAntigravityUsage([...antigravityDirs], windowStartMs), + readAntigravityUsage([...antigravityDirs], windowStartMs, antigravityCache), ); for (const dir of antigravityDirs) { const exists = yield* fileSystem diff --git a/apps/server/src/usage/antigravityUsageReader.ts b/apps/server/src/usage/antigravityUsageReader.ts index 34a88114f42eab0595aa74782ded89482855c51c..8b1559cfedd79ec09e1ec59a5b3316164d111c6c 100644 --- a/apps/server/src/usage/antigravityUsageReader.ts +++ b/apps/server/src/usage/antigravityUsageReader.ts @@ -257,10 +257,28 @@ async function readDatabase(path: string, fallbackTimestamp: number): Promise new Map(); + +/** + * Reads and merges aliases across every configured Antigravity store before date + * filtering. With a cache, databases unchanged since the previous read are not + * decoded again. + */ export async function readAntigravityUsage( conversationsDirectories: string | readonly string[], sinceMs: number, + cache?: Map, ) { const roots = typeof conversationsDirectories === "string" @@ -351,7 +369,18 @@ export async function readAntigravityUsage( if (visited.has(canonical)) continue; visited.add(canonical); const stat = await NodeFSP.stat(path); - const candidates = await readDatabase(path, stat.mtimeMs); + const wal = await NodeFSP.stat(`${path}-wal`).catch(() => null); + const fingerprint = [stat, wal] + .map((file) => (file ? `${file.size}:${file.mtimeMs}:${file.ctimeMs}` : "-")) + .join("/"); + const cached = cache?.get(canonical); + let candidates: readonly UsageCandidate[]; + if (cached?.fingerprint === fingerprint) { + candidates = cached.candidates; + } else { + candidates = await readDatabase(path, stat.mtimeMs); + cache?.set(canonical, { fingerprint, candidates }); + } const fileIndex = files.length; files.push({ root, path, records: [] }); for (const [index, candidate] of candidates.entries()) { @@ -365,6 +394,9 @@ export async function readAntigravityUsage( } }; for (const root of roots) await walk(root, root); + if (cache !== undefined) { + for (const key of cache.keys()) if (!visited.has(key)) cache.delete(key); + } for (const [index, group] of groups.entries()) { if (group.parent === index && group.record.timestampMs >= sinceMs) { files[group.fileIndex]!.records.push(group.record); diff --git a/apps/server/src/usage/usageTranscriptReader.test.ts b/apps/server/src/usage/usageTranscriptReader.test.ts index e55c01ad491924d70af94cd184d9b64e99b1b517..e6f19d50718a5a2f1f6a3f0440ee794ceb621cd2 100644 --- a/apps/server/src/usage/usageTranscriptReader.test.ts +++ b/apps/server/src/usage/usageTranscriptReader.test.ts @@ -11,7 +11,7 @@ import { afterEach, assert, beforeEach, describe, it } from "@effect/vitest"; import { readTranscriptRecords } from "./usageTranscriptReader.ts"; import { readOpenCodeUsage } from "./opencodeUsageReader.ts"; import { readCursorAccountUsage } from "./cursorUsageReader.ts"; -import { readAntigravityUsage } from "./antigravityUsageReader.ts"; +import { makeAntigravityUsageCache, readAntigravityUsage } from "./antigravityUsageReader.ts"; let dir: string; @@ -738,6 +738,105 @@ describe("SQLite usage readers", () => { ); }); + const antigravityGeneration = (responseId: string) => + new Uint8Array( + protoBytes(1, [ + ...protoBytes(4, [ + ...protoNumber(2, 100), + ...protoNumber(3, 40), + ...protoText(11, responseId), + ]), + ...protoText(19, "Gemini 3 Pro"), + ...protoBytes(9, protoBytes(4, protoNumber(1, 1780000000))), + ]), + ); + + it("reuses an unchanged Antigravity database instead of decoding it again", async () => { + const path = NodePath.join(dir, "session-1.db"); + const db = new NodeSqlite.DatabaseSync(path); + try { + db.exec("CREATE TABLE gen_metadata (idx INTEGER, data BLOB)"); + db.prepare("INSERT INTO gen_metadata VALUES (?, ?)").run(0, antigravityGeneration("r-1")); + const cache = makeAntigravityUsageCache(); + assert.deepStrictEqual((await readAntigravityUsage(dir, 0, cache)).errors, []); + + // An exclusive lock makes a fresh read fail without touching the file, so + // only a cache hit can still return the earlier records. + db.exec("BEGIN EXCLUSIVE"); + assert.strictEqual((await readAntigravityUsage(dir, 0)).errors.length, 1); + const cached = await readAntigravityUsage(dir, 0, cache); + assert.deepStrictEqual(cached.errors, []); + assert.deepStrictEqual( + cached.files.flatMap((file) => file.records).map((record) => record.dedupeKey), + ["antigravity:11:r-1"], + ); + db.exec("ROLLBACK"); + } finally { + db.close(); + } + }); + + it("rereads an Antigravity database rewritten with its size and mtime restored", async () => { + const path = NodePath.join(dir, "session-1.db"); + const db = new NodeSqlite.DatabaseSync(path); + try { + db.exec("CREATE TABLE gen_metadata (idx INTEGER, data BLOB)"); + db.prepare("INSERT INTO gen_metadata VALUES (?, ?)").run(0, antigravityGeneration("r-1")); + } finally { + db.close(); + } + await NodeFSP.utimes(path, 1780000000, 1780000000); + const cache = makeAntigravityUsageCache(); + assert.deepStrictEqual((await readAntigravityUsage(dir, 0, cache)).errors, []); + + // ctime has the kernel's timestamp granularity, which can be a few + // milliseconds, so repeat the forged rewrite until it lands on a later tick + // than the cached read, as any real rewrite does. + const cached = await NodeFSP.stat(path); + do { + await NodeFSP.writeFile(path, Buffer.alloc(cached.size)); + await NodeFSP.utimes(path, 1780000000, 1780000000); + } while ((await NodeFSP.stat(path)).ctimeMs === cached.ctimeMs); + const restored = await NodeFSP.stat(path); + assert.strictEqual(restored.size, cached.size); + assert.strictEqual(restored.mtimeMs, cached.mtimeMs); + const next = await readAntigravityUsage(dir, 0, cache); + assert.deepStrictEqual(next.errors, [path]); + assert.deepStrictEqual( + next.files.flatMap((file) => file.records), + [], + ); + }); + + it("rereads an Antigravity database when only its WAL changed", async () => { + const path = NodePath.join(dir, "session-1.db"); + const db = new NodeSqlite.DatabaseSync(path); + try { + db.exec( + "PRAGMA journal_mode = WAL; PRAGMA wal_autocheckpoint = 0; CREATE TABLE gen_metadata (idx INTEGER, data BLOB)", + ); + const insert = db.prepare("INSERT INTO gen_metadata VALUES (?, ?)"); + insert.run(0, antigravityGeneration("r-1")); + const cache = makeAntigravityUsageCache(); + const first = await readAntigravityUsage(dir, 0, cache); + assert.strictEqual(first.files.flatMap((file) => file.records).length, 1); + + const before = await NodeFSP.stat(path); + insert.run(1, antigravityGeneration("r-2")); + const after = await NodeFSP.stat(path); + assert.strictEqual(after.size, before.size); + assert.strictEqual(after.mtimeMs, before.mtimeMs); + + const next = await readAntigravityUsage(dir, 0, cache); + assert.deepStrictEqual( + next.files.flatMap((file) => file.records).map((record) => record.dedupeKey), + ["antigravity:11:r-1", "antigravity:11:r-2"], + ); + } finally { + db.close(); + } + }); + it("reads Antigravity step-only stores and reports malformed databases", async () => { const db = new NodeSqlite.DatabaseSync(NodePath.join(dir, "steps.db")); try { -- 2.54.0