authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-10-09 15:26:19-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-09 15:26:19-07:00
log9a0766b0238f2c044a965dcba32c6cb7ea0b63cd
treedd6e04cba8f4c69a3f2adc0190569994a800f49b
parent5a9c664b3ec8214cc5847d561efa130ba213eb38
signature Signed by PGP key B5690EEEBB952194

refactor(usage): Antigravity usage is a reader on its driver (#17579)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

11 files changed, 970 insertions(+), 891 deletions(-)

apps/server/src/provider/Drivers/AntigravityDriver.ts+7-1
...@@ -26,6 +26,7 @@ import {...@@ -26,6 +26,7 @@ import {
26} from "../../textGeneration/AntigravityTextGeneration.ts";26} from "../../textGeneration/AntigravityTextGeneration.ts";
27import { makeAntigravityAuth, type AntigravityAuth } from "../AntigravityAuth.ts";27import { makeAntigravityAuth, type AntigravityAuth } from "../AntigravityAuth.ts";
28import * as AntigravityInstallation from "../AntigravityInstallation.ts";28import * as AntigravityInstallation from "../AntigravityInstallation.ts";
29import * as AntigravityUsage from "./AntigravityUsage.ts";
29import {30import {
30 antigravityAuthConfigIssue,31 antigravityAuthConfigIssue,
31 antigravityAuthLabel,32 antigravityAuthLabel,
...@@ -80,11 +81,16 @@ export type AntigravityDriverEnv =...@@ -80,11 +81,16 @@ export type AntigravityDriverEnv =
80 | ProviderEventLoggers.ProviderEventLoggers;81 | ProviderEventLoggers.ProviderEventLoggers;
8182
82/** Each instance owns its Google profile. Executable releases are shared by the environment. */83/** Each instance owns its Google profile. Executable releases are shared by the environment. */
83export const AntigravityDriver: ProviderDriver<AntigravitySettings, AntigravityDriverEnv> = {84export const AntigravityDriver: ProviderDriver<
85 AntigravitySettings,
86 AntigravityDriverEnv,
87 AntigravityUsage.AntigravityUsage
88> = {
84 driverKind: DRIVER,89 driverKind: DRIVER,
85 metadata: { displayName: "Antigravity", supportsMultipleInstances: true },90 metadata: { displayName: "Antigravity", supportsMultipleInstances: true },
86 configSchema: AntigravitySettings,91 configSchema: AntigravitySettings,
87 defaultConfig: () => decodeSettings({}),92 defaultConfig: () => decodeSettings({}),
93 usage: AntigravityUsage.antigravityUsageReader,
88 create: ({ instanceId, displayName, accentColor, environment, enabled, config }) =>94 create: ({ instanceId, displayName, accentColor, environment, enabled, config }) =>
89 Effect.gen(function* () {95 Effect.gen(function* () {
90 const crypto = yield* Crypto.Crypto;96 const crypto = yield* Crypto.Crypto;
apps/server/src/provider/Drivers/AntigravityUsage.ts created+140
...@@ -0,0 +1,140 @@
1/**
2 * AntigravityUsage - Antigravity's usage history as a usage source.
3 *
4 * Antigravity keeps conversations in SQLite databases under its data
5 * directories and each instance's profile. Parsed databases stay in memory
6 * while a database and its WAL keep the same `(size, mtime, ctime)`, so a
7 * warm scan decodes only what changed.
8 *
9 * @module provider/Drivers/AntigravityUsage
10 */
11import * as NodeOS from "node:os";
12
13import { type AntigravitySettings, UsageReadError } from "@t3tools/contracts";
14import { expandHomePath } from "@t3tools/provider-core/server/pathExpansion";
15import * as ProviderHost from "@t3tools/provider-core/server/ProviderHost";
16import type {
17 ProviderUsageInstance,
18 ProviderUsageReader,
19 ProviderUsageScan,
20} from "@t3tools/provider-core/server/usage";
21import { HostProcessEnvironment } from "@t3tools/shared/hostProcess";
22import * as Context from "effect/Context";
23import * as Crypto from "effect/Crypto";
24import * as Effect from "effect/Effect";
25import * as FileSystem from "effect/FileSystem";
26import * as Layer from "effect/Layer";
27import * as Path from "effect/Path";
28
29import { resolveAntigravityInstanceDirectories } from "../antigravityAuthSupport.ts";
30import { makeAntigravityUsageCache, readAntigravityUsage } from "./antigravityUsageReader.ts";
31
32export class AntigravityUsage extends Context.Service<
33 AntigravityUsage,
34 {
35 /** Every Antigravity store: the data directories and each configured instance's profile. */
36 readonly scan: (input: {
37 readonly instances: ReadonlyArray<ProviderUsageInstance<AntigravitySettings>>;
38 readonly windowStartMs: number;
39 }) => Effect.Effect<ReadonlyArray<ProviderUsageScan>, UsageReadError>;
40 }
41>()("t3/provider/Drivers/AntigravityUsage") {}
42
43const make = Effect.gen(function* () {
44 const crypto = yield* Crypto.Crypto;
45 const fileSystem = yield* FileSystem.FileSystem;
46 const path = yield* Path.Path;
47 const host = yield* ProviderHost.ProviderHost;
48 const hostEnvironment = yield* HostProcessEnvironment;
49 const cache = makeAntigravityUsageCache();
50
51 /** `ANTIGRAVITY_DATA_DIR` (comma-separated) or the defaults, canonicalized. */
52 const dataRoots = Effect.gen(function* () {
53 const home = NodeOS.homedir();
54 const configured = hostEnvironment["ANTIGRAVITY_DATA_DIR"]
55 ?.split(",")
56 .map((value) => value.trim())
57 .filter(Boolean);
58 const defaults = [
59 ...["antigravity", "antigravity-cli", "antigravity-ide", "antigravity-backup"].map((name) =>
60 path.join(home, ".gemini", name),
61 ),
62 path.join(home, ".config", "antigravity"),
63 ];
64 const canonical = new Set<string>();
65 for (const root of configured?.length ? configured : defaults) {
66 const resolved = path.resolve(expandHomePath(root));
67 canonical.add(
68 yield* fileSystem.realPath(resolved).pipe(Effect.orElseSucceed(() => resolved)),
69 );
70 }
71 return [...canonical];
72 });
73
74 const scan: AntigravityUsage["Service"]["scan"] = Effect.fn("AntigravityUsage.scan")(function* ({
75 instances,
76 windowStartMs,
77 }) {
78 const roots = yield* dataRoots;
79 // Only configured instances have a profile; the implicit default never ran.
80 for (const { instanceId } of instances.filter((instance) => instance.configured)) {
81 const directories = yield* resolveAntigravityInstanceDirectories(
82 host.paths.stateDir,
83 instanceId,
84 ).pipe(
85 Effect.provideService(Crypto.Crypto, crypto),
86 Effect.provideService(Path.Path, path),
87 Effect.mapError(
88 (cause) =>
89 new UsageReadError({
90 reason: "scanFailed",
91 detail: "Antigravity profile directory could not be resolved.",
92 cause,
93 }),
94 ),
95 );
96 roots.push(path.join(directories.profile, "antigravity-acp"));
97 }
98 const conversationDirs = new Set<string>();
99 for (const root of roots) {
100 const resolvedRoot = yield* fileSystem.realPath(root).pipe(Effect.orElseSucceed(() => root));
101 const nested = path.join(resolvedRoot, "conversations");
102 const dir = (yield* fileSystem
103 .exists(nested)
104 .pipe(Effect.catchCause(() => Effect.succeed(false))))
105 ? nested
106 : resolvedRoot;
107 conversationDirs.add(yield* fileSystem.realPath(dir).pipe(Effect.orElseSucceed(() => dir)));
108 }
109 const result = yield* Effect.promise(() =>
110 readAntigravityUsage([...conversationDirs], windowStartMs, cache),
111 );
112 const scanned: ProviderUsageScan[] = [];
113 for (const dir of conversationDirs) {
114 const exists = yield* fileSystem
115 .exists(dir)
116 .pipe(Effect.catchCause(() => Effect.succeed(false)));
117 const failed = result.errors.some(
118 (error) => error === dir || error.startsWith(`${dir}${path.sep}`),
119 );
120 scanned.push({
121 dir,
122 files: !exists && !failed ? null : result.files.filter((file) => file.root === dir),
123 status: failed ? "partial" : "ok",
124 ...(failed ? { message: "Some Antigravity history could not be read." } : {}),
125 });
126 }
127 return scanned;
128 });
129
130 return AntigravityUsage.of({ scan });
131});
132
133export const layer = Layer.effect(AntigravityUsage, make);
134
135export const antigravityUsageReader: ProviderUsageReader<AntigravitySettings, AntigravityUsage> = {
136 kind: "scan",
137 provider: "antigravity",
138 scan: ({ instances, windowStartMs }) =>
139 AntigravityUsage.pipe(Effect.flatMap((usage) => usage.scan({ instances, windowStartMs }))),
140};
apps/server/src/provider/Drivers/antigravityUsageReader.test.ts created+391
...@@ -0,0 +1,391 @@
1// @effect-diagnostics nodeBuiltinImport:off - the reader under test opens real
2// SQLite databases, mirroring the reader's own deliberate node:sqlite usage.
3import * as NodeFSP from "node:fs/promises";
4import * as NodeOS from "node:os";
5import * as NodePath from "node:path";
6import * as NodeSqlite from "node:sqlite";
7
8import { afterEach, assert, beforeEach, describe, it } from "@effect/vitest";
9
10import { makeAntigravityUsageCache, readAntigravityUsage } from "./antigravityUsageReader.ts";
11
12let dir: string;
13
14function protoNumber(field: number, value: number): number[] {
15 const varint = (number: number) => {
16 const bytes: number[] = [];
17 do {
18 const byte = number % 128;
19 number = Math.floor(number / 128);
20 bytes.push(byte + (number > 0 ? 128 : 0));
21 } while (number > 0);
22 return bytes;
23 };
24 return [...varint(field * 8), ...varint(value)];
25}
26
27function protoBytes(field: number, bytes: readonly number[]): number[] {
28 const encoded = protoNumber(field, bytes.length);
29 encoded[0] = encoded[0]! + 2;
30 return [...encoded, ...bytes];
31}
32
33function protoText(field: number, value: string): number[] {
34 return protoBytes(field, [...Buffer.from(value)]);
35}
36
37beforeEach(async () => {
38 dir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "usage-reader-test-"));
39});
40
41afterEach(async () => {
42 await NodeFSP.rm(dir, { recursive: true, force: true });
43});
44
45describe("readAntigravityUsage", () => {
46 it("deduplicates Antigravity generation and step usage while preserving retry model and token buckets", async () => {
47 const db = new NodeSqlite.DatabaseSync(NodePath.join(dir, "session-1.db"));
48 const stamp = protoNumber(1, 1780000000);
49 const usage = [
50 ...protoNumber(2, 100),
51 ...protoNumber(3, 40),
52 ...protoNumber(4, 5),
53 ...protoNumber(5, 20),
54 ...protoNumber(9, 10),
55 ...protoText(11, "response-1"),
56 ];
57 const retry = [
58 ...protoNumber(1, 1026),
59 ...protoNumber(2, 12),
60 ...protoNumber(3, 3),
61 ...protoText(11, "retry-1"),
62 ];
63 const generation = protoBytes(1, [
64 ...protoBytes(4, usage),
65 ...protoText(19, "Gemini 3 Pro"),
66 ...protoBytes(9, protoBytes(4, stamp)),
67 ]);
68 const step = [
69 ...protoBytes(9, usage),
70 ...protoBytes(8, stamp),
71 ...protoBytes(28, protoBytes(2, retry)),
72 ];
73 try {
74 db.exec(
75 "CREATE TABLE gen_metadata (idx INTEGER, data BLOB); CREATE TABLE steps (idx INTEGER, metadata BLOB)",
76 );
77 db.prepare("INSERT INTO gen_metadata VALUES (?, ?)").run(0, new Uint8Array(generation));
78 db.prepare("INSERT INTO steps VALUES (?, ?)").run(0, new Uint8Array(step));
79 } finally {
80 db.close();
81 }
82 const result = await readAntigravityUsage(dir, 0);
83 assert.deepStrictEqual(result.errors, []);
84 const records = result.files.flatMap((file) => file.records);
85 assert.strictEqual(records.length, 2);
86 const main = records.find((record) => record.model === "gemini-3-pro");
87 assert.isDefined(main);
88 assert.strictEqual(main?.timestampMs, 1780000000000);
89 assert.strictEqual(main?.sessionId, "session-1");
90 assert.deepStrictEqual(main?.totals, {
91 uncachedInputTokens: 100,
92 cachedInputTokens: 20,
93 cacheCreationTokens: 5,
94 outputTokens: 40,
95 reasoningTokens: 10,
96 });
97 assert.strictEqual(
98 records.find((record) => record.model === "claude-opus-4-6")?.totals.uncachedInputTokens,
99 12,
100 );
101 assert.deepStrictEqual(
102 (await readAntigravityUsage(dir, 1780000000001)).files.flatMap((file) => file.records),
103 [],
104 );
105 });
106
107 it("uses the matching Antigravity generation model for each model-less step", async () => {
108 const db = new NodeSqlite.DatabaseSync(NodePath.join(dir, "model-switch.db"));
109 try {
110 db.exec(
111 "CREATE TABLE gen_metadata (idx INTEGER, data BLOB); CREATE TABLE steps (idx INTEGER, metadata BLOB)",
112 );
113 const generation = db.prepare("INSERT INTO gen_metadata VALUES (?, ?)");
114 const step = db.prepare("INSERT INTO steps VALUES (?, ?)");
115 for (const [idx, name] of ["Gemini 3 Pro", "Claude Opus 4.6"].entries()) {
116 generation.run(idx, new Uint8Array(protoBytes(1, protoText(19, name))));
117 step.run(idx, new Uint8Array(protoBytes(9, protoNumber(2, 10 + idx))));
118 }
119 } finally {
120 db.close();
121 }
122 const result = await readAntigravityUsage(dir, 0);
123 assert.deepStrictEqual(result.errors, []);
124 assert.deepStrictEqual(
125 result.files.flatMap((file) => file.records).map((record) => record.model),
126 ["gemini-3-pro", "claude-opus-4-6"],
127 );
128 });
129
130 it("merges Antigravity aliases that bridge previously separate step records", async () => {
131 const db = new NodeSqlite.DatabaseSync(NodePath.join(dir, "bridge.db"));
132 try {
133 db.exec(
134 "CREATE TABLE gen_metadata (idx INTEGER, data BLOB); CREATE TABLE steps (idx INTEGER, metadata BLOB)",
135 );
136 const step = db.prepare("INSERT INTO steps VALUES (?, ?)");
137 step.run(
138 0,
139 new Uint8Array(protoBytes(9, [...protoNumber(2, 100), ...protoText(11, "response")])),
140 );
141 step.run(
142 1,
143 new Uint8Array(protoBytes(9, [...protoNumber(3, 40), ...protoText(12, "provider")])),
144 );
145 db.prepare("INSERT INTO gen_metadata VALUES (?, ?)").run(
146 0,
147 new Uint8Array(
148 protoBytes(1, [
149 ...protoText(19, "Gemini 3 Pro"),
150 ...protoBytes(4, [
151 ...protoNumber(2, 50),
152 ...protoNumber(5, 20),
153 ...protoText(11, "response"),
154 ...protoText(12, "provider"),
155 ]),
156 ]),
157 ),
158 );
159 } finally {
160 db.close();
161 }
162 const result = await readAntigravityUsage(dir, 0);
163 assert.deepStrictEqual(result.errors, []);
164 const records = result.files.flatMap((file) => file.records);
165 assert.strictEqual(records.length, 1);
166 assert.deepStrictEqual(records[0]?.totals, {
167 uncachedInputTokens: 100,
168 cachedInputTokens: 20,
169 cacheCreationTokens: 0,
170 outputTokens: 40,
171 reasoningTokens: 0,
172 });
173 });
174
175 it("merges Antigravity provider and message aliases across configured roots while keeping original ownership", async () => {
176 const roots = [NodePath.join(dir, "first"), NodePath.join(dir, "second")];
177 for (const [index, root] of roots.entries()) {
178 await NodeFSP.mkdir(root);
179 const db = new NodeSqlite.DatabaseSync(NodePath.join(root, `session-${index}.db`));
180 try {
181 db.exec("CREATE TABLE steps (idx INTEGER, metadata BLOB)");
182 for (const identity of [7, 12]) {
183 const usage = [
184 ...protoNumber(1, 246),
185 ...protoNumber(2, index === 0 ? 100 : 150),
186 ...protoText(11, `response-${index}-${identity}`),
187 ...protoText(identity, `shared-${identity}`),
188 ];
189 db.prepare("INSERT INTO steps VALUES (?, ?)").run(
190 identity,
191 new Uint8Array(protoBytes(9, usage)),
192 );
193 }
194 } finally {
195 db.close();
196 }
197 }
198 const result = await readAntigravityUsage(roots, 0);
199 assert.deepStrictEqual(result.errors, []);
200 assert.strictEqual(result.files.length, 2);
201 assert.strictEqual(result.files[0]?.root, roots[0]);
202 assert.strictEqual(result.files[0]?.records.length, 2);
203 assert.strictEqual(result.files[1]?.records.length, 0);
204 assert.deepStrictEqual(
205 result.files[0]?.records.map((record) => record.totals.uncachedInputTokens),
206 [150, 150],
207 );
208 assert.isTrue(result.files[0]?.records.every((record) => record.sessionId === "session-0"));
209 });
210
211 it("upgrades Antigravity fallback timestamps before applying the date window", async () => {
212 for (const fallback of ["mtime", "trajectory"]) {
213 const path = NodePath.join(dir, `${fallback}.db`);
214 const db = new NodeSqlite.DatabaseSync(path);
215 try {
216 db.exec(
217 "CREATE TABLE gen_metadata (idx INTEGER, data BLOB); CREATE TABLE steps (idx INTEGER, metadata BLOB)",
218 );
219 if (fallback === "trajectory") {
220 db.exec("CREATE TABLE trajectory_metadata_blob (data BLOB)");
221 db.prepare("INSERT INTO trajectory_metadata_blob VALUES (?)").run(
222 new Uint8Array(protoBytes(2, protoNumber(1, 1780000200))),
223 );
224 }
225 for (const [index, seconds] of [1780000000, 1780000200].entries()) {
226 const usage = [...protoNumber(2, 10), ...protoText(11, `${fallback}-${index}`)];
227 db.prepare("INSERT INTO steps VALUES (?, ?)").run(
228 index,
229 new Uint8Array(protoBytes(9, usage)),
230 );
231 db.prepare("INSERT INTO gen_metadata VALUES (?, ?)").run(
232 index,
233 new Uint8Array(
234 protoBytes(1, [
235 ...protoBytes(4, usage),
236 ...protoBytes(9, protoBytes(4, protoNumber(1, seconds))),
237 ]),
238 ),
239 );
240 }
241 } finally {
242 db.close();
243 }
244 await NodeFSP.utimes(path, 1780000000, 1780000000);
245 }
246 const result = await readAntigravityUsage(dir, 1780000100000);
247 assert.deepStrictEqual(result.errors, []);
248 const records = result.files.flatMap((file) => file.records);
249 assert.strictEqual(records.length, 2);
250 assert.deepStrictEqual(
251 records.map((record) => record.timestampMs),
252 [1780000200000, 1780000200000],
253 );
254 });
255
256 const antigravityGeneration = (responseId: string) =>
257 new Uint8Array(
258 protoBytes(1, [
259 ...protoBytes(4, [
260 ...protoNumber(2, 100),
261 ...protoNumber(3, 40),
262 ...protoText(11, responseId),
263 ]),
264 ...protoText(19, "Gemini 3 Pro"),
265 ...protoBytes(9, protoBytes(4, protoNumber(1, 1780000000))),
266 ]),
267 );
268
269 it("reuses an unchanged Antigravity database instead of decoding it again", async () => {
270 const path = NodePath.join(dir, "session-1.db");
271 const db = new NodeSqlite.DatabaseSync(path);
272 try {
273 db.exec("CREATE TABLE gen_metadata (idx INTEGER, data BLOB)");
274 db.prepare("INSERT INTO gen_metadata VALUES (?, ?)").run(0, antigravityGeneration("r-1"));
275 const cache = makeAntigravityUsageCache();
276 assert.deepStrictEqual((await readAntigravityUsage(dir, 0, cache)).errors, []);
277
278 // An exclusive lock makes a fresh read fail without touching the file, so
279 // only a cache hit can still return the earlier records.
280 db.exec("BEGIN EXCLUSIVE");
281 assert.strictEqual((await readAntigravityUsage(dir, 0)).errors.length, 1);
282 const cached = await readAntigravityUsage(dir, 0, cache);
283 assert.deepStrictEqual(cached.errors, []);
284 assert.deepStrictEqual(
285 cached.files.flatMap((file) => file.records).map((record) => record.dedupeKey),
286 ["antigravity:11:r-1"],
287 );
288 db.exec("ROLLBACK");
289 } finally {
290 db.close();
291 }
292 });
293
294 it("rereads an Antigravity database rewritten with its size and mtime restored", async () => {
295 const path = NodePath.join(dir, "session-1.db");
296 const db = new NodeSqlite.DatabaseSync(path);
297 try {
298 db.exec("CREATE TABLE gen_metadata (idx INTEGER, data BLOB)");
299 db.prepare("INSERT INTO gen_metadata VALUES (?, ?)").run(0, antigravityGeneration("r-1"));
300 } finally {
301 db.close();
302 }
303 await NodeFSP.utimes(path, 1780000000, 1780000000);
304 const cache = makeAntigravityUsageCache();
305 assert.deepStrictEqual((await readAntigravityUsage(dir, 0, cache)).errors, []);
306
307 // ctime has the kernel's timestamp granularity, which can be a few
308 // milliseconds, so repeat the forged rewrite until it lands on a later tick
309 // than the cached read, as any real rewrite does.
310 const cached = await NodeFSP.stat(path);
311 do {
312 await NodeFSP.writeFile(path, Buffer.alloc(cached.size));
313 await NodeFSP.utimes(path, 1780000000, 1780000000);
314 } while ((await NodeFSP.stat(path)).ctimeMs === cached.ctimeMs);
315 const restored = await NodeFSP.stat(path);
316 assert.strictEqual(restored.size, cached.size);
317 assert.strictEqual(restored.mtimeMs, cached.mtimeMs);
318 const next = await readAntigravityUsage(dir, 0, cache);
319 assert.deepStrictEqual(next.errors, [path]);
320 assert.deepStrictEqual(
321 next.files.flatMap((file) => file.records),
322 [],
323 );
324 });
325
326 it("rereads an Antigravity database when only its WAL changed", async () => {
327 const path = NodePath.join(dir, "session-1.db");
328 const db = new NodeSqlite.DatabaseSync(path);
329 try {
330 db.exec(
331 "PRAGMA journal_mode = WAL; PRAGMA wal_autocheckpoint = 0; CREATE TABLE gen_metadata (idx INTEGER, data BLOB)",
332 );
333 const insert = db.prepare("INSERT INTO gen_metadata VALUES (?, ?)");
334 insert.run(0, antigravityGeneration("r-1"));
335 const cache = makeAntigravityUsageCache();
336 const first = await readAntigravityUsage(dir, 0, cache);
337 assert.strictEqual(first.files.flatMap((file) => file.records).length, 1);
338
339 const before = await NodeFSP.stat(path);
340 insert.run(1, antigravityGeneration("r-2"));
341 const after = await NodeFSP.stat(path);
342 assert.strictEqual(after.size, before.size);
343 assert.strictEqual(after.mtimeMs, before.mtimeMs);
344
345 const next = await readAntigravityUsage(dir, 0, cache);
346 assert.deepStrictEqual(
347 next.files.flatMap((file) => file.records).map((record) => record.dedupeKey),
348 ["antigravity:11:r-1", "antigravity:11:r-2"],
349 );
350 } finally {
351 db.close();
352 }
353 });
354
355 it("reads Antigravity step-only stores and reports malformed databases", async () => {
356 const db = new NodeSqlite.DatabaseSync(NodePath.join(dir, "steps.db"));
357 try {
358 db.exec("CREATE TABLE steps (idx INTEGER, metadata BLOB)");
359 const usage = [...protoNumber(1, 246), ...protoNumber(2, 10), ...protoNumber(3, 5)];
360 db.prepare("INSERT INTO steps VALUES (?, ?)").run(
361 0,
362 new Uint8Array([...protoBytes(9, usage), ...protoBytes(8, protoNumber(1, 1780000000))]),
363 );
364 } finally {
365 db.close();
366 }
367 await NodeFSP.writeFile(NodePath.join(dir, "broken.db"), "not a sqlite database");
368 const result = await readAntigravityUsage(dir, 0);
369 assert.strictEqual(result.errors.length, 1);
370 assert.strictEqual(result.files.flatMap((file) => file.records)[0]?.model, "gemini-2.5-pro");
371 assert.strictEqual(result.files.flatMap((file) => file.records)[0]?.totals.outputTokens, 5);
372 });
373
374 it("ignores large values in unused Antigravity protobuf fields", async () => {
375 const db = new NodeSqlite.DatabaseSync(NodePath.join(dir, "large-varint.db"));
376 try {
377 db.exec("CREATE TABLE steps (idx INTEGER, metadata BLOB)");
378 const unusedField = [...protoNumber(99, 0).slice(0, -1), ...Array(9).fill(0xff), 0x01];
379 const usage = [...protoNumber(1, 246), ...protoNumber(2, 10), ...unusedField];
380 db.prepare("INSERT INTO steps VALUES (?, ?)").run(0, new Uint8Array(protoBytes(9, usage)));
381 } finally {
382 db.close();
383 }
384 const result = await readAntigravityUsage(dir, 0);
385 assert.deepStrictEqual(result.errors, []);
386 assert.strictEqual(
387 result.files.flatMap((file) => file.records)[0]?.totals.uncachedInputTokens,
388 10,
389 );
390 });
391});
apps/server/src/provider/Drivers/antigravityUsageReader.ts created+406
...@@ -0,0 +1,406 @@
1// node:sqlite reads live conversation databases while Node fs discovers them.
2// @effect-diagnostics nodeBuiltinImport:off
3import * as NodeFSP from "node:fs/promises";
4import * as NodePath from "node:path";
5import * as NodeSqlite from "node:sqlite";
6import * as NodeTimersPromises from "node:timers/promises";
7
8import type { UsageRecord } from "@t3tools/provider-core/server/usage";
9
10type FieldValue = number | bigint | Uint8Array;
11type Fields = Map<number, FieldValue[]>;
12
13/** Antigravity stores usage metadata as protobuf, independently of conversation text. */
14function fields(bytes: Uint8Array): Fields {
15 let offset = 0;
16 const result: Fields = new Map();
17 const varint = () => {
18 let value = 0n;
19 for (let shift = 0n; shift < 70n; shift += 7n) {
20 const byte = bytes[offset++];
21 if (byte === undefined || (shift === 63n && byte > 1)) {
22 throw new Error("Invalid Antigravity protobuf varint");
23 }
24 value |= BigInt(byte & 127) << shift;
25 if (byte < 128) {
26 return value > BigInt(Number.MAX_SAFE_INTEGER) ? value : Number(value);
27 }
28 }
29 throw new Error("Invalid Antigravity protobuf varint");
30 };
31 while (offset < bytes.length) {
32 const tag = varint();
33 if (typeof tag !== "number") throw new Error("Invalid protobuf field");
34 const number = Math.floor(tag / 8);
35 const wire = tag % 8;
36 if (number === 0) throw new Error("Invalid protobuf field");
37 let value: FieldValue;
38 if (wire === 0) {
39 value = varint();
40 } else if (wire === 1 || wire === 5 || wire === 2) {
41 const length = wire === 2 ? varint() : wire === 1 ? 8 : 4;
42 if (typeof length !== "number") throw new Error("Invalid protobuf field length");
43 if (length > bytes.length - offset) throw new Error("Truncated protobuf field");
44 value = bytes.subarray(offset, offset + length);
45 offset += length;
46 if (wire !== 2) continue;
47 } else {
48 throw new Error("Unsupported protobuf wire type");
49 }
50 const entries = result.get(number) ?? [];
51 entries.push(value);
52 result.set(number, entries);
53 }
54 return result;
55}
56
57const numberAt = (value: Fields, key: number) => {
58 const entry = value.get(key)?.[0];
59 return typeof entry === "number" ? entry : 0;
60};
61const bytesAt = (value: Fields, key: number) => {
62 const entry = value.get(key)?.[0];
63 return entry instanceof Uint8Array ? entry : undefined;
64};
65const nested = (value: Fields, key: number) => {
66 const bytes = bytesAt(value, key);
67 return bytes === undefined ? new Map<number, FieldValue[]>() : fields(bytes);
68};
69const textAt = (value: Fields, key: number) => {
70 const bytes = bytesAt(value, key);
71 return bytes === undefined ? "" : new TextDecoder("utf-8", { fatal: true }).decode(bytes).trim();
72};
73const timestamp = (value: Fields) => {
74 const seconds = numberAt(value, 1);
75 return seconds > 0 ? seconds * 1000 + Math.floor(numberAt(value, 2) / 1_000_000) : null;
76};
77
78const MODEL_IDS: Record<number, string> = {
79 246: "gemini-2.5-pro",
80 312: "gemini-2.5-flash",
81 313: "gemini-2.5-flash-thinking",
82 329: "gemini-2.5-flash-thinking",
83 330: "gemini-2.5-flash-lite",
84 281: "claude-sonnet-4",
85 282: "claude-sonnet-4",
86 290: "claude-opus-4",
87 291: "claude-opus-4",
88 333: "claude-sonnet-4-5",
89 334: "claude-sonnet-4-5",
90 340: "claude-haiku-4-5",
91 341: "claude-haiku-4-5",
92 1026: "claude-opus-4-6",
93 1035: "claude-sonnet-4-6",
94 1016: "gemini-3.1-pro",
95 1036: "gemini-3.1-pro",
96 1037: "gemini-3.1-pro",
97 1018: "gemini-3-flash-preview",
98 1084: "gemini-3-flash-preview",
99 1047: "gemini-3-flash-preview",
100};
101
102function modelName(name: string, id: number): string {
103 if (name) {
104 const normalized = name
105 .toLowerCase()
106 .replace(/\s*\([^)]*\)\s*$/, "")
107 .replaceAll(" ", "-");
108 if (normalized.startsWith("claude-")) {
109 return normalized
110 .replace(/^claude-(4(?:\.\d+)?)-(sonnet|opus|haiku)/, "claude-$2-$1")
111 .replaceAll(".", "-");
112 }
113 return normalized;
114 }
115 return MODEL_IDS[id] ?? (id > 0 ? `antigravity-model-${id}` : "");
116}
117
118interface Metadata {
119 model: string;
120 timestampMs: number | null;
121 usages: Fields[];
122}
123
124function metadata(bytes: Uint8Array, step: boolean): Metadata {
125 const root = fields(bytes);
126 if (!step && bytesAt(root, 1) === undefined) {
127 throw new Error("Missing Antigravity generation metadata");
128 }
129 const data = step ? root : nested(root, 1);
130 const model = step ? nested(data, 24) : data;
131 const usage = bytesAt(data, step ? 9 : 4);
132 const usages = usage === undefined ? [] : [fields(usage)];
133 for (const retry of data.get(step ? 28 : 17) ?? []) {
134 if (!(retry instanceof Uint8Array)) throw new Error("Invalid retry metadata");
135 const retryUsage = bytesAt(fields(retry), 2);
136 if (retryUsage !== undefined) usages.push(fields(retryUsage));
137 }
138 return {
139 model: modelName(
140 textAt(model, step ? 12 : 19) || textAt(model, step ? 8 : 21),
141 numberAt(model, step ? 1 : 3),
142 ),
143 timestampMs: step
144 ? (timestamp(nested(data, 8)) ?? timestamp(nested(data, 1)))
145 : timestamp(nested(nested(data, 9), 4)),
146 usages,
147 };
148}
149
150function blob(value: unknown): Uint8Array {
151 if (!(value instanceof Uint8Array)) throw new Error("Invalid Antigravity metadata blob");
152 return value;
153}
154
155interface UsageCandidate {
156 record: UsageRecord;
157 keys: readonly string[];
158 timestampQuality: number;
159}
160
161async function readDatabase(path: string, fallbackTimestamp: number): Promise<UsageCandidate[]> {
162 const db = new NodeSqlite.DatabaseSync(path, { readOnly: true });
163 try {
164 db.exec("PRAGMA busy_timeout = 100; BEGIN");
165 const tables = new Set(
166 db
167 .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
168 .all()
169 .map((row) => row.name),
170 );
171 if (!tables.has("gen_metadata") && !tables.has("steps")) {
172 throw new Error("Missing Antigravity usage tables");
173 }
174 const readMetadata = async (query: string, column: string, step: boolean) => {
175 const entries: Array<{ idx: number; entry: Metadata }> = [];
176 for (const row of db.prepare(query).iterate()) {
177 if (typeof row.idx !== "number") throw new Error("Invalid Antigravity metadata index");
178 entries.push({ idx: row.idx, entry: metadata(blob(row[column]), step) });
179 if (entries.length % 256 === 0) await NodeTimersPromises.setImmediate();
180 }
181 return entries;
182 };
183 const generations = tables.has("gen_metadata")
184 ? await readMetadata("SELECT idx, data FROM gen_metadata ORDER BY idx", "data", false)
185 : [];
186 let trajectoryTimestamp: number | null = null;
187 if (tables.has("trajectory_metadata_blob")) {
188 for (const row of db.prepare("SELECT data FROM trajectory_metadata_blob").iterate()) {
189 trajectoryTimestamp ??= timestamp(nested(fields(blob(row.data)), 2));
190 }
191 }
192 const steps = tables.has("steps")
193 ? await readMetadata(
194 "SELECT idx, metadata FROM steps WHERE metadata IS NOT NULL ORDER BY idx",
195 "metadata",
196 true,
197 )
198 : [];
199 const sessionId = NodePath.basename(path, ".db");
200 const records: UsageCandidate[] = [];
201 const generationModels = new Map(generations.map(({ idx, entry }) => [idx, entry.model]));
202 for (const [source, entries] of [
203 ["step", steps],
204 ["generation", generations],
205 ] as const) {
206 for (const [index, { idx, entry }] of entries.entries()) {
207 for (const [usageIndex, usage] of entry.usages.entries()) {
208 const outputTokens = Math.max(
209 numberAt(usage, 3),
210 numberAt(usage, 9) + numberAt(usage, 10),
211 );
212 const totals = {
213 uncachedInputTokens: numberAt(usage, 2),
214 cachedInputTokens: numberAt(usage, 5),
215 cacheCreationTokens: numberAt(usage, 4),
216 outputTokens,
217 reasoningTokens: Math.min(outputTokens, numberAt(usage, 9)),
218 };
219 if (
220 totals.uncachedInputTokens +
221 totals.cachedInputTokens +
222 totals.cacheCreationTokens +
223 outputTokens ===
224 0
225 )
226 continue;
227 const keys = ([11, 12, 7] as const).flatMap((key) => {
228 const id = textAt(usage, key);
229 return id ? [`antigravity:${key}:${id}`] : [];
230 });
231 const record: UsageRecord = {
232 provider: "antigravity",
233 sessionId,
234 timestampMs: entry.timestampMs ?? trajectoryTimestamp ?? fallbackTimestamp,
235 model:
236 MODEL_IDS[numberAt(usage, 1)] ||
237 entry.model ||
238 (source === "step" ? generationModels.get(idx) : "") ||
239 modelName("", numberAt(usage, 1)) ||
240 "antigravity-unknown",
241 totals,
242 reportedCostUsd: null,
243 speed: "standard",
244 dedupeKey: keys[0] ?? `antigravity:${sessionId}:${source}:${index}:${usageIndex}`,
245 };
246 records.push({
247 record,
248 keys,
249 timestampQuality: entry.timestampMs !== null ? 2 : trajectoryTimestamp !== null ? 1 : 0,
250 });
251 }
252 }
253 }
254 return records;
255 } finally {
256 db.close();
257 }
258}
259
260interface CachedDatabase {
261 readonly fingerprint: string;
262 readonly candidates: readonly UsageCandidate[];
263}
264
265/**
266 * Parsed databases keyed by canonical path. An entry is reused only while both
267 * the database and its `-wal` sidecar keep the same size, mtime and ctime: new
268 * rows land in the WAL without touching the main file until a checkpoint, and
269 * ctime moves on any content write even when mtime is restored.
270 */
271export const makeAntigravityUsageCache = () => new Map<string, CachedDatabase>();
272
273/**
274 * Reads and merges aliases across every configured Antigravity store before date
275 * filtering. With a cache, databases unchanged since the previous read are not
276 * decoded again.
277 */
278export async function readAntigravityUsage(
279 conversationsDirectories: string | readonly string[],
280 sinceMs: number,
281 cache?: Map<string, CachedDatabase>,
282) {
283 const roots =
284 typeof conversationsDirectories === "string"
285 ? [conversationsDirectories]
286 : conversationsDirectories;
287 const files: Array<{ root: string; path: string; records: UsageRecord[] }> = [];
288 const errors: string[] = [];
289 const identities = new Map<string, number>();
290 const groups: Array<
291 UsageCandidate & { parent: number; size: number; owner: number; fileIndex: number }
292 > = [];
293 const find = (index: number): number => {
294 let root = index;
295 while (groups[root]!.parent !== root) root = groups[root]!.parent;
296 while (index !== root) {
297 const parent = groups[index]!.parent;
298 groups[index]!.parent = root;
299 index = parent;
300 }
301 return root;
302 };
303 const merge = (left: number, right: number): number => {
304 let a = find(left);
305 let b = find(right);
306 if (a === b) return a;
307 if (groups[a]!.size < groups[b]!.size) [a, b] = [b, a];
308 const target = groups[a]!;
309 const source = groups[b]!;
310 const first = target.owner < source.owner ? target : source;
311 const bestTime =
312 source.timestampQuality > target.timestampQuality ||
313 (source.timestampQuality === target.timestampQuality &&
314 source.record.timestampMs < target.record.timestampMs)
315 ? source
316 : target;
317 const x = target.record.totals;
318 const y = source.record.totals;
319 target.record = {
320 ...first.record,
321 model:
322 first.record.model === "antigravity-unknown"
323 ? first === target
324 ? source.record.model
325 : target.record.model
326 : first.record.model,
327 timestampMs: bestTime.record.timestampMs,
328 totals: {
329 uncachedInputTokens: Math.max(x.uncachedInputTokens, y.uncachedInputTokens),
330 cachedInputTokens: Math.max(x.cachedInputTokens, y.cachedInputTokens),
331 cacheCreationTokens: Math.max(x.cacheCreationTokens, y.cacheCreationTokens),
332 outputTokens: Math.max(x.outputTokens, y.outputTokens),
333 reasoningTokens: Math.max(x.reasoningTokens, y.reasoningTokens),
334 },
335 };
336 target.timestampQuality = bestTime.timestampQuality;
337 target.owner = first.owner;
338 target.fileIndex = first.fileIndex;
339 target.size += source.size;
340 source.parent = a;
341 return a;
342 };
343 const append = (candidate: UsageCandidate, fileIndex: number) => {
344 const index = groups.length;
345 groups.push({ ...candidate, parent: index, size: 1, owner: index, fileIndex });
346 for (const key of candidate.keys) {
347 const existing = identities.get(key);
348 if (existing !== undefined) merge(index, existing);
349 identities.set(key, index);
350 }
351 };
352 const visited = new Set<string>();
353 const walk = async (directory: string, root: string): Promise<void> => {
354 let entries;
355 try {
356 entries = await NodeFSP.readdir(directory, { withFileTypes: true });
357 } catch (error) {
358 if ((error as NodeJS.ErrnoException).code !== "ENOENT") errors.push(directory);
359 return;
360 }
361 entries.sort((a, b) => a.name.localeCompare(b.name));
362 for (const entry of entries) {
363 const path = NodePath.join(directory, entry.name);
364 if (entry.isDirectory()) {
365 await walk(path, root);
366 } else if (entry.isFile() && entry.name.endsWith(".db")) {
367 try {
368 const canonical = await NodeFSP.realpath(path);
369 if (visited.has(canonical)) continue;
370 visited.add(canonical);
371 const stat = await NodeFSP.stat(path);
372 const wal = await NodeFSP.stat(`${path}-wal`).catch(() => null);
373 const fingerprint = [stat, wal]
374 .map((file) => (file ? `${file.size}:${file.mtimeMs}:${file.ctimeMs}` : "-"))
375 .join("/");
376 const cached = cache?.get(canonical);
377 let candidates: readonly UsageCandidate[];
378 if (cached?.fingerprint === fingerprint) {
379 candidates = cached.candidates;
380 } else {
381 candidates = await readDatabase(path, stat.mtimeMs);
382 cache?.set(canonical, { fingerprint, candidates });
383 }
384 const fileIndex = files.length;
385 files.push({ root, path, records: [] });
386 for (const [index, candidate] of candidates.entries()) {
387 append(candidate, fileIndex);
388 if (index % 256 === 255) await NodeTimersPromises.setImmediate();
389 }
390 } catch {
391 errors.push(path);
392 }
393 }
394 }
395 };
396 for (const root of roots) await walk(root, root);
397 if (cache !== undefined) {
398 for (const key of cache.keys()) if (!visited.has(key)) cache.delete(key);
399 }
400 for (const [index, group] of groups.entries()) {
401 if (group.parent === index && group.record.timestampMs >= sinceMs) {
402 files[group.fileIndex]!.records.push(group.record);
403 }
404 }
405 return { files, errors };
406}
apps/server/src/provider/builtInDrivers.ts+2-1
...@@ -76,6 +76,7 @@ export type BuiltInUsageReadersEnv =...@@ -76,6 +76,7 @@ export type BuiltInUsageReadersEnv =
76 | ProviderUsageReaderEnv<typeof CodexDriver>76 | ProviderUsageReaderEnv<typeof CodexDriver>
77 | ProviderUsageReaderEnv<typeof GrokDriver>77 | ProviderUsageReaderEnv<typeof GrokDriver>
78 | ProviderUsageReaderEnv<typeof OpenCodeDriver>78 | ProviderUsageReaderEnv<typeof OpenCodeDriver>
79 | ProviderUsageReaderEnv<typeof AntigravityDriver>
79 | ProviderUsageReaderEnv<typeof CursorDriver>;80 | ProviderUsageReaderEnv<typeof CursorDriver>;
8081
81/**82/**
...@@ -85,4 +86,4 @@ export type BuiltInUsageReadersEnv =...@@ -85,4 +86,4 @@ export type BuiltInUsageReadersEnv =
85 */86 */
86export const BUILT_IN_USAGE_DRIVERS: ReadonlyArray<87export const BUILT_IN_USAGE_DRIVERS: ReadonlyArray<
87 AnyProviderDriver<BuiltInDriversEnv, BuiltInUsageReadersEnv>88 AnyProviderDriver<BuiltInDriversEnv, BuiltInUsageReadersEnv>
88> = [ClaudeDriver, CodexDriver, GrokDriver, OpenCodeDriver, CursorDriver];89> = [ClaudeDriver, CodexDriver, GrokDriver, OpenCodeDriver, AntigravityDriver, CursorDriver];
apps/server/src/server.ts+7-4
...@@ -154,6 +154,7 @@ import * as NativeTelemetryClient from "./resourceTelemetry/NativeTelemetryClien...@@ -154,6 +154,7 @@ import * as NativeTelemetryClient from "./resourceTelemetry/NativeTelemetryClien
154import * as ResourceAttribution from "./resourceTelemetry/ResourceAttribution.ts";154import * as ResourceAttribution from "./resourceTelemetry/ResourceAttribution.ts";
155import * as ResourceMonitorBinary from "./resourceTelemetry/ResourceMonitorBinary.ts";155import * as ResourceMonitorBinary from "./resourceTelemetry/ResourceMonitorBinary.ts";
156import * as ResourceTelemetry from "./resourceTelemetry/ResourceTelemetry.ts";156import * as ResourceTelemetry from "./resourceTelemetry/ResourceTelemetry.ts";
157import * as AntigravityUsage from "./provider/Drivers/AntigravityUsage.ts";
157import * as CursorAccountReader from "@t3tools/provider-cursor/server/CursorAccountReader";158import * as CursorAccountReader from "@t3tools/provider-cursor/server/CursorAccountReader";
158import * as CursorKeychain from "@t3tools/provider-cursor/server/CursorKeychain";159import * as CursorKeychain from "@t3tools/provider-cursor/server/CursorKeychain";
159import * as CursorUsageAccounts from "@t3tools/provider-cursor/server/CursorUsageAccounts";160import * as CursorUsageAccounts from "@t3tools/provider-cursor/server/CursorUsageAccounts";
...@@ -255,10 +256,12 @@ const layerBackground = BackgroundPolicy.layer.pipe(...@@ -255,10 +256,12 @@ const layerBackground = BackgroundPolicy.layer.pipe(
255256
256const layerUsage = UsageService.layer.pipe(257const layerUsage = UsageService.layer.pipe(
257 Layer.provide(258 Layer.provide(
258 CursorUsageAccounts.layer.pipe(259 Layer.mergeAll(
259 Layer.provide(CursorAccountReader.layer.pipe(Layer.provide(CursorKeychain.layer))),260 AntigravityUsage.layer,
260 Layer.provide(ProviderHostLive.layer.pipe(Layer.provide(ServerSecretStore.layer))),261 CursorUsageAccounts.layer.pipe(
261 ),262 Layer.provide(CursorAccountReader.layer.pipe(Layer.provide(CursorKeychain.layer))),
263 ),
264 ).pipe(Layer.provide(ProviderHostLive.layer.pipe(Layer.provide(ServerSecretStore.layer)))),
262 ),265 ),
263 Layer.provide(layerServerSettings),266 Layer.provide(layerServerSettings),
264);267);
apps/server/src/usage/UsageService.test.ts+2
...@@ -38,6 +38,7 @@ import * as CursorKeychain from "@t3tools/provider-cursor/server/CursorKeychain"...@@ -38,6 +38,7 @@ import * as CursorKeychain from "@t3tools/provider-cursor/server/CursorKeychain"
38import * as CursorUsageAccounts from "@t3tools/provider-cursor/server/CursorUsageAccounts";38import * as CursorUsageAccounts from "@t3tools/provider-cursor/server/CursorUsageAccounts";
39import * as ServerSecretStore from "../auth/ServerSecretStore.ts";39import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
40import * as BackgroundPolicy from "../background/BackgroundPolicy.ts";40import * as BackgroundPolicy from "../background/BackgroundPolicy.ts";
41import * as AntigravityUsage from "../provider/Drivers/AntigravityUsage.ts";
41import * as ProviderHostLive from "../provider/ProviderHostLive.ts";42import * as ProviderHostLive from "../provider/ProviderHostLive.ts";
42import type { UsageRecord } from "@t3tools/provider-core/server/usage";43import type { UsageRecord } from "@t3tools/provider-core/server/usage";
43import * as UsageService from "./UsageService.ts";44import * as UsageService from "./UsageService.ts";
...@@ -127,6 +128,7 @@ const layerService = (input: {...@@ -127,6 +128,7 @@ const layerService = (input: {
127 layerCursorUsageAccounts(128 layerCursorUsageAccounts(
128 CursorAccountReader.layer.pipe(Layer.provide(CursorKeychain.layer)),129 CursorAccountReader.layer.pipe(Layer.provide(CursorKeychain.layer)),
129 ).pipe(130 ).pipe(
131 Layer.provideMerge(AntigravityUsage.layer),
130 Layer.provideMerge(ProviderHostLive.layer),132 Layer.provideMerge(ProviderHostLive.layer),
131 Layer.provideMerge(Layer.mock(BackgroundPolicy.BackgroundPolicy)({})),133 Layer.provideMerge(Layer.mock(BackgroundPolicy.BackgroundPolicy)({})),
132 Layer.provideMerge(Layer.mock(ServerSecretStore.ServerSecretStore)({})),134 Layer.provideMerge(Layer.mock(ServerSecretStore.ServerSecretStore)({})),
apps/server/src/usage/UsageService.ts+12-105
...@@ -1,16 +1,15 @@...@@ -1,16 +1,15 @@
1/**1/**
2 * UsageService - scans provider transcripts and returns priced usage buckets.2 * UsageService - reads every driver's usage history and returns priced usage
3 * buckets.
3 *4 *
4 * The scan reads native session files and databases, including work driven5 * Each built-in driver with a `usage` reader contributes its sources, including
5 * outside T3 Code. Cursor's local records provide only partial coverage.6 * work driven outside T3 Code. A `transcripts` reader names JSONL directories
7 * that this service streams itself; a `scan` reader reads its own sources.
6 *8 *
7 * JSONL transcripts are append-only, so parsed records are memoised per file by9 * JSONL transcripts are append-only, so parsed records are memoised per file by
8 * `(size, mtime)`. A cold 30-day scan of ~1.4 GB lands around 2-3 seconds; warm10 * `(size, mtime)`. A cold 30-day scan of ~1.4 GB lands around 2-3 seconds; warm
9 * scans only reparse files that changed, and a file that merely grew resumes11 * scans only reparse files that changed, and a file that merely grew resumes
10 * from its cached parse position so only the appended bytes are read.12 * from its cached parse position so only the appended bytes are read.
11 * OpenCode's SQLite reader queries the live database each scan so WAL writes
12 * remain visible. Antigravity databases are memoised in memory while the
13 * database and its WAL keep the same `(size, mtime, ctime)`.
14 *13 *
15 * A scan reader with a slow source (an account API) may answer from its own14 * A scan reader with a slow source (an account API) may answer from its own
16 * cache and mark itself `refreshing` while a background refresh runs;15 * cache and mark itself `refreshing` while a background refresh runs;
...@@ -36,7 +35,6 @@ import { HostProcessEnvironment } from "@t3tools/shared/hostProcess";...@@ -36,7 +35,6 @@ import { HostProcessEnvironment } from "@t3tools/shared/hostProcess";
36import * as Cause from "effect/Cause";35import * as Cause from "effect/Cause";
37import * as Clock from "effect/Clock";36import * as Clock from "effect/Clock";
38import * as Context from "effect/Context";37import * as Context from "effect/Context";
39import * as Crypto from "effect/Crypto";
40import * as DateTime from "effect/DateTime";38import * as DateTime from "effect/DateTime";
41import * as Deferred from "effect/Deferred";39import * as Deferred from "effect/Deferred";
42import * as Effect from "effect/Effect";40import * as Effect from "effect/Effect";
...@@ -52,9 +50,7 @@ import { HttpClient, HttpClientResponse } from "effect/http";...@@ -52,9 +50,7 @@ import { HttpClient, HttpClientResponse } from "effect/http";
5250
53import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite";51import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite";
54import * as ServerConfig from "../config.ts";52import * as ServerConfig from "../config.ts";
55import { expandHomePath } from "@t3tools/provider-core/server/pathExpansion";
56import * as ServerSettings from "../serverSettings.ts";53import * as ServerSettings from "../serverSettings.ts";
57import { resolveAntigravityInstanceDirectories } from "../provider/antigravityAuthSupport.ts";
58import { BUILT_IN_USAGE_DRIVERS, type BuiltInUsageReadersEnv } from "../provider/builtInDrivers.ts";54import { BUILT_IN_USAGE_DRIVERS, type BuiltInUsageReadersEnv } from "../provider/builtInDrivers.ts";
59import type { ProviderDriver } from "@t3tools/provider-core/server/driver";55import type { ProviderDriver } from "@t3tools/provider-core/server/driver";
60import { mergeProviderInstanceEnvironment } from "@t3tools/provider-core/server/instanceEnvironment";56import { mergeProviderInstanceEnvironment } from "@t3tools/provider-core/server/instanceEnvironment";
...@@ -63,7 +59,6 @@ import type {...@@ -63,7 +59,6 @@ import type {
63 TranscriptUsageFormat,59 TranscriptUsageFormat,
64 UsageRecord,60 UsageRecord,
65} from "@t3tools/provider-core/server/usage";61} from "@t3tools/provider-core/server/usage";
66import { makeAntigravityUsageCache, readAntigravityUsage } from "./antigravityUsageReader.ts";
67import { resolveModelAliases, UsageAggregator } from "./usageAggregation.ts";62import { resolveModelAliases, UsageAggregator } from "./usageAggregation.ts";
68import { createOverrideRateTable, parseRateTable, type RateTable } from "./usagePricing.ts";63import { createOverrideRateTable, parseRateTable, type RateTable } from "./usagePricing.ts";
69import {64import {
...@@ -227,7 +222,6 @@ const layerTest = Layer.succeed(...@@ -227,7 +222,6 @@ const layerTest = Layer.succeed(
227);222);
228223
229export const make = Effect.gen(function* () {224export const make = Effect.gen(function* () {
230 const crypto = yield* Crypto.Crypto;
231 const fileSystem = yield* FileSystem.FileSystem;225 const fileSystem = yield* FileSystem.FileSystem;
232 const path = yield* Path.Path;226 const path = yield* Path.Path;
233 const config = yield* ServerConfig.ServerConfig;227 const config = yield* ServerConfig.ServerConfig;
...@@ -238,7 +232,6 @@ export const make = Effect.gen(function* () {...@@ -238,7 +232,6 @@ export const make = Effect.gen(function* () {
238 const readerContext = yield* Effect.context<BuiltInUsageReadersEnv>();232 const readerContext = yield* Effect.context<BuiltInUsageReadersEnv>();
239233
240 const fileCache: ScanCache = new Map();234 const fileCache: ScanCache = new Map();
241 const antigravityCache = makeAntigravityUsageCache();
242 const sourceCache = new Map<string, typeof CachedSource.Type>();235 const sourceCache = new Map<string, typeof CachedSource.Type>();
243 let cacheDirty = false;236 let cacheDirty = false;
244 const isWithinDirectory = (filePath: string, dir: string) => {237 const isWithinDirectory = (filePath: string, dir: string) => {
...@@ -639,22 +632,6 @@ export const make = Effect.gen(function* () {...@@ -639,22 +632,6 @@ export const make = Effect.gen(function* () {
639 Effect.provideContext(readerContext),632 Effect.provideContext(readerContext),
640 );633 );
641634
642 const home = NodeOS.homedir();
643 const envRoots = Effect.fnUntraced(function* (key: string, defaults: readonly string[]) {
644 const roots = hostEnvironment[key]
645 ?.split(",")
646 .map((value) => value.trim())
647 .filter(Boolean);
648 const canonical = new Set<string>();
649 for (const root of roots?.length ? roots : defaults) {
650 const resolved = path.resolve(expandHomePath(root));
651 canonical.add(
652 yield* fileSystem.realPath(resolved).pipe(Effect.orElseSucceed(() => resolved)),
653 );
654 }
655 return [...canonical];
656 });
657
658 const scans = Effect.forEach(635 const scans = Effect.forEach(
659 scanReaders,636 scanReaders,
660 ({ driver, reader }) =>637 ({ driver, reader }) =>
...@@ -686,83 +663,14 @@ export const make = Effect.gen(function* () {...@@ -686,83 +663,14 @@ export const make = Effect.gen(function* () {
686 { concurrency: "unbounded" },663 { concurrency: "unbounded" },
687 );664 );
688665
689 const antigravity = Effect.gen(function* () {
690 const antigravityRoots = yield* envRoots("ANTIGRAVITY_DATA_DIR", [
691 ...["antigravity", "antigravity-cli", "antigravity-ide", "antigravity-backup"].map((name) =>
692 path.join(home, ".gemini", name),
693 ),
694 path.join(home, ".config", "antigravity"),
695 ]);
696 for (const [instanceId, instance] of Object.entries(settings.providerInstances)) {
697 if (instance.driver === "antigravity") {
698 const directories = yield* resolveAntigravityInstanceDirectories(
699 config.stateDir,
700 ProviderInstanceId.make(instanceId),
701 ).pipe(
702 Effect.provideService(Crypto.Crypto, crypto),
703 Effect.provideService(Path.Path, path),
704 Effect.mapError(
705 (cause) =>
706 new UsageReadError({
707 reason: "scanFailed",
708 detail: "Antigravity profile directory could not be resolved.",
709 cause,
710 }),
711 ),
712 );
713 antigravityRoots.push(path.join(directories.profile, "antigravity-acp"));
714 }
715 }
716 const antigravityDirs = new Set<string>();
717 for (const root of antigravityRoots) {
718 const resolvedRoot = yield* fileSystem
719 .realPath(root)
720 .pipe(Effect.orElseSucceed(() => root));
721 const nested = path.join(resolvedRoot, "conversations");
722 const dir = (yield* fileSystem
723 .exists(nested)
724 .pipe(Effect.catchCause(() => Effect.succeed(false))))
725 ? nested
726 : resolvedRoot;
727 antigravityDirs.add(yield* fileSystem.realPath(dir).pipe(Effect.orElseSucceed(() => dir)));
728 }
729 const result = yield* Effect.promise(() =>
730 readAntigravityUsage([...antigravityDirs], windowStartMs, antigravityCache),
731 );
732 const scanned: ScannedDir[] = [];
733 for (const dir of antigravityDirs) {
734 const exists = yield* fileSystem
735 .exists(dir)
736 .pipe(Effect.catchCause(() => Effect.succeed(false)));
737 const failed = result.errors.some(
738 (error) => error === dir || error.startsWith(`${dir}${path.sep}`),
739 );
740 scanned.push({
741 provider: "antigravity",
742 dir,
743 volumeId: yield* Effect.promise(() => readDirectoryVolumeId(dir)),
744 files: !exists && !failed ? null : result.files.filter((file) => file.root === dir),
745 status: failed ? "partial" : "ok",
746 ...(failed ? { message: "Some Antigravity history could not be read." } : {}),
747 });
748 }
749 return scanned;
750 });
751
752 // Independent sources scan together. Transcript directories go one at a666 // Independent sources scan together. Transcript directories go one at a
753 // time, so open files stay at `TRANSCRIPT_READ_CONCURRENCY`. The result667 // time, so open files stay at `TRANSCRIPT_READ_CONCURRENCY`. The result
754 // keeps this order, since aggregation keeps the first copy of a duplicate.668 // keeps this order, since aggregation keeps the first copy of a duplicate.
755 const [transcripts, scanDirs, antigravityDirs] = yield* Effect.all(669 const [transcripts, scanDirs] = yield* Effect.all(
756 [Effect.forEach(dirs, (dir) => scanTranscriptDir(dir, windowStartMs)), scans, antigravity],670 [Effect.forEach(dirs, (dir) => scanTranscriptDir(dir, windowStartMs)), scans],
757 { concurrency: "unbounded" },671 { concurrency: "unbounded" },
758 );672 );
759 // Antigravity reads between the other scan readers until it has a reader of its own.673 const scanned: readonly ScannedDir[] = [...transcripts, ...scanDirs.flat()];
760 const scanned: readonly ScannedDir[] = [
761 ...transcripts,
762 ...scanDirs.slice(0, -1).flat(),
763 ...antigravityDirs,
764 ...(scanDirs.at(-1) ?? []),
765 ];
766 return scanned;674 return scanned;
767 });675 });
768676
...@@ -944,9 +852,10 @@ export const make = Effect.gen(function* () {...@@ -944,9 +852,10 @@ export const make = Effect.gen(function* () {
944 });852 });
945853
946 /**854 /**
947 * In-flight scans by window and usage settings, so concurrent identical requests (the usage855 * In-flight scans by window and settings, so concurrent identical requests (the usage
948 * page open on two clients at once) share one scan instead of racing over856 * page open on two clients at once) share one scan instead of racing over
949 * the same corpus twice.857 * the same corpus twice. Readers read settings of their own, so a scan is
858 * shared only under the same settings snapshot.
950 */859 */
951 const inflightScans = new Map<string, Deferred.Deferred<UsageSummary, UsageReadError>>();860 const inflightScans = new Map<string, Deferred.Deferred<UsageSummary, UsageReadError>>();
952861
...@@ -958,9 +867,7 @@ export const make = Effect.gen(function* () {...@@ -958,9 +867,7 @@ export const make = Effect.gen(function* () {
958 input.resolution ?? "day",867 input.resolution ?? "day",
959 input.sinceTime ?? null,868 input.sinceTime ?? null,
960 input.untilTime ?? null,869 input.untilTime ?? null,
961 settings.usagePriceOverrides,870 settings,
962 settings.usageModelAliases,
963 settings.cursorKeychainUsageEnabled,
964 // A waiting read must never share a scan that answers with `refreshing`.871 // A waiting read must never share a scan that answers with `refreshing`.
965 input.awaitRefresh === true,872 input.awaitRefresh === true,
966 ]);873 ]);
apps/server/src/usage/antigravityUsageReader.ts deleted-406
...@@ -1,406 +0,0 @@
1// node:sqlite reads live conversation databases while Node fs discovers them.
2// @effect-diagnostics nodeBuiltinImport:off
3import * as NodeFSP from "node:fs/promises";
4import * as NodePath from "node:path";
5import * as NodeSqlite from "node:sqlite";
6import * as NodeTimersPromises from "node:timers/promises";
7
8import type { UsageRecord } from "@t3tools/provider-core/server/usage";
9
10type FieldValue = number | bigint | Uint8Array;
11type Fields = Map<number, FieldValue[]>;
12
13/** Antigravity stores usage metadata as protobuf, independently of conversation text. */
14function fields(bytes: Uint8Array): Fields {
15 let offset = 0;
16 const result: Fields = new Map();
17 const varint = () => {
18 let value = 0n;
19 for (let shift = 0n; shift < 70n; shift += 7n) {
20 const byte = bytes[offset++];
21 if (byte === undefined || (shift === 63n && byte > 1)) {
22 throw new Error("Invalid Antigravity protobuf varint");
23 }
24 value |= BigInt(byte & 127) << shift;
25 if (byte < 128) {
26 return value > BigInt(Number.MAX_SAFE_INTEGER) ? value : Number(value);
27 }
28 }
29 throw new Error("Invalid Antigravity protobuf varint");
30 };
31 while (offset < bytes.length) {
32 const tag = varint();
33 if (typeof tag !== "number") throw new Error("Invalid protobuf field");
34 const number = Math.floor(tag / 8);
35 const wire = tag % 8;
36 if (number === 0) throw new Error("Invalid protobuf field");
37 let value: FieldValue;
38 if (wire === 0) {
39 value = varint();
40 } else if (wire === 1 || wire === 5 || wire === 2) {
41 const length = wire === 2 ? varint() : wire === 1 ? 8 : 4;
42 if (typeof length !== "number") throw new Error("Invalid protobuf field length");
43 if (length > bytes.length - offset) throw new Error("Truncated protobuf field");
44 value = bytes.subarray(offset, offset + length);
45 offset += length;
46 if (wire !== 2) continue;
47 } else {
48 throw new Error("Unsupported protobuf wire type");
49 }
50 const entries = result.get(number) ?? [];
51 entries.push(value);
52 result.set(number, entries);
53 }
54 return result;
55}
56
57const numberAt = (value: Fields, key: number) => {
58 const entry = value.get(key)?.[0];
59 return typeof entry === "number" ? entry : 0;
60};
61const bytesAt = (value: Fields, key: number) => {
62 const entry = value.get(key)?.[0];
63 return entry instanceof Uint8Array ? entry : undefined;
64};
65const nested = (value: Fields, key: number) => {
66 const bytes = bytesAt(value, key);
67 return bytes === undefined ? new Map<number, FieldValue[]>() : fields(bytes);
68};
69const textAt = (value: Fields, key: number) => {
70 const bytes = bytesAt(value, key);
71 return bytes === undefined ? "" : new TextDecoder("utf-8", { fatal: true }).decode(bytes).trim();
72};
73const timestamp = (value: Fields) => {
74 const seconds = numberAt(value, 1);
75 return seconds > 0 ? seconds * 1000 + Math.floor(numberAt(value, 2) / 1_000_000) : null;
76};
77
78const MODEL_IDS: Record<number, string> = {
79 246: "gemini-2.5-pro",
80 312: "gemini-2.5-flash",
81 313: "gemini-2.5-flash-thinking",
82 329: "gemini-2.5-flash-thinking",
83 330: "gemini-2.5-flash-lite",
84 281: "claude-sonnet-4",
85 282: "claude-sonnet-4",
86 290: "claude-opus-4",
87 291: "claude-opus-4",
88 333: "claude-sonnet-4-5",
89 334: "claude-sonnet-4-5",
90 340: "claude-haiku-4-5",
91 341: "claude-haiku-4-5",
92 1026: "claude-opus-4-6",
93 1035: "claude-sonnet-4-6",
94 1016: "gemini-3.1-pro",
95 1036: "gemini-3.1-pro",
96 1037: "gemini-3.1-pro",
97 1018: "gemini-3-flash-preview",
98 1084: "gemini-3-flash-preview",
99 1047: "gemini-3-flash-preview",
100};
101
102function modelName(name: string, id: number): string {
103 if (name) {
104 const normalized = name
105 .toLowerCase()
106 .replace(/\s*\([^)]*\)\s*$/, "")
107 .replaceAll(" ", "-");
108 if (normalized.startsWith("claude-")) {
109 return normalized
110 .replace(/^claude-(4(?:\.\d+)?)-(sonnet|opus|haiku)/, "claude-$2-$1")
111 .replaceAll(".", "-");
112 }
113 return normalized;
114 }
115 return MODEL_IDS[id] ?? (id > 0 ? `antigravity-model-${id}` : "");
116}
117
118interface Metadata {
119 model: string;
120 timestampMs: number | null;
121 usages: Fields[];
122}
123
124function metadata(bytes: Uint8Array, step: boolean): Metadata {
125 const root = fields(bytes);
126 if (!step && bytesAt(root, 1) === undefined) {
127 throw new Error("Missing Antigravity generation metadata");
128 }
129 const data = step ? root : nested(root, 1);
130 const model = step ? nested(data, 24) : data;
131 const usage = bytesAt(data, step ? 9 : 4);
132 const usages = usage === undefined ? [] : [fields(usage)];
133 for (const retry of data.get(step ? 28 : 17) ?? []) {
134 if (!(retry instanceof Uint8Array)) throw new Error("Invalid retry metadata");
135 const retryUsage = bytesAt(fields(retry), 2);
136 if (retryUsage !== undefined) usages.push(fields(retryUsage));
137 }
138 return {
139 model: modelName(
140 textAt(model, step ? 12 : 19) || textAt(model, step ? 8 : 21),
141 numberAt(model, step ? 1 : 3),
142 ),
143 timestampMs: step
144 ? (timestamp(nested(data, 8)) ?? timestamp(nested(data, 1)))
145 : timestamp(nested(nested(data, 9), 4)),
146 usages,
147 };
148}
149
150function blob(value: unknown): Uint8Array {
151 if (!(value instanceof Uint8Array)) throw new Error("Invalid Antigravity metadata blob");
152 return value;
153}
154
155interface UsageCandidate {
156 record: UsageRecord;
157 keys: readonly string[];
158 timestampQuality: number;
159}
160
161async function readDatabase(path: string, fallbackTimestamp: number): Promise<UsageCandidate[]> {
162 const db = new NodeSqlite.DatabaseSync(path, { readOnly: true });
163 try {
164 db.exec("PRAGMA busy_timeout = 100; BEGIN");
165 const tables = new Set(
166 db
167 .prepare("SELECT name FROM sqlite_master WHERE type = 'table'")
168 .all()
169 .map((row) => row.name),
170 );
171 if (!tables.has("gen_metadata") && !tables.has("steps")) {
172 throw new Error("Missing Antigravity usage tables");
173 }
174 const readMetadata = async (query: string, column: string, step: boolean) => {
175 const entries: Array<{ idx: number; entry: Metadata }> = [];
176 for (const row of db.prepare(query).iterate()) {
177 if (typeof row.idx !== "number") throw new Error("Invalid Antigravity metadata index");
178 entries.push({ idx: row.idx, entry: metadata(blob(row[column]), step) });
179 if (entries.length % 256 === 0) await NodeTimersPromises.setImmediate();
180 }
181 return entries;
182 };
183 const generations = tables.has("gen_metadata")
184 ? await readMetadata("SELECT idx, data FROM gen_metadata ORDER BY idx", "data", false)
185 : [];
186 let trajectoryTimestamp: number | null = null;
187 if (tables.has("trajectory_metadata_blob")) {
188 for (const row of db.prepare("SELECT data FROM trajectory_metadata_blob").iterate()) {
189 trajectoryTimestamp ??= timestamp(nested(fields(blob(row.data)), 2));
190 }
191 }
192 const steps = tables.has("steps")
193 ? await readMetadata(
194 "SELECT idx, metadata FROM steps WHERE metadata IS NOT NULL ORDER BY idx",
195 "metadata",
196 true,
197 )
198 : [];
199 const sessionId = NodePath.basename(path, ".db");
200 const records: UsageCandidate[] = [];
201 const generationModels = new Map(generations.map(({ idx, entry }) => [idx, entry.model]));
202 for (const [source, entries] of [
203 ["step", steps],
204 ["generation", generations],
205 ] as const) {
206 for (const [index, { idx, entry }] of entries.entries()) {
207 for (const [usageIndex, usage] of entry.usages.entries()) {
208 const outputTokens = Math.max(
209 numberAt(usage, 3),
210 numberAt(usage, 9) + numberAt(usage, 10),
211 );
212 const totals = {
213 uncachedInputTokens: numberAt(usage, 2),
214 cachedInputTokens: numberAt(usage, 5),
215 cacheCreationTokens: numberAt(usage, 4),
216 outputTokens,
217 reasoningTokens: Math.min(outputTokens, numberAt(usage, 9)),
218 };
219 if (
220 totals.uncachedInputTokens +
221 totals.cachedInputTokens +
222 totals.cacheCreationTokens +
223 outputTokens ===
224 0
225 )
226 continue;
227 const keys = ([11, 12, 7] as const).flatMap((key) => {
228 const id = textAt(usage, key);
229 return id ? [`antigravity:${key}:${id}`] : [];
230 });
231 const record: UsageRecord = {
232 provider: "antigravity",
233 sessionId,
234 timestampMs: entry.timestampMs ?? trajectoryTimestamp ?? fallbackTimestamp,
235 model:
236 MODEL_IDS[numberAt(usage, 1)] ||
237 entry.model ||
238 (source === "step" ? generationModels.get(idx) : "") ||
239 modelName("", numberAt(usage, 1)) ||
240 "antigravity-unknown",
241 totals,
242 reportedCostUsd: null,
243 speed: "standard",
244 dedupeKey: keys[0] ?? `antigravity:${sessionId}:${source}:${index}:${usageIndex}`,
245 };
246 records.push({
247 record,
248 keys,
249 timestampQuality: entry.timestampMs !== null ? 2 : trajectoryTimestamp !== null ? 1 : 0,
250 });
251 }
252 }
253 }
254 return records;
255 } finally {
256 db.close();
257 }
258}
259
260interface CachedDatabase {
261 readonly fingerprint: string;
262 readonly candidates: readonly UsageCandidate[];
263}
264
265/**
266 * Parsed databases keyed by canonical path. An entry is reused only while both
267 * the database and its `-wal` sidecar keep the same size, mtime and ctime: new
268 * rows land in the WAL without touching the main file until a checkpoint, and
269 * ctime moves on any content write even when mtime is restored.
270 */
271export const makeAntigravityUsageCache = () => new Map<string, CachedDatabase>();
272
273/**
274 * Reads and merges aliases across every configured Antigravity store before date
275 * filtering. With a cache, databases unchanged since the previous read are not
276 * decoded again.
277 */
278export async function readAntigravityUsage(
279 conversationsDirectories: string | readonly string[],
280 sinceMs: number,
281 cache?: Map<string, CachedDatabase>,
282) {
283 const roots =
284 typeof conversationsDirectories === "string"
285 ? [conversationsDirectories]
286 : conversationsDirectories;
287 const files: Array<{ root: string; path: string; records: UsageRecord[] }> = [];
288 const errors: string[] = [];
289 const identities = new Map<string, number>();
290 const groups: Array<
291 UsageCandidate & { parent: number; size: number; owner: number; fileIndex: number }
292 > = [];
293 const find = (index: number): number => {
294 let root = index;
295 while (groups[root]!.parent !== root) root = groups[root]!.parent;
296 while (index !== root) {
297 const parent = groups[index]!.parent;
298 groups[index]!.parent = root;
299 index = parent;
300 }
301 return root;
302 };
303 const merge = (left: number, right: number): number => {
304 let a = find(left);
305 let b = find(right);
306 if (a === b) return a;
307 if (groups[a]!.size < groups[b]!.size) [a, b] = [b, a];
308 const target = groups[a]!;
309 const source = groups[b]!;
310 const first = target.owner < source.owner ? target : source;
311 const bestTime =
312 source.timestampQuality > target.timestampQuality ||
313 (source.timestampQuality === target.timestampQuality &&
314 source.record.timestampMs < target.record.timestampMs)
315 ? source
316 : target;
317 const x = target.record.totals;
318 const y = source.record.totals;
319 target.record = {
320 ...first.record,
321 model:
322 first.record.model === "antigravity-unknown"
323 ? first === target
324 ? source.record.model
325 : target.record.model
326 : first.record.model,
327 timestampMs: bestTime.record.timestampMs,
328 totals: {
329 uncachedInputTokens: Math.max(x.uncachedInputTokens, y.uncachedInputTokens),
330 cachedInputTokens: Math.max(x.cachedInputTokens, y.cachedInputTokens),
331 cacheCreationTokens: Math.max(x.cacheCreationTokens, y.cacheCreationTokens),
332 outputTokens: Math.max(x.outputTokens, y.outputTokens),
333 reasoningTokens: Math.max(x.reasoningTokens, y.reasoningTokens),
334 },
335 };
336 target.timestampQuality = bestTime.timestampQuality;
337 target.owner = first.owner;
338 target.fileIndex = first.fileIndex;
339 target.size += source.size;
340 source.parent = a;
341 return a;
342 };
343 const append = (candidate: UsageCandidate, fileIndex: number) => {
344 const index = groups.length;
345 groups.push({ ...candidate, parent: index, size: 1, owner: index, fileIndex });
346 for (const key of candidate.keys) {
347 const existing = identities.get(key);
348 if (existing !== undefined) merge(index, existing);
349 identities.set(key, index);
350 }
351 };
352 const visited = new Set<string>();
353 const walk = async (directory: string, root: string): Promise<void> => {
354 let entries;
355 try {
356 entries = await NodeFSP.readdir(directory, { withFileTypes: true });
357 } catch (error) {
358 if ((error as NodeJS.ErrnoException).code !== "ENOENT") errors.push(directory);
359 return;
360 }
361 entries.sort((a, b) => a.name.localeCompare(b.name));
362 for (const entry of entries) {
363 const path = NodePath.join(directory, entry.name);
364 if (entry.isDirectory()) {
365 await walk(path, root);
366 } else if (entry.isFile() && entry.name.endsWith(".db")) {
367 try {
368 const canonical = await NodeFSP.realpath(path);
369 if (visited.has(canonical)) continue;
370 visited.add(canonical);
371 const stat = await NodeFSP.stat(path);
372 const wal = await NodeFSP.stat(`${path}-wal`).catch(() => null);
373 const fingerprint = [stat, wal]
374 .map((file) => (file ? `${file.size}:${file.mtimeMs}:${file.ctimeMs}` : "-"))
375 .join("/");
376 const cached = cache?.get(canonical);
377 let candidates: readonly UsageCandidate[];
378 if (cached?.fingerprint === fingerprint) {
379 candidates = cached.candidates;
380 } else {
381 candidates = await readDatabase(path, stat.mtimeMs);
382 cache?.set(canonical, { fingerprint, candidates });
383 }
384 const fileIndex = files.length;
385 files.push({ root, path, records: [] });
386 for (const [index, candidate] of candidates.entries()) {
387 append(candidate, fileIndex);
388 if (index % 256 === 255) await NodeTimersPromises.setImmediate();
389 }
390 } catch {
391 errors.push(path);
392 }
393 }
394 }
395 };
396 for (const root of roots) await walk(root, root);
397 if (cache !== undefined) {
398 for (const key of cache.keys()) if (!visited.has(key)) cache.delete(key);
399 }
400 for (const [index, group] of groups.entries()) {
401 if (group.parent === index && group.record.timestampMs >= sinceMs) {
402 files[group.fileIndex]!.records.push(group.record);
403 }
404 }
405 return { files, errors };
406}
apps/server/src/usage/usageTranscriptReader.test.ts-373
...@@ -4,39 +4,14 @@...@@ -4,39 +4,14 @@
4import * as NodeFSP from "node:fs/promises";4import * as NodeFSP from "node:fs/promises";
5import * as NodeOS from "node:os";5import * as NodeOS from "node:os";
6import * as NodePath from "node:path";6import * as NodePath from "node:path";
7import * as NodeSqlite from "node:sqlite";
87
9import { afterEach, assert, beforeEach, describe, it } from "@effect/vitest";8import { afterEach, assert, beforeEach, describe, it } from "@effect/vitest";
109
11import { TEST_FORMATS } from "./usageTestFormats.ts";10import { TEST_FORMATS } from "./usageTestFormats.ts";
12import { readTranscriptRecords } from "./usageTranscriptReader.ts";11import { readTranscriptRecords } from "./usageTranscriptReader.ts";
13import { makeAntigravityUsageCache, readAntigravityUsage } from "./antigravityUsageReader.ts";
1412
15let dir: string;13let dir: string;
1614
17function protoNumber(field: number, value: number): number[] {
18 const varint = (number: number) => {
19 const bytes: number[] = [];
20 do {
21 const byte = number % 128;
22 number = Math.floor(number / 128);
23 bytes.push(byte + (number > 0 ? 128 : 0));
24 } while (number > 0);
25 return bytes;
26 };
27 return [...varint(field * 8), ...varint(value)];
28}
29
30function protoBytes(field: number, bytes: readonly number[]): number[] {
31 const encoded = protoNumber(field, bytes.length);
32 encoded[0] = encoded[0]! + 2;
33 return [...encoded, ...bytes];
34}
35
36function protoText(field: number, value: string): number[] {
37 return protoBytes(field, [...Buffer.from(value)]);
38}
39
40beforeEach(async () => {15beforeEach(async () => {
41 dir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "usage-reader-test-"));16 dir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "usage-reader-test-"));
42});17});
...@@ -249,351 +224,3 @@ describe("readTranscriptRecords resume", () => {...@@ -249,351 +224,3 @@ describe("readTranscriptRecords resume", () => {
249 );224 );
250 });225 });
251});226});
252
253describe("SQLite usage readers", () => {
254 it("deduplicates Antigravity generation and step usage while preserving retry model and token buckets", async () => {
255 const db = new NodeSqlite.DatabaseSync(NodePath.join(dir, "session-1.db"));
256 const stamp = protoNumber(1, 1780000000);
257 const usage = [
258 ...protoNumber(2, 100),
259 ...protoNumber(3, 40),
260 ...protoNumber(4, 5),
261 ...protoNumber(5, 20),
262 ...protoNumber(9, 10),
263 ...protoText(11, "response-1"),
264 ];
265 const retry = [
266 ...protoNumber(1, 1026),
267 ...protoNumber(2, 12),
268 ...protoNumber(3, 3),
269 ...protoText(11, "retry-1"),
270 ];
271 const generation = protoBytes(1, [
272 ...protoBytes(4, usage),
273 ...protoText(19, "Gemini 3 Pro"),
274 ...protoBytes(9, protoBytes(4, stamp)),
275 ]);
276 const step = [
277 ...protoBytes(9, usage),
278 ...protoBytes(8, stamp),
279 ...protoBytes(28, protoBytes(2, retry)),
280 ];
281 try {
282 db.exec(
283 "CREATE TABLE gen_metadata (idx INTEGER, data BLOB); CREATE TABLE steps (idx INTEGER, metadata BLOB)",
284 );
285 db.prepare("INSERT INTO gen_metadata VALUES (?, ?)").run(0, new Uint8Array(generation));
286 db.prepare("INSERT INTO steps VALUES (?, ?)").run(0, new Uint8Array(step));
287 } finally {
288 db.close();
289 }
290 const result = await readAntigravityUsage(dir, 0);
291 assert.deepStrictEqual(result.errors, []);
292 const records = result.files.flatMap((file) => file.records);
293 assert.strictEqual(records.length, 2);
294 const main = records.find((record) => record.model === "gemini-3-pro");
295 assert.isDefined(main);
296 assert.strictEqual(main?.timestampMs, 1780000000000);
297 assert.strictEqual(main?.sessionId, "session-1");
298 assert.deepStrictEqual(main?.totals, {
299 uncachedInputTokens: 100,
300 cachedInputTokens: 20,
301 cacheCreationTokens: 5,
302 outputTokens: 40,
303 reasoningTokens: 10,
304 });
305 assert.strictEqual(
306 records.find((record) => record.model === "claude-opus-4-6")?.totals.uncachedInputTokens,
307 12,
308 );
309 assert.deepStrictEqual(
310 (await readAntigravityUsage(dir, 1780000000001)).files.flatMap((file) => file.records),
311 [],
312 );
313 });
314
315 it("uses the matching Antigravity generation model for each model-less step", async () => {
316 const db = new NodeSqlite.DatabaseSync(NodePath.join(dir, "model-switch.db"));
317 try {
318 db.exec(
319 "CREATE TABLE gen_metadata (idx INTEGER, data BLOB); CREATE TABLE steps (idx INTEGER, metadata BLOB)",
320 );
321 const generation = db.prepare("INSERT INTO gen_metadata VALUES (?, ?)");
322 const step = db.prepare("INSERT INTO steps VALUES (?, ?)");
323 for (const [idx, name] of ["Gemini 3 Pro", "Claude Opus 4.6"].entries()) {
324 generation.run(idx, new Uint8Array(protoBytes(1, protoText(19, name))));
325 step.run(idx, new Uint8Array(protoBytes(9, protoNumber(2, 10 + idx))));
326 }
327 } finally {
328 db.close();
329 }
330 const result = await readAntigravityUsage(dir, 0);
331 assert.deepStrictEqual(result.errors, []);
332 assert.deepStrictEqual(
333 result.files.flatMap((file) => file.records).map((record) => record.model),
334 ["gemini-3-pro", "claude-opus-4-6"],
335 );
336 });
337
338 it("merges Antigravity aliases that bridge previously separate step records", async () => {
339 const db = new NodeSqlite.DatabaseSync(NodePath.join(dir, "bridge.db"));
340 try {
341 db.exec(
342 "CREATE TABLE gen_metadata (idx INTEGER, data BLOB); CREATE TABLE steps (idx INTEGER, metadata BLOB)",
343 );
344 const step = db.prepare("INSERT INTO steps VALUES (?, ?)");
345 step.run(
346 0,
347 new Uint8Array(protoBytes(9, [...protoNumber(2, 100), ...protoText(11, "response")])),
348 );
349 step.run(
350 1,
351 new Uint8Array(protoBytes(9, [...protoNumber(3, 40), ...protoText(12, "provider")])),
352 );
353 db.prepare("INSERT INTO gen_metadata VALUES (?, ?)").run(
354 0,
355 new Uint8Array(
356 protoBytes(1, [
357 ...protoText(19, "Gemini 3 Pro"),
358 ...protoBytes(4, [
359 ...protoNumber(2, 50),
360 ...protoNumber(5, 20),
361 ...protoText(11, "response"),
362 ...protoText(12, "provider"),
363 ]),
364 ]),
365 ),
366 );
367 } finally {
368 db.close();
369 }
370 const result = await readAntigravityUsage(dir, 0);
371 assert.deepStrictEqual(result.errors, []);
372 const records = result.files.flatMap((file) => file.records);
373 assert.strictEqual(records.length, 1);
374 assert.deepStrictEqual(records[0]?.totals, {
375 uncachedInputTokens: 100,
376 cachedInputTokens: 20,
377 cacheCreationTokens: 0,
378 outputTokens: 40,
379 reasoningTokens: 0,
380 });
381 });
382
383 it("merges Antigravity provider and message aliases across configured roots while keeping original ownership", async () => {
384 const roots = [NodePath.join(dir, "first"), NodePath.join(dir, "second")];
385 for (const [index, root] of roots.entries()) {
386 await NodeFSP.mkdir(root);
387 const db = new NodeSqlite.DatabaseSync(NodePath.join(root, `session-${index}.db`));
388 try {
389 db.exec("CREATE TABLE steps (idx INTEGER, metadata BLOB)");
390 for (const identity of [7, 12]) {
391 const usage = [
392 ...protoNumber(1, 246),
393 ...protoNumber(2, index === 0 ? 100 : 150),
394 ...protoText(11, `response-${index}-${identity}`),
395 ...protoText(identity, `shared-${identity}`),
396 ];
397 db.prepare("INSERT INTO steps VALUES (?, ?)").run(
398 identity,
399 new Uint8Array(protoBytes(9, usage)),
400 );
401 }
402 } finally {
403 db.close();
404 }
405 }
406 const result = await readAntigravityUsage(roots, 0);
407 assert.deepStrictEqual(result.errors, []);
408 assert.strictEqual(result.files.length, 2);
409 assert.strictEqual(result.files[0]?.root, roots[0]);
410 assert.strictEqual(result.files[0]?.records.length, 2);
411 assert.strictEqual(result.files[1]?.records.length, 0);
412 assert.deepStrictEqual(
413 result.files[0]?.records.map((record) => record.totals.uncachedInputTokens),
414 [150, 150],
415 );
416 assert.isTrue(result.files[0]?.records.every((record) => record.sessionId === "session-0"));
417 });
418
419 it("upgrades Antigravity fallback timestamps before applying the date window", async () => {
420 for (const fallback of ["mtime", "trajectory"]) {
421 const path = NodePath.join(dir, `${fallback}.db`);
422 const db = new NodeSqlite.DatabaseSync(path);
423 try {
424 db.exec(
425 "CREATE TABLE gen_metadata (idx INTEGER, data BLOB); CREATE TABLE steps (idx INTEGER, metadata BLOB)",
426 );
427 if (fallback === "trajectory") {
428 db.exec("CREATE TABLE trajectory_metadata_blob (data BLOB)");
429 db.prepare("INSERT INTO trajectory_metadata_blob VALUES (?)").run(
430 new Uint8Array(protoBytes(2, protoNumber(1, 1780000200))),
431 );
432 }
433 for (const [index, seconds] of [1780000000, 1780000200].entries()) {
434 const usage = [...protoNumber(2, 10), ...protoText(11, `${fallback}-${index}`)];
435 db.prepare("INSERT INTO steps VALUES (?, ?)").run(
436 index,
437 new Uint8Array(protoBytes(9, usage)),
438 );
439 db.prepare("INSERT INTO gen_metadata VALUES (?, ?)").run(
440 index,
441 new Uint8Array(
442 protoBytes(1, [
443 ...protoBytes(4, usage),
444 ...protoBytes(9, protoBytes(4, protoNumber(1, seconds))),
445 ]),
446 ),
447 );
448 }
449 } finally {
450 db.close();
451 }
452 await NodeFSP.utimes(path, 1780000000, 1780000000);
453 }
454 const result = await readAntigravityUsage(dir, 1780000100000);
455 assert.deepStrictEqual(result.errors, []);
456 const records = result.files.flatMap((file) => file.records);
457 assert.strictEqual(records.length, 2);
458 assert.deepStrictEqual(
459 records.map((record) => record.timestampMs),
460 [1780000200000, 1780000200000],
461 );
462 });
463
464 const antigravityGeneration = (responseId: string) =>
465 new Uint8Array(
466 protoBytes(1, [
467 ...protoBytes(4, [
468 ...protoNumber(2, 100),
469 ...protoNumber(3, 40),
470 ...protoText(11, responseId),
471 ]),
472 ...protoText(19, "Gemini 3 Pro"),
473 ...protoBytes(9, protoBytes(4, protoNumber(1, 1780000000))),
474 ]),
475 );
476
477 it("reuses an unchanged Antigravity database instead of decoding it again", async () => {
478 const path = NodePath.join(dir, "session-1.db");
479 const db = new NodeSqlite.DatabaseSync(path);
480 try {
481 db.exec("CREATE TABLE gen_metadata (idx INTEGER, data BLOB)");
482 db.prepare("INSERT INTO gen_metadata VALUES (?, ?)").run(0, antigravityGeneration("r-1"));
483 const cache = makeAntigravityUsageCache();
484 assert.deepStrictEqual((await readAntigravityUsage(dir, 0, cache)).errors, []);
485
486 // An exclusive lock makes a fresh read fail without touching the file, so
487 // only a cache hit can still return the earlier records.
488 db.exec("BEGIN EXCLUSIVE");
489 assert.strictEqual((await readAntigravityUsage(dir, 0)).errors.length, 1);
490 const cached = await readAntigravityUsage(dir, 0, cache);
491 assert.deepStrictEqual(cached.errors, []);
492 assert.deepStrictEqual(
493 cached.files.flatMap((file) => file.records).map((record) => record.dedupeKey),
494 ["antigravity:11:r-1"],
495 );
496 db.exec("ROLLBACK");
497 } finally {
498 db.close();
499 }
500 });
501
502 it("rereads an Antigravity database rewritten with its size and mtime restored", async () => {
503 const path = NodePath.join(dir, "session-1.db");
504 const db = new NodeSqlite.DatabaseSync(path);
505 try {
506 db.exec("CREATE TABLE gen_metadata (idx INTEGER, data BLOB)");
507 db.prepare("INSERT INTO gen_metadata VALUES (?, ?)").run(0, antigravityGeneration("r-1"));
508 } finally {
509 db.close();
510 }
511 await NodeFSP.utimes(path, 1780000000, 1780000000);
512 const cache = makeAntigravityUsageCache();
513 assert.deepStrictEqual((await readAntigravityUsage(dir, 0, cache)).errors, []);
514
515 // ctime has the kernel's timestamp granularity, which can be a few
516 // milliseconds, so repeat the forged rewrite until it lands on a later tick
517 // than the cached read, as any real rewrite does.
518 const cached = await NodeFSP.stat(path);
519 do {
520 await NodeFSP.writeFile(path, Buffer.alloc(cached.size));
521 await NodeFSP.utimes(path, 1780000000, 1780000000);
522 } while ((await NodeFSP.stat(path)).ctimeMs === cached.ctimeMs);
523 const restored = await NodeFSP.stat(path);
524 assert.strictEqual(restored.size, cached.size);
525 assert.strictEqual(restored.mtimeMs, cached.mtimeMs);
526 const next = await readAntigravityUsage(dir, 0, cache);
527 assert.deepStrictEqual(next.errors, [path]);
528 assert.deepStrictEqual(
529 next.files.flatMap((file) => file.records),
530 [],
531 );
532 });
533
534 it("rereads an Antigravity database when only its WAL changed", async () => {
535 const path = NodePath.join(dir, "session-1.db");
536 const db = new NodeSqlite.DatabaseSync(path);
537 try {
538 db.exec(
539 "PRAGMA journal_mode = WAL; PRAGMA wal_autocheckpoint = 0; CREATE TABLE gen_metadata (idx INTEGER, data BLOB)",
540 );
541 const insert = db.prepare("INSERT INTO gen_metadata VALUES (?, ?)");
542 insert.run(0, antigravityGeneration("r-1"));
543 const cache = makeAntigravityUsageCache();
544 const first = await readAntigravityUsage(dir, 0, cache);
545 assert.strictEqual(first.files.flatMap((file) => file.records).length, 1);
546
547 const before = await NodeFSP.stat(path);
548 insert.run(1, antigravityGeneration("r-2"));
549 const after = await NodeFSP.stat(path);
550 assert.strictEqual(after.size, before.size);
551 assert.strictEqual(after.mtimeMs, before.mtimeMs);
552
553 const next = await readAntigravityUsage(dir, 0, cache);
554 assert.deepStrictEqual(
555 next.files.flatMap((file) => file.records).map((record) => record.dedupeKey),
556 ["antigravity:11:r-1", "antigravity:11:r-2"],
557 );
558 } finally {
559 db.close();
560 }
561 });
562
563 it("reads Antigravity step-only stores and reports malformed databases", async () => {
564 const db = new NodeSqlite.DatabaseSync(NodePath.join(dir, "steps.db"));
565 try {
566 db.exec("CREATE TABLE steps (idx INTEGER, metadata BLOB)");
567 const usage = [...protoNumber(1, 246), ...protoNumber(2, 10), ...protoNumber(3, 5)];
568 db.prepare("INSERT INTO steps VALUES (?, ?)").run(
569 0,
570 new Uint8Array([...protoBytes(9, usage), ...protoBytes(8, protoNumber(1, 1780000000))]),
571 );
572 } finally {
573 db.close();
574 }
575 await NodeFSP.writeFile(NodePath.join(dir, "broken.db"), "not a sqlite database");
576 const result = await readAntigravityUsage(dir, 0);
577 assert.strictEqual(result.errors.length, 1);
578 assert.strictEqual(result.files.flatMap((file) => file.records)[0]?.model, "gemini-2.5-pro");
579 assert.strictEqual(result.files.flatMap((file) => file.records)[0]?.totals.outputTokens, 5);
580 });
581
582 it("ignores large values in unused Antigravity protobuf fields", async () => {
583 const db = new NodeSqlite.DatabaseSync(NodePath.join(dir, "large-varint.db"));
584 try {
585 db.exec("CREATE TABLE steps (idx INTEGER, metadata BLOB)");
586 const unusedField = [...protoNumber(99, 0).slice(0, -1), ...Array(9).fill(0xff), 0x01];
587 const usage = [...protoNumber(1, 246), ...protoNumber(2, 10), ...unusedField];
588 db.prepare("INSERT INTO steps VALUES (?, ?)").run(0, new Uint8Array(protoBytes(9, usage)));
589 } finally {
590 db.close();
591 }
592 const result = await readAntigravityUsage(dir, 0);
593 assert.deepStrictEqual(result.errors, []);
594 assert.strictEqual(
595 result.files.flatMap((file) => file.records)[0]?.totals.uncachedInputTokens,
596 10,
597 );
598 });
599});
packages/provider-core/src/server/usage.ts+3-1
...@@ -13,6 +13,7 @@ import type {...@@ -13,6 +13,7 @@ import type {
13 ProviderInstanceId,13 ProviderInstanceId,
14 ServerSettings,14 ServerSettings,
15 UsageProviderKind,15 UsageProviderKind,
16 UsageReadError,
16 UsageSource,17 UsageSource,
17 UsageTokenTotals,18 UsageTokenTotals,
18} from "@t3tools/contracts";19} from "@t3tools/contracts";
...@@ -177,7 +178,8 @@ export type ProviderUsageReader<Config, R> =...@@ -177,7 +178,8 @@ export type ProviderUsageReader<Config, R> =
177 | {178 | {
178 readonly kind: "scan";179 readonly kind: "scan";
179 readonly provider: UsageProviderKind;180 readonly provider: UsageProviderKind;
181 /** Fails only when the whole usage read cannot go on; an unreadable source is a status. */
180 readonly scan: (182 readonly scan: (
181 input: ProviderUsageScanInput<Config>,183 input: ProviderUsageScanInput<Config>,
182 ) => Effect.Effect<ReadonlyArray<ProviderUsageScan>, never, R>;184 ) => Effect.Effect<ReadonlyArray<ProviderUsageScan>, UsageReadError, R>;
183 };185 };