authorgravatar for admin@snappeh.comTristan Knight <admin@snappeh.com> 2026-10-11 02:52:10+01:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-10-10 18:52:10-07:00
log9ec472c7d9345a0c9528183d918f5fbc7f73be75
tree2efa9d1789b3569a5bdd050822580760473b6ad1
parentfebe1b2420c850aa17c12539a5a479c0438506f8
signature Signed by PGP key B5690EEEBB952194

perf(usage): OpenCode legacy walk stats entries before checking for symlinks (#17638)

Co-authored-by: Theo Browne <me@t3.gg> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

2 files changed, 161 insertions(+), 30 deletions(-)

packages/provider-opencode/src/server/usage.test.ts+90
...@@ -182,5 +182,95 @@ describe("readOpenCodeUsage", () => {...@@ -182,5 +182,95 @@ describe("readOpenCodeUsage", () => {
182 );182 );
183 }).pipe(Effect.provide(NodeServices.layer)),183 }).pipe(Effect.provide(NodeServices.layer)),
184 );184 );
185
186 it.effect("never follows symlinks in the legacy OpenCode store", () =>
187 Effect.gen(function* () {
188 const fileSystem = yield* FileSystem.FileSystem;
189 const path = yield* Path.Path;
190 const dir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "usage-reader-test-" });
191 const outside = yield* fileSystem.makeTempDirectoryScoped({ prefix: "usage-reader-test-" });
192 const sinceMs = 1780000000000;
193 const store = path.join(dir, "storage", "message");
194 const session = path.join(store, "session-1");
195 yield* fileSystem.makeDirectory(session, { recursive: true });
196 const write = Effect.fn(function* (directory: string, id: string, mtimeMs: number) {
197 const file = path.join(directory, `${id}.json`);
198 yield* fileSystem.writeFileString(
199 file,
200 JSON.stringify({
201 id,
202 sessionID: "session-1",
203 role: "assistant",
204 modelID: "claude-sonnet-4-5",
205 time: { created: sinceMs + 1000 },
206 tokens: { input: 100, output: 20 },
207 }),
208 );
209 yield* fileSystem.utimes(file, mtimeMs / 1000, mtimeMs / 1000);
210 return file;
211 });
212 const target = yield* write(outside, "msg-linked", sinceMs + 60_000);
213 const stale = yield* write(outside, "msg-stale", sinceMs - 60_000);
214 yield* fileSystem.symlink(target, path.join(session, "msg-linked.json"));
215 yield* fileSystem.symlink(stale, path.join(session, "msg-stale.json"));
216 yield* fileSystem.symlink(outside, path.join(store, "session-linked"));
217 yield* fileSystem.symlink(store, path.join(session, "cycle"));
218 yield* fileSystem.symlink(path.join(outside, "gone.json"), path.join(session, "gone.json"));
219 yield* fileSystem.symlink(path.join(session, "loop.json"), path.join(session, "loop.json"));
220
221 // Links alone are not a store, even when one points at a stale message.
222 const linksOnly = yield* readOpenCodeUsage(dir, sinceMs);
223 assert.deepStrictEqual(linksOnly, { files: [], missing: true, error: false });
224
225 const real = yield* write(session, "msg-real", sinceMs + 60_000);
226 const result = yield* readOpenCodeUsage(dir, sinceMs);
227 assert.isFalse(result.missing);
228 assert.isFalse(result.error);
229 assert.deepStrictEqual(
230 result.files.map((file) => file.path),
231 [real],
232 );
233 assert.deepStrictEqual(
234 result.files.flatMap((file) => file.records).map((record) => record.dedupeKey),
235 ["opencode:msg-real"],
236 );
237 }).pipe(Effect.provide(NodeServices.layer)),
238 );
239
240 it.effect("reads a large legacy OpenCode session in directory order", () =>
241 Effect.gen(function* () {
242 const fileSystem = yield* FileSystem.FileSystem;
243 const path = yield* Path.Path;
244 const dir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "usage-reader-test-" });
245 const session = path.join(dir, "storage", "message", "session-1");
246 yield* fileSystem.makeDirectory(session, { recursive: true });
247 // More entries than the walk stats at once.
248 yield* Effect.forEach(
249 Array.from({ length: 600 }, (_, index) => `msg-${index}`),
250 (id) =>
251 fileSystem.writeFileString(
252 path.join(session, `${id}.json`),
253 JSON.stringify({
254 id,
255 sessionID: "session-1",
256 role: "assistant",
257 modelID: "claude-sonnet-4-5",
258 time: { created: 1780000000000 },
259 tokens: { input: 100, output: 20 },
260 }),
261 ),
262 { concurrency: 16, discard: true },
263 );
264 const names = yield* fileSystem.readDirectory(session);
265
266 const result = yield* readOpenCodeUsage(dir, 0);
267 assert.isFalse(result.error);
268 assert.deepStrictEqual(
269 result.files.map((file) => file.path),
270 names.map((name) => path.join(session, name)),
271 );
272 assert.strictEqual(result.files.flatMap((file) => file.records).length, 600);
273 }).pipe(Effect.provide(NodeServices.layer)),
274 );
185});275});
186import * as NodeSqlite from "node:sqlite";276import * as NodeSqlite from "node:sqlite";
packages/provider-opencode/src/server/usage.ts+71-30
...@@ -14,7 +14,6 @@ import {...@@ -14,7 +14,6 @@ import {
14import * as HostProcess from "@t3tools/shared/HostProcess";14import * as HostProcess from "@t3tools/shared/HostProcess";
15import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";15import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
16import * as Effect from "effect/Effect";16import * as Effect from "effect/Effect";
17import * as Exit from "effect/Exit";
18import * as FileSystem from "effect/FileSystem";17import * as FileSystem from "effect/FileSystem";
19import * as Option from "effect/Option";18import * as Option from "effect/Option";
20import * as Path from "effect/Path";19import * as Path from "effect/Path";
...@@ -94,14 +93,29 @@ export interface OpenCodeUsageReadResult {...@@ -94,14 +93,29 @@ export interface OpenCodeUsageReadResult {
94const isNotFound = (cause: PlatformError.PlatformError) => cause.reason._tag === "NotFound";93const isNotFound = (cause: PlatformError.PlatformError) => cause.reason._tag === "NotFound";
95const MESSAGE_PAGE_SIZE = 64;94const MESSAGE_PAGE_SIZE = 64;
9695
96/** Whether the path is itself a symlink, without following it. */
97const isSymbolicLink = Effect.fnUntraced(function* (path: string) {
98 const fileSystem = yield* FileSystem.FileSystem;
99 return yield* Effect.isSuccess(fileSystem.readLink(path));
100});
101
97/** Stat of a regular file or directory; a symlink reports as itself and is never followed. */102/** Stat of a regular file or directory; a symlink reports as itself and is never followed. */
98const entryInfo = Effect.fn("entryInfo")(function* (path: string) {103const entryInfo = Effect.fn("entryInfo")(function* (path: string) {
99 const fileSystem = yield* FileSystem.FileSystem;104 const fileSystem = yield* FileSystem.FileSystem;
100 const link = yield* Effect.exit(fileSystem.readLink(path));105 if (yield* isSymbolicLink(path))
101 if (Exit.isSuccess(link)) return { type: "SymbolicLink" as const, mtime: Option.none<Date>() };106 return { type: "SymbolicLink" as const, mtime: Option.none<Date>() };
102 return yield* fileSystem.stat(path);107 return yield* fileSystem.stat(path);
103});108});
104109
110/**
111 * Stats run in the libuv thread pool, so a few at a time cut the walk's wall
112 * time without lengthening any event-loop turn.
113 */
114const LEGACY_STAT_CONCURRENCY = 4;
115
116/** Entries stat'd before any is processed, so one huge directory is not held in memory at once. */
117const LEGACY_STAT_BATCH = 256;
118
105/** Reads current SQLite and pre-migration JSON stores without modifying either. */119/** Reads current SQLite and pre-migration JSON stores without modifying either. */
106export const readOpenCodeUsage = Effect.fn("readOpenCodeUsage")(function* (120export const readOpenCodeUsage = Effect.fn("readOpenCodeUsage")(function* (
107 root: string,121 root: string,
...@@ -234,34 +248,61 @@ export const readOpenCodeUsage = Effect.fn("readOpenCodeUsage")(function* (...@@ -234,34 +248,61 @@ export const readOpenCodeUsage = Effect.fn("readOpenCodeUsage")(function* (
234 while (directories.length > 0) {248 while (directories.length > 0) {
235 const directory = directories.pop()!;249 const directory = directories.pop()!;
236 yield* Effect.gen(function* () {250 yield* Effect.gen(function* () {
237 for (const name of yield* fileSystem.readDirectory(directory)) {251 const names = yield* fileSystem.readDirectory(directory);
238 const entry = path.join(directory, name);252 for (let start = 0; start < names.length; start += LEGACY_STAT_BATCH) {
239 const info = yield* entryInfo(entry).pipe(253 const batch = names.slice(start, start + LEGACY_STAT_BATCH);
240 Effect.catchTags({254 const entries = batch.map((name) => path.join(directory, name));
241 PlatformError: (cause) =>255 // `stat` follows symlinks and `readLink` fails, expensively, for
242 isNotFound(cause) ? Effect.succeed(null) : Effect.fail(cause),256 // everything else. Stat a batch, then ask whether an entry is a symlink
243 }),257 // only where the answer changes the result.
258 const infos = yield* Effect.forEach(
259 entries,
260 (entry) =>
261 fileSystem.stat(entry).pipe(
262 Effect.catchTags({
263 // A symlink whose target cannot be stat'd is skipped, not an error.
264 PlatformError: (cause) =>
265 isNotFound(cause)
266 ? Effect.succeed(null)
267 : isSymbolicLink(entry).pipe(
268 Effect.flatMap((link) =>
269 link ? Effect.succeed(null) : Effect.fail(cause),
270 ),
271 ),
272 }),
273 Effect.exit,
274 ),
275 { concurrency: LEGACY_STAT_CONCURRENCY },
244 );276 );
245 if (info?.type === "Directory") {277 for (const [index, name] of batch.entries()) {
246 directories.push(entry);278 const entry = entries[index]!;
247 } else if (info?.type === "File" && name.endsWith(".json")) {279 const info = yield* infos[index]!;
248 found = true;280 if (info?.type === "Directory") {
249 const id = name.slice(0, -5);281 if (!(yield* isSymbolicLink(entry))) directories.push(entry);
250 if (seen.has(`opencode:${id}`)) continue;282 } else if (info?.type === "File" && name.endsWith(".json")) {
251 // A message cannot be created after its file was last written, so a283 const id = name.slice(0, -5);
252 // file untouched since the window opened holds nothing in range.284 // A message cannot be created after its file was last written, so a
253 if (Option.exists(info.mtime, (mtime) => mtime.getTime() < sinceMs)) continue;285 // file untouched since the window opened holds nothing in range.
254 const file = { path: entry, records: [] as UsageRecord[] };286 const skip =
255 files.push(file);287 seen.has(`opencode:${id}`) ||
256 yield* fileSystem.readFileString(entry).pipe(288 Option.exists(info.mtime, (mtime) => mtime.getTime() < sinceMs);
257 Effect.map((source) => append(file.records, parseOpenCodeMessage(source, { id }))),289 // A skipped file only matters as proof that the store exists.
258 Effect.catchTags({290 if (found && skip) continue;
259 PlatformError: (cause) => {291 if (yield* isSymbolicLink(entry)) continue;
260 if (!isNotFound(cause)) error = true;292 found = true;
261 return Effect.void;293 if (skip) continue;
262 },294 const file = { path: entry, records: [] as UsageRecord[] };
263 }),295 files.push(file);
264 );296 yield* fileSystem.readFileString(entry).pipe(
297 Effect.map((source) => append(file.records, parseOpenCodeMessage(source, { id }))),
298 Effect.catchTags({
299 PlatformError: (cause) => {
300 if (!isNotFound(cause)) error = true;
301 return Effect.void;
302 },
303 }),
304 );
305 }
265 }306 }
266 }307 }
267 }).pipe(308 }).pipe(