From 9ec472c7d9345a0c9528183d918f5fbc7f73be75 Mon Sep 17 00:00:00 2001 From: Tristan Knight Date: Sun, 11 Oct 2026 02:52:10 +0100 Subject: [PATCH] perf(usage): OpenCode legacy walk stats entries before checking for symlinks (#17638) Co-authored-by: Theo Browne Co-authored-by: Claude Opus 5.5 --- .../src/server/usage.test.ts | 90 ++++++++++++++++ .../provider-opencode/src/server/usage.ts | 101 ++++++++++++------ 2 files changed, 161 insertions(+), 30 deletions(-) diff --git a/packages/provider-opencode/src/server/usage.test.ts b/packages/provider-opencode/src/server/usage.test.ts index b11abcd49720583fa9eda0d00b6b74e81e85fa2f..4dc7ec99486c6d5bb41bc4259fa5f7cd7377d725 100644 --- a/packages/provider-opencode/src/server/usage.test.ts +++ b/packages/provider-opencode/src/server/usage.test.ts @@ -182,5 +182,95 @@ describe("readOpenCodeUsage", () => { ); }).pipe(Effect.provide(NodeServices.layer)), ); + + it.effect("never follows symlinks in the legacy OpenCode store", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "usage-reader-test-" }); + const outside = yield* fileSystem.makeTempDirectoryScoped({ prefix: "usage-reader-test-" }); + const sinceMs = 1780000000000; + const store = path.join(dir, "storage", "message"); + const session = path.join(store, "session-1"); + yield* fileSystem.makeDirectory(session, { recursive: true }); + const write = Effect.fn(function* (directory: string, id: string, mtimeMs: number) { + const file = path.join(directory, `${id}.json`); + yield* fileSystem.writeFileString( + file, + JSON.stringify({ + id, + sessionID: "session-1", + role: "assistant", + modelID: "claude-sonnet-4-5", + time: { created: sinceMs + 1000 }, + tokens: { input: 100, output: 20 }, + }), + ); + yield* fileSystem.utimes(file, mtimeMs / 1000, mtimeMs / 1000); + return file; + }); + const target = yield* write(outside, "msg-linked", sinceMs + 60_000); + const stale = yield* write(outside, "msg-stale", sinceMs - 60_000); + yield* fileSystem.symlink(target, path.join(session, "msg-linked.json")); + yield* fileSystem.symlink(stale, path.join(session, "msg-stale.json")); + yield* fileSystem.symlink(outside, path.join(store, "session-linked")); + yield* fileSystem.symlink(store, path.join(session, "cycle")); + yield* fileSystem.symlink(path.join(outside, "gone.json"), path.join(session, "gone.json")); + yield* fileSystem.symlink(path.join(session, "loop.json"), path.join(session, "loop.json")); + + // Links alone are not a store, even when one points at a stale message. + const linksOnly = yield* readOpenCodeUsage(dir, sinceMs); + assert.deepStrictEqual(linksOnly, { files: [], missing: true, error: false }); + + const real = yield* write(session, "msg-real", sinceMs + 60_000); + const result = yield* readOpenCodeUsage(dir, sinceMs); + assert.isFalse(result.missing); + assert.isFalse(result.error); + assert.deepStrictEqual( + result.files.map((file) => file.path), + [real], + ); + assert.deepStrictEqual( + result.files.flatMap((file) => file.records).map((record) => record.dedupeKey), + ["opencode:msg-real"], + ); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("reads a large legacy OpenCode session in directory order", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "usage-reader-test-" }); + const session = path.join(dir, "storage", "message", "session-1"); + yield* fileSystem.makeDirectory(session, { recursive: true }); + // More entries than the walk stats at once. + yield* Effect.forEach( + Array.from({ length: 600 }, (_, index) => `msg-${index}`), + (id) => + fileSystem.writeFileString( + path.join(session, `${id}.json`), + JSON.stringify({ + id, + sessionID: "session-1", + role: "assistant", + modelID: "claude-sonnet-4-5", + time: { created: 1780000000000 }, + tokens: { input: 100, output: 20 }, + }), + ), + { concurrency: 16, discard: true }, + ); + const names = yield* fileSystem.readDirectory(session); + + const result = yield* readOpenCodeUsage(dir, 0); + assert.isFalse(result.error); + assert.deepStrictEqual( + result.files.map((file) => file.path), + names.map((name) => path.join(session, name)), + ); + assert.strictEqual(result.files.flatMap((file) => file.records).length, 600); + }).pipe(Effect.provide(NodeServices.layer)), + ); }); import * as NodeSqlite from "node:sqlite"; diff --git a/packages/provider-opencode/src/server/usage.ts b/packages/provider-opencode/src/server/usage.ts index 07700109c6860558e4479488b256f706bb456350..63b5568f05285f88bbab622705c0987fc9dda09c 100644 --- a/packages/provider-opencode/src/server/usage.ts +++ b/packages/provider-opencode/src/server/usage.ts @@ -14,7 +14,6 @@ import { import * as HostProcess from "@t3tools/shared/HostProcess"; import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; import * as Effect from "effect/Effect"; -import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; @@ -94,14 +93,29 @@ export interface OpenCodeUsageReadResult { const isNotFound = (cause: PlatformError.PlatformError) => cause.reason._tag === "NotFound"; const MESSAGE_PAGE_SIZE = 64; +/** Whether the path is itself a symlink, without following it. */ +const isSymbolicLink = Effect.fnUntraced(function* (path: string) { + const fileSystem = yield* FileSystem.FileSystem; + return yield* Effect.isSuccess(fileSystem.readLink(path)); +}); + /** Stat of a regular file or directory; a symlink reports as itself and is never followed. */ const entryInfo = Effect.fn("entryInfo")(function* (path: string) { const fileSystem = yield* FileSystem.FileSystem; - const link = yield* Effect.exit(fileSystem.readLink(path)); - if (Exit.isSuccess(link)) return { type: "SymbolicLink" as const, mtime: Option.none() }; + if (yield* isSymbolicLink(path)) + return { type: "SymbolicLink" as const, mtime: Option.none() }; return yield* fileSystem.stat(path); }); +/** + * Stats run in the libuv thread pool, so a few at a time cut the walk's wall + * time without lengthening any event-loop turn. + */ +const LEGACY_STAT_CONCURRENCY = 4; + +/** Entries stat'd before any is processed, so one huge directory is not held in memory at once. */ +const LEGACY_STAT_BATCH = 256; + /** Reads current SQLite and pre-migration JSON stores without modifying either. */ export const readOpenCodeUsage = Effect.fn("readOpenCodeUsage")(function* ( root: string, @@ -234,34 +248,61 @@ export const readOpenCodeUsage = Effect.fn("readOpenCodeUsage")(function* ( while (directories.length > 0) { const directory = directories.pop()!; yield* Effect.gen(function* () { - for (const name of yield* fileSystem.readDirectory(directory)) { - const entry = path.join(directory, name); - const info = yield* entryInfo(entry).pipe( - Effect.catchTags({ - PlatformError: (cause) => - isNotFound(cause) ? Effect.succeed(null) : Effect.fail(cause), - }), + const names = yield* fileSystem.readDirectory(directory); + for (let start = 0; start < names.length; start += LEGACY_STAT_BATCH) { + const batch = names.slice(start, start + LEGACY_STAT_BATCH); + const entries = batch.map((name) => path.join(directory, name)); + // `stat` follows symlinks and `readLink` fails, expensively, for + // everything else. Stat a batch, then ask whether an entry is a symlink + // only where the answer changes the result. + const infos = yield* Effect.forEach( + entries, + (entry) => + fileSystem.stat(entry).pipe( + Effect.catchTags({ + // A symlink whose target cannot be stat'd is skipped, not an error. + PlatformError: (cause) => + isNotFound(cause) + ? Effect.succeed(null) + : isSymbolicLink(entry).pipe( + Effect.flatMap((link) => + link ? Effect.succeed(null) : Effect.fail(cause), + ), + ), + }), + Effect.exit, + ), + { concurrency: LEGACY_STAT_CONCURRENCY }, ); - if (info?.type === "Directory") { - directories.push(entry); - } else if (info?.type === "File" && name.endsWith(".json")) { - found = true; - const id = name.slice(0, -5); - if (seen.has(`opencode:${id}`)) continue; - // A message cannot be created after its file was last written, so a - // file untouched since the window opened holds nothing in range. - if (Option.exists(info.mtime, (mtime) => mtime.getTime() < sinceMs)) continue; - const file = { path: entry, records: [] as UsageRecord[] }; - files.push(file); - yield* fileSystem.readFileString(entry).pipe( - Effect.map((source) => append(file.records, parseOpenCodeMessage(source, { id }))), - Effect.catchTags({ - PlatformError: (cause) => { - if (!isNotFound(cause)) error = true; - return Effect.void; - }, - }), - ); + for (const [index, name] of batch.entries()) { + const entry = entries[index]!; + const info = yield* infos[index]!; + if (info?.type === "Directory") { + if (!(yield* isSymbolicLink(entry))) directories.push(entry); + } else if (info?.type === "File" && name.endsWith(".json")) { + const id = name.slice(0, -5); + // A message cannot be created after its file was last written, so a + // file untouched since the window opened holds nothing in range. + const skip = + seen.has(`opencode:${id}`) || + Option.exists(info.mtime, (mtime) => mtime.getTime() < sinceMs); + // A skipped file only matters as proof that the store exists. + if (found && skip) continue; + if (yield* isSymbolicLink(entry)) continue; + found = true; + if (skip) continue; + const file = { path: entry, records: [] as UsageRecord[] }; + files.push(file); + yield* fileSystem.readFileString(entry).pipe( + Effect.map((source) => append(file.records, parseOpenCodeMessage(source, { id }))), + Effect.catchTags({ + PlatformError: (cause) => { + if (!isNotFound(cause)) error = true; + return Effect.void; + }, + }), + ); + } } } }).pipe( -- 2.54.0