From 5d61b8b14f7f80e6183a7cc9b51a1c48802ead34 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 17 Feb 2026 16:01:13 -0800 Subject: [PATCH] Add tests for chunking git check-ignore in workspace entry search - Introduce a new test file to validate chunking behavior of git check-ignore to prevent oversized input strings. - Adjust workspaceEntries.ts to implement chunking logic for git check-ignore, improving performance with large ignored path lists. - Set a maximum byte limit for stdin in git check-ignore to enhance stability and prevent errors during processing. --- .../src/workspaceEntries.chunking.test.ts | 80 ++++++++++++++++ apps/server/src/workspaceEntries.ts | 96 +++++++++++++------ 2 files changed, 145 insertions(+), 31 deletions(-) create mode 100644 apps/server/src/workspaceEntries.chunking.test.ts diff --git a/apps/server/src/workspaceEntries.chunking.test.ts b/apps/server/src/workspaceEntries.chunking.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..6644d608015d2d3cac619a7936eaf7b30ddf0054 --- /dev/null +++ b/apps/server/src/workspaceEntries.chunking.test.ts @@ -0,0 +1,80 @@ +import { assert, beforeEach, describe, it, vi } from "vitest"; +import type { ProcessRunOptions, ProcessRunResult } from "./processRunner"; + +const { runProcessMock } = vi.hoisted(() => ({ + runProcessMock: vi.fn< + ( + command: string, + args: readonly string[], + options?: ProcessRunOptions, + ) => Promise + >(), +})); + +vi.mock("./processRunner", () => ({ + runProcess: runProcessMock, +})); + +function processResult( + overrides: Partial & Pick, +): ProcessRunResult { + return { + stdout: overrides.stdout, + code: overrides.code, + stderr: overrides.stderr ?? "", + signal: overrides.signal ?? null, + timedOut: overrides.timedOut ?? false, + stdoutTruncated: overrides.stdoutTruncated ?? false, + stderrTruncated: overrides.stderrTruncated ?? false, + }; +} + +describe("searchWorkspaceEntries git-ignore chunking", () => { + beforeEach(() => { + runProcessMock.mockReset(); + vi.resetModules(); + }); + + it("chunks git check-ignore stdin to avoid building giant strings", async () => { + const ignoredPaths = Array.from( + { length: 5000 }, + (_, index) => `ignored/${index.toString().padStart(5, "0")}/${"x".repeat(80)}.ts`, + ); + const keptPaths = ["src/keep.ts", "docs/readme.md"]; + const listedPaths = [...ignoredPaths, ...keptPaths]; + let checkIgnoreCalls = 0; + + runProcessMock.mockImplementation(async (_command, args, options) => { + if (args[0] === "rev-parse") { + return processResult({ code: 0, stdout: "true\n" }); + } + + if (args[0] === "ls-files") { + return processResult({ code: 0, stdout: `${listedPaths.join("\0")}\0` }); + } + + if (args[0] === "check-ignore") { + checkIgnoreCalls += 1; + const chunkPaths = (options?.stdin ?? "").split("\0").filter((value) => value.length > 0); + const chunkIgnored = chunkPaths.filter((value) => value.startsWith("ignored/")); + return processResult({ + code: chunkIgnored.length > 0 ? 0 : 1, + stdout: chunkIgnored.length > 0 ? `${chunkIgnored.join("\0")}\0` : "", + }); + } + + throw new Error(`Unexpected command: git ${args.join(" ")}`); + }); + + const { searchWorkspaceEntries } = await import("./workspaceEntries"); + const result = await searchWorkspaceEntries({ + cwd: "/virtual/workspace", + query: "", + limit: 100, + }); + + assert.isAbove(checkIgnoreCalls, 1); + assert.isFalse(result.entries.some((entry) => entry.path.startsWith("ignored/"))); + assert.isTrue(result.entries.some((entry) => entry.path === "src/keep.ts")); + }); +}); diff --git a/apps/server/src/workspaceEntries.ts b/apps/server/src/workspaceEntries.ts index 5c0ab3cfaee350e99236e3d100334f7f23bab079..4815353a95c1fe6c710dbf1614909448cd95011b 100644 --- a/apps/server/src/workspaceEntries.ts +++ b/apps/server/src/workspaceEntries.ts @@ -14,17 +14,8 @@ const WORKSPACE_CACHE_TTL_MS = 15_000; const WORKSPACE_CACHE_MAX_KEYS = 4; const WORKSPACE_INDEX_MAX_ENTRIES = 25_000; const WORKSPACE_SCAN_READDIR_CONCURRENCY = 32; -const IGNORED_DIRECTORY_NAMES = new Set([ - ".git", - ".convex", - "node_modules", - ".next", - ".turbo", - "dist", - "build", - "out", - ".cache", -]); +const GIT_CHECK_IGNORE_MAX_STDIN_BYTES = 256 * 1024; +const IGNORED_DIRECTORY_NAMES = new Set([".git", "node_modules"]); interface WorkspaceIndex { scannedAt: number; @@ -56,7 +47,10 @@ function basenameOf(input: string): string { } function normalizeQuery(input: string): string { - return input.trim().replace(/^[@./]+/, "").toLowerCase(); + return input + .trim() + .replace(/^[@./]+/, "") + .toLowerCase(); } function scoreEntry(entry: ProjectEntry, query: string): number { @@ -145,26 +139,67 @@ async function filterGitIgnoredPaths(cwd: string, relativePaths: string[]): Prom return relativePaths; } - const checkIgnore = await runProcess("git", ["check-ignore", "--no-index", "-z", "--stdin"], { - cwd, - allowNonZeroExit: true, - timeoutMs: 20_000, - maxBufferBytes: 16 * 1024 * 1024, - outputMode: "truncate", - stdin: `${relativePaths.join("\0")}\0`, - }).catch(() => null); - if (!checkIgnore) { - return relativePaths; + const ignoredPaths = new Set(); + let chunk: string[] = []; + let chunkBytes = 0; + + const flushChunk = async (): Promise => { + if (chunk.length === 0) { + return true; + } + + const checkIgnore = await runProcess("git", ["check-ignore", "--no-index", "-z", "--stdin"], { + cwd, + allowNonZeroExit: true, + timeoutMs: 20_000, + maxBufferBytes: 16 * 1024 * 1024, + outputMode: "truncate", + stdin: `${chunk.join("\0")}\0`, + }).catch(() => null); + chunk = []; + chunkBytes = 0; + + if (!checkIgnore) { + return false; + } + + // git-check-ignore exits with 1 when no paths match. + if (checkIgnore.code !== 0 && checkIgnore.code !== 1) { + return false; + } + + const matchedIgnoredPaths = splitNullSeparatedPaths( + checkIgnore.stdout, + Boolean(checkIgnore.stdoutTruncated), + ); + for (const ignoredPath of matchedIgnoredPaths) { + ignoredPaths.add(ignoredPath); + } + return true; + }; + + for (const relativePath of relativePaths) { + const relativePathBytes = Buffer.byteLength(relativePath) + 1; + if ( + chunk.length > 0 && + chunkBytes + relativePathBytes > GIT_CHECK_IGNORE_MAX_STDIN_BYTES && + !(await flushChunk()) + ) { + return relativePaths; + } + + chunk.push(relativePath); + chunkBytes += relativePathBytes; + + if (chunkBytes >= GIT_CHECK_IGNORE_MAX_STDIN_BYTES && !(await flushChunk())) { + return relativePaths; + } } - // git-check-ignore exits with 1 when no paths match. - if (checkIgnore.code !== 0 && checkIgnore.code !== 1) { + if (!(await flushChunk())) { return relativePaths; } - const ignoredPaths = new Set( - splitNullSeparatedPaths(checkIgnore.stdout, Boolean(checkIgnore.stdoutTruncated)), - ); if (ignoredPaths.size === 0) { return relativePaths; } @@ -210,14 +245,14 @@ async function buildWorkspaceIndexFromGit(cwd: string): Promise left.localeCompare(right)) + .toSorted((left, right) => left.localeCompare(right)) .map((directoryPath) => ({ path: directoryPath, kind: "directory", parentPath: parentPathOf(directoryPath), })); const fileEntries: ProjectEntry[] = [...new Set(filePaths)] - .sort((left, right) => left.localeCompare(right)) + .toSorted((left, right) => left.localeCompare(right)) .map((filePath) => ({ path: filePath, kind: "file", @@ -228,8 +263,7 @@ async function buildWorkspaceIndexFromGit(cwd: string): Promise WORKSPACE_INDEX_MAX_ENTRIES, + truncated: Boolean(listedFiles.stdoutTruncated) || entries.length > WORKSPACE_INDEX_MAX_ENTRIES, }; } -- 2.54.0