authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-02-17 16:01:13-08:00
committergravatar for jmarminge@gmail.comJulius Marminge <jmarminge@gmail.com> 2026-02-17 19:47:09-08:00
log5d61b8b14f7f80e6183a7cc9b51a1c48802ead34
tree80c50918c36834e909bb358e895ce61398814156
parent2743e45dbed2895e47aed18bbabb36e8dae54247

Add tests for chunking git check-ignore in workspace entry search

- Introduce a new test file to validate chunking behavior of git check-ignore to prevent oversized input strings. - Adjust workspaceEntries.ts to implement chunking logic for git check-ignore, improving performance with large ignored path lists. - Set a maximum byte limit for stdin in git check-ignore to enhance stability and prevent errors during processing.

2 files changed, 145 insertions(+), 31 deletions(-)

apps/server/src/workspaceEntries.chunking.test.ts created+80
......@@ -0,0 +1,80 @@
1import { assert, beforeEach, describe, it, vi } from "vitest";
2import type { ProcessRunOptions, ProcessRunResult } from "./processRunner";
3
4const { runProcessMock } = vi.hoisted(() => ({
5 runProcessMock: vi.fn<
6 (
7 command: string,
8 args: readonly string[],
9 options?: ProcessRunOptions,
10 ) => Promise<ProcessRunResult>
11 >(),
12}));
13
14vi.mock("./processRunner", () => ({
15 runProcess: runProcessMock,
16}));
17
18function processResult(
19 overrides: Partial<ProcessRunResult> & Pick<ProcessRunResult, "stdout" | "code">,
20): ProcessRunResult {
21 return {
22 stdout: overrides.stdout,
23 code: overrides.code,
24 stderr: overrides.stderr ?? "",
25 signal: overrides.signal ?? null,
26 timedOut: overrides.timedOut ?? false,
27 stdoutTruncated: overrides.stdoutTruncated ?? false,
28 stderrTruncated: overrides.stderrTruncated ?? false,
29 };
30}
31
32describe("searchWorkspaceEntries git-ignore chunking", () => {
33 beforeEach(() => {
34 runProcessMock.mockReset();
35 vi.resetModules();
36 });
37
38 it("chunks git check-ignore stdin to avoid building giant strings", async () => {
39 const ignoredPaths = Array.from(
40 { length: 5000 },
41 (_, index) => `ignored/${index.toString().padStart(5, "0")}/${"x".repeat(80)}.ts`,
42 );
43 const keptPaths = ["src/keep.ts", "docs/readme.md"];
44 const listedPaths = [...ignoredPaths, ...keptPaths];
45 let checkIgnoreCalls = 0;
46
47 runProcessMock.mockImplementation(async (_command, args, options) => {
48 if (args[0] === "rev-parse") {
49 return processResult({ code: 0, stdout: "true\n" });
50 }
51
52 if (args[0] === "ls-files") {
53 return processResult({ code: 0, stdout: `${listedPaths.join("\0")}\0` });
54 }
55
56 if (args[0] === "check-ignore") {
57 checkIgnoreCalls += 1;
58 const chunkPaths = (options?.stdin ?? "").split("\0").filter((value) => value.length > 0);
59 const chunkIgnored = chunkPaths.filter((value) => value.startsWith("ignored/"));
60 return processResult({
61 code: chunkIgnored.length > 0 ? 0 : 1,
62 stdout: chunkIgnored.length > 0 ? `${chunkIgnored.join("\0")}\0` : "",
63 });
64 }
65
66 throw new Error(`Unexpected command: git ${args.join(" ")}`);
67 });
68
69 const { searchWorkspaceEntries } = await import("./workspaceEntries");
70 const result = await searchWorkspaceEntries({
71 cwd: "/virtual/workspace",
72 query: "",
73 limit: 100,
74 });
75
76 assert.isAbove(checkIgnoreCalls, 1);
77 assert.isFalse(result.entries.some((entry) => entry.path.startsWith("ignored/")));
78 assert.isTrue(result.entries.some((entry) => entry.path === "src/keep.ts"));
79 });
80});
apps/server/src/workspaceEntries.ts+65-31
......@@ -14,17 +14,8 @@ const WORKSPACE_CACHE_TTL_MS = 15_000;
1414const WORKSPACE_CACHE_MAX_KEYS = 4;
1515const WORKSPACE_INDEX_MAX_ENTRIES = 25_000;
1616const WORKSPACE_SCAN_READDIR_CONCURRENCY = 32;
17const IGNORED_DIRECTORY_NAMES = new Set([
18 ".git",
19 ".convex",
20 "node_modules",
21 ".next",
22 ".turbo",
23 "dist",
24 "build",
25 "out",
26 ".cache",
27]);
17const GIT_CHECK_IGNORE_MAX_STDIN_BYTES = 256 * 1024;
18const IGNORED_DIRECTORY_NAMES = new Set([".git", "node_modules"]);
2819
2920interface WorkspaceIndex {
3021 scannedAt: number;
......@@ -56,7 +47,10 @@ function basenameOf(input: string): string {
5647}
5748
5849function normalizeQuery(input: string): string {
59 return input.trim().replace(/^[@./]+/, "").toLowerCase();
50 return input
51 .trim()
52 .replace(/^[@./]+/, "")
53 .toLowerCase();
6054}
6155
6256function scoreEntry(entry: ProjectEntry, query: string): number {
......@@ -145,26 +139,67 @@ async function filterGitIgnoredPaths(cwd: string, relativePaths: string[]): Prom
145139 return relativePaths;
146140 }
147141
148 const checkIgnore = await runProcess("git", ["check-ignore", "--no-index", "-z", "--stdin"], {
149 cwd,
150 allowNonZeroExit: true,
151 timeoutMs: 20_000,
152 maxBufferBytes: 16 * 1024 * 1024,
153 outputMode: "truncate",
154 stdin: `${relativePaths.join("\0")}\0`,
155 }).catch(() => null);
156 if (!checkIgnore) {
157 return relativePaths;
142 const ignoredPaths = new Set<string>();
143 let chunk: string[] = [];
144 let chunkBytes = 0;
145
146 const flushChunk = async (): Promise<boolean> => {
147 if (chunk.length === 0) {
148 return true;
149 }
150
151 const checkIgnore = await runProcess("git", ["check-ignore", "--no-index", "-z", "--stdin"], {
152 cwd,
153 allowNonZeroExit: true,
154 timeoutMs: 20_000,
155 maxBufferBytes: 16 * 1024 * 1024,
156 outputMode: "truncate",
157 stdin: `${chunk.join("\0")}\0`,
158 }).catch(() => null);
159 chunk = [];
160 chunkBytes = 0;
161
162 if (!checkIgnore) {
163 return false;
164 }
165
166 // git-check-ignore exits with 1 when no paths match.
167 if (checkIgnore.code !== 0 && checkIgnore.code !== 1) {
168 return false;
169 }
170
171 const matchedIgnoredPaths = splitNullSeparatedPaths(
172 checkIgnore.stdout,
173 Boolean(checkIgnore.stdoutTruncated),
174 );
175 for (const ignoredPath of matchedIgnoredPaths) {
176 ignoredPaths.add(ignoredPath);
177 }
178 return true;
179 };
180
181 for (const relativePath of relativePaths) {
182 const relativePathBytes = Buffer.byteLength(relativePath) + 1;
183 if (
184 chunk.length > 0 &&
185 chunkBytes + relativePathBytes > GIT_CHECK_IGNORE_MAX_STDIN_BYTES &&
186 !(await flushChunk())
187 ) {
188 return relativePaths;
189 }
190
191 chunk.push(relativePath);
192 chunkBytes += relativePathBytes;
193
194 if (chunkBytes >= GIT_CHECK_IGNORE_MAX_STDIN_BYTES && !(await flushChunk())) {
195 return relativePaths;
196 }
158197 }
159198
160 // git-check-ignore exits with 1 when no paths match.
161 if (checkIgnore.code !== 0 && checkIgnore.code !== 1) {
199 if (!(await flushChunk())) {
162200 return relativePaths;
163201 }
164202
165 const ignoredPaths = new Set(
166 splitNullSeparatedPaths(checkIgnore.stdout, Boolean(checkIgnore.stdoutTruncated)),
167 );
168203 if (ignoredPaths.size === 0) {
169204 return relativePaths;
170205 }
......@@ -210,14 +245,14 @@ async function buildWorkspaceIndexFromGit(cwd: string): Promise<WorkspaceIndex |
210245 }
211246
212247 const directoryEntries: ProjectEntry[] = [...directorySet]
213 .sort((left, right) => left.localeCompare(right))
248 .toSorted((left, right) => left.localeCompare(right))
214249 .map((directoryPath) => ({
215250 path: directoryPath,
216251 kind: "directory",
217252 parentPath: parentPathOf(directoryPath),
218253 }));
219254 const fileEntries: ProjectEntry[] = [...new Set(filePaths)]
220 .sort((left, right) => left.localeCompare(right))
255 .toSorted((left, right) => left.localeCompare(right))
221256 .map((filePath) => ({
222257 path: filePath,
223258 kind: "file",
......@@ -228,8 +263,7 @@ async function buildWorkspaceIndexFromGit(cwd: string): Promise<WorkspaceIndex |
228263 return {
229264 scannedAt: Date.now(),
230265 entries: entries.slice(0, WORKSPACE_INDEX_MAX_ENTRIES),
231 truncated:
232 Boolean(listedFiles.stdoutTruncated) || entries.length > WORKSPACE_INDEX_MAX_ENTRIES,
266 truncated: Boolean(listedFiles.stdoutTruncated) || entries.length > WORKSPACE_INDEX_MAX_ENTRIES,
233267 };
234268}
235269