1import * as NodeModule from "node:module";
2
3import type {
4 DirItem,
5 DirSearchResult,
6 FileItem,
7 FileFinder as FileFinderType,
8 GrepCursor,
9 MixedItem,
10 MixedSearchResult,
11 Result,
12 SearchResult,
13} from "@ff-labs/fff-node";
14import * as Context from "effect/Context";
15import * as Effect from "effect/Effect";
16import * as Layer from "effect/Layer";
17import * as LayerMap from "effect/LayerMap";
18import * as Schema from "effect/Schema";
19
20import type {
21 ProjectEntry,
22 ProjectEntryKind,
23 ProjectListEntriesResult,
24 ProjectSearchContentsInput,
25 ProjectSearchContentsResult,
26 ProjectSearchEntriesResult,
27} from "@t3tools/contracts";
28import { isProjectFaviconPath } from "@t3tools/shared/projectFavicon";
29
30// fff-node stays external to the CLI bundle because it dlopens a native
31// library. A static `import` of an external package is a hard error inside a
32// Node single-executable (only built-ins resolve there), so load it through
33// `require`, which reads from the real filesystem in every runtime.
34const requireForFff = NodeModule.createRequire(import.meta.url);
35const { FileFinder } = requireForFff("@ff-labs/fff-node") as typeof import("@ff-labs/fff-node");
36
37const WORKSPACE_INDEX_MAX_ENTRIES = 25_000;
38const WORKSPACE_INDEX_PAGE_SIZE = WORKSPACE_INDEX_MAX_ENTRIES + 2;
39const WORKSPACE_INDEX_SCAN_TIMEOUT = "15 seconds";
40const WORKSPACE_INDEX_SCAN_TIMEOUT_MS = 15_000;
41const WORKSPACE_INDEX_IDLE_TTL = "15 minutes";
42const CONTENT_SEARCH_TIME_BUDGET_MS = 250;
43const CONTENT_SEARCH_MAX_MATCHES_PER_FILE = 100;
44
45export class WorkspaceSearchIndexCreateFailed extends Schema.TaggedError<WorkspaceSearchIndexCreateFailed>()(
46 "WorkspaceSearchIndexCreateFailed",
47 {
48 cwd: Schema.String,
49 reason: Schema.String,
50 cause: Schema.optional(Schema.Defect()),
51 },
52) {
53 override get message(): string {
54 return `Failed to create the workspace search index for '${this.cwd}'.`;
55 }
56}
57
58export class WorkspaceSearchIndexScanTimedOut extends Schema.TaggedError<WorkspaceSearchIndexScanTimedOut>()(
59 "WorkspaceSearchIndexScanTimedOut",
60 {
61 cwd: Schema.String,
62 timeout: Schema.String,
63 },
64) {
65 override get message(): string {
66 return `Workspace search index for '${this.cwd}' did not finish scanning within ${this.timeout}`;
67 }
68}
69
70export class WorkspaceSearchIndexSearchFailed extends Schema.TaggedError<WorkspaceSearchIndexSearchFailed>()(
71 "WorkspaceSearchIndexSearchFailed",
72 {
73 cwd: Schema.String,
74 queryLength: Schema.Number,
75 pageSize: Schema.Number,
76 reason: Schema.String,
77 cause: Schema.optional(Schema.Defect()),
78 },
79) {
80 override get message(): string {
81 return `Workspace search failed for '${this.cwd}'.`;
82 }
83}
84
85export class WorkspaceSearchIndexRefreshFailed extends Schema.TaggedError<WorkspaceSearchIndexRefreshFailed>()(
86 "WorkspaceSearchIndexRefreshFailed",
87 {
88 cwd: Schema.String,
89 reason: Schema.String,
90 cause: Schema.optional(Schema.Defect()),
91 },
92) {
93 override get message(): string {
94 return `Failed to refresh the workspace search index for '${this.cwd}'.`;
95 }
96}
97
98export class WorkspaceSearchIndexDestroyFailed extends Schema.TaggedError<WorkspaceSearchIndexDestroyFailed>()(
99 "WorkspaceSearchIndexDestroyFailed",
100 {
101 cwd: Schema.String,
102 cause: Schema.Defect(),
103 },
104) {
105 override get message(): string {
106 return `Failed to destroy the workspace search index for '${this.cwd}'.`;
107 }
108}
109
110export class WorkspaceSearchIndex extends Context.Service<
111 WorkspaceSearchIndex,
112 {
113 readonly list: () => Effect.Effect<ProjectListEntriesResult, WorkspaceSearchIndexSearchFailed>;
114 readonly search: (
115 query: string,
116 limit: number,
117 kind?: ProjectEntryKind,
118 imageOnly?: boolean,
119 ) => Effect.Effect<ProjectSearchEntriesResult, WorkspaceSearchIndexSearchFailed>;
120 readonly searchContents: (
121 input: Omit<ProjectSearchContentsInput, "cwd">,
122 ) => Effect.Effect<ProjectSearchContentsResult, WorkspaceSearchIndexSearchFailed>;
123 readonly refresh: () => Effect.Effect<
124 void,
125 WorkspaceSearchIndexRefreshFailed | WorkspaceSearchIndexScanTimedOut
126 >;
127 }
128>()("t3/workspace/WorkspaceSearchIndex") {}
129
130function toPosixPath(input: string): string {
131 return input.replaceAll("\\", "/");
132}
133
134function trimDirectorySeparator(input: string): string {
135 return input.endsWith("/") ? input.slice(0, -1) : input;
136}
137
138function parentPathOf(input: string): string | undefined {
139 const separatorIndex = input.lastIndexOf("/");
140 return separatorIndex === -1 ? undefined : input.slice(0, separatorIndex);
141}
142
143function toProjectEntry(item: MixedItem): ProjectEntry | null {
144 const normalizedPath = trimDirectorySeparator(toPosixPath(item.item.relativePath));
145 if (!normalizedPath) {
146 return null;
147 }
148
149 return {
150 path: normalizedPath,
151 kind: item.type,
152 };
153}
154
155function toFileEntry(item: FileItem): ProjectEntry | null {
156 const normalizedPath = trimDirectorySeparator(toPosixPath(item.relativePath));
157 return normalizedPath ? { path: normalizedPath, kind: "file" } : null;
158}
159
160function toDirectoryEntry(item: DirItem): ProjectEntry | null {
161 const normalizedPath = trimDirectorySeparator(toPosixPath(item.relativePath));
162 return normalizedPath ? { path: normalizedPath, kind: "directory" } : null;
163}
164
165function mapFileSearchResult(
166 result: SearchResult,
167 limit: number,
168 imageOnly = false,
169): ProjectSearchEntriesResult {
170 const entries = result.items.flatMap((item) => {
171 const entry = toFileEntry(item);
172 return entry && (!imageOnly || isProjectFaviconPath(entry.path)) ? [entry] : [];
173 });
174 return {
175 entries: entries.slice(0, limit),
176 truncated: entries.length > limit || result.totalMatched > result.items.length,
177 };
178}
179
180function mapDirectorySearchResult(
181 result: DirSearchResult,
182 limit: number,
183): ProjectSearchEntriesResult {
184 const entries = result.items.flatMap((item) => {
185 const entry = toDirectoryEntry(item);
186 return entry ? [entry] : [];
187 });
188 const rootDirectoryCount = result.items.some((item) => item.relativePath.length === 0) ? 1 : 0;
189 return {
190 entries: entries.slice(0, limit),
191 truncated: result.totalMatched - rootDirectoryCount > limit,
192 };
193}
194
195function mapMixedSearchResult(
196 result: MixedSearchResult,
197 limit: number,
198): { readonly entries: ProjectEntry[]; readonly truncated: boolean } {
199 const entries: ProjectEntry[] = [];
200 for (const item of result.items) {
201 const entry = toProjectEntry(item);
202 if (entry) {
203 entries.push(entry);
204 }
205 if (entries.length >= limit) {
206 break;
207 }
208 }
209
210 const rootDirectoryCount = result.items.some(
211 (item) => item.type === "directory" && item.item.relativePath.length === 0,
212 )
213 ? 1
214 : 0;
215 return {
216 entries,
217 truncated: result.totalMatched - rootDirectoryCount > limit,
218 };
219}
220
221const WORD_CHARACTER = /[\p{Letter}\p{Mark}\p{Number}_]/u;
222
223function codePointAt(line: string, index: number): string | undefined {
224 const codePoint = line.codePointAt(index);
225 return codePoint === undefined ? undefined : String.fromCodePoint(codePoint);
226}
227
228function codePointBefore(line: string, index: number): string | undefined {
229 if (index <= 0) return undefined;
230 const previousCodeUnit = line.charCodeAt(index - 1);
231 const previousIndex =
232 previousCodeUnit >= 0xdc00 && previousCodeUnit <= 0xdfff ? index - 2 : index - 1;
233 return codePointAt(line, previousIndex);
234}
235
236function buildContentSearchQuery(input: Omit<ProjectSearchContentsInput, "cwd">): {
237 readonly searchQuery: string;
238 readonly regexMode: boolean;
239} {
240 if (input.caseSensitive) {
241 return { searchQuery: input.query, regexMode: input.useRegex };
242 }
243 // Plain mode relies on smart case: an all-lowercase needle matches
244 // case-insensitively. Regex mode needs an explicit inline flag instead.
245 return input.useRegex
246 ? { searchQuery: `(?i)${input.query}`, regexMode: true }
247 : { searchQuery: input.query.toLowerCase(), regexMode: false };
248}
249
250function mapContentMatchRanges(
251 line: string,
252 byteRanges: ReadonlyArray<readonly [number, number]>,
253): Array<{ readonly start: number; readonly end: number }> {
254 const lineBytes = Buffer.from(line);
255 const toStringIndex = (byteOffset: number) => lineBytes.subarray(0, byteOffset).toString().length;
256 return byteRanges.map(([startByte, endByte]) => ({
257 start: toStringIndex(startByte),
258 end: toStringIndex(endByte),
259 }));
260}
261
262/**
263 * Whole-word filtering happens after the grep rather than by wrapping the
264 * pattern in boundary regex: consuming boundaries such as `(?:^|\W)` swallow
265 * the separator between adjacent matches and widen the reported ranges, and
266 * `\b` cannot match punctuation-edged queries at all. Matching VS Code, a
267 * match edge is a word boundary when it touches the line edge, the
268 * neighbouring character is not a word character, or the match's own edge
269 * character is not a word character.
270 */
271function isWholeWordRange(
272 line: string,
273 range: { readonly start: number; readonly end: number },
274): boolean {
275 if (range.end <= range.start) return false;
276 const isWord = (character: string | undefined) =>
277 character !== undefined && WORD_CHARACTER.test(character);
278 const leftIsBoundary =
279 range.start === 0 ||
280 !isWord(codePointBefore(line, range.start)) ||
281 !isWord(codePointAt(line, range.start));
282 const rightIsBoundary =
283 range.end >= line.length ||
284 !isWord(codePointAt(line, range.end)) ||
285 !isWord(codePointBefore(line, range.end));
286 return leftIsBoundary && rightIsBoundary;
287}
288
289function withDirectoryAncestors(entries: ReadonlyArray<ProjectEntry>): ProjectEntry[] {
290 const entryByPath = new Map(entries.map((entry) => [entry.path, entry]));
291 for (const entry of entries) {
292 let parentPath = parentPathOf(entry.path);
293 while (parentPath) {
294 if (!entryByPath.has(parentPath)) {
295 entryByPath.set(parentPath, { path: parentPath, kind: "directory" });
296 }
297 parentPath = parentPathOf(parentPath);
298 }
299 }
300 return [...entryByPath.values()];
301}
302
303const createFinder = Effect.fn("WorkspaceSearchIndex.createFinder")(function* (
304 cwd: string,
305 variant: WorkspaceSearchIndexVariant,
306) {
307 const result = yield* Effect.try({
308 try: () =>
309 FileFinder.create({
310 basePath: cwd,
311 disableMmapCache: true,
312 // Content indexing costs scan CPU and memory, so only the on-demand
313 // content-search index pays for it; path-only consumers (file tree,
314 // composer path search, file picker) keep the lightweight index.
315 disableContentIndexing: variant !== "content",
316 aiMode: false,
317 enableFsRootScanning: true,
318 enableHomeDirScanning: true,
319 }),
320 catch: (cause) =>
321 new WorkspaceSearchIndexCreateFailed({
322 cwd,
323 reason: "FileFinder.create threw unexpectedly.",
324 cause,
325 }),
326 });
327 if (result.ok) return result.value;
328 return yield* new WorkspaceSearchIndexCreateFailed({
329 cwd,
330 reason: result.error,
331 });
332});
333
334const waitForIndexReady = Effect.fn("WorkspaceSearchIndex.waitForIndexReady")(function* <E>(
335 cwd: string,
336 finder: FileFinderType,
337 onFailure: (input: { readonly reason: string; readonly cause?: unknown }) => E,
338): Effect.fn.Return<void, E | WorkspaceSearchIndexScanTimedOut> {
339 const result = yield* Effect.tryPromise({
340 try: () => finder.waitForIndexReady(WORKSPACE_INDEX_SCAN_TIMEOUT_MS),
341 catch: (cause) =>
342 onFailure({
343 reason: "FileFinder.waitForIndexReady rejected unexpectedly.",
344 cause,
345 }),
346 });
347 if (!result.ok) {
348 return yield* Effect.fail(onFailure({ reason: result.error }));
349 }
350 if (!result.value) {
351 return yield* new WorkspaceSearchIndexScanTimedOut({
352 cwd,
353 timeout: WORKSPACE_INDEX_SCAN_TIMEOUT,
354 });
355 }
356});
357
358export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
359 cwd: string,
360 variant: WorkspaceSearchIndexVariant = "paths",
361) {
362 const finder = yield* Effect.acquireRelease(createFinder(cwd, variant), (finder) =>
363 Effect.try({
364 try: () => finder.destroy(),
365 catch: (cause) => new WorkspaceSearchIndexDestroyFailed({ cwd, cause }),
366 }).pipe(Effect.orDie),
367 );
368 let initialScanTimedOut = false;
369 yield* waitForIndexReady(
370 cwd,
371 finder,
372 ({ reason, cause }) =>
373 new WorkspaceSearchIndexCreateFailed({
374 cwd,
375 reason,
376 cause,
377 }),
378 ).pipe(
379 Effect.catchTags({
380 WorkspaceSearchIndexScanTimedOut: (error) =>
381 variant === "paths"
382 ? Effect.sync(() => {
383 initialScanTimedOut = true;
384 })
385 : Effect.fail(error),
386 }),
387 );
388
389 const hasIncompleteInitialScan = () => initialScanTimedOut && finder.isScanning();
390
391 const runSearch = Effect.fn("WorkspaceSearchIndex.runSearch")(function* <A>(
392 query: string,
393 pageSize: number,
394 operation: "directorySearch" | "fileSearch" | "grep" | "mixedSearch",
395 execute: () => Result<A>,
396 ): Effect.fn.Return<A, WorkspaceSearchIndexSearchFailed> {
397 const result = yield* Effect.try({
398 try: execute,
399 catch: (cause) =>
400 new WorkspaceSearchIndexSearchFailed({
401 cwd,
402 queryLength: query.length,
403 pageSize,
404 reason: `FileFinder.${operation} threw unexpectedly.`,
405 cause,
406 }),
407 });
408 if (!result.ok) {
409 return yield* new WorkspaceSearchIndexSearchFailed({
410 cwd,
411 queryLength: query.length,
412 pageSize,
413 reason: result.error,
414 });
415 }
416 return result.value;
417 });
418
419 const refresh: WorkspaceSearchIndex["Service"]["refresh"] = Effect.fn(
420 "WorkspaceSearchIndex.refresh",
421 )(function* () {
422 const result = yield* Effect.try({
423 try: () => finder.scanFiles(),
424 catch: (cause) =>
425 new WorkspaceSearchIndexRefreshFailed({
426 cwd,
427 reason: "FileFinder.scanFiles threw unexpectedly.",
428 cause,
429 }),
430 });
431 if (!result.ok) {
432 return yield* new WorkspaceSearchIndexRefreshFailed({
433 cwd,
434 reason: result.error,
435 });
436 }
437 yield* waitForIndexReady(
438 cwd,
439 finder,
440 ({ reason, cause }) =>
441 new WorkspaceSearchIndexRefreshFailed({
442 cwd,
443 reason,
444 cause,
445 }),
446 );
447 });
448
449 const list: WorkspaceSearchIndex["Service"]["list"] = Effect.fn("WorkspaceSearchIndex.list")(
450 function* () {
451 const incompleteBeforeQuery = hasIncompleteInitialScan();
452 const result = yield* runSearch("", WORKSPACE_INDEX_PAGE_SIZE, "mixedSearch", () =>
453 finder.mixedSearch("", { pageSize: WORKSPACE_INDEX_PAGE_SIZE }),
454 );
455 const mapped = mapMixedSearchResult(result, WORKSPACE_INDEX_MAX_ENTRIES);
456 const sortedEntries = withDirectoryAncestors(mapped.entries).toSorted((left, right) =>
457 left.path.localeCompare(right.path),
458 );
459 const entries = sortedEntries.slice(0, WORKSPACE_INDEX_MAX_ENTRIES);
460 return {
461 entries,
462 truncated:
463 incompleteBeforeQuery || mapped.truncated || entries.length < sortedEntries.length,
464 };
465 },
466 );
467
468 const search: WorkspaceSearchIndex["Service"]["search"] = Effect.fn(
469 "WorkspaceSearchIndex.search",
470 )(function* (query, limit, kind, imageOnly) {
471 const pageSize = imageOnly ? WORKSPACE_INDEX_PAGE_SIZE : Math.max(1, limit + 1);
472 const incompleteBeforeQuery = hasIncompleteInitialScan();
473 if (kind === "file" || imageOnly) {
474 const result = yield* runSearch(query, pageSize, "fileSearch", () =>
475 finder.fileSearch(query, { pageSize }),
476 );
477 const mapped = mapFileSearchResult(result, limit, imageOnly);
478 return { ...mapped, truncated: incompleteBeforeQuery || mapped.truncated };
479 }
480 if (kind === "directory") {
481 const result = yield* runSearch(query, pageSize, "directorySearch", () =>
482 finder.directorySearch(query, { pageSize }),
483 );
484 const mapped = mapDirectorySearchResult(result, limit);
485 return { ...mapped, truncated: incompleteBeforeQuery || mapped.truncated };
486 }
487 const result = yield* runSearch(query, pageSize, "mixedSearch", () =>
488 finder.mixedSearch(query, { pageSize }),
489 );
490 const mapped = mapMixedSearchResult(result, limit);
491 return { ...mapped, truncated: incompleteBeforeQuery || mapped.truncated };
492 });
493
494 const searchContents: WorkspaceSearchIndex["Service"]["searchContents"] = Effect.fn(
495 "WorkspaceSearchIndex.searchContents",
496 )(function* (input) {
497 const { searchQuery, regexMode } = buildContentSearchQuery(input);
498 const deadline = performance.now() + CONTENT_SEARCH_TIME_BUDGET_MS;
499 // Grep cursors advance by file, so whole-word post-filtering needs enough
500 // raw candidates from the current file before moving to the next one.
501 const rawPageSize = input.wholeWord
502 ? Math.max(input.limit, CONTENT_SEARCH_MAX_MATCHES_PER_FILE)
503 : input.limit;
504 const matches: Array<ProjectSearchContentsResult["matches"][number]> = [];
505 let nextCursor: GrepCursor | null = null;
506 let regexFallbackError: string | undefined;
507
508 do {
509 const remainingTimeBudgetMs = Math.max(1, Math.ceil(deadline - performance.now()));
510 const result = yield* runSearch(input.query, input.limit, "grep", () =>
511 finder.grep(searchQuery, {
512 mode: regexMode ? "regex" : "plain",
513 smartCase: !input.caseSensitive && !regexMode,
514 // A single dense file must not consume the whole result page.
515 maxMatchesPerFile: Math.min(CONTENT_SEARCH_MAX_MATCHES_PER_FILE, rawPageSize),
516 pageSize: rawPageSize,
517 cursor: nextCursor,
518 timeBudgetMs: remainingTimeBudgetMs,
519 }),
520 );
521
522 for (const match of result.items) {
523 const matchRanges = mapContentMatchRanges(match.lineContent, match.matchRanges).filter(
524 (range) => !input.wholeWord || isWholeWordRange(match.lineContent, range),
525 );
526 if (matchRanges.length === 0) continue;
527 matches.push({
528 path: toPosixPath(match.relativePath),
529 lineNumber: match.lineNumber,
530 lineContent: match.lineContent,
531 matchRanges,
532 });
533 }
534 nextCursor = result.nextCursor;
535 regexFallbackError ??= result.regexFallbackError;
536 } while (matches.length < input.limit && nextCursor !== null && performance.now() < deadline);
537
538 return {
539 matches: matches.slice(0, input.limit),
540 truncated: matches.length > input.limit || nextCursor !== null,
541 ...(regexFallbackError !== undefined ? { regexFallbackError } : {}),
542 };
543 });
544
545 return WorkspaceSearchIndex.of({ list, refresh, search, searchContents });
546});
547
548export const WORKSPACE_SEARCH_INDEX_VARIANTS = ["paths", "content"] as const;
549export type WorkspaceSearchIndexVariant = (typeof WORKSPACE_SEARCH_INDEX_VARIANTS)[number];
550
551/**
552 * Composite LayerMap key so the lightweight path index and the on-demand
553 * content-search index of the same workspace are separate resources with
554 * independent lifecycles. "\n" cannot appear in a filesystem path.
555 */
556export const workspaceSearchIndexKey = (cwd: string, variant: WorkspaceSearchIndexVariant) =>
557 `${variant}\n${cwd}`;
558
559function parseWorkspaceSearchIndexKey(key: string): {
560 readonly cwd: string;
561 readonly variant: WorkspaceSearchIndexVariant;
562} {
563 const separatorIndex = key.indexOf("\n");
564 return {
565 variant: key.slice(0, separatorIndex) as WorkspaceSearchIndexVariant,
566 cwd: key.slice(separatorIndex + 1),
567 };
568}
569
570/**
571 * A layer factory is required because every index is scoped to a concrete
572 * workspace root and variant. WorkspaceSearchIndexMap owns memoization and
573 * idle cleanup; using a default cwd here would mix resources from different
574 * workspaces.
575 *
576 * @public Service construction is part of the canonical Effect module API.
577 */
578export const layer = (key: string) => {
579 const { cwd, variant } = parseWorkspaceSearchIndexKey(key);
580 return Layer.effect(WorkspaceSearchIndex, make(cwd, variant));
581};
582
583export class WorkspaceSearchIndexMap extends LayerMap.Service<WorkspaceSearchIndexMap>()(
584 "t3/workspace/WorkspaceSearchIndexMap",
585 {
586 lookup: layer,
587 idleTimeToLive: WORKSPACE_INDEX_IDLE_TTL,
588 },
589) {}