authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-09-04 22:56:36-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2026-09-04 22:56:36-07:00
log2fa5ef4c7bf3aafabe98392d25be7eb86847ce8f
tree2bfc3bafd73f054a53853f287141b705217cec48
parent7a089b2b2449c5c146e1a7d554a31e6d55924d3f
signature Signed by PGP key B5690EEEBB952194

fix(web): load file grammar before enabling edits (#9947)

3 files changed, 219 insertions(+), 5 deletions(-)

apps/web/src/components/files/fileEditorLanguageReadiness.test.ts created+186
...@@ -0,0 +1,186 @@
1import {
2 FileRenderer,
3 disposeHighlighter,
4 getSharedHighlighter,
5 type BaseCodeOptions,
6 type DiffsHighlighter,
7 type FileContents,
8 type HighlightedToken,
9} from "@pierre/diffs";
10import { TextDocument } from "@pierre/diffs/editor";
11import { WorkerPoolManager, type WorkerRequest, type WorkerResponse } from "@pierre/diffs/worker";
12import * as NodeWorkerThreads from "node:worker_threads";
13import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test";
14
15type DocumentChange = NonNullable<ReturnType<TextDocument<unknown>["applyEdits"]>>;
16interface Tokenizer {
17 tokenize(change: DocumentChange): Map<number, HighlightedToken[]>;
18 cleanUp(): void;
19}
20
21const tokenizerUrl = new URL("./editor/tokenizer.js", import.meta.resolve("@pierre/diffs"));
22const { EditorTokenizer } = (await import(/* @vite-ignore */ tokenizerUrl.href)) as {
23 EditorTokenizer: new (options: {
24 codeOptions: BaseCodeOptions;
25 highlighter: DiffsHighlighter;
26 textDocument: TextDocument<unknown>;
27 setStyle: (style: string) => void;
28 onDeferTokenize: () => void;
29 }) => Tokenizer;
30};
31
32const workerModule = import.meta.resolve("@pierre/diffs/worker/worker.js");
33const options = {
34 theme: "pierre-dark",
35 themeType: "dark",
36 preferredHighlighter: "shiki-wasm",
37 useTokenTransformer: true,
38} as const;
39const source = "export const View = () => <div>Ready</div>;";
40let pool: WorkerPoolManager;
41let renderer: FileRenderer;
42let terminationPromises: Promise<number>[];
43
44class WorkerTransport {
45 private readonly worker = new NodeWorkerThreads.Worker(
46 `const { parentPort, workerData } = require("node:worker_threads");
47 globalThis.self = {
48 addEventListener(type, listener) {
49 if (type === "message") parentPort.on("message", data => listener({ data }));
50 if (type === "error") process.on("uncaughtException", listener);
51 }
52 };
53 globalThis.postMessage = data => parentPort.postMessage(data);
54 import(workerData.moduleUrl);`,
55 { eval: true, workerData: { moduleUrl: workerModule }, execArgv: [] },
56 );
57
58 addEventListener(
59 type: "message" | "error",
60 listener: (event: { data: WorkerResponse } | Error) => void,
61 ) {
62 if (type === "error") this.worker.on("error", listener);
63 else this.worker.on("message", (data: WorkerResponse) => listener({ data }));
64 }
65
66 postMessage(message: WorkerRequest) {
67 this.worker.postMessage(message, []);
68 }
69
70 terminate() {
71 terminationPromises.push(this.worker.terminate());
72 }
73}
74
75function firstEnter(highlighter: DiffsHighlighter, file: FileContents, language: string) {
76 const document = new TextDocument(file.name, file.contents, language);
77 const tokenizer = new EditorTokenizer({
78 codeOptions: options,
79 highlighter,
80 textDocument: document,
81 setStyle: () => {},
82 onDeferTokenize: () => {},
83 });
84 try {
85 const end = document.positionAt(file.contents.length);
86 const change = document.applyEdits([{ range: { start: end, end }, newText: "\n" }]);
87 expect(change).toBeDefined();
88 // This is the synchronous first edit, before the tokenizer's debounced prebuild.
89 const dirtyLines = tokenizer.tokenize(change!);
90 expect([...dirtyLines.keys()]).toEqual([0, 1]);
91 expect(document.getText()).toBe(`${file.contents}\n`);
92 } finally {
93 tokenizer.cleanUp();
94 }
95}
96
97beforeEach(async () => {
98 terminationPromises = [];
99 vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) =>
100 setImmediate(() => callback(0)),
101 );
102 vi.stubGlobal("cancelAnimationFrame", clearImmediate);
103 vi.stubGlobal("window", { matchMedia: () => ({ matches: true }) });
104 await disposeHighlighter();
105 pool = new WorkerPoolManager(
106 // Adapt transport only. The installed Pierre worker resolves and highlights the file.
107 { workerFactory: () => new WorkerTransport() as unknown as globalThis.Worker, poolSize: 1 },
108 options,
109 );
110 await pool.initialize();
111 renderer = new FileRenderer(options, undefined, pool);
112});
113
114afterEach(async () => {
115 renderer?.cleanUp();
116 pool?.terminate();
117 await Promise.all(terminationPromises);
118 await disposeHighlighter();
119 vi.unstubAllGlobals();
120});
121
122describe("editable file language readiness", () => {
123 it.each(["hydrate", "renderFile"] as const)(
124 "%s prepares the inferred language before the first edit of a worker-highlighted file",
125 async (method) => {
126 const file = { name: "cold.tsx", contents: source, cacheKey: "cold-tsx" };
127 await pool.primeFileHighlightCache(file);
128 expect(pool.getFileResultCache(file)).toBeDefined();
129 const mainHighlighter = await getSharedHighlighter({
130 themes: ["pierre-dark"],
131 langs: ["text"],
132 });
133 expect(mainHighlighter.getLoadedLanguages()).not.toContain("tsx");
134 renderer[method](file);
135 // Read-only worker rendering must not load editor grammars on the main thread.
136 expect(mainHighlighter.getLoadedLanguages()).not.toContain("tsx");
137 const highlighter = await renderer.initializeHighlighter();
138 firstEnter(highlighter, file, "tsx");
139 },
140 );
141
142 it.each(["hydrate", "renderFile"] as const)(
143 "%s respects an explicit language when the filename suggests plain text",
144 async (method) => {
145 const file: FileContents = {
146 name: "source.txt",
147 lang: "tsx",
148 contents: source,
149 cacheKey: "explicit-tsx",
150 };
151 await pool.primeFileHighlightCache(file);
152 renderer[method](file);
153 firstEnter(await renderer.initializeHighlighter(), file, "tsx");
154 },
155 );
156
157 it("loads a newly opened language after reusing a worker-backed renderer", async () => {
158 const previousFile: FileContents = {
159 name: "previous.ts",
160 contents: "export const value = 1;",
161 cacheKey: "previous-ts",
162 };
163 await getSharedHighlighter({ themes: ["pierre-dark"], langs: ["typescript"] });
164 renderer.renderFile(previousFile);
165 firstEnter(await renderer.initializeHighlighter(), previousFile, "typescript");
166 const nextFile = { name: "next.tsx", contents: source, cacheKey: "next-tsx" };
167 renderer.renderFile(nextFile);
168 firstEnter(await renderer.initializeHighlighter(), nextFile, "tsx");
169 });
170
171 it("prepares a hydrated non-worker file even when its theme was already loaded", async () => {
172 renderer.cleanUp();
173 renderer = new FileRenderer(options);
174 const file = { name: "local.tsx", contents: source, cacheKey: "local-tsx" };
175 renderer.hydrate(file);
176 firstEnter(await renderer.initializeHighlighter(), file, "tsx");
177 });
178
179 it("keeps plain text editable without loading an unrelated grammar", async () => {
180 const file = { name: "notes.txt", contents: "Plain text", cacheKey: "plain-text" };
181 renderer.renderFile(file);
182 const highlighter = await renderer.initializeHighlighter();
183 firstEnter(highlighter, file, "text");
184 expect(highlighter.getLoadedLanguages()).not.toContain("tsx");
185 });
186});
patches/@pierre%2Fdiffs@1.3.0-beta.10.patch+28
...@@ -63,6 +63,18 @@ index e9f62f5..af82a46 100644...@@ -63,6 +63,18 @@ index e9f62f5..af82a46 100644
63diff --git a/dist/renderers/FileRenderer.js b/dist/renderers/FileRenderer.js63diff --git a/dist/renderers/FileRenderer.js b/dist/renderers/FileRenderer.js
64--- a/dist/renderers/FileRenderer.js64--- a/dist/renderers/FileRenderer.js
65+++ b/dist/renderers/FileRenderer.js65+++ b/dist/renderers/FileRenderer.js
66@@ -107,10 +107,10 @@
67 result: massiveFile ? void 0 : cache?.result,
68 renderRange: void 0
69 };
70+ this.computedLang = file.lang ?? getFiletypeFromFileName(file.name);
71 if (this.workerManager?.isWorkingPool() === true) {
72 if (this.renderCache.result == null && !massiveFile) this.workerManager.highlightFileAST(this, file);
73 } else if (this.highlighter == null) {
74- this.computedLang = file.lang ?? getFiletypeFromFileName(file.name);
75 this.initializeHighlighter();
76 }
77 }
66@@ -163,6 +163,8 @@78@@ -163,6 +163,8 @@
67 if (this.renderCache == null) return;79 if (this.renderCache == null) return;
68 const { file, result } = this.renderCache;80 const { file, result } = this.renderCache;
...@@ -72,6 +84,22 @@ diff --git a/dist/renderers/FileRenderer.js b/dist/renderers/FileRenderer.js...@@ -72,6 +84,22 @@ diff --git a/dist/renderers/FileRenderer.js b/dist/renderers/FileRenderer.js
72 const lineCache = this.lineCache != null && isLineCacheForFile(this.lineCache, file) ? this.lineCache : void 0;84 const lineCache = this.lineCache != null && isLineCacheForFile(this.lineCache, file) ? this.lineCache : void 0;
73 for (const [line, tokens] of dirtyLines) {85 for (const [line, tokens] of dirtyLines) {
74 if (lineCache != null && line < lineCache.lines.length) {86 if (lineCache != null && line < lineCache.lines.length) {
87@@ -268,6 +270,7 @@
88 const forcePlainText = !hasContent || isFilePlainText(file) || isFileMassive(lines.length, this.getTokenizeMaxLength());
89 const newContent = !areFilesEqual(file, this.renderCache.file);
90 const newRenderRange = !areRenderRangesEqual(this.renderCache.renderRange, renderRange);
91+ this.computedLang = file.lang ?? getFiletypeFromFileName(file.name);
92 if (this.workerManager?.isWorkingPool() === true) {
93 if (forcePlainText || this.renderCache.result == null || !this.renderCache.highlighted && (newContent || newRenderRange)) {
94 this.renderCache.file = file;
95@@ -278,7 +281,6 @@
96 }
97 if (!forcePlainText && hasContent && (!this.renderCache.highlighted || forceHighlight)) this.workerManager.highlightFileAST(this, file);
98 } else {
99- this.computedLang = file.lang ?? getFiletypeFromFileName(file.name);
100 const hasThemes = this.highlighter != null && areThemesAttached(options.theme);
101 const hasLangs = this.highlighter != null && areLanguagesAttached(this.computedLang);
102 const canHighlight = !forcePlainText && hasLangs;
75diff --git a/package.json b/package.json103diff --git a/package.json b/package.json
76index ff61c90..1e170e5 100644104index ff61c90..1e170e5 100644
77--- a/package.json105--- a/package.json
pnpm-lock.yaml+5-5
...@@ -92,7 +92,7 @@ patchedDependencies:...@@ -92,7 +92,7 @@ patchedDependencies:
92 '@expo/metro-config@57.0.12': 96f1a75347e6ea02dc4b7034ace815d8ee39e18b8166ebfb573d9e58328f0dc292 '@expo/metro-config@57.0.12': 96f1a75347e6ea02dc4b7034ace815d8ee39e18b8166ebfb573d9e58328f0dc2
93 '@ff-labs/fff-node@0.9.4': ab9ff544009e1891cfe3930105862d3699007f38922a79f3c98d90018deca36893 '@ff-labs/fff-node@0.9.4': ab9ff544009e1891cfe3930105862d3699007f38922a79f3c98d90018deca368
94 '@legendapp/list@3.3.5': 03ec41339cd915ecb9a774a6b90cc2197c29038f7db67c4d2e55cd3971e5be4394 '@legendapp/list@3.3.5': 03ec41339cd915ecb9a774a6b90cc2197c29038f7db67c4d2e55cd3971e5be43
95 '@pierre/diffs@1.3.0-beta.10': c087c47b657c44ddce2b0ec8237d675086414134e91829c4b4edd009ec86893e95 '@pierre/diffs@1.3.0-beta.10': c2ea3a9821addf19641e88074a6d8896c62b0f7cfa762899b0e9e80ff5e8add4
96 '@react-native-ai/apple@0.12.0': 2d09870c2848d185cb05b53ed823a46e12dba519324d8dd8e584e28731990f9d96 '@react-native-ai/apple@0.12.0': 2d09870c2848d185cb05b53ed823a46e12dba519324d8dd8e584e28731990f9d
97 '@react-native-menu/menu@2.0.0': f63d256bf6a97a873b5e628eb595bd6ef0075ddd5bdd890fc920f7a6024290dd97 '@react-native-menu/menu@2.0.0': f63d256bf6a97a873b5e628eb595bd6ef0075ddd5bdd890fc920f7a6024290dd
98 '@react-navigation/native-stack@7.17.6': e667c3cef8c78bb9ff4882ee5bd23a432247b843060a9499eb5f07e9e229555298 '@react-navigation/native-stack@7.17.6': e667c3cef8c78bb9ff4882ee5bd23a432247b843060a9499eb5f07e9e2295552
...@@ -245,7 +245,7 @@ importers:...@@ -245,7 +245,7 @@ importers:
245 version: 1.9.1245 version: 1.9.1
246 '@pierre/diffs':246 '@pierre/diffs':
247 specifier: 'catalog:'247 specifier: 'catalog:'
248 version: 1.3.0-beta.10(patch_hash=c087c47b657c44ddce2b0ec8237d675086414134e91829c4b4edd009ec86893e)(@shikijs/themes@4.2.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)248 version: 1.3.0-beta.10(patch_hash=c2ea3a9821addf19641e88074a6d8896c62b0f7cfa762899b0e9e80ff5e8add4)(@shikijs/themes@4.2.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)
249 '@react-native-ai/apple':249 '@react-native-ai/apple':
250 specifier: 0.12.0250 specifier: 0.12.0
251 version: 0.12.0(patch_hash=2d09870c2848d185cb05b53ed823a46e12dba519324d8dd8e584e28731990f9d)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))251 version: 0.12.0(patch_hash=2d09870c2848d185cb05b53ed823a46e12dba519324d8dd8e584e28731990f9d)(react-native@0.86.3(@babel/core@7.29.7)(@react-native/metro-config@0.86.3(@babel/core@7.29.7)(bufferutil@4.1.0)(utf-8-validate@6.0.6))(@types/react@19.2.16)(bufferutil@4.1.0)(react@19.2.3)(utf-8-validate@6.0.6))
...@@ -586,7 +586,7 @@ importers:...@@ -586,7 +586,7 @@ importers:
586 version: 1.8.0586 version: 1.8.0
587 '@pierre/diffs':587 '@pierre/diffs':
588 specifier: 'catalog:'588 specifier: 'catalog:'
589 version: 1.3.0-beta.10(patch_hash=c087c47b657c44ddce2b0ec8237d675086414134e91829c4b4edd009ec86893e)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)589 version: 1.3.0-beta.10(patch_hash=c2ea3a9821addf19641e88074a6d8896c62b0f7cfa762899b0e9e80ff5e8add4)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)
590 '@pierre/trees':590 '@pierre/trees':
591 specifier: 1.0.0-beta.4591 specifier: 1.0.0-beta.4
592 version: 1.0.0-beta.4(react-dom@19.2.6(react@19.2.6))(react@19.2.6)592 version: 1.0.0-beta.4(react-dom@19.2.6(react@19.2.6))(react@19.2.6)
...@@ -13613,7 +13613,7 @@ snapshots:...@@ -13613,7 +13613,7 @@ snapshots:
13613 tslib: 2.8.113613 tslib: 2.8.1
13614 webcrypto-core: 1.9.213614 webcrypto-core: 1.9.2
1361513615
13616 '@pierre/diffs@1.3.0-beta.10(patch_hash=c087c47b657c44ddce2b0ec8237d675086414134e91829c4b4edd009ec86893e)(@shikijs/themes@4.2.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)':13616 '@pierre/diffs@1.3.0-beta.10(patch_hash=c2ea3a9821addf19641e88074a6d8896c62b0f7cfa762899b0e9e80ff5e8add4)(@shikijs/themes@4.2.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)':
13617 dependencies:13617 dependencies:
13618 '@pierre/theme': 1.1.013618 '@pierre/theme': 1.1.0
13619 '@pierre/theming': 0.0.2(@pierre/theme@1.1.0)(@shikijs/themes@4.2.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(shiki@4.2.0)13619 '@pierre/theming': 0.0.2(@pierre/theme@1.1.0)(@shikijs/themes@4.2.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)(shiki@4.2.0)
...@@ -13627,7 +13627,7 @@ snapshots:...@@ -13627,7 +13627,7 @@ snapshots:
13627 transitivePeerDependencies:13627 transitivePeerDependencies:
13628 - '@shikijs/themes'13628 - '@shikijs/themes'
1362913629
13630 '@pierre/diffs@1.3.0-beta.10(patch_hash=c087c47b657c44ddce2b0ec8237d675086414134e91829c4b4edd009ec86893e)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)':13630 '@pierre/diffs@1.3.0-beta.10(patch_hash=c2ea3a9821addf19641e88074a6d8896c62b0f7cfa762899b0e9e80ff5e8add4)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)':
13631 dependencies:13631 dependencies:
13632 '@pierre/theme': 1.1.013632 '@pierre/theme': 1.1.0
13633 '@pierre/theming': 0.0.2(@pierre/theme@1.1.0)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(shiki@4.2.0)13633 '@pierre/theming': 0.0.2(@pierre/theme@1.1.0)(@shikijs/themes@4.3.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)(shiki@4.2.0)