1const languages = [
2 "astro",
3 "css",
4 "diff",
5 "dosbatch",
6 "html",
7 "js",
8 "json",
9 "jsx",
10 "lua",
11 "mdx",
12 "php",
13 "powershell",
14 "python",
15 "shell",
16 "toml",
17 "ts",
18 "tsx",
19 "xml",
20 "yaml",
21 "zig",
22] as const;
23const altScopes: Record<string, string> = {
24 html: "text.html.basic",
25 astro: "text.html.astro",
26 xml: "text.xml",
27 php: "text.html.php",
28};
29export type Language = (typeof languages)[number];
30
31const scopes = [
32 // HTML
33 ["meta.tag.metadata.doctype.html", "builtin"],
34 ["meta.tag.structure", "keyword"],
35 ["meta.tag.metadata.script", "keyword"],
36
37 // CSS
38 ["punctuation.definition.keyword", "keyword", "css"],
39 ["entity.name.tag.css", "class", "css"],
40 ["meta.selector.css", "method", "css"],
41 ["entity.other.attribute-name.class.css", "builtin", "css"],
42 ["punctuation.definition.entity", "builtin", "css"],
43 ["variable.css", "parameter", "css"],
44
45 // JSON
46 ["support.type.property-name.json", "variable", "json"],
47 ["constant.numeric", "method", "json"],
48 ["constant", "class", "json"],
49
50 // Lua
51 ["entity.name", "class", "lua"],
52
53 // Diff
54 ["punctuation.definition.deleted", "variable", "diff"],
55 ["markup.deleted", "variable", "diff"],
56 ["punctuation.definition.inserted", "method", "diff"],
57 ["markup.inserted", "method", "diff"],
58 ["meta.diff.range", "string", "diff"],
59 ["punctuation.definition.range", "string", "diff"],
60 ["meta.toc-list-line.number", "keyword", "diff"],
61 ["meta.diff", "comment", "diff"],
62
63 // General
64 ["meta.object-literal.key", "property"],
65 ["comment", "comment"],
66 ["string", "string"],
67 ["storage", "keyword"],
68 ["keyword", "keyword"],
69 ["variable.parameter", "parameter"],
70 ["entity.name.function", "method"],
71 ["support.class.component", "method"],
72 ["support.type.primitive", "builtin"],
73 ["entity.name.type", "class"],
74 ["support.type", "class"],
75 ["support.class", "class"],
76 ["constant.language", "builtin"],
77 ["constant", "constant"],
78 ["support.constant", "constant"],
79 ["meta.parameters", "parameter"],
80 ["support.function", "method"],
81 ["entity.name.tag", "builtin"],
82 ["variable", "variable"],
83 ["meta.tag.attributes", "property"],
84 ["punctuation.definition.tag.begin.tsx", "keyword"],
85 ["punctuation.definition.tag.end.tsx", "keyword"],
86 ["punctuation", null],
87 ["meta.function-call", "method"],
88] as const;
89
90interface HighlightLinesOptions {
91 lines: string[];
92 grammar: textmate.IGrammar;
93 state: textmate.StateStack;
94 language: Language;
95}
96
97export function getStyle(scopesToCheck: string[], langugage: Language) {
98 for (const scope of scopes) {
99 if (scope[2] && scope[2] !== langugage) continue;
100 const find = scopesToCheck.find((s) => s.startsWith(scope[0]));
101 if (find) {
102 return scope[1];
103 }
104 }
105 return null;
106}
107
108function highlightLines({
109 lines,
110 grammar,
111 state,
112 language,
113}: HighlightLinesOptions) {
114 let html = "";
115 let lastHtmlStyle: string | null = null;
116 for (const line of lines) {
117 const { tokens, ruleStack, stoppedEarly } = grammar.tokenizeLine(
118 line,
119 state,
120 );
121 if (stoppedEarly) throw new Error("TODO: Tokenization stopped early?");
122 state = ruleStack;
123
124 for (const token of tokens) {
125 const str = line.slice(token.startIndex, token.endIndex);
126 if (str.trim().length === 0) {
127 // Emit but do not consider scope changes
128 html += string.escapeHtmlContent(str);
129 continue;
130 }
131
132 const style = getStyle(token.scopes, language);
133 if (style !== lastHtmlStyle) {
134 if (lastHtmlStyle) html += "</span>";
135 if (style) html += `<span class='${style}'>`;
136 }
137 html += string.escapeHtmlContent(str);
138 lastHtmlStyle = style;
139 }
140 html += "\n";
141 }
142
143 if (lastHtmlStyle) html += "</span>";
144
145 return { state, html };
146}
147
148export const getRegistry = async.once(async () => {
149 const wasmBin = await fs.readFile(
150 require.resolve("vscode-oniguruma/release/onig.wasm"),
151 );
152 await oniguruma.loadWASM(wasmBin);
153
154 return new textmate.Registry({
155 onigLib: Promise.resolve({
156 createOnigScanner: (patterns) => new oniguruma.OnigScanner(patterns),
157 createOnigString: (s) => new oniguruma.OnigString(s),
158 }),
159 loadGrammar: async (scopeName: string) => {
160 for (const lang of languages) {
161 if (
162 altScopes[lang]
163 ? scopeName === altScopes[lang]
164 : scopeName.endsWith(`.${lang}`)
165 ) {
166 const file = await fs.readFile(
167 path.join(import.meta.dirname, `highlight-grammar/${lang}.plist`),
168 "utf-8",
169 );
170 return textmate.parseRawGrammar(file);
171 }
172 }
173 return null;
174 },
175 });
176});
177
178export async function highlightCode(code: string, language: Language) {
179 const registry = await getRegistry();
180 const grammar = await registry.loadGrammar(
181 altScopes[language] ?? "source." + language,
182 );
183 if (!grammar) {
184 throw new Error(`No grammar found for language: ${language}`);
185 }
186 let state = textmate.INITIAL;
187 const { html } = highlightLines({
188 lines: code.split("\n"),
189 grammar,
190 state,
191 language,
192 });
193 return html;
194}
195
196// export async function main() {
197// validate exts
198// for (const ext of languages) {
199// if (
200// !fs.existsSync(
201// path.join(import.meta.dirname, `highlight-grammar/${ext}.plist`),
202// )
203// ) {
204// console.error(`Missing grammar for ${ext}`);
205// }
206// // Sanity check
207// await highlightCode("wwwwwwwwwwwaaaaaaaaaaaaaaaa", ext);
208// }
209// }
210//
211// await highlightCode(
212// `return <Snow attr={'meow'}>snow!</Snow>`,
213// "tsx",
214// );
215//
216import * as fs from "#sitegen/fs";
217import * as async from "@clo/lib/async";
218import * as string from "@clo/lib/string";
219import * as path from "node:path";
220import * as oniguruma from "vscode-oniguruma";
221import * as textmate from "vscode-textmate";