1/* oxlint-disable jest/expect-expect, jest/no-conditional-expect */
2import { describe, expect, test } from "vitest";
3import { Predict } from "../src/Predict.ts";
4
5describe.each(["stateless", "stateful"] as const)("%s processing", (mode) => {
6 function stateless(text: string) {
7 return new Predict().update(text);
8 }
9
10 function splitBySizes(text: string, sizes: number[]) {
11 const chunks: string[] = [];
12 let index = 0;
13
14 for (const size of sizes) {
15 if (index >= text.length) break;
16
17 const nextIndex = Math.min(index + size, text.length);
18 chunks.push(text.slice(index, nextIndex));
19 index = nextIndex;
20 }
21
22 if (index < text.length) {
23 chunks.push(text.slice(index));
24 }
25
26 return chunks;
27 }
28
29 function addChunking(text: string, sizes: number[], seen: Set<string>, chunkings: string[][]) {
30 const chunks = splitBySizes(text, sizes);
31 const key = JSON.stringify(chunks);
32
33 if (seen.has(key)) return;
34 seen.add(key);
35 chunkings.push(chunks);
36 }
37
38 function buildChunkings(text: string) {
39 if (text.length === 0) return [[]];
40
41 const seen = new Set<string>();
42 const chunkings: string[][] = [];
43
44 if (text.length <= 10) {
45 const splitCount = text.length - 1;
46 const totalMasks = 1 << splitCount;
47
48 for (let mask = 0; mask < totalMasks; mask++) {
49 const sizes: number[] = [];
50 let last = 0;
51
52 for (let bit = 0; bit < splitCount; bit++) {
53 if ((mask & (1 << bit)) === 0) continue;
54
55 const next = bit + 1;
56 sizes.push(next - last);
57 last = next;
58 }
59
60 sizes.push(text.length - last);
61 addChunking(text, sizes, seen, chunkings);
62 }
63
64 return chunkings;
65 }
66
67 for (let size = 1; size <= text.length; size++) {
68 const sizes: number[] = [];
69
70 for (let remaining = text.length; remaining > 0; remaining -= size) {
71 sizes.push(Math.min(size, remaining));
72 }
73
74 addChunking(text, sizes, seen, chunkings);
75 }
76
77 const maxOffsetSize = Math.min(text.length, 8);
78
79 for (let size = 2; size <= maxOffsetSize; size++) {
80 for (let head = 1; head < size && head < text.length; head++) {
81 const sizes = [head];
82
83 for (let remaining = text.length - head; remaining > 0; remaining -= size) {
84 sizes.push(Math.min(size, remaining));
85 }
86
87 addChunking(text, sizes, seen, chunkings);
88 }
89
90 for (let tail = 1; tail < size && tail < text.length; tail++) {
91 const sizes: number[] = [];
92
93 for (let remaining = text.length - tail; remaining > 0; remaining -= size) {
94 sizes.push(Math.min(size, remaining));
95 }
96
97 sizes.push(tail);
98 addChunking(text, sizes, seen, chunkings);
99 }
100 }
101
102 return chunkings;
103 }
104
105 function assertStatefulMatchesStateless(text: string) {
106 for (const chunks of buildChunkings(text)) {
107 const prediction = new Predict();
108 let prefix = "";
109
110 for (const chunk of chunks) {
111 prefix += chunk;
112
113 const actual = prediction.update(prefix);
114 const expected = stateless(prefix);
115
116 if (actual !== expected) {
117 throw new Error(
118 [
119 "stateful prediction diverged from stateless parsing",
120 `input: ${JSON.stringify(text)}`,
121 `chunks: ${JSON.stringify(chunks)}`,
122 `prefix: ${JSON.stringify(prefix)}`,
123 `expected: ${JSON.stringify(expected)}`,
124 `received: ${JSON.stringify(actual)}`,
125 ].join("\n"),
126 );
127 }
128 }
129 }
130 }
131
132 function check(text: string, expected: string) {
133 if (mode === "stateless") {
134 expect(stateless(text)).toBe(expected);
135 } else {
136 assertStatefulMatchesStateless(text);
137 }
138 }
139
140 test.each([
141 ["plain text", "plain text", "plain text"],
142 ["italic", "a *hello world", "a *hello world*"],
143 ["bold", "a **hello world", "a **hello world**"],
144 ["bold italic", "a ***hello world", "a ***hello world***"],
145 ["code", "a `hello world", "a `hello world`"],
146 ["strike", "a ~~hello world", "a ~~hello world~~"],
147 ["katex", "something\n\n$$\nE =", "something\n\n$$\nE =\n$$"],
148
149 ["bold partial close", "a **hello world*", "a **hello world**"],
150 ["bold italic partial close 1", "a ***hello world*", "a ***hello world***"],
151 ["bold italic partial close 2", "a ***hello world**", "a ***hello world***"],
152 ["code", "a `hello world", "a `hello world`"],
153 ["strike", "a ~~hello world", "a ~~hello world~~"],
154 ["strike partial close", "a ~~hello world~", "a ~~hello world~~"],
155 ["katex", "something\n\n$$\nE =", "something\n\n$$\nE =\n$$"],
156 ["katex partial close", "something\n\n$$\nE =\n$", "something\n\n$$\nE =\n$$"],
157
158 ["balanced markdown stays unchanged", "a *hello* world", "a *hello* world"],
159 ["balanced bold stays unchanged", "a **hello** world", "a **hello** world"],
160 ["balanced double underscore stays unchanged", "a __hello__ world", "a __hello__ world"],
161 ["balanced strike stays unchanged", "a ~~hello~~ world", "a ~~hello~~ world"],
162 ["balanced code stays unchanged", "a `hello` world", "a `hello` world"],
163 ["balanced inline math stays unchanged", "a $x$ world", "a $x$ world"],
164 [
165 "balanced display math stays unchanged",
166 "something\n\n$$\nE = mc^2\n$$",
167 "something\n\n$$\nE = mc^2\n$$",
168 ],
169 ["escaped markdown stays escaped", "a \\*hello", "a \\*hello"],
170 ["escaped underscore stays escaped", "a \\_hello", "a \\_hello"],
171 ["escaped strike stays escaped", "a \\~~hello", "a \\~~hello"],
172 ["escaped code stays escaped", "a \\`hello", "a \\`hello"],
173 ["escaped link bracket stays escaped", "a \\[hello", "a \\[hello"],
174 ["escaped inline math stays escaped", "a \\$x", "a \\$x"],
175 ["code suppresses other inline markdown", "a `code *not emphasis", "a `code *not emphasis`"],
176 ["paragraph break resets open formatting", "a **hello\n\nworld", "a **hello\n\nworld"],
177 ])("%s", (_name, input, expected) => check(input, expected));
178
179 describe("whitespace around emphasis delimiters", () => {
180 test.each(["* ", "** ", "_ ", "__ ", "~~ "])("keeps disqualified opener %j visible", (input) =>
181 check(input, input),
182 );
183
184 test.each([
185 "hello * world",
186 "hello* world",
187 "hello _ world",
188 "hello_ world",
189 "hello ** world",
190 "hello** world",
191 "hello __ world",
192 "hello__ world",
193 ])("does not predict for %j", (input) => check(input, input));
194 });
195
196 describe("flanking around emphasis delimiters", () => {
197 test.each([
198 ["hello *world", "hello *world*"],
199 ["hello **world", "hello **world**"],
200 ["hello _world", "hello _world_"],
201 ["hello __world", "hello __world__"],
202 ])("predicts for left-flanking opener %j", (input, expected) => check(input, expected));
203
204 test.each(["snake_case", "foo*bar", "foo**bar", "foo__bar"])(
205 "does not predict inside words for %j",
206 (input) => check(input, input),
207 );
208
209 test.each([
210 ["hello *world ", "hello *world* "],
211 ["hello **world ", "hello **world** "],
212 ["hello _world ", "hello _world_ "],
213 ["hello __world ", "hello __world__ "],
214 ])("closes before trailing whitespace for %j", (input, expected) => check(input, expected));
215 });
216
217 describe("soft line breaks", () => {
218 test.each([
219 ["italic continues across single newline", "a *hello\nworld", "a *hello\nworld*"],
220 ["bold continues across single newline", "a **hello\nworld", "a **hello\nworld**"],
221 ["strike continues across single newline", "a ~~hello\nworld", "a ~~hello\nworld~~"],
222 ["code continues across single newline", "a `hello\nworld", "a `hello\nworld`"],
223 ])("%s", (_name, input, expected) => check(input, expected));
224 });
225
226 describe("table prediction", () => {
227 test.each([
228 ["bare pipe is hidden", "|", ""],
229 ["partial header", "| hello", "| hello |\n| - |"],
230 ["single header predicts separator", "| hello |", "| hello |\n| - |"],
231 [
232 "unfinished second header cell closes before separator",
233 "| hello | **world",
234 "| hello | **world** |\n| - | - |",
235 ],
236 ["trailing newline keeps predicted separator", "| hello |\n", "| hello |\n| - |"],
237 ["partial separator line is completed", "| hello |\n|", "| hello |\n| - |"],
238 ["partial hyphen separator is completed", "| hello |\n| -", "| hello |\n| - |"],
239 ["partial left-aligned separator is completed", "| hello |\n| :", "| hello |\n| :- |"],
240 ["partial right-aligned separator is completed", "| hello |\n| -:", "| hello |\n| -: |"],
241 ["partial centered separator is completed", "| hello |\n| :-:", "| hello |\n| :-: |"],
242 ["synthetic separator stays ahead of body rows", "| hello |\n| a", "| hello |\n| - |\n| a"],
243 [
244 "partial multi-column separator is completed",
245 "| hello | world\n| - |",
246 "| hello | world |\n| - | - |",
247 ],
248 ["real separator row stops prediction", "| hello |\n| - |\n| a", "| hello |\n| - |\n| a"],
249 [
250 "complete left-aligned separator stops prediction",
251 "| hello |\n| :- |\n| a",
252 "| hello |\n| :- |\n| a",
253 ],
254 [
255 "complete right-aligned separator stops prediction",
256 "| hello |\n| -: |\n| a",
257 "| hello |\n| -: |\n| a",
258 ],
259 [
260 "complete centered separator stops prediction",
261 "| hello |\n| :-: |\n| a",
262 "| hello |\n| :-: |\n| a",
263 ],
264 ])("%s", (_name, input, expected) => check(input, expected));
265 });
266
267 describe("code span variants", () => {
268 test.each([
269 ["double backtick code span", "a ``hello world", "a ``hello world``"],
270 ["double backtick can contain a single backtick", "a ``hello ` world", "a ``hello ` world``"],
271 ])("%s", (_name, input, expected) => check(input, expected));
272 });
273
274 describe("stuff in code blocks aren't completed", () => {
275 test.each([
276 ["backtick code basic", "hello\n\n```tsx\nconst y = '**bold"],
277 ["tilde code block", "hello\n\n~~~tsx\nconst y = '**bold"],
278
279 ["italic", "hello\n\n```tsx\nconst y = '*italic"],
280 ["strike", "hello\n\n```tsx\nconst y = '~~strike"],
281 ["link 1", "hello\n\n```tsx\nconst y = '[link"],
282 ["link 2", "hello\n\n```tsx\nconst y = '[link]"],
283 ["link 3", "hello\n\n```tsx\nconst y = '[link]("],
284 ["link 4", "hello\n\n```tsx\nconst y = '[link](meow"],
285 ["underscore", "hello\n\n```tsx\nconst y = '__this"],
286 ["inline code", "hello\n\n```tsx\nconst y = `template"],
287
288 ["backtick code extra line", "hello\n\n```tsx\nmeow\n\nconst y = '**bold"],
289 ["tilde code extra line", "hello\n\n~~~tsx\nmeow\n\nconst y = '**bold"],
290
291 ["italic", "hello\n\n```tsx\nmeow\n\nconst y = '*italic"],
292 ["strike", "hello\n\n```tsx\nmeow\n\nconst y = '~~strike"],
293 ["link 1", "hello\n\n```tsx\nmeow\n\nconst y = '[link"],
294 ["link 2", "hello\n\n```tsx\nmeow\n\nconst y = '[link]"],
295 ["link 3", "hello\n\n```tsx\nmeow\n\nconst y = '[link]("],
296 ["link 4", "hello\n\n```tsx\nmeow\n\nconst y = '[link](meow"],
297 ["underscore", "hello\n\n```tsx\nmeow\n\nconst y = '__this"],
298 ["inline code", "hello\n\n```tsx\nmeow\n\nconst y = `template"],
299 ])("%s", (_name, input) => check(input, input));
300 });
301
302 describe("math handling", () => {
303 test.each([
304 ["inline math", "a $x + y", "a $x + y$"],
305 ["inline math suppresses emphasis markers", "a $x * y", "a $x * y$"],
306 [
307 "display math suppresses emphasis markers",
308 "something\n\n$$\na * b",
309 "something\n\n$$\na * b\n$$",
310 ],
311 ])("%s", (_name, input, expected) => check(input, expected));
312 });
313
314 describe("nested emphasis handling", () => {
315 test.each([
316 ["bold containing italic", "one **two *three", "one **two *three***"],
317 ["italic containing bold", "one *two **three", "one *two **three***"],
318 ["bold containing code", "**hello `world", "**hello `world`**"],
319 ["italic containing code", "*hello `world", "*hello `world`*"],
320 ["mixed double underscores inside bold", "one **two __three", "one **two __three__**"],
321 ["mixed bold inside double underscores", "one __two **three", "one __two **three**__"],
322 ["mixed italic underscore inside bold", "one **two _three", "one **two _three_**"],
323 ["mixed bold inside italic underscore", "one _two **three", "one _two **three**_"],
324 ["mixed italic markers", "one *two _three", "one *two _three_*"],
325 ["mixed italic markers reversed", "one _two *three", "one _two *three*_"],
326 ["bold containing code with trailing space", "**hello `world ", "**hello `world `**"],
327 ["italic containing code with trailing space", "*hello `world ", "*hello `world `*"],
328 ["bold containing code opener", "one **two `three", "one **two `three`**"],
329 ["star in code block", "one `two *three", "one `two *three`"],
330 ])("%s", (_name, input, expected) => check(input, expected));
331
332 // these cases unsupported
333 test.each([
334 ["strike inside bold", "one **two ~~three", "one **two ~~three~~"],
335 ["bold inside strike", "one ~~two **three", "one ~~two **three**"],
336 // NOTE: if these close just the last one, its ok to edit the test. that behavior is better
337 ["double underscore containing double underscore", "one __two __three", "one __two __three"],
338 ["bold containing bold", "one **two **three", "one **two **three"],
339 ["italic containing italic", "one *two *three", "one *two *three"],
340 ["strike containing strike", "one ~~two ~~three", "one ~~two ~~three"],
341 ])("%s", (_name, input, expected) => check(input, expected));
342 });
343
344 test.each([
345 ["link 1", "a [hello world", "a hello world"],
346 ["link 2", "a [hello world](incomplete", "a hello world"],
347 ["link 3", "a [hello] tail", "a hello tail"],
348 ["link 4", "a [x](", "a x"],
349 ["link full", "a [hello world](complete)", "a [hello world](complete)"],
350 ["link full 2", "a [hello world](complete) xyz", "a [hello world](complete) xyz"],
351 ["link with parentheses in url", "a [hello](path_(x))", "a [hello](path_(x))"],
352 ["link with title", 'a [hello](url "title")', 'a [hello](url "title")'],
353 ["link with emphasis in label", "a [*hello*](url)", "a [*hello*](url)"],
354 ["link with code in label", "a [`hello`](url)", "a [`hello`](url)"],
355 ["link full then open link", "a [link](done) and [open", "a [link](done) and open"],
356 ["link full then open emphasis", "a [link](done) and *open", "a [link](done) and *open*"],
357 ])("%s", (_name, input, expected) => check(input, expected));
358
359 describe("lossy link edge cases", () => {
360 test.each([
361 ["nested open bracket keeps the outer bracket visible", "[a [b", "[a b"],
362 [
363 "multiple open links keep the earliest unmatched bracket visible",
364 "a [one [two",
365 "a [one two",
366 ],
367 ["unfinished link text can swallow emphasis markers", "[a **b", "a **b"],
368 ["unfinished link text can swallow italic markers", "[a *b", "a *b"],
369 ["unfinished url drops the url entirely", "hello [label](url and more", "hello label"],
370 ["unfinished url with spaces still drops to text", "[x](y z", "x"],
371 ["completed label without url degrades to text", "hello [label] tail", "hello label tail"],
372 ])("%s", (_name, input, expected) => check(input, expected));
373 });
374
375 describe("html fragments", () => {
376 test.each([
377 ["bare angle gets hidden", "<", ""],
378 ["partial tag gets hidden", "<hello", ""],
379 ["complete opening tag passes through", "<hello-element>", "<hello-element>"],
380 ["incomplete self-closing tag gets hidden", "<hello-element /", ""],
381 ["incomplete attribute list gets hidden", '<hello-element attribute="meow"', ""],
382 ["complete self-closing tag passes through", "<hello-element />", "<hello-element />"],
383 ["completed tag followed by bare angle hides", "<hello-element><", ""],
384 ["completed tag followed by partial tag hides", "<hello-element><other-not-checked", ""],
385 ["completed tag followed by partial tag hides it", "<hello-element><hello-element/", ""],
386 [
387 "completed tags pass through together",
388 "<hello-element><hello-element/>",
389 "<hello-element><hello-element/>",
390 ],
391 ["text after a completed tag stays visible", "<hello-element>hello", "<hello-element>hello"],
392 [
393 "text before an incomplete tag stays visible",
394 "<hello-element>hello<",
395 "<hello-element>hello",
396 ],
397 [
398 "text before a completed tag stays visible",
399 "<hello-element>hello<hello-element/>",
400 "<hello-element>hello<hello-element/>",
401 ],
402 [
403 "mismatched completed tag still passes through",
404 "<hello-element>hello<wrong-tag/>",
405 "<hello-element>hello<wrong-tag/>",
406 ],
407 ["comparison text stays visible", "1 < 2", "1 < 2"],
408 ["comparison text with letters stays visible", "a < b", "a < b"],
409 ["markdown still closes before a hidden html tail", "**bold <hello", "**bold** "],
410 ])("%s", (_name, input, expected) => check(input, expected));
411 });
412
413 describe("opener-only inputs", () => {
414 test.each([
415 ["lone italic delimiter", "*", ""],
416 ["lone bold delimiter", "**", ""],
417 ["lone underscore delimiter", "_", ""],
418 ["lone double underscore delimiter", "__", ""],
419 ["lone strike delimiter", "~~", ""],
420 ["lone code delimiter", "`", ""],
421 ["lone inline math delimiter", "$", ""],
422 ["bare open bracket", "[", ""],
423 ["completed label without url at bol", "[x]", "x"],
424 ["bare url opener at bol", "[x](", "x"],
425 ])("%s", (_name, input, expected) => check(input, expected));
426 });
427});