authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-03-20 16:57:18-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-03-20 18:52:07-07:00
log80fb1789cd75eb917d402b010325e5d446b4b997
treee4f8d9ebcbedaa3fb261e323d388d2c680e0c7af
parentc2f5400c509fa622eb21a7c227b2d180fdbbc694
signaturebadge-check Signed by SSH key SHA256:cOKiuRFOeSRxne6EWgHtdQQSlBxjOXm2hOCFnCdLQbQ

fix: html tag prediction was cooked

6 files changed, 68 insertions(+), 68 deletions(-)

src/Memoizer.ts+1-1
......@@ -182,7 +182,7 @@ export class Memoizer {
182182 i !== end;
183183 i += direction
184184 ) {
185 let previousIndex: number = -1;
185 let previousIndex: number = -1;
186186 if (i < blockStart) {
187187 previousIndex = i;
188188 } else if (i >= nextMiddleEnd) {
src/Predict.ts+29-53
......@@ -50,10 +50,8 @@ function parseTail(tail: string): TailState {
5050 pendingDelim: null,
5151 pendingHtml: null,
5252 };
53
5453 for (let i = 0; i < tail.length; i++) {
5554 const char = tail[i]!;
56
5755 if (char === "\n" && tail[i - 1] === "\n") {
5856 resetParagraphState(state);
5957 if (!state.exclusive) state.commitIndex = i + 1;
......@@ -65,31 +63,25 @@ function parseTail(tail: string): TailState {
6563 i = exclusiveEnd - 1;
6664 continue;
6765 }
68
6966 if (state.exclusive) continue;
7067
7168 if (inLinkUrlMode(state)) {
7269 updateLink(state, i, char);
7370 continue;
7471 }
75
7672 const htmlEnd = consumeHtml(state, tail, i);
7773 if (htmlEnd > i) {
7874 i = htmlEnd - 1;
7975 continue;
8076 }
81
8277 const delimEnd = consumeDelim(state, tail, i);
8378 if (delimEnd > i) {
8479 i = delimEnd - 1;
8580 continue;
8681 }
87
8882 if (isEscapedAt(tail, i)) continue;
89
9083 updateLink(state, i, char);
9184 }
92
9385 return state;
9486}
9587
......@@ -105,7 +97,6 @@ function consumeBackticks(state: TailState, tail: string, start: number) {
10597 const length = runLengthAt(tail, start);
10698 const token = "`".repeat(length);
10799 const exclusive = state.exclusive;
108
109100 if (exclusive?.kind === "fence") {
110101 if (
111102 exclusive.token[0] === "`" &&
......@@ -117,22 +108,18 @@ function consumeBackticks(state: TailState, tail: string, start: number) {
117108 }
118109 return start + length;
119110 }
120
121111 if (exclusive?.kind === "code") {
122112 if (!isEscapedAt(tail, start) && length >= exclusive.token.length) {
123113 state.exclusive = null;
124114 }
125115 return start + length;
126116 }
127
128117 if (exclusive) return start + length;
129118 if (isEscapedAt(tail, start)) return start + length;
130
131119 if (length >= 3 && isLineStart(tail, start)) {
132120 state.exclusive = { kind: "fence", start, token };
133121 return start + length;
134122 }
135
136123 state.exclusive = { kind: "code", start, token };
137124 return start + length;
138125}
......@@ -140,17 +127,14 @@ function consumeBackticks(state: TailState, tail: string, start: number) {
140127function consumeMath(state: TailState, tail: string, start: number) {
141128 const length = runLengthAt(tail, start);
142129 const exclusive = state.exclusive;
143
144130 if (exclusive?.kind === "math") {
145131 if (!isEscapedAt(tail, start) && length >= exclusive.token.length) {
146132 state.exclusive = null;
147133 }
148134 return start + length;
149135 }
150
151136 if (exclusive) return start + length;
152137 if (isEscapedAt(tail, start)) return start + length;
153
154138 state.exclusive = {
155139 kind: "math",
156140 start,
......@@ -162,27 +146,22 @@ function consumeMath(state: TailState, tail: string, start: number) {
162146function consumeTildes(state: TailState, tail: string, start: number) {
163147 const length = runLengthAt(tail, start);
164148 const exclusive = state.exclusive;
165
166149 if (exclusive?.kind === "fence" && exclusive.token[0] === "~") {
167150 if (isLineStart(tail, start) && !isEscapedAt(tail, start) && length >= exclusive.token.length) {
168151 state.exclusive = null;
169152 }
170153 return start + length;
171154 }
172
173155 if (exclusive) return start;
174156 if (length < 3 || !isLineStart(tail, start) || isEscapedAt(tail, start)) return start;
175
176157 state.exclusive = { kind: "fence", start, token: "~".repeat(length) };
177158 return start + length;
178159}
179160
180161function consumeHtml(state: TailState, tail: string, start: number) {
181162 if (tail[start] !== "<" || isEscapedAt(tail, start)) return start;
182
183163 const next = tail[start + 1];
184164 if (next !== undefined && !isAsciiLetter(next)) return start;
185
186165 state.pendingHtml = start;
187166 for (let i = start + 1; i < tail.length; i++) {
188167 if (tail[i] === ">" || tail[i] === "\n") {
......@@ -190,47 +169,38 @@ function consumeHtml(state: TailState, tail: string, start: number) {
190169 return i + 1;
191170 }
192171 }
193
194172 return tail.length;
195173}
196174
197175function consumeDelim(state: TailState, tail: string, start: number) {
198176 const token = matchDelim(tail, start);
199177 if (!token) return start;
200
201178 if (isEscapedAt(tail, start)) return start + token.length;
202
203179 const existingIndex = state.delims.findLastIndex((delim) => delim.token === token);
204180 if (existingIndex !== -1) {
205181 state.delims.splice(existingIndex, 1);
206182 return start + token.length;
207183 }
208
209184 if (start + token.length === tail.length) {
210185 state.pendingDelim = { start, token };
211186 return start + token.length;
212187 }
213
214188 if (!canOpenDelim(tail, start, token)) return start + token.length;
215
216189 state.delims.push({ start, token });
217190 return start + token.length;
218191}
219192
220193function matchDelim(tail: string, start: number): DelimToken | undefined {
221194 const char = tail[start];
222
223195 if (char === "*") {
224196 if (tail.startsWith("***", start)) return "***";
225197 if (tail.startsWith("**", start)) return "**";
226198 return "*";
227199 }
228
229200 if (char === "_") {
230201 if (tail.startsWith("__", start)) return "__";
231202 return "_";
232203 }
233
234204 if (char === "~" && tail.startsWith("~~", start)) {
235205 return "~~";
236206 }
......@@ -239,19 +209,16 @@ function matchDelim(tail: string, start: number): DelimToken | undefined {
239209function canOpenDelim(tail: string, start: number, token: DelimToken) {
240210 const next = tail[start + token.length];
241211 if (!next || /\s/.test(next)) return false;
242
243212 const prev = tail[start - 1];
244213 return !isWordChar(prev) || !isWordChar(next);
245214}
246215
247216function updateLink(state: TailState, index: number, char: string) {
248217 const top = state.links.at(-1);
249
250218 if (char === "[") {
251219 state.links.push({ phase: "text", start: index });
252220 return;
253221 }
254
255222 if (char === "]" && top?.phase === "text") {
256223 state.links[state.links.length - 1] = {
257224 phase: "url_wait",
......@@ -260,7 +227,6 @@ function updateLink(state: TailState, index: number, char: string) {
260227 };
261228 return;
262229 }
263
264230 if (char === "(" && top?.phase === "url_wait") {
265231 state.links[state.links.length - 1] = {
266232 phase: "url",
......@@ -270,18 +236,15 @@ function updateLink(state: TailState, index: number, char: string) {
270236 };
271237 return;
272238 }
273
274239 if (char === "(" && top?.phase === "url") {
275240 top.parenDepth += 1;
276241 return;
277242 }
278
279243 if (char === ")" && top?.phase === "url") {
280244 if (top.parenDepth > 0) {
281245 top.parenDepth -= 1;
282246 return;
283247 }
284
285248 state.links.pop();
286249 }
287250}
......@@ -289,10 +252,8 @@ function updateLink(state: TailState, index: number, char: string) {
289252function resetParagraphState(state: TailState) {
290253 state.delims.length = 0;
291254 state.links.length = 0;
292
293255 if (state.exclusive?.kind === "fence") return;
294256 if (state.exclusive?.kind === "math" && state.exclusive.token === "$$") return;
295
296257 state.exclusive = null;
297258}
298259
......@@ -302,22 +263,18 @@ function inLinkUrlMode(state: TailState) {
302263}
303264
304265function renderTail(tail: string, state: TailState) {
305 if (state.pendingHtml !== null) tail = tail.slice(0, state.pendingHtml);
306
266 if (state.pendingHtml !== null)
267 tail = tail.slice(0, findPendingHtmlCutoff(tail, state.pendingHtml));
307268 const link = state.links.at(-1);
308269 if (link) return renderOpenLink(tail, link);
309
310270 const nestedClosers = renderNestedClosers(state);
311271 if (nestedClosers) return appendCloser(tail, nestedClosers);
312
313272 const open = findLastOpen(state);
314273 if (!open) return state.pendingDelim ? tail.slice(0, state.pendingDelim.start) : tail;
315
316274 if (open.kind === "delim") {
317275 if (!hasContentAfter(tail, open.start, open.token.length)) return tail.slice(0, open.start);
318276 return appendCloserBeforeTrailingInlineWhitespace(tail, open.token);
319277 }
320
321278 if (!hasContentAfter(tail, open.start, open.token.length)) {
322279 return open.kind === "fence" ? tail : tail.slice(0, open.start);
323280 }
......@@ -330,12 +287,10 @@ function renderTail(tail: string, state: TailState) {
330287
331288function renderOpenLink(tail: string, state: LinkState) {
332289 const before = tail.slice(0, state.start);
333
334290 if (state.phase === "text") {
335291 if (!hasContentAfter(tail, state.start, 1)) return before;
336292 return before + tail.slice(state.start + 1);
337293 }
338
339294 const text = tail.slice(state.start + 1, state.textEnd);
340295 if (state.phase === "url_wait") return before + text + tail.slice(state.textEnd + 1);
341296 return before + text;
......@@ -343,22 +298,46 @@ function renderOpenLink(tail: string, state: LinkState) {
343298
344299function renderNestedClosers(state: TailState) {
345300 const closers: string[] = [];
346
347301 if (state.exclusive) {
348302 if (state.exclusive.kind !== "code") return undefined;
349303 closers.push(state.exclusive.token);
350304 }
351
352305 for (let i = state.delims.length - 1; i >= 0; i--) {
353306 const token = state.delims[i]!.token;
354307 if (!NESTABLE_DELIMS.has(token)) return undefined;
355308 closers.push(token);
356309 }
357
358310 if (closers.length < 2) return undefined;
359311 return closers.join("");
360312}
361313
314function findPendingHtmlCutoff(tail: string, pendingStart: number) {
315 let cutoff = pendingStart;
316 let cursor = pendingStart;
317
318 while (cursor > 0) {
319 const candidateStart = tail.lastIndexOf("<", cursor - 1);
320 if (candidateStart === -1) break;
321 if (!isCompleteHtmlTag(tail, candidateStart, cursor)) break;
322 cutoff = candidateStart;
323 cursor = candidateStart;
324 }
325
326 return cutoff;
327}
328
329function isCompleteHtmlTag(tail: string, start: number, end: number) {
330 if (tail[end - 1] !== ">") return false;
331 if (isEscapedAt(tail, start)) return false;
332 const next = tail[start + 1];
333 if (next !== undefined && !isAsciiLetter(next)) return false;
334 for (let i = start + 1; i < end - 1; i++) {
335 const char = tail[i];
336 if (char === ">" || char === "\n") return false;
337 }
338 return true;
339}
340
362341function appendCloser(tail: string, closer: string) {
363342 return tail + closer.slice(overlapLength(tail, closer));
364343}
......@@ -366,7 +345,6 @@ function appendCloser(tail: string, closer: string) {
366345function appendCloserBeforeTrailingInlineWhitespace(tail: string, closer: string) {
367346 const trailingWhitespace = tail.match(/[^\S\n]+$/)?.[0];
368347 if (!trailingWhitespace) return appendCloser(tail, closer);
369
370348 const body = tail.slice(0, -trailingWhitespace.length);
371349 return appendCloser(body, closer) + trailingWhitespace;
372350}
......@@ -401,12 +379,10 @@ function isLineStart(text: string, index: number) {
401379
402380function isEscapedAt(text: string, index: number) {
403381 let count = 0;
404
405382 for (let i = index - 1; i >= 0; i--) {
406383 if (text[i] !== "\\") break;
407384 count += 1;
408385 }
409
410386 return count % 2 === 1;
411387}
412388
src/hast.ts+29-6
......@@ -61,14 +61,31 @@ function recursiveMemoRender(
6161 const nextChildren: RenderState[] = [];
6262 const children: ReactNode[] = [];
6363 const countsByName = new Map<string, number>();
64 const usedKeys = new Set(previousChildren.map((child) => child.key).filter((key) => key !== undefined));
64 const usedKeys = new Set(
65 previousChildren.map((child) => child.key).filter((key) => key !== undefined),
66 );
6567 let prefixLength = 0;
66 for (; prefixLength < node.children.length && prefixLength < previousChildren.length; prefixLength += 1) {
68 for (
69 ;
70 prefixLength < node.children.length && prefixLength < previousChildren.length;
71 prefixLength += 1
72 ) {
6773 if (!nodeDeepEquals(node.children[prefixLength], previousChildren[prefixLength]?.node)) break;
6874 }
6975 let suffixLength = 0;
70 for (; suffixLength < node.children.length - prefixLength && suffixLength < previousChildren.length - prefixLength; suffixLength += 1) {
71 if (!nodeDeepEquals(node.children[node.children.length - 1 - suffixLength], previousChildren[previousChildren.length - 1 - suffixLength]?.node)) break;
76 for (
77 ;
78 suffixLength < node.children.length - prefixLength &&
79 suffixLength < previousChildren.length - prefixLength;
80 suffixLength += 1
81 ) {
82 if (
83 !nodeDeepEquals(
84 node.children[node.children.length - 1 - suffixLength],
85 previousChildren[previousChildren.length - 1 - suffixLength]?.node,
86 )
87 )
88 break;
7289 }
7390
7491 for (let i = 0; i < node.children.length; i += 1) {
......@@ -85,9 +102,15 @@ function recursiveMemoRender(
85102 previousIndex = previousChildren.length - (node.children.length - i);
86103 }
87104 if (previousIndex >= 0) childKey = previousChildren[previousIndex]?.key ?? childKey;
88 while (childKey && usedKeys.has(childKey) && previousChildren[previousIndex]?.key !== childKey) childKey = `${childKey}+`;
105 while (childKey && usedKeys.has(childKey) && previousChildren[previousIndex]?.key !== childKey)
106 childKey = `${childKey}+`;
89107 if (childKey) usedKeys.add(childKey);
90 const childState = recursiveMemoRender(child, previousChildren[previousIndex] ?? null, options, childKey);
108 const childState = recursiveMemoRender(
109 child,
110 previousChildren[previousIndex] ?? null,
111 options,
112 childKey,
113 );
91114 nextChildren.push(childState);
92115 if (childState.react !== undefined) children.push(childState.react);
93116 }
tests/Memoizer.test.tsx+3-1
......@@ -31,7 +31,9 @@ const documentStateProcessor = unified()
3131 ?.slice("suffix: ".length);
3232 if (!suffix) return tree;
3333
34 const target = paragraphs.find((paragraph) => !paragraphText(paragraph).startsWith("suffix: "));
34 const target = paragraphs.find(
35 (paragraph) => !paragraphText(paragraph).startsWith("suffix: "),
36 );
3537 const firstChild = target?.children[0];
3638 if (firstChild?.type === "text") {
3739 const textNode = firstChild as Literal;
tests/memoizedHastToReact.test.tsx+3-1
......@@ -185,7 +185,9 @@ it("reuses an equal subtree when a sibling of another tag is inserted above it",
185185});
186186
187187it("preserves an unchanged list item when a sibling item is inserted above it", () => {
188 const first = renderTree(root(element("ul", [element("li", [text("stable")]), element("li", [text("tail")])])));
188 const first = renderTree(
189 root(element("ul", [element("li", [text("stable")]), element("li", [text("tail")])])),
190 );
189191 const firstList = childAt(first.react, 0);
190192 const firstStable = childAt(firstList, 0);
191193
tests/prediction.test.ts+3-6
......@@ -339,12 +339,9 @@ describe.each(["stateless", "stateful"] as const)("%s processing", (mode) => {
339339 ["incomplete self-closing tag gets hidden", "<hello-element /", ""],
340340 ["incomplete attribute list gets hidden", '<hello-element attribute="meow"', ""],
341341 ["complete self-closing tag passes through", "<hello-element />", "<hello-element />"],
342 ["completed tag followed by bare angle hides the tail", "<hello-element><", "<hello-element>"],
343 [
344 "completed tag followed by partial tag hides the tail",
345 "<hello-element><hello-element/",
346 "<hello-element>",
347 ],
342 ["completed tag followed by bare angle hides", "<hello-element><", ""],
343 ["completed tag followed by partial tag hides", "<hello-element><other-not-checked", ""],
344 ["completed tag followed by partial tag hides it", "<hello-element><hello-element/", ""],
348345 [
349346 "completed tags pass through together",
350347 "<hello-element><hello-element/>",