From a5ec1eef9bc66643cc4a597bbcac83d7b713bd3d Mon Sep 17 00:00:00 2001 From: clover caruso Date: Thu, 19 Mar 2026 14:23:28 -0700 Subject: [PATCH] feat: prediction --- README.md | 8 +- TODO.md | 12 - example/App.tsx | 153 +++++++++-- example/index.css | 2 +- example/markdown-components.tsx | 6 +- example/markdown-demo.ts | 3 +- package.json | 4 +- pnpm-lock.yaml | 27 ++ src/Markdown.ts | 25 +- src/Memoizer.ts | 17 +- src/Prediction.ts | 376 ++++++++++++++++++++++++++++ src/strings.ts | 5 - tests/Markdown.memoization.test.tsx | 19 +- tests/prediction.test.ts | 343 +++++++++++++++++++++++++ 14 files changed, 935 insertions(+), 65 deletions(-) delete mode 100644 TODO.md create mode 100644 src/Prediction.ts delete mode 100644 src/strings.ts create mode 100644 tests/prediction.test.ts diff --git a/README.md b/README.md index 88e5617e6c6b87aecab533240f3a76c0a8726bb7..7ec87946dbe6c31454673b58c26f746d009d408a 100644 --- a/README.md +++ b/README.md @@ -12,10 +12,8 @@ such as LLM chat interfaces. you can get started with just ``. Configuration is done either with a provider, or right at the component level. - **Handle Partial Markdown**: When the `predict` prop is set, sequences like - `hello **world` will be emitted as `hello world`. - + `hello **world` will be emitted as `hello world`. This + behavior can be used standlone via the `@clo/react-markdown/Predict` class. The motivation for this package is to have an easy to understand version of [Streamdown]. With me banning all Vercel software at the company I work at, @@ -29,7 +27,7 @@ to Streamdown: - **Headless UI**: No built in styles or components, bring your own CSS to blend your markdown with your existing theme. `@clo/react-markdown` simply takes in your existing `unified` pipeline and works off of that. -- **Easy to Audit**: Under 450 lines of precise, highly commented TypeScript. +- **Easy to Audit**: Under 700 lines of precise, highly commented TypeScript. [Streamdown]: https://streamdown.ai [unified]: https://unifiedjs.com/ diff --git a/TODO.md b/TODO.md deleted file mode 100644 index 4175707526b90794d8dd11f382e45bfba89de1f4..0000000000000000000000000000000000000000 --- a/TODO.md +++ /dev/null @@ -1,12 +0,0 @@ -- readme claims are not true - - no `predict` implementation - - bold `**text` - - italic `*text` or `_text` - - bold italic `***text` - - inline code `` `code`` - - strike `~~text` - - link `[hello` or `[hello](world` - - block katex `$$\nE = m` - - custom handling maybe -- animate in paragraph text -- smoothly interpolate to handle bumpy connections diff --git a/example/App.tsx b/example/App.tsx index 667260df80b6b60cb0fe2a0cf23c1a4caa23bed4..26451b9801ddb5962a466d28494e24c86f487d50 100644 --- a/example/App.tsx +++ b/example/App.tsx @@ -1,32 +1,112 @@ -import { useDeferredValue, useState } from "react"; +import { type ReactNode, useDeferredValue, useId, useState } from "react"; +import ReactMarkdown from "react-markdown"; import { Streamdown } from "streamdown"; import { Markdown } from "../src/Markdown.ts"; -import { components, streamdownComponents } from "./markdown-components"; -import { initialMarkdown, processor } from "./markdown-demo"; +import { + components, + reactMarkdownComponents, + streamdownComponents, +} from "./markdown-components"; +import { initialMarkdown, processor, reactMarkdownRemarkPlugins } from "./markdown-demo"; + +type Renderer = "memo" | "streamdown" | "react-markdown"; -type Renderer = "memo" | "streamdown"; const rendererOptions: Array<{ label: string; value: Renderer }> = [ - { label: "MemoMarkdown", value: "memo" }, - { label: "Streamdown", value: "streamdown" }, + { + label: "@clo/react-markdown", + value: "memo", + }, + { + label: "streamdown", + value: "streamdown", + }, + { + label: "react-markdown", + value: "react-markdown", + }, ]; +type ToggleSwitchProps = { + checked: boolean; + label: string; + onChange: (checked: boolean) => void; +}; + +function ToggleSwitch({ checked, label, onChange }: ToggleSwitchProps) { + const id = useId(); + + return ( + + ); +} + function getRendererButtonClass(isActive: boolean) { return isActive ? "rounded bg-white px-2.5 py-1 text-xs font-medium text-stone-900 shadow-sm" : "rounded px-2.5 py-1 text-xs text-stone-600 hover:text-stone-900"; } +type PreviewPanelProps = { + children: ReactNode; + header: ReactNode; + toolbar: ReactNode; +}; + +function PreviewPanel({ children, header, toolbar }: PreviewPanelProps) { + return ( +
+
+ {header} +
{toolbar}
+
+
{children}
+
+ ); +} + export function App() { const [content, setContent] = useState(initialMarkdown); const deferredContent = useDeferredValue(content); + const [predictionEnabled, setPredictionEnabled] = useState(true); const [renderer, setRenderer] = useState("memo"); + const previewToolbar = + renderer === "memo" ? ( + + ) : renderer === "streamdown" ? ( + + ) : ( + + No prediction support + + ); + return ( -
-
-
-
-

Source

+
+
+
+
+
-
-
-

Preview

+ ))}
-
-
- {renderer === "memo" ? ( -
- -
- ) : ( - + } + toolbar={ + previewToolbar + } + > + {renderer === "memo" ? ( +
+ +
+ ) : renderer === "streamdown" ? ( + + {deferredContent} + + ) : ( +
+ {deferredContent} - - )} -
-
+ +
+ )} +
); diff --git a/example/index.css b/example/index.css index ef4c9102b1a4cc0883c30b6083c16cfc08f773cc..08bab9f2c62e2f0d4421b2da49c8ff778de7b150 100644 --- a/example/index.css +++ b/example/index.css @@ -20,7 +20,7 @@ body { .preview-prose { @apply prose prose-stone max-w-none; - @apply prose-headings:text-stone-900 prose-p:text-stone-700 + @apply prose-headings:mt-0 prose-headings:text-stone-900 prose-p:text-stone-700 prose-li:text-stone-700; @apply prose-a:text-blue-700; @apply prose-pre:overflow-x-auto; diff --git a/example/markdown-components.tsx b/example/markdown-components.tsx index 10f0bc3268c66452defd0e9c5cdce7ecff6e01e8..2c10a2d9b2b4dbf8f562eb4311627b6c417342b1 100644 --- a/example/markdown-components.tsx +++ b/example/markdown-components.tsx @@ -1,5 +1,6 @@ -import { type ComponentPropsWithoutRef, type JSX, memo } from "react"; +import { type ComponentPropsWithoutRef, type JSX } from "react"; import { type Components as MemoComponents } from "rehype-react"; +import { type Components as ReactMarkdownComponents } from "react-markdown"; import { type Components as StreamdownComponents } from "streamdown"; type MarkdownProps = ComponentPropsWithoutRef & { @@ -134,7 +135,7 @@ export const components: Partial = { input: MarkdownInput, li: MarkdownListItem, ol: MarkdownOrderedList, - p: memo(MarkdownParagraph), + p: MarkdownParagraph, pre: MarkdownPre, strong: MarkdownStrong, table: MarkdownTable, @@ -146,4 +147,5 @@ export const components: Partial = { ul: MarkdownUnorderedList, }; +export const reactMarkdownComponents = components as ReactMarkdownComponents; export const streamdownComponents = components as StreamdownComponents; diff --git a/example/markdown-demo.ts b/example/markdown-demo.ts index 73060e2b799be418158fc0aad1e6505ba63edf70..747e5cb56da01a01f4bc18e89494d6b8df7b18cd 100644 --- a/example/markdown-demo.ts +++ b/example/markdown-demo.ts @@ -1,7 +1,8 @@ import remarkGfm from "remark-gfm"; import remarkParse from "remark-parse"; -import { unified } from "unified"; +import { type PluggableList, unified } from "unified"; import projectReadme from "../README.md?raw"; export const initialMarkdown = projectReadme; export const processor = unified().use(remarkParse).use(remarkGfm); +export const reactMarkdownRemarkPlugins: PluggableList = [remarkGfm]; diff --git a/package.json b/package.json index 0381a1c145dd3f65626702120015f5cf1b2b7000..3b787004fabb3fe4fa8b0b647985f494f7ca5cf2 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,8 @@ ], "type": "module", "exports": { - ".": "./src/Markdown.ts" + ".": "./src/Markdown.ts", + "./Predict": "./src/Predict.ts" }, "publishConfig": { "access": "public" @@ -35,6 +36,7 @@ "hast-util-to-jsx-runtime": "2.3.6", "react": "^19.2.4", "react-dom": "^19.2.4", + "react-markdown": "^10.1.0", "react-scan": "^0.5.3", "rehype-react": "^8.0.0", "remark-gfm": "^4.0.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e6a5d56df5d880cdd13c234b65acc1aa08f6b3fd..d3947033f856f0d471901ec6d3a9930d5d0a28cb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -58,6 +58,9 @@ importers: react-dom: specifier: ^19.2.4 version: 19.2.4(react@19.2.4) + react-markdown: + specifier: ^10.1.0 + version: 10.1.0(@types/react@19.2.14)(react@19.2.4) react-scan: specifier: ^0.5.3 version: 0.5.3(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -2033,6 +2036,12 @@ packages: react-is@17.0.2: resolution: {integrity: sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==} + react-markdown@10.1.0: + resolution: {integrity: sha512-qKxVopLT/TyA6BX3Ue5NwabOsAzm0Q7kAPwq6L+wWDwisYs7R8vZ0nRXqq6rkueboxpkjvLGU9fWifiX/ZZFxQ==} + peerDependencies: + '@types/react': '>=18' + react: '>=18' + react-scan@0.5.3: resolution: {integrity: sha512-qde9PupmUf0L3MU1H6bjmoukZNbCXdMyTEwP4Gh8RQ4rZPd2GGNBgEKWszwLm96E8k+sGtMpc0B9P0KyFDP6Bw==} hasBin: true @@ -4316,6 +4325,24 @@ snapshots: react-is@17.0.2: {} + react-markdown@10.1.0(@types/react@19.2.14)(react@19.2.4): + dependencies: + '@types/hast': 3.0.4 + '@types/mdast': 4.0.4 + '@types/react': 19.2.14 + devlop: 1.1.0 + hast-util-to-jsx-runtime: 2.3.6 + html-url-attributes: 3.0.1 + mdast-util-to-hast: 13.2.1 + react: 19.2.4 + remark-parse: 11.0.0 + remark-rehype: 11.1.2 + unified: 11.0.5 + unist-util-visit: 5.1.0 + vfile: 6.0.3 + transitivePeerDependencies: + - supports-color + react-scan@0.5.3(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4): dependencies: '@babel/core': 7.29.0 diff --git a/src/Markdown.ts b/src/Markdown.ts index 02708a2ebfd954522048fe4dd1398be2800b977c..9edfe3f7e636e4662c991a767e0af2d0fed57107 100644 --- a/src/Markdown.ts +++ b/src/Markdown.ts @@ -4,6 +4,7 @@ import { type Processor } from "unified"; import { type Components } from "rehype-react"; import { jsx } from "react/jsx-runtime"; +/** Options for {@linkcode Markdown} */ export interface MarkdownOptions { /** Markdown source content. */ content: string; @@ -11,6 +12,8 @@ export interface MarkdownOptions { processor?: Processor | null | undefined; /** Customize component rendering. */ components?: Partial | null | undefined; + /** Automatically predict closing tags, useful for LLM streaming. */ + predict?: boolean; } /** @@ -23,14 +26,19 @@ export const Markdown = memo(function Markdown({ content, processor, components, + predict, }: MarkdownOptions) { const context = useContext(Context); const ref = useRef(null); const memoizer = (ref.current ??= new Memoizer()); - memoizer.reconfigure(processor ?? context.processor ?? defaultProcessor, { - ...components, - ...context.components, - }); + memoizer.reconfigure( + processor ?? context.processor ?? defaultProcessor, + predict ?? context.predict ?? false, + { + ...components, + ...context.components, + }, + ); return memoizer.update(content); }, propsAreEqual); @@ -45,17 +53,22 @@ export const MarkdownOptionsProvider = memo(function MarkdownOptionsProvider({ processor, components, children, + predict, }: PropsWithChildren) { - const value = useMemo(() => ({ processor, components }), [processor, components]); + const value = useMemo( + () => ({ processor, components, predict }), + [processor, components, predict], + ); return jsx(Context.Provider, { value, children }); }, propsAreEqual); function propsAreEqual(prev: T, next: T) { if (prev.content !== next.content) return false; if (prev.processor !== next.processor) return false; + if (prev.predict !== next.predict) return false; if (prev.components === next.components) return true; if (prev.components && next.components) { return componentsAreEqual(prev.components, next.components); } - return false; + return !prev.components && !next.components; } diff --git a/src/Memoizer.ts b/src/Memoizer.ts index 06af8be568ff96d80ae92bfc3075dcfba050ea1d..e2c909324961a28b86832e8afbc21a3684715709 100644 --- a/src/Memoizer.ts +++ b/src/Memoizer.ts @@ -1,5 +1,4 @@ import { ASSERT, UNWRAP } from "@clo/lib/assert.ts"; -import { indexOfDiff } from "./strings.ts"; import { type JSX, type Key, memo, type ReactNode } from "react"; import { type Components } from "rehype-react"; import remarkRehype from "remark-rehype"; @@ -9,6 +8,7 @@ import type { Root as HastRoot } from "hast"; import type { Literal, Node, Parent } from "unist"; import { memoizedHastToReact, type RenderState } from "./hast.ts"; import remarkParse from "remark-parse"; +import { Predict } from "./Prediction.ts"; export const defaultProcessor = unified().use(remarkParse); @@ -35,6 +35,7 @@ export class Memoizer { #previousComponents: Partial = {}; // incremental parsing graph, structure of arrays + #predict: Predict | null = null; #content: string = ""; #positions: number[] = []; #parsed: Node[][] = []; @@ -43,12 +44,14 @@ export class Memoizer { reconfigure( processor: Processor, + predict: boolean, components: Partial, ) { if ( this.#astProcessor !== null && this.#baseProcessor === processor && - componentsAreEqual(this.#previousComponents, components) + componentsAreEqual(this.#previousComponents, components) && + !!this.#predict === predict ) { return; } @@ -73,10 +76,12 @@ export class Memoizer { this.#parsed = []; this.#renderStates = []; this.#reactNodes = []; + this.#predict = predict ? new Predict() : null; } update(content: string): readonly ReactNode[] { - const previous = this.#content; + if (this.#predict) content = this.#predict.update(content); + let previous = this.#content; if (content === previous) return this.#reactNodes; // Locate the last block that has changed content. This is done quickly by finding @@ -166,6 +171,12 @@ export class Memoizer { } } +export function indexOfDiff(a: string, b: string) { + var i = 0; + while (a[i] === b[i]) i++; + return i; +} + export function componentsAreEqual(left: Partial, right: Partial) { const leftRecord = left as Record; const rightRecord = right as Record; diff --git a/src/Prediction.ts b/src/Prediction.ts new file mode 100644 index 0000000000000000000000000000000000000000..8ab28aeab2a6d55b9498980c4248841813457bce --- /dev/null +++ b/src/Prediction.ts @@ -0,0 +1,376 @@ +/** + * Incremental parser for predicting closing tags in Markdown text. Works best + * when continuously appending text to the end of the string; this behavior + * specifically triggers a fast parsing path. + */ +export class Predict { + #stable = ""; + + /** Incrementally reparses the tail. */ + update(text: string) { + if (!text.startsWith(this.#stable)) this.#stable = ""; + let tail = text.slice(this.#stable.length); + let state = parseTail(tail); + if (state.commitIndex > 0) { + this.#stable += tail.slice(0, state.commitIndex); + tail = tail.slice(state.commitIndex); + state = parseTail(tail); + } + return this.#stable + renderTail(tail, state); + } +} + +type DelimToken = "***" | "**" | "__" | "~~" | "*" | "_"; +type DelimState = { start: number; token: DelimToken }; +type LinkState = + | { phase: "text"; start: number } + | { phase: "url_wait"; start: number; textEnd: number } + | { phase: "url"; start: number; textEnd: number; parenDepth: number }; +type ExclusiveState = + | { kind: "code"; start: number; token: string } + | { kind: "fence"; start: number; token: string } + | { kind: "math"; start: number; token: "$" | "$$" }; +interface TailState { + commitIndex: number; + delims: DelimState[]; + exclusive: ExclusiveState | null; + links: LinkState[]; +}; + +const NESTABLE_DELIMS = new Set(["*", "**", "_", "__"]); + +function parseTail(tail: string): TailState { + const state: TailState = { + commitIndex: 0, + delims: [], + exclusive: null, + links: [], + }; + + for (let i = 0; i < tail.length; i++) { + const char = tail[i]!; + + if (char === "\n" && tail[i - 1] === "\n") { + resetParagraphState(state); + if (!state.exclusive) state.commitIndex = i + 1; + continue; + } + + const exclusiveEnd = consumeExclusive(state, tail, i); + if (exclusiveEnd > i) { + i = exclusiveEnd - 1; + continue; + } + + if (state.exclusive) continue; + + if (inLinkUrlMode(state)) { + updateLink(state, i, char); + continue; + } + + const delimEnd = consumeDelim(state, tail, i); + if (delimEnd > i) { + i = delimEnd - 1; + continue; + } + + if (isEscapedAt(tail, i)) continue; + + updateLink(state, i, char); + } + + return state; +} + +function consumeExclusive(state: TailState, tail: string, start: number) { + const char = tail[start]; + if (char === "`") return consumeBackticks(state, tail, start); + if (char === "$") return consumeMath(state, tail, start); + if (char === "~") return consumeTildes(state, tail, start); + return start; +} + +function consumeBackticks(state: TailState, tail: string, start: number) { + const length = runLengthAt(tail, start); + const token = "`".repeat(length); + const exclusive = state.exclusive; + + if (exclusive?.kind === "fence") { + if ( + exclusive.token[0] === "`" && + isLineStart(tail, start) && + !isEscapedAt(tail, start) && + length >= exclusive.token.length + ) { + state.exclusive = null; + } + return start + length; + } + + if (exclusive?.kind === "code") { + if (!isEscapedAt(tail, start) && length >= exclusive.token.length) { + state.exclusive = null; + } + return start + length; + } + + if (exclusive) return start + length; + if (isEscapedAt(tail, start)) return start + length; + + if (length >= 3 && isLineStart(tail, start)) { + state.exclusive = { kind: "fence", start, token }; + return start + length; + } + + state.exclusive = { kind: "code", start, token }; + return start + length; +} + +function consumeMath(state: TailState, tail: string, start: number) { + const length = runLengthAt(tail, start); + const exclusive = state.exclusive; + + if (exclusive?.kind === "math") { + if (!isEscapedAt(tail, start) && length >= exclusive.token.length) { + state.exclusive = null; + } + return start + length; + } + + if (exclusive) return start + length; + if (isEscapedAt(tail, start)) return start + length; + + state.exclusive = { + kind: "math", + start, + token: length >= 2 ? "$$" : "$", + }; + return start + length; +} + +function consumeTildes(state: TailState, tail: string, start: number) { + const length = runLengthAt(tail, start); + const exclusive = state.exclusive; + + if (exclusive?.kind === "fence" && exclusive.token[0] === "~") { + if ( + isLineStart(tail, start) && + !isEscapedAt(tail, start) && + length >= exclusive.token.length + ) { + state.exclusive = null; + } + return start + length; + } + + if (exclusive) return start; + if (length < 3 || !isLineStart(tail, start) || isEscapedAt(tail, start)) return start; + + state.exclusive = { kind: "fence", start, token: "~".repeat(length) }; + return start + length; +} + +function consumeDelim(state: TailState, tail: string, start: number) { + const token = matchDelim(tail, start); + if (!token) return start; + + if (isEscapedAt(tail, start)) return start + token.length; + + const existingIndex = state.delims.findLastIndex((delim) => delim.token === token); + if (existingIndex !== -1) { + state.delims.splice(existingIndex, 1); + return start + token.length; + } + + if (!canOpenDelim(tail, start, token)) return start + token.length; + + state.delims.push({ start, token }); + return start + token.length; +} + +function matchDelim(tail: string, start: number): DelimToken | undefined { + const char = tail[start]; + + if (char === "*") { + if (tail.startsWith("***", start)) return "***"; + if (tail.startsWith("**", start)) return "**"; + return "*"; + } + + if (char === "_") { + if (tail.startsWith("__", start)) return "__"; + return "_"; + } + + if (char === "~" && tail.startsWith("~~", start)) { + return "~~"; + } +} + +function canOpenDelim(tail: string, start: number, token: DelimToken) { + const next = tail[start + token.length]; + if (!next || /\s/.test(next)) return false; + + const prev = tail[start - 1]; + return !isWordChar(prev) || !isWordChar(next); +} + +function updateLink(state: TailState, index: number, char: string) { + const top = state.links.at(-1); + + if (char === "[") { + state.links.push({ phase: "text", start: index }); + return; + } + + if (char === "]" && top?.phase === "text") { + state.links[state.links.length - 1] = { + phase: "url_wait", + start: top.start, + textEnd: index, + }; + return; + } + + if (char === "(" && top?.phase === "url_wait") { + state.links[state.links.length - 1] = { + phase: "url", + start: top.start, + textEnd: top.textEnd, + parenDepth: 0, + }; + return; + } + + if (char === "(" && top?.phase === "url") { + top.parenDepth += 1; + return; + } + + if (char === ")" && top?.phase === "url") { + if (top.parenDepth > 0) { + top.parenDepth -= 1; + return; + } + + state.links.pop(); + } +} + +function resetParagraphState(state: TailState) { + state.delims.length = 0; + state.links.length = 0; + + if (state.exclusive?.kind === "fence") return; + if (state.exclusive?.kind === "math" && state.exclusive.token === "$$") return; + + state.exclusive = null; +} + +function inLinkUrlMode(state: TailState) { + const phase = state.links.at(-1)?.phase; + return phase === "url_wait" || phase === "url"; +} + +function renderTail(tail: string, state: TailState) { + const link = state.links.at(-1); + if (link) return renderOpenLink(tail, link); + + const nestedClosers = renderNestedClosers(state); + if (nestedClosers) return appendCloser(tail, nestedClosers); + + const open = findLastOpen(state); + if (!open) return tail; + + if (open.kind === "delim") { + if (!hasContentAfter(tail, open.start, open.token.length)) return tail; + if (/\s/.test(tail[tail.length - 1] ?? "")) return tail; + return appendCloser(tail, open.token); + } + + if (!hasContentAfter(tail, open.start, open.token.length)) return tail; + if (open.kind === "fence") return tail; + if (open.kind === "code") return appendCloser(tail, open.token); + if (open.token === "$$") return appendCloser(tail, (tail.endsWith("\n") ? "" : "\n") + "$$"); + if (/\s/.test(tail[tail.length - 1] ?? "")) return tail; + return appendCloser(tail, "$"); +} + +function renderOpenLink(tail: string, state: LinkState) { + const before = tail.slice(0, state.start); + + if (state.phase === "text") { + if (!hasContentAfter(tail, state.start, 1)) return tail; + return before + tail.slice(state.start + 1); + } + + const text = tail.slice(state.start + 1, state.textEnd); + if (state.phase === "url_wait") return before + text + tail.slice(state.textEnd + 1); + return before + text; +} + +function renderNestedClosers(state: TailState) { + const closers: string[] = []; + + if (state.exclusive) { + if (state.exclusive.kind !== "code") return undefined; + closers.push(state.exclusive.token); + } + + for (let i = state.delims.length - 1; i >= 0; i--) { + const token = state.delims[i]!.token; + if (!NESTABLE_DELIMS.has(token)) return undefined; + closers.push(token); + } + + if (closers.length < 2) return undefined; + return closers.join(""); +} + +function appendCloser(tail: string, closer: string) { + return tail + closer.slice(overlapLength(tail, closer)); +} + +function overlapLength(tail: string, closer: string) { + for (let length = Math.min(tail.length, closer.length); length > 0; length -= 1) { + if (tail.endsWith(closer.slice(0, length))) return length; + } + return 0; +} + +function findLastOpen(state: TailState) { + const delim = state.delims.at(-1); + if (state.exclusive && (!delim || state.exclusive.start > delim.start)) return state.exclusive; + if (delim) return { kind: "delim" as const, start: delim.start, token: delim.token }; + return state.exclusive; +} + +function runLengthAt(text: string, start: number) { + let end = start + 1; + while (end < text.length && text[end] === text[start]) end += 1; + return end - start; +} + +function hasContentAfter(text: string, start: number, tokenLength: number) { + return text.length > start + tokenLength; +} + +function isLineStart(text: string, index: number) { + return index === 0 || text[index - 1] === "\n"; +} + +function isEscapedAt(text: string, index: number) { + let count = 0; + + for (let i = index - 1; i >= 0; i--) { + if (text[i] !== "\\") break; + count += 1; + } + + return count % 2 === 1; +} + +function isWordChar(char?: string) { + return !!char && /[A-Za-z0-9]/.test(char); +} diff --git a/src/strings.ts b/src/strings.ts deleted file mode 100644 index 33e7eeee50d6a4c1f6f5643028ff8ef29db6ffac..0000000000000000000000000000000000000000 --- a/src/strings.ts +++ /dev/null @@ -1,5 +0,0 @@ -export function indexOfDiff(a: string, b: string) { - var i = 0; - while (a[i] === b[i]) i++; - return i; -} diff --git a/tests/Markdown.memoization.test.tsx b/tests/Markdown.memoization.test.tsx index 1a5fcadd3b0b75ad07c8d659a87946289a900f3d..47bf06fdf50ea290879c84bef9fe6ce7a2f2f85b 100644 --- a/tests/Markdown.memoization.test.tsx +++ b/tests/Markdown.memoization.test.tsx @@ -1,7 +1,7 @@ import { cleanup, render, screen } from "@testing-library/react"; import type { Components } from "rehype-react"; import type { ComponentPropsWithoutRef, JSX } from "react"; -import { afterEach, describe, expect, it, vi } from "vite-plus/test"; +import { afterEach, expect, it, vi } from "vite-plus/test"; import { Markdown, MarkdownOptionsProvider } from "../src/Markdown.ts"; import { Memoizer } from "../src/Memoizer.ts"; @@ -107,7 +107,7 @@ it("does not rerender an unchanged inline custom renderer when only adjacent tex expect(Link).toHaveBeenCalledTimes(1); }); -it.only("does not rerender an unchanged inline custom renderer with formatted children when only adjacent text changes", () => { +it("does not rerender an unchanged inline custom renderer with formatted children when only adjacent text changes", () => { const Link = vi.fn(function Link(props: MarkdownProps<"a">) { return ; }); @@ -154,3 +154,18 @@ it("does not rerender Markdown through the provider when the effective component expect(Link).toHaveBeenCalledTimes(1); expect(screen.getByTestId("provider-link").textContent).toBe("stable"); }); + +it("reprocesses the document when prediction changes", () => { + const { container, rerender } = render( + , + renderOptions, + ); + + expect(container.querySelector("em")).toBeNull(); + expect(container.textContent).toBe("hello *world"); + + rerender(); + + expect(container.querySelector("em")?.textContent).toBe("world"); + expect(container.textContent).toBe("hello world"); +}); diff --git a/tests/prediction.test.ts b/tests/prediction.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..ddec48eab62e7b8d0f095b5aa18ab030255df8ca --- /dev/null +++ b/tests/prediction.test.ts @@ -0,0 +1,343 @@ +import { test, describe, expect } from "vite-plus/test"; +import { Predict } from "../src/Prediction.ts"; + +describe.each(["stateless", "stateful"] as const)("%s processing", (mode) => { + function stateless(text: string) { + return new Predict().update(text); + } + + function splitBySizes(text: string, sizes: number[]) { + const chunks: string[] = []; + let index = 0; + + for (const size of sizes) { + if (index >= text.length) break; + + const nextIndex = Math.min(index + size, text.length); + chunks.push(text.slice(index, nextIndex)); + index = nextIndex; + } + + if (index < text.length) { + chunks.push(text.slice(index)); + } + + return chunks; + } + + function addChunking(text: string, sizes: number[], seen: Set, chunkings: string[][]) { + const chunks = splitBySizes(text, sizes); + const key = JSON.stringify(chunks); + + if (seen.has(key)) return; + seen.add(key); + chunkings.push(chunks); + } + + function buildChunkings(text: string) { + if (text.length === 0) return [[]]; + + const seen = new Set(); + const chunkings: string[][] = []; + + if (text.length <= 10) { + const splitCount = text.length - 1; + const totalMasks = 1 << splitCount; + + for (let mask = 0; mask < totalMasks; mask++) { + const sizes: number[] = []; + let last = 0; + + for (let bit = 0; bit < splitCount; bit++) { + if ((mask & (1 << bit)) === 0) continue; + + const next = bit + 1; + sizes.push(next - last); + last = next; + } + + sizes.push(text.length - last); + addChunking(text, sizes, seen, chunkings); + } + + return chunkings; + } + + for (let size = 1; size <= text.length; size++) { + const sizes: number[] = []; + + for (let remaining = text.length; remaining > 0; remaining -= size) { + sizes.push(Math.min(size, remaining)); + } + + addChunking(text, sizes, seen, chunkings); + } + + const maxOffsetSize = Math.min(text.length, 8); + + for (let size = 2; size <= maxOffsetSize; size++) { + for (let head = 1; head < size && head < text.length; head++) { + const sizes = [head]; + + for (let remaining = text.length - head; remaining > 0; remaining -= size) { + sizes.push(Math.min(size, remaining)); + } + + addChunking(text, sizes, seen, chunkings); + } + + for (let tail = 1; tail < size && tail < text.length; tail++) { + const sizes: number[] = []; + + for (let remaining = text.length - tail; remaining > 0; remaining -= size) { + sizes.push(Math.min(size, remaining)); + } + + sizes.push(tail); + addChunking(text, sizes, seen, chunkings); + } + } + + return chunkings; + } + + function assertStatefulMatchesStateless(text: string) { + for (const chunks of buildChunkings(text)) { + const prediction = new Predict(); + let prefix = ""; + + for (const chunk of chunks) { + prefix += chunk; + + const actual = prediction.update(prefix); + const expected = stateless(prefix); + + if (actual !== expected) { + throw new Error( + [ + "stateful prediction diverged from stateless parsing", + `input: ${JSON.stringify(text)}`, + `chunks: ${JSON.stringify(chunks)}`, + `prefix: ${JSON.stringify(prefix)}`, + `expected: ${JSON.stringify(expected)}`, + `received: ${JSON.stringify(actual)}`, + ].join("\n"), + ); + } + } + } + } + + function check(text: string, expected: string) { + if (mode === "stateless") { + expect(stateless(text)).toBe(expected); + } else { + assertStatefulMatchesStateless(text); + } + } + + test.each([ + ["plain text", "plain text", "plain text"], + ["italic", "a *hello world", "a *hello world*"], + ["bold", "a **hello world", "a **hello world**"], + ["bold italic", "a ***hello world", "a ***hello world***"], + ["code", "a `hello world", "a `hello world`"], + ["strike", "a ~~hello world", "a ~~hello world~~"], + ["katex", "something\n\n$$\nE =", "something\n\n$$\nE =\n$$"], + + + ["bold partial close", "a **hello world*", "a **hello world**"], + ["bold italic partial close 1", "a ***hello world*", "a ***hello world***"], + ["bold italic partial close 2", "a ***hello world**", "a ***hello world***"], + ["code", "a `hello world", "a `hello world`"], + ["strike", "a ~~hello world", "a ~~hello world~~"], + ["strike partial close", "a ~~hello world~", "a ~~hello world~~"], + ["katex", "something\n\n$$\nE =", "something\n\n$$\nE =\n$$"], + ["katex partial close", "something\n\n$$\nE =\n$", "something\n\n$$\nE =\n$$"], + + ["balanced markdown stays unchanged", "a *hello* world", "a *hello* world"], + ["balanced bold stays unchanged", "a **hello** world", "a **hello** world"], + ["balanced double underscore stays unchanged", "a __hello__ world", "a __hello__ world"], + ["balanced strike stays unchanged", "a ~~hello~~ world", "a ~~hello~~ world"], + ["balanced code stays unchanged", "a `hello` world", "a `hello` world"], + ["balanced inline math stays unchanged", "a $x$ world", "a $x$ world"], + [ + "balanced display math stays unchanged", + "something\n\n$$\nE = mc^2\n$$", + "something\n\n$$\nE = mc^2\n$$", + ], + ["escaped markdown stays escaped", "a \\*hello", "a \\*hello"], + ["escaped underscore stays escaped", "a \\_hello", "a \\_hello"], + ["escaped strike stays escaped", "a \\~~hello", "a \\~~hello"], + ["escaped code stays escaped", "a \\`hello", "a \\`hello"], + ["escaped link bracket stays escaped", "a \\[hello", "a \\[hello"], + ["escaped inline math stays escaped", "a \\$x", "a \\$x"], + ["code suppresses other inline markdown", "a `code *not emphasis", "a `code *not emphasis`"], + ["paragraph break resets open formatting", "a **hello\n\nworld", "a **hello\n\nworld"], + ])("%s", (_name, input, expected) => check(input, expected)); + + describe("whitespace around emphasis delimiters", () => { + test.each([ + "hello * world", + "hello* world", + "hello _ world", + "hello_ world", + "hello ** world", + "hello** world", + "hello __ world", + "hello__ world", + ])("does not predict for %j", (input) => check(input, input)); + }); + + describe("flanking around emphasis delimiters", () => { + test.each([ + ["hello *world", "hello *world*"], + ["hello **world", "hello **world**"], + ["hello _world", "hello _world_"], + ["hello __world", "hello __world__"], + ])("predicts for left-flanking opener %j", (input, expected) => check(input, expected)); + + test.each(["snake_case", "foo*bar", "foo**bar", "foo__bar"])( + "does not predict inside words for %j", + (input) => check(input, input), + ); + + // TODO: these are dubious + test.each(["hello *world ", "hello **world ", "hello _world ", "hello __world "])( + "does not add a closer after trailing whitespace for %j", + (input) => check(input, input), + ); + }); + + describe("soft line breaks", () => { + test.each([ + ["italic continues across single newline", "a *hello\nworld", "a *hello\nworld*"], + ["bold continues across single newline", "a **hello\nworld", "a **hello\nworld**"], + ["strike continues across single newline", "a ~~hello\nworld", "a ~~hello\nworld~~"], + ["code continues across single newline", "a `hello\nworld", "a `hello\nworld`"], + ])("%s", (_name, input, expected) => check(input, expected)); + }); + + describe("code span variants", () => { + test.each([ + ["double backtick code span", "a ``hello world", "a ``hello world``"], + ["double backtick can contain a single backtick", "a ``hello ` world", "a ``hello ` world``"], + ])("%s", (_name, input, expected) => check(input, expected)); + }); + + describe("stuff in code blocks aren't completed", () => { + test.each([ + ["backtick code basic", "hello\n\n```tsx\nconst y = '**bold"], + ["tilde code block", "hello\n\n~~~tsx\nconst y = '**bold"], + + ["italic", "hello\n\n```tsx\nconst y = '*italic"], + ["strike", "hello\n\n```tsx\nconst y = '~~strike"], + ["link 1", "hello\n\n```tsx\nconst y = '[link"], + ["link 2", "hello\n\n```tsx\nconst y = '[link]"], + ["link 3", "hello\n\n```tsx\nconst y = '[link]("], + ["link 4", "hello\n\n```tsx\nconst y = '[link](meow"], + ["underscore", "hello\n\n```tsx\nconst y = '__this"], + ["inline code", "hello\n\n```tsx\nconst y = `template"], + + ["backtick code extra line", "hello\n\n```tsx\nmeow\n\nconst y = '**bold"], + ["tilde code extra line", "hello\n\n~~~tsx\nmeow\n\nconst y = '**bold"], + + ["italic", "hello\n\n```tsx\nmeow\n\nconst y = '*italic"], + ["strike", "hello\n\n```tsx\nmeow\n\nconst y = '~~strike"], + ["link 1", "hello\n\n```tsx\nmeow\n\nconst y = '[link"], + ["link 2", "hello\n\n```tsx\nmeow\n\nconst y = '[link]"], + ["link 3", "hello\n\n```tsx\nmeow\n\nconst y = '[link]("], + ["link 4", "hello\n\n```tsx\nmeow\n\nconst y = '[link](meow"], + ["underscore", "hello\n\n```tsx\nmeow\n\nconst y = '__this"], + ["inline code", "hello\n\n```tsx\nmeow\n\nconst y = `template"], + ])("%s", (_name, input) => check(input, input)); + }); + + describe("math handling", () => { + test.each([ + ["inline math", "a $x + y", "a $x + y$"], + ["inline math suppresses emphasis markers", "a $x * y", "a $x * y$"], + [ + "display math suppresses emphasis markers", + "something\n\n$$\na * b", + "something\n\n$$\na * b\n$$", + ], + ])("%s", (_name, input, expected) => check(input, expected)); + }); + + describe("nested emphasis handling", () => { + test.each([ + ["bold containing italic", "one **two *three", "one **two *three***"], + ["italic containing bold", "one *two **three", "one *two **three***"], + ["bold containing code", "**hello `world", "**hello `world`**"], + ["italic containing code", "*hello `world", "*hello `world`*"], + ["mixed double underscores inside bold", "one **two __three", "one **two __three__**"], + ["mixed bold inside double underscores", "one __two **three", "one __two **three**__"], + ["mixed italic underscore inside bold", "one **two _three", "one **two _three_**"], + ["mixed bold inside italic underscore", "one _two **three", "one _two **three**_"], + ["mixed italic markers", "one *two _three", "one *two _three_*"], + ["mixed italic markers reversed", "one _two *three", "one _two *three*_"], + ["bold containing code with trailing space", "**hello `world ", "**hello `world `**"], + ["italic containing code with trailing space", "*hello `world ", "*hello `world `*"], + ["bold containing code opener", "one **two `three", "one **two `three`**"], + ["star in code block", "one `two *three", "one `two *three`"], + ])("%s", (_name, input, expected) => check(input, expected)); + + // these cases unsupported + test.each([ + ["strike inside bold", "one **two ~~three", "one **two ~~three~~"], + ["bold inside strike", "one ~~two **three", "one ~~two **three**"], + // NOTE: if these close just the last one, its ok to edit the test. that behavior is better + ["double underscore containing double underscore", "one __two __three", "one __two __three"], + ["bold containing bold", "one **two **three", "one **two **three"], + ["italic containing italic", "one *two *three", "one *two *three"], + ["strike containing strike", "one ~~two ~~three", "one ~~two ~~three"], + ])("%s", (_name, input, expected) => check(input, expected)); + }); + + test.each([ + ["link 1", "a [hello world", "a hello world"], + ["link 2", "a [hello world](incomplete", "a hello world"], + ["link 3", "a [hello] tail", "a hello tail"], + ["link 4", "a [x](", "a x"], + ["link full", "a [hello world](complete)", "a [hello world](complete)"], + ["link full 2", "a [hello world](complete) xyz", "a [hello world](complete) xyz"], + ["link with parentheses in url", "a [hello](path_(x))", "a [hello](path_(x))"], + ["link with title", 'a [hello](url "title")', 'a [hello](url "title")'], + ["link with emphasis in label", "a [*hello*](url)", "a [*hello*](url)"], + ["link with code in label", "a [`hello`](url)", "a [`hello`](url)"], + ["link full then open link", "a [link](done) and [open", "a [link](done) and open"], + ["link full then open emphasis", "a [link](done) and *open", "a [link](done) and *open*"], + ])("%s", (_name, input, expected) => check(input, expected)); + + describe("lossy link edge cases", () => { + test.each([ + ["nested open bracket keeps the outer bracket visible", "[a [b", "[a b"], + [ + "multiple open links keep the earliest unmatched bracket visible", + "a [one [two", + "a [one two", + ], + ["unfinished link text can swallow emphasis markers", "[a **b", "a **b"], + ["unfinished link text can swallow italic markers", "[a *b", "a *b"], + ["unfinished url drops the url entirely", "hello [label](url and more", "hello label"], + ["unfinished url with spaces still drops to text", "[x](y z", "x"], + ["completed label without url degrades to text", "hello [label] tail", "hello label tail"], + ])("%s", (_name, input, expected) => check(input, expected)); + }); + + describe("opener-only inputs", () => { + test.each([ + ["lone italic delimiter", "*", "*"], + ["lone bold delimiter", "**", "**"], + ["lone underscore delimiter", "_", "_"], + ["lone double underscore delimiter", "__", "__"], + ["lone strike delimiter", "~~", "~~"], + ["lone code delimiter", "`", "`"], + ["lone inline math delimiter", "$", "$"], + ["bare open bracket", "[", "["], + ["completed label without url at bol", "[x]", "x"], + ["bare url opener at bol", "[x](", "x"], + ])("%s", (_name, input, expected) => check(input, expected)); + }); +}); -- 2.54.0