{toolbar}
+ {children}
+
- = ComponentPropsWithoutRef & {
@@ -134,7 +135,7 @@ export const components: Partial = {
input: MarkdownInput,
li: MarkdownListItem,
ol: MarkdownOrderedList,
- p: memo(MarkdownParagraph),
+ p: MarkdownParagraph,
pre: MarkdownPre,
strong: MarkdownStrong,
table: MarkdownTable,
@@ -146,4 +147,5 @@ export const components: Partial = {
ul: MarkdownUnorderedList,
};
+export const reactMarkdownComponents = components as ReactMarkdownComponents;
export const streamdownComponents = components as StreamdownComponents;
diff --git a/example/markdown-demo.ts b/example/markdown-demo.ts
index 73060e2b799be418158fc0aad1e6505ba63edf70..747e5cb56da01a01f4bc18e89494d6b8df7b18cd 100644
--- a/example/markdown-demo.ts
+++ b/example/markdown-demo.ts
@@ -1,7 +1,8 @@
import remarkGfm from "remark-gfm";
import remarkParse from "remark-parse";
-import { unified } from "unified";
+import { type PluggableList, unified } from "unified";
import projectReadme from "../README.md?raw";
export const initialMarkdown = projectReadme;
export const processor = unified().use(remarkParse).use(remarkGfm);
+export const reactMarkdownRemarkPlugins: PluggableList = [remarkGfm];
diff --git a/package.json b/package.json
index 0381a1c145dd3f65626702120015f5cf1b2b7000..3b787004fabb3fe4fa8b0b647985f494f7ca5cf2 100644
--- a/package.json
+++ b/package.json
@@ -5,7 +5,8 @@
],
"type": "module",
"exports": {
- ".": "./src/Markdown.ts"
+ ".": "./src/Markdown.ts",
+ "./Predict": "./src/Predict.ts"
},
"publishConfig": {
"access": "public"
@@ -35,6 +36,7 @@
"hast-util-to-jsx-runtime": "2.3.6",
"react": "^19.2.4",
"react-dom": "^19.2.4",
+ "react-markdown": "^10.1.0",
"react-scan": "^0.5.3",
"rehype-react": "^8.0.0",
"remark-gfm": "^4.0.1",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index e6a5d56df5d880cdd13c234b65acc1aa08f6b3fd..d3947033f856f0d471901ec6d3a9930d5d0a28cb 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -58,6 +58,9 @@ importers:
react-dom:
specifier: ^19.2.4
version: 19.2.4(react@19.2.4)
+ react-markdown:
+ specifier: ^10.1.0
+ version: 10.1.0(@types/react@19.2.14)(react@19.2.4)
react-scan:
specifier: ^0.5.3
version: 0.5.3(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)
@@ -2033,6 +2036,12 @@ packages:
react-is@17.0.2:
resolution: {integrity: sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==}
+ react-markdown@10.1.0:
+ resolution: {integrity: sha512-qKxVopLT/TyA6BX3Ue5NwabOsAzm0Q7kAPwq6L+wWDwisYs7R8vZ0nRXqq6rkueboxpkjvLGU9fWifiX/ZZFxQ==}
+ peerDependencies:
+ '@types/react': '>=18'
+ react: '>=18'
+
react-scan@0.5.3:
resolution: {integrity: sha512-qde9PupmUf0L3MU1H6bjmoukZNbCXdMyTEwP4Gh8RQ4rZPd2GGNBgEKWszwLm96E8k+sGtMpc0B9P0KyFDP6Bw==}
hasBin: true
@@ -4316,6 +4325,24 @@ snapshots:
react-is@17.0.2: {}
+ react-markdown@10.1.0(@types/react@19.2.14)(react@19.2.4):
+ dependencies:
+ '@types/hast': 3.0.4
+ '@types/mdast': 4.0.4
+ '@types/react': 19.2.14
+ devlop: 1.1.0
+ hast-util-to-jsx-runtime: 2.3.6
+ html-url-attributes: 3.0.1
+ mdast-util-to-hast: 13.2.1
+ react: 19.2.4
+ remark-parse: 11.0.0
+ remark-rehype: 11.1.2
+ unified: 11.0.5
+ unist-util-visit: 5.1.0
+ vfile: 6.0.3
+ transitivePeerDependencies:
+ - supports-color
+
react-scan@0.5.3(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4):
dependencies:
'@babel/core': 7.29.0
diff --git a/src/Markdown.ts b/src/Markdown.ts
index 02708a2ebfd954522048fe4dd1398be2800b977c..9edfe3f7e636e4662c991a767e0af2d0fed57107 100644
--- a/src/Markdown.ts
+++ b/src/Markdown.ts
@@ -4,6 +4,7 @@ import { type Processor } from "unified";
import { type Components } from "rehype-react";
import { jsx } from "react/jsx-runtime";
+/** Options for {@linkcode Markdown} */
export interface MarkdownOptions {
/** Markdown source content. */
content: string;
@@ -11,6 +12,8 @@ export interface MarkdownOptions {
processor?: Processor | null | undefined;
/** Customize component rendering. */
components?: Partial | null | undefined;
+ /** Automatically predict closing tags, useful for LLM streaming. */
+ predict?: boolean;
}
/**
@@ -23,14 +26,19 @@ export const Markdown = memo(function Markdown({
content,
processor,
components,
+ predict,
}: MarkdownOptions) {
const context = useContext(Context);
const ref = useRef(null);
const memoizer = (ref.current ??= new Memoizer());
- memoizer.reconfigure(processor ?? context.processor ?? defaultProcessor, {
- ...components,
- ...context.components,
- });
+ memoizer.reconfigure(
+ processor ?? context.processor ?? defaultProcessor,
+ predict ?? context.predict ?? false,
+ {
+ ...components,
+ ...context.components,
+ },
+ );
return memoizer.update(content);
}, propsAreEqual);
@@ -45,17 +53,22 @@ export const MarkdownOptionsProvider = memo(function MarkdownOptionsProvider({
processor,
components,
children,
+ predict,
}: PropsWithChildren) {
- const value = useMemo(() => ({ processor, components }), [processor, components]);
+ const value = useMemo(
+ () => ({ processor, components, predict }),
+ [processor, components, predict],
+ );
return jsx(Context.Provider, { value, children });
}, propsAreEqual);
function propsAreEqual(prev: T, next: T) {
if (prev.content !== next.content) return false;
if (prev.processor !== next.processor) return false;
+ if (prev.predict !== next.predict) return false;
if (prev.components === next.components) return true;
if (prev.components && next.components) {
return componentsAreEqual(prev.components, next.components);
}
- return false;
+ return !prev.components && !next.components;
}
diff --git a/src/Memoizer.ts b/src/Memoizer.ts
index 06af8be568ff96d80ae92bfc3075dcfba050ea1d..e2c909324961a28b86832e8afbc21a3684715709 100644
--- a/src/Memoizer.ts
+++ b/src/Memoizer.ts
@@ -1,5 +1,4 @@
import { ASSERT, UNWRAP } from "@clo/lib/assert.ts";
-import { indexOfDiff } from "./strings.ts";
import { type JSX, type Key, memo, type ReactNode } from "react";
import { type Components } from "rehype-react";
import remarkRehype from "remark-rehype";
@@ -9,6 +8,7 @@ import type { Root as HastRoot } from "hast";
import type { Literal, Node, Parent } from "unist";
import { memoizedHastToReact, type RenderState } from "./hast.ts";
import remarkParse from "remark-parse";
+import { Predict } from "./Prediction.ts";
export const defaultProcessor = unified().use(remarkParse);
@@ -35,6 +35,7 @@ export class Memoizer {
#previousComponents: Partial = {};
// incremental parsing graph, structure of arrays
+ #predict: Predict | null = null;
#content: string = "";
#positions: number[] = [];
#parsed: Node[][] = [];
@@ -43,12 +44,14 @@ export class Memoizer {
reconfigure(
processor: Processor,
+ predict: boolean,
components: Partial,
) {
if (
this.#astProcessor !== null &&
this.#baseProcessor === processor &&
- componentsAreEqual(this.#previousComponents, components)
+ componentsAreEqual(this.#previousComponents, components) &&
+ !!this.#predict === predict
) {
return;
}
@@ -73,10 +76,12 @@ export class Memoizer {
this.#parsed = [];
this.#renderStates = [];
this.#reactNodes = [];
+ this.#predict = predict ? new Predict() : null;
}
update(content: string): readonly ReactNode[] {
- const previous = this.#content;
+ if (this.#predict) content = this.#predict.update(content);
+ let previous = this.#content;
if (content === previous) return this.#reactNodes;
// Locate the last block that has changed content. This is done quickly by finding
@@ -166,6 +171,12 @@ export class Memoizer {
}
}
+export function indexOfDiff(a: string, b: string) {
+ var i = 0;
+ while (a[i] === b[i]) i++;
+ return i;
+}
+
export function componentsAreEqual(left: Partial, right: Partial) {
const leftRecord = left as Record;
const rightRecord = right as Record;
diff --git a/src/Prediction.ts b/src/Prediction.ts
new file mode 100644
index 0000000000000000000000000000000000000000..8ab28aeab2a6d55b9498980c4248841813457bce
--- /dev/null
+++ b/src/Prediction.ts
@@ -0,0 +1,376 @@
+/**
+ * Incremental parser for predicting closing tags in Markdown text. Works best
+ * when continuously appending text to the end of the string; this behavior
+ * specifically triggers a fast parsing path.
+ */
+export class Predict {
+ #stable = "";
+
+ /** Incrementally reparses the tail. */
+ update(text: string) {
+ if (!text.startsWith(this.#stable)) this.#stable = "";
+ let tail = text.slice(this.#stable.length);
+ let state = parseTail(tail);
+ if (state.commitIndex > 0) {
+ this.#stable += tail.slice(0, state.commitIndex);
+ tail = tail.slice(state.commitIndex);
+ state = parseTail(tail);
+ }
+ return this.#stable + renderTail(tail, state);
+ }
+}
+
+type DelimToken = "***" | "**" | "__" | "~~" | "*" | "_";
+type DelimState = { start: number; token: DelimToken };
+type LinkState =
+ | { phase: "text"; start: number }
+ | { phase: "url_wait"; start: number; textEnd: number }
+ | { phase: "url"; start: number; textEnd: number; parenDepth: number };
+type ExclusiveState =
+ | { kind: "code"; start: number; token: string }
+ | { kind: "fence"; start: number; token: string }
+ | { kind: "math"; start: number; token: "$" | "$$" };
+interface TailState {
+ commitIndex: number;
+ delims: DelimState[];
+ exclusive: ExclusiveState | null;
+ links: LinkState[];
+};
+
+const NESTABLE_DELIMS = new Set(["*", "**", "_", "__"]);
+
+function parseTail(tail: string): TailState {
+ const state: TailState = {
+ commitIndex: 0,
+ delims: [],
+ exclusive: null,
+ links: [],
+ };
+
+ for (let i = 0; i < tail.length; i++) {
+ const char = tail[i]!;
+
+ if (char === "\n" && tail[i - 1] === "\n") {
+ resetParagraphState(state);
+ if (!state.exclusive) state.commitIndex = i + 1;
+ continue;
+ }
+
+ const exclusiveEnd = consumeExclusive(state, tail, i);
+ if (exclusiveEnd > i) {
+ i = exclusiveEnd - 1;
+ continue;
+ }
+
+ if (state.exclusive) continue;
+
+ if (inLinkUrlMode(state)) {
+ updateLink(state, i, char);
+ continue;
+ }
+
+ const delimEnd = consumeDelim(state, tail, i);
+ if (delimEnd > i) {
+ i = delimEnd - 1;
+ continue;
+ }
+
+ if (isEscapedAt(tail, i)) continue;
+
+ updateLink(state, i, char);
+ }
+
+ return state;
+}
+
+function consumeExclusive(state: TailState, tail: string, start: number) {
+ const char = tail[start];
+ if (char === "`") return consumeBackticks(state, tail, start);
+ if (char === "$") return consumeMath(state, tail, start);
+ if (char === "~") return consumeTildes(state, tail, start);
+ return start;
+}
+
+function consumeBackticks(state: TailState, tail: string, start: number) {
+ const length = runLengthAt(tail, start);
+ const token = "`".repeat(length);
+ const exclusive = state.exclusive;
+
+ if (exclusive?.kind === "fence") {
+ if (
+ exclusive.token[0] === "`" &&
+ isLineStart(tail, start) &&
+ !isEscapedAt(tail, start) &&
+ length >= exclusive.token.length
+ ) {
+ state.exclusive = null;
+ }
+ return start + length;
+ }
+
+ if (exclusive?.kind === "code") {
+ if (!isEscapedAt(tail, start) && length >= exclusive.token.length) {
+ state.exclusive = null;
+ }
+ return start + length;
+ }
+
+ if (exclusive) return start + length;
+ if (isEscapedAt(tail, start)) return start + length;
+
+ if (length >= 3 && isLineStart(tail, start)) {
+ state.exclusive = { kind: "fence", start, token };
+ return start + length;
+ }
+
+ state.exclusive = { kind: "code", start, token };
+ return start + length;
+}
+
+function consumeMath(state: TailState, tail: string, start: number) {
+ const length = runLengthAt(tail, start);
+ const exclusive = state.exclusive;
+
+ if (exclusive?.kind === "math") {
+ if (!isEscapedAt(tail, start) && length >= exclusive.token.length) {
+ state.exclusive = null;
+ }
+ return start + length;
+ }
+
+ if (exclusive) return start + length;
+ if (isEscapedAt(tail, start)) return start + length;
+
+ state.exclusive = {
+ kind: "math",
+ start,
+ token: length >= 2 ? "$$" : "$",
+ };
+ return start + length;
+}
+
+function consumeTildes(state: TailState, tail: string, start: number) {
+ const length = runLengthAt(tail, start);
+ const exclusive = state.exclusive;
+
+ if (exclusive?.kind === "fence" && exclusive.token[0] === "~") {
+ if (
+ isLineStart(tail, start) &&
+ !isEscapedAt(tail, start) &&
+ length >= exclusive.token.length
+ ) {
+ state.exclusive = null;
+ }
+ return start + length;
+ }
+
+ if (exclusive) return start;
+ if (length < 3 || !isLineStart(tail, start) || isEscapedAt(tail, start)) return start;
+
+ state.exclusive = { kind: "fence", start, token: "~".repeat(length) };
+ return start + length;
+}
+
+function consumeDelim(state: TailState, tail: string, start: number) {
+ const token = matchDelim(tail, start);
+ if (!token) return start;
+
+ if (isEscapedAt(tail, start)) return start + token.length;
+
+ const existingIndex = state.delims.findLastIndex((delim) => delim.token === token);
+ if (existingIndex !== -1) {
+ state.delims.splice(existingIndex, 1);
+ return start + token.length;
+ }
+
+ if (!canOpenDelim(tail, start, token)) return start + token.length;
+
+ state.delims.push({ start, token });
+ return start + token.length;
+}
+
+function matchDelim(tail: string, start: number): DelimToken | undefined {
+ const char = tail[start];
+
+ if (char === "*") {
+ if (tail.startsWith("***", start)) return "***";
+ if (tail.startsWith("**", start)) return "**";
+ return "*";
+ }
+
+ if (char === "_") {
+ if (tail.startsWith("__", start)) return "__";
+ return "_";
+ }
+
+ if (char === "~" && tail.startsWith("~~", start)) {
+ return "~~";
+ }
+}
+
+function canOpenDelim(tail: string, start: number, token: DelimToken) {
+ const next = tail[start + token.length];
+ if (!next || /\s/.test(next)) return false;
+
+ const prev = tail[start - 1];
+ return !isWordChar(prev) || !isWordChar(next);
+}
+
+function updateLink(state: TailState, index: number, char: string) {
+ const top = state.links.at(-1);
+
+ if (char === "[") {
+ state.links.push({ phase: "text", start: index });
+ return;
+ }
+
+ if (char === "]" && top?.phase === "text") {
+ state.links[state.links.length - 1] = {
+ phase: "url_wait",
+ start: top.start,
+ textEnd: index,
+ };
+ return;
+ }
+
+ if (char === "(" && top?.phase === "url_wait") {
+ state.links[state.links.length - 1] = {
+ phase: "url",
+ start: top.start,
+ textEnd: top.textEnd,
+ parenDepth: 0,
+ };
+ return;
+ }
+
+ if (char === "(" && top?.phase === "url") {
+ top.parenDepth += 1;
+ return;
+ }
+
+ if (char === ")" && top?.phase === "url") {
+ if (top.parenDepth > 0) {
+ top.parenDepth -= 1;
+ return;
+ }
+
+ state.links.pop();
+ }
+}
+
+function resetParagraphState(state: TailState) {
+ state.delims.length = 0;
+ state.links.length = 0;
+
+ if (state.exclusive?.kind === "fence") return;
+ if (state.exclusive?.kind === "math" && state.exclusive.token === "$$") return;
+
+ state.exclusive = null;
+}
+
+function inLinkUrlMode(state: TailState) {
+ const phase = state.links.at(-1)?.phase;
+ return phase === "url_wait" || phase === "url";
+}
+
+function renderTail(tail: string, state: TailState) {
+ const link = state.links.at(-1);
+ if (link) return renderOpenLink(tail, link);
+
+ const nestedClosers = renderNestedClosers(state);
+ if (nestedClosers) return appendCloser(tail, nestedClosers);
+
+ const open = findLastOpen(state);
+ if (!open) return tail;
+
+ if (open.kind === "delim") {
+ if (!hasContentAfter(tail, open.start, open.token.length)) return tail;
+ if (/\s/.test(tail[tail.length - 1] ?? "")) return tail;
+ return appendCloser(tail, open.token);
+ }
+
+ if (!hasContentAfter(tail, open.start, open.token.length)) return tail;
+ if (open.kind === "fence") return tail;
+ if (open.kind === "code") return appendCloser(tail, open.token);
+ if (open.token === "$$") return appendCloser(tail, (tail.endsWith("\n") ? "" : "\n") + "$$");
+ if (/\s/.test(tail[tail.length - 1] ?? "")) return tail;
+ return appendCloser(tail, "$");
+}
+
+function renderOpenLink(tail: string, state: LinkState) {
+ const before = tail.slice(0, state.start);
+
+ if (state.phase === "text") {
+ if (!hasContentAfter(tail, state.start, 1)) return tail;
+ return before + tail.slice(state.start + 1);
+ }
+
+ const text = tail.slice(state.start + 1, state.textEnd);
+ if (state.phase === "url_wait") return before + text + tail.slice(state.textEnd + 1);
+ return before + text;
+}
+
+function renderNestedClosers(state: TailState) {
+ const closers: string[] = [];
+
+ if (state.exclusive) {
+ if (state.exclusive.kind !== "code") return undefined;
+ closers.push(state.exclusive.token);
+ }
+
+ for (let i = state.delims.length - 1; i >= 0; i--) {
+ const token = state.delims[i]!.token;
+ if (!NESTABLE_DELIMS.has(token)) return undefined;
+ closers.push(token);
+ }
+
+ if (closers.length < 2) return undefined;
+ return closers.join("");
+}
+
+function appendCloser(tail: string, closer: string) {
+ return tail + closer.slice(overlapLength(tail, closer));
+}
+
+function overlapLength(tail: string, closer: string) {
+ for (let length = Math.min(tail.length, closer.length); length > 0; length -= 1) {
+ if (tail.endsWith(closer.slice(0, length))) return length;
+ }
+ return 0;
+}
+
+function findLastOpen(state: TailState) {
+ const delim = state.delims.at(-1);
+ if (state.exclusive && (!delim || state.exclusive.start > delim.start)) return state.exclusive;
+ if (delim) return { kind: "delim" as const, start: delim.start, token: delim.token };
+ return state.exclusive;
+}
+
+function runLengthAt(text: string, start: number) {
+ let end = start + 1;
+ while (end < text.length && text[end] === text[start]) end += 1;
+ return end - start;
+}
+
+function hasContentAfter(text: string, start: number, tokenLength: number) {
+ return text.length > start + tokenLength;
+}
+
+function isLineStart(text: string, index: number) {
+ return index === 0 || text[index - 1] === "\n";
+}
+
+function isEscapedAt(text: string, index: number) {
+ let count = 0;
+
+ for (let i = index - 1; i >= 0; i--) {
+ if (text[i] !== "\\") break;
+ count += 1;
+ }
+
+ return count % 2 === 1;
+}
+
+function isWordChar(char?: string) {
+ return !!char && /[A-Za-z0-9]/.test(char);
+}
diff --git a/src/strings.ts b/src/strings.ts
deleted file mode 100644
index 33e7eeee50d6a4c1f6f5643028ff8ef29db6ffac..0000000000000000000000000000000000000000
--- a/src/strings.ts
+++ /dev/null
@@ -1,5 +0,0 @@
-export function indexOfDiff(a: string, b: string) {
- var i = 0;
- while (a[i] === b[i]) i++;
- return i;
-}
diff --git a/tests/Markdown.memoization.test.tsx b/tests/Markdown.memoization.test.tsx
index 1a5fcadd3b0b75ad07c8d659a87946289a900f3d..47bf06fdf50ea290879c84bef9fe6ce7a2f2f85b 100644
--- a/tests/Markdown.memoization.test.tsx
+++ b/tests/Markdown.memoization.test.tsx
@@ -1,7 +1,7 @@
import { cleanup, render, screen } from "@testing-library/react";
import type { Components } from "rehype-react";
import type { ComponentPropsWithoutRef, JSX } from "react";
-import { afterEach, describe, expect, it, vi } from "vite-plus/test";
+import { afterEach, expect, it, vi } from "vite-plus/test";
import { Markdown, MarkdownOptionsProvider } from "../src/Markdown.ts";
import { Memoizer } from "../src/Memoizer.ts";
@@ -107,7 +107,7 @@ it("does not rerender an unchanged inline custom renderer when only adjacent tex
expect(Link).toHaveBeenCalledTimes(1);
});
-it.only("does not rerender an unchanged inline custom renderer with formatted children when only adjacent text changes", () => {
+it("does not rerender an unchanged inline custom renderer with formatted children when only adjacent text changes", () => {
const Link = vi.fn(function Link(props: MarkdownProps<"a">) {
return ;
});
@@ -154,3 +154,18 @@ it("does not rerender Markdown through the provider when the effective component
expect(Link).toHaveBeenCalledTimes(1);
expect(screen.getByTestId("provider-link").textContent).toBe("stable");
});
+
+it("reprocesses the document when prediction changes", () => {
+ const { container, rerender } = render(
+ , chunkings: string[][]) {
+ const chunks = splitBySizes(text, sizes);
+ const key = JSON.stringify(chunks);
+
+ if (seen.has(key)) return;
+ seen.add(key);
+ chunkings.push(chunks);
+ }
+
+ function buildChunkings(text: string) {
+ if (text.length === 0) return [[]];
+
+ const seen = new Set();
+ const chunkings: string[][] = [];
+
+ if (text.length <= 10) {
+ const splitCount = text.length - 1;
+ const totalMasks = 1 << splitCount;
+
+ for (let mask = 0; mask < totalMasks; mask++) {
+ const sizes: number[] = [];
+ let last = 0;
+
+ for (let bit = 0; bit < splitCount; bit++) {
+ if ((mask & (1 << bit)) === 0) continue;
+
+ const next = bit + 1;
+ sizes.push(next - last);
+ last = next;
+ }
+
+ sizes.push(text.length - last);
+ addChunking(text, sizes, seen, chunkings);
+ }
+
+ return chunkings;
+ }
+
+ for (let size = 1; size <= text.length; size++) {
+ const sizes: number[] = [];
+
+ for (let remaining = text.length; remaining > 0; remaining -= size) {
+ sizes.push(Math.min(size, remaining));
+ }
+
+ addChunking(text, sizes, seen, chunkings);
+ }
+
+ const maxOffsetSize = Math.min(text.length, 8);
+
+ for (let size = 2; size <= maxOffsetSize; size++) {
+ for (let head = 1; head < size && head < text.length; head++) {
+ const sizes = [head];
+
+ for (let remaining = text.length - head; remaining > 0; remaining -= size) {
+ sizes.push(Math.min(size, remaining));
+ }
+
+ addChunking(text, sizes, seen, chunkings);
+ }
+
+ for (let tail = 1; tail < size && tail < text.length; tail++) {
+ const sizes: number[] = [];
+
+ for (let remaining = text.length - tail; remaining > 0; remaining -= size) {
+ sizes.push(Math.min(size, remaining));
+ }
+
+ sizes.push(tail);
+ addChunking(text, sizes, seen, chunkings);
+ }
+ }
+
+ return chunkings;
+ }
+
+ function assertStatefulMatchesStateless(text: string) {
+ for (const chunks of buildChunkings(text)) {
+ const prediction = new Predict();
+ let prefix = "";
+
+ for (const chunk of chunks) {
+ prefix += chunk;
+
+ const actual = prediction.update(prefix);
+ const expected = stateless(prefix);
+
+ if (actual !== expected) {
+ throw new Error(
+ [
+ "stateful prediction diverged from stateless parsing",
+ `input: ${JSON.stringify(text)}`,
+ `chunks: ${JSON.stringify(chunks)}`,
+ `prefix: ${JSON.stringify(prefix)}`,
+ `expected: ${JSON.stringify(expected)}`,
+ `received: ${JSON.stringify(actual)}`,
+ ].join("\n"),
+ );
+ }
+ }
+ }
+ }
+
+ function check(text: string, expected: string) {
+ if (mode === "stateless") {
+ expect(stateless(text)).toBe(expected);
+ } else {
+ assertStatefulMatchesStateless(text);
+ }
+ }
+
+ test.each([
+ ["plain text", "plain text", "plain text"],
+ ["italic", "a *hello world", "a *hello world*"],
+ ["bold", "a **hello world", "a **hello world**"],
+ ["bold italic", "a ***hello world", "a ***hello world***"],
+ ["code", "a `hello world", "a `hello world`"],
+ ["strike", "a ~~hello world", "a ~~hello world~~"],
+ ["katex", "something\n\n$$\nE =", "something\n\n$$\nE =\n$$"],
+
+
+ ["bold partial close", "a **hello world*", "a **hello world**"],
+ ["bold italic partial close 1", "a ***hello world*", "a ***hello world***"],
+ ["bold italic partial close 2", "a ***hello world**", "a ***hello world***"],
+ ["code", "a `hello world", "a `hello world`"],
+ ["strike", "a ~~hello world", "a ~~hello world~~"],
+ ["strike partial close", "a ~~hello world~", "a ~~hello world~~"],
+ ["katex", "something\n\n$$\nE =", "something\n\n$$\nE =\n$$"],
+ ["katex partial close", "something\n\n$$\nE =\n$", "something\n\n$$\nE =\n$$"],
+
+ ["balanced markdown stays unchanged", "a *hello* world", "a *hello* world"],
+ ["balanced bold stays unchanged", "a **hello** world", "a **hello** world"],
+ ["balanced double underscore stays unchanged", "a __hello__ world", "a __hello__ world"],
+ ["balanced strike stays unchanged", "a ~~hello~~ world", "a ~~hello~~ world"],
+ ["balanced code stays unchanged", "a `hello` world", "a `hello` world"],
+ ["balanced inline math stays unchanged", "a $x$ world", "a $x$ world"],
+ [
+ "balanced display math stays unchanged",
+ "something\n\n$$\nE = mc^2\n$$",
+ "something\n\n$$\nE = mc^2\n$$",
+ ],
+ ["escaped markdown stays escaped", "a \\*hello", "a \\*hello"],
+ ["escaped underscore stays escaped", "a \\_hello", "a \\_hello"],
+ ["escaped strike stays escaped", "a \\~~hello", "a \\~~hello"],
+ ["escaped code stays escaped", "a \\`hello", "a \\`hello"],
+ ["escaped link bracket stays escaped", "a \\[hello", "a \\[hello"],
+ ["escaped inline math stays escaped", "a \\$x", "a \\$x"],
+ ["code suppresses other inline markdown", "a `code *not emphasis", "a `code *not emphasis`"],
+ ["paragraph break resets open formatting", "a **hello\n\nworld", "a **hello\n\nworld"],
+ ])("%s", (_name, input, expected) => check(input, expected));
+
+ describe("whitespace around emphasis delimiters", () => {
+ test.each([
+ "hello * world",
+ "hello* world",
+ "hello _ world",
+ "hello_ world",
+ "hello ** world",
+ "hello** world",
+ "hello __ world",
+ "hello__ world",
+ ])("does not predict for %j", (input) => check(input, input));
+ });
+
+ describe("flanking around emphasis delimiters", () => {
+ test.each([
+ ["hello *world", "hello *world*"],
+ ["hello **world", "hello **world**"],
+ ["hello _world", "hello _world_"],
+ ["hello __world", "hello __world__"],
+ ])("predicts for left-flanking opener %j", (input, expected) => check(input, expected));
+
+ test.each(["snake_case", "foo*bar", "foo**bar", "foo__bar"])(
+ "does not predict inside words for %j",
+ (input) => check(input, input),
+ );
+
+ // TODO: these are dubious
+ test.each(["hello *world ", "hello **world ", "hello _world ", "hello __world "])(
+ "does not add a closer after trailing whitespace for %j",
+ (input) => check(input, input),
+ );
+ });
+
+ describe("soft line breaks", () => {
+ test.each([
+ ["italic continues across single newline", "a *hello\nworld", "a *hello\nworld*"],
+ ["bold continues across single newline", "a **hello\nworld", "a **hello\nworld**"],
+ ["strike continues across single newline", "a ~~hello\nworld", "a ~~hello\nworld~~"],
+ ["code continues across single newline", "a `hello\nworld", "a `hello\nworld`"],
+ ])("%s", (_name, input, expected) => check(input, expected));
+ });
+
+ describe("code span variants", () => {
+ test.each([
+ ["double backtick code span", "a ``hello world", "a ``hello world``"],
+ ["double backtick can contain a single backtick", "a ``hello ` world", "a ``hello ` world``"],
+ ])("%s", (_name, input, expected) => check(input, expected));
+ });
+
+ describe("stuff in code blocks aren't completed", () => {
+ test.each([
+ ["backtick code basic", "hello\n\n```tsx\nconst y = '**bold"],
+ ["tilde code block", "hello\n\n~~~tsx\nconst y = '**bold"],
+
+ ["italic", "hello\n\n```tsx\nconst y = '*italic"],
+ ["strike", "hello\n\n```tsx\nconst y = '~~strike"],
+ ["link 1", "hello\n\n```tsx\nconst y = '[link"],
+ ["link 2", "hello\n\n```tsx\nconst y = '[link]"],
+ ["link 3", "hello\n\n```tsx\nconst y = '[link]("],
+ ["link 4", "hello\n\n```tsx\nconst y = '[link](meow"],
+ ["underscore", "hello\n\n```tsx\nconst y = '__this"],
+ ["inline code", "hello\n\n```tsx\nconst y = `template"],
+
+ ["backtick code extra line", "hello\n\n```tsx\nmeow\n\nconst y = '**bold"],
+ ["tilde code extra line", "hello\n\n~~~tsx\nmeow\n\nconst y = '**bold"],
+
+ ["italic", "hello\n\n```tsx\nmeow\n\nconst y = '*italic"],
+ ["strike", "hello\n\n```tsx\nmeow\n\nconst y = '~~strike"],
+ ["link 1", "hello\n\n```tsx\nmeow\n\nconst y = '[link"],
+ ["link 2", "hello\n\n```tsx\nmeow\n\nconst y = '[link]"],
+ ["link 3", "hello\n\n```tsx\nmeow\n\nconst y = '[link]("],
+ ["link 4", "hello\n\n```tsx\nmeow\n\nconst y = '[link](meow"],
+ ["underscore", "hello\n\n```tsx\nmeow\n\nconst y = '__this"],
+ ["inline code", "hello\n\n```tsx\nmeow\n\nconst y = `template"],
+ ])("%s", (_name, input) => check(input, input));
+ });
+
+ describe("math handling", () => {
+ test.each([
+ ["inline math", "a $x + y", "a $x + y$"],
+ ["inline math suppresses emphasis markers", "a $x * y", "a $x * y$"],
+ [
+ "display math suppresses emphasis markers",
+ "something\n\n$$\na * b",
+ "something\n\n$$\na * b\n$$",
+ ],
+ ])("%s", (_name, input, expected) => check(input, expected));
+ });
+
+ describe("nested emphasis handling", () => {
+ test.each([
+ ["bold containing italic", "one **two *three", "one **two *three***"],
+ ["italic containing bold", "one *two **three", "one *two **three***"],
+ ["bold containing code", "**hello `world", "**hello `world`**"],
+ ["italic containing code", "*hello `world", "*hello `world`*"],
+ ["mixed double underscores inside bold", "one **two __three", "one **two __three__**"],
+ ["mixed bold inside double underscores", "one __two **three", "one __two **three**__"],
+ ["mixed italic underscore inside bold", "one **two _three", "one **two _three_**"],
+ ["mixed bold inside italic underscore", "one _two **three", "one _two **three**_"],
+ ["mixed italic markers", "one *two _three", "one *two _three_*"],
+ ["mixed italic markers reversed", "one _two *three", "one _two *three*_"],
+ ["bold containing code with trailing space", "**hello `world ", "**hello `world `**"],
+ ["italic containing code with trailing space", "*hello `world ", "*hello `world `*"],
+ ["bold containing code opener", "one **two `three", "one **two `three`**"],
+ ["star in code block", "one `two *three", "one `two *three`"],
+ ])("%s", (_name, input, expected) => check(input, expected));
+
+ // these cases unsupported
+ test.each([
+ ["strike inside bold", "one **two ~~three", "one **two ~~three~~"],
+ ["bold inside strike", "one ~~two **three", "one ~~two **three**"],
+ // NOTE: if these close just the last one, its ok to edit the test. that behavior is better
+ ["double underscore containing double underscore", "one __two __three", "one __two __three"],
+ ["bold containing bold", "one **two **three", "one **two **three"],
+ ["italic containing italic", "one *two *three", "one *two *three"],
+ ["strike containing strike", "one ~~two ~~three", "one ~~two ~~three"],
+ ])("%s", (_name, input, expected) => check(input, expected));
+ });
+
+ test.each([
+ ["link 1", "a [hello world", "a hello world"],
+ ["link 2", "a [hello world](incomplete", "a hello world"],
+ ["link 3", "a [hello] tail", "a hello tail"],
+ ["link 4", "a [x](", "a x"],
+ ["link full", "a [hello world](complete)", "a [hello world](complete)"],
+ ["link full 2", "a [hello world](complete) xyz", "a [hello world](complete) xyz"],
+ ["link with parentheses in url", "a [hello](path_(x))", "a [hello](path_(x))"],
+ ["link with title", 'a [hello](url "title")', 'a [hello](url "title")'],
+ ["link with emphasis in label", "a [*hello*](url)", "a [*hello*](url)"],
+ ["link with code in label", "a [`hello`](url)", "a [`hello`](url)"],
+ ["link full then open link", "a [link](done) and [open", "a [link](done) and open"],
+ ["link full then open emphasis", "a [link](done) and *open", "a [link](done) and *open*"],
+ ])("%s", (_name, input, expected) => check(input, expected));
+
+ describe("lossy link edge cases", () => {
+ test.each([
+ ["nested open bracket keeps the outer bracket visible", "[a [b", "[a b"],
+ [
+ "multiple open links keep the earliest unmatched bracket visible",
+ "a [one [two",
+ "a [one two",
+ ],
+ ["unfinished link text can swallow emphasis markers", "[a **b", "a **b"],
+ ["unfinished link text can swallow italic markers", "[a *b", "a *b"],
+ ["unfinished url drops the url entirely", "hello [label](url and more", "hello label"],
+ ["unfinished url with spaces still drops to text", "[x](y z", "x"],
+ ["completed label without url degrades to text", "hello [label] tail", "hello label tail"],
+ ])("%s", (_name, input, expected) => check(input, expected));
+ });
+
+ describe("opener-only inputs", () => {
+ test.each([
+ ["lone italic delimiter", "*", "*"],
+ ["lone bold delimiter", "**", "**"],
+ ["lone underscore delimiter", "_", "_"],
+ ["lone double underscore delimiter", "__", "__"],
+ ["lone strike delimiter", "~~", "~~"],
+ ["lone code delimiter", "`", "`"],
+ ["lone inline math delimiter", "$", "$"],
+ ["bare open bracket", "[", "["],
+ ["completed label without url at bol", "[x]", "x"],
+ ["bare url opener at bol", "[x](", "x"],
+ ])("%s", (_name, input, expected) => check(input, expected));
+ });
+});
-
+
+
+
+
-
+ )}
+
);
diff --git a/example/index.css b/example/index.css
index ef4c9102b1a4cc0883c30b6083c16cfc08f773cc..08bab9f2c62e2f0d4421b2da49c8ff778de7b150 100644
--- a/example/index.css
+++ b/example/index.css
@@ -20,7 +20,7 @@ body {
.preview-prose {
@apply prose prose-stone max-w-none;
- @apply prose-headings:text-stone-900 prose-p:text-stone-700
+ @apply prose-headings:mt-0 prose-headings:text-stone-900 prose-p:text-stone-700
prose-li:text-stone-700;
@apply prose-a:text-blue-700;
@apply prose-pre:overflow-x-auto;
diff --git a/example/markdown-components.tsx b/example/markdown-components.tsx
index 10f0bc3268c66452defd0e9c5cdce7ecff6e01e8..2c10a2d9b2b4dbf8f562eb4311627b6c417342b1 100644
--- a/example/markdown-components.tsx
+++ b/example/markdown-components.tsx
@@ -1,5 +1,6 @@
-import { type ComponentPropsWithoutRef, type JSX, memo } from "react";
+import { type ComponentPropsWithoutRef, type JSX } from "react";
import { type Components as MemoComponents } from "rehype-react";
+import { type Components as ReactMarkdownComponents } from "react-markdown";
import { type Components as StreamdownComponents } from "streamdown";
type MarkdownPropsSource
+
-
-
-
))}
- Preview
+
- {renderer === "memo" ? (
-
+ }
+ toolbar={
+ previewToolbar
+ }
+ >
+ {renderer === "memo" ? (
+
+ {deferredContent}
+
+ ) : (
+
+
+
-
- ) : (
-
+
+ ) : renderer === "streamdown" ? (
+
+
{deferredContent}
-
- )}
-
-