diff --git a/README.md b/README.md index ad9645a578faf85284f331a3003f0aba80746a3b..1386c75860fadff1e7ccc91127df29073843d7fe 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # `@clo/react-markdown` -This package exports a React component to render markdown using the [unified] -ecosystem's markdown tools ([remark], [rehype]). The exported `` +This package exports a React component to render Markdown using the [unified] +ecosystem's Markdown tools (remark, rehype). The exported `` component is extremely memoized, making it suitable for streaming situations such as LLM chat interfaces. @@ -32,7 +32,6 @@ with their library: [Streamdown]: https://streamdown.ai [unified]: https://unifiedjs.com/ -[remark]: https://remark.js.org/ ## Getting Started diff --git a/package.json b/package.json index 4adcb23979aac04ba242155e89b6feeaa3f9f007..a15095e80a2537471aa61c127bd0ed753e68d343 100644 --- a/package.json +++ b/package.json @@ -18,16 +18,7 @@ "prepublishOnly": "vp run build" }, "dependencies": { - "@clo/lib": "jsr:^3.0.0", - "@types/unist": "^3.0.3", - "mdast-util-from-markdown": "^2.0.3", - "rehype-react": "^8.0.0", - "rehype-stringify": "^10.0.1", - "remark-gfm": "^4.0.1", - "remark-parse": "^11.0.0", - "remark-rehype": "^11.1.2", - "remark-stringify": "^11.0.0", - "unified": "^11.0.5" + "@clo/lib": "jsr:^3.0.0" }, "devDependencies": { "@tailwindcss/typography": "^0.5.19", @@ -35,11 +26,16 @@ "@types/node": "^25.5.0", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", + "@types/unist": "^3.0.3", "@typescript/native-preview": "7.0.0-dev.20260318.1", "@vitejs/plugin-react": "^6.0.1", "react": "^19.2.4", "react-dom": "^19.2.4", "react-scan": "^0.5.3", + "rehype-react": "^8.0.0", + "remark-gfm": "^4.0.1", + "remark-parse": "^11.0.0", + "remark-rehype": "^11.1.2", "streamdown": "^2.5.0", "tailwindcss": "^4.2.2", "typescript": "^5.9.3", @@ -47,7 +43,11 @@ "vite-plus": "^0.1.11" }, "peerDependencies": { - "react": "*" + "react": "*", + "rehype-react": "^8.0.0", + "remark-parse": "^11.0.0", + "remark-rehype": "^11.0.0", + "unified": "^11.0.0" }, "packageManager": "pnpm@10.26.1", "pnpm": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 32e6388e64ba0fcc4f9ad95e847fff7572d61a36..e48967cfe7245802a98eea4d00cc304b0b7c747e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,36 +15,6 @@ importers: '@clo/lib': specifier: jsr:^3.0.0 version: '@jsr/clo__lib@3.0.0' - '@types/unist': - specifier: ^3.0.3 - version: 3.0.3 - mdast-util-from-markdown: - specifier: ^2.0.3 - version: 2.0.3 - rehype-react: - specifier: ^8.0.0 - version: 8.0.0 - rehype-stringify: - specifier: ^10.0.1 - version: 10.0.1 - remark-gfm: - specifier: ^4.0.1 - version: 4.0.1 - remark-parse: - specifier: ^11.0.0 - version: 11.0.0 - remark-rehype: - specifier: ^11.1.2 - version: 11.1.2 - remark-stringify: - specifier: ^11.0.0 - version: 11.0.0 - typescript: - specifier: ^5.9.3 - version: 5.9.3 - unified: - specifier: ^11.0.5 - version: 11.0.5 devDependencies: '@tailwindcss/typography': specifier: ^0.5.19 @@ -61,6 +31,9 @@ importers: '@types/react-dom': specifier: ^19.2.3 version: 19.2.3(@types/react@19.2.14) + '@types/unist': + specifier: ^3.0.3 + version: 3.0.3 '@typescript/native-preview': specifier: 7.0.0-dev.20260318.1 version: 7.0.0-dev.20260318.1 @@ -76,12 +49,30 @@ importers: react-scan: specifier: ^0.5.3 version: 0.5.3(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + rehype-react: + specifier: ^8.0.0 + version: 8.0.0 + remark-gfm: + specifier: ^4.0.1 + version: 4.0.1 + remark-parse: + specifier: ^11.0.0 + version: 11.0.0 + remark-rehype: + specifier: ^11.1.2 + version: 11.1.2 streamdown: specifier: ^2.5.0 version: 2.5.0(react-dom@19.2.4(react@19.2.4))(react@19.2.4) tailwindcss: specifier: ^4.2.2 version: 4.2.2 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + unified: + specifier: ^11.0.5 + version: 11.0.5 vite-plus: specifier: ^0.1.11 version: 0.1.12(@types/node@25.5.0)(happy-dom@20.8.4)(jiti@2.6.1)(typescript@5.9.3)(vite@8.0.0(@types/node@25.5.0)(jiti@2.6.1)(yaml@2.8.2))(yaml@2.8.2) @@ -1557,9 +1548,6 @@ packages: hast-util-sanitize@5.0.2: resolution: {integrity: sha512-3yTWghByc50aGS7JlGhk61SPenfE/p1oaFeNwkOOyrscaOkMGrcW9+Cy/QAIOBpZxP1yqDIzFMR0+Np0i0+usg==} - hast-util-to-html@9.0.5: - resolution: {integrity: sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==} - hast-util-to-jsx-runtime@2.3.6: resolution: {integrity: sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==} @@ -2005,9 +1993,6 @@ packages: rehype-sanitize@6.0.0: resolution: {integrity: sha512-CsnhKNsyI8Tub6L4sm5ZFsme4puGfc6pYylvXo1AeqaGbjOYyzNv3qZPwvs0oMJ39eryyeOdmxwUIo94IpEhqg==} - rehype-stringify@10.0.1: - resolution: {integrity: sha512-k9ecfXHmIPuFVI61B9DeLPN0qFHfawM6RsuX48hoqlaKSF61RskNjSm1lI8PhBEM0MRdLxVVm4WmTqJQccH9mA==} - remark-gfm@4.0.1: resolution: {integrity: sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg==} @@ -3546,20 +3531,6 @@ snapshots: '@ungap/structured-clone': 1.3.0 unist-util-position: 5.0.0 - hast-util-to-html@9.0.5: - dependencies: - '@types/hast': 3.0.4 - '@types/unist': 3.0.3 - ccount: 2.0.1 - comma-separated-tokens: 2.0.3 - hast-util-whitespace: 3.0.0 - html-void-elements: 3.0.0 - mdast-util-to-hast: 13.2.1 - property-information: 7.1.0 - space-separated-tokens: 2.0.2 - stringify-entities: 4.0.4 - zwitch: 2.0.4 - hast-util-to-jsx-runtime@2.3.6: dependencies: '@types/estree': 1.0.8 @@ -4286,12 +4257,6 @@ snapshots: '@types/hast': 3.0.4 hast-util-sanitize: 5.0.2 - rehype-stringify@10.0.1: - dependencies: - '@types/hast': 3.0.4 - hast-util-to-html: 9.0.5 - unified: 11.0.5 - remark-gfm@4.0.1: dependencies: '@types/mdast': 4.0.4 diff --git a/src/Memoizer.ts b/src/Memoizer.ts index 28c64701f2b572033a1989e06dfaa297f61f6cb3..3ceef7ee3e460b87883b1a2b219a8bc86ed21a1f 100644 --- a/src/Memoizer.ts +++ b/src/Memoizer.ts @@ -1,40 +1,14 @@ import { ASSERT, UNWRAP } from "@clo/lib/assert.ts"; -import { fromMarkdown } from "mdast-util-from-markdown"; import { indexOfDiff } from "./strings.ts"; import { memo, type JSX, type Key, type ReactNode } from "react"; import rehypeReact, { type Components } from "rehype-react"; import remarkRehype from "remark-rehype"; import { jsx, jsxs, Fragment } from "react/jsx-runtime"; import { unified, type Processor } from "unified"; -import type { Node } from "unist"; +import type { Node, Parent, Literal } from "unist"; import remarkParse from "remark-parse"; export const defaultProcessor = unified().use(remarkParse); -const extractBlockOptions = { - extensions: [ - { - disable: { - null: [ - "attention", - "autolink", - "characterEscape", - "characterReference", - "codeText", - "hardBreakEscape", - "htmlText", - "labelStartImage", - "labelStartLink", - "labelEnd", - ], - }, - }, - ], -}; - -function extractBlockPositions(source: string, offset: number) { - const tree = fromMarkdown(source, extractBlockOptions); - return tree.children.map((child) => UNWRAP(UNWRAP(child.position).start.offset) + offset); -} export type BaseProcessor = Processor< Node, @@ -43,27 +17,13 @@ export type BaseProcessor = Processor< undefined, undefined | JSX.Element >; -export type ReactProcessor = Processor; - -function componentsAreEqual(left: Partial, right: Partial) { - const leftRecord = left as Record; - const rightRecord = right as Record; - const leftKeys = Object.keys(leftRecord); - if (leftKeys.length !== Object.keys(rightRecord).length) return false; - for (const key of leftKeys) { - if (leftRecord[key] !== rightRecord[key]) return false; - } - return true; -} - -function setReactKey(element: JSX.Element, key: Key) { - const { ...props } = element.props; - return jsx(element.type, props, key); -} +type AstProcessor = Processor; +type ReactCompiler = Processor; export class Memoizer { #baseProcessor: BaseProcessor = defaultProcessor; - #processor: ReactProcessor | null = null; + #astProcessor: AstProcessor | null = null; + #compiler: ReactCompiler | null = null; #previousComponents: Partial = {}; #previous: string = ""; #blockPositions: number[] = []; @@ -74,6 +34,7 @@ export class Memoizer { reset() { this.#previous = ""; this.#blockPositions = []; + this.#blockSources = []; this.#reactNodes = []; } @@ -82,19 +43,22 @@ export class Memoizer { components: Partial, ) { if ( - this.#processor !== null && + this.#astProcessor !== null && this.#baseProcessor === processor && componentsAreEqual(this.#previousComponents, components) ) return; this.#baseProcessor = processor; - if (!processor.attachers.some((plugin) => plugin[0] === rehypeReact)) { + if (!processor.attachers.some((plugin) => plugin[0] === remarkRehype)) { processor = processor(); - if (!processor.attachers.some((plugin) => plugin[0] === remarkRehype)) { - processor.use(remarkRehype); - } - processor.use(rehypeReact, { + processor.use(remarkRehype); + processor.freeze(); + } + + this.#astProcessor = processor as unknown as AstProcessor; + this.#compiler = unified() + .use(rehypeReact, { jsx, jsxs, Fragment, @@ -104,13 +68,9 @@ export class Memoizer { components: Object.fromEntries( Object.entries(components).map(([k, v]) => [k, typeof v === "function" ? memo(v) : v]), ), - }); - processor.freeze(); - } else { - // TODO: troll and force memoize all components - } + }) + .freeze() as ReactCompiler; this.#previousComponents = components; - this.#processor = processor as ReactProcessor; this.reset(); } @@ -130,12 +90,10 @@ export class Memoizer { if (blockIndexToReplace === -1) blockIndexToReplace = 0; // Compute and replace changed block regions. This behavior relys on the - // fact that this block extraction process can be isolated across blocks. - // For the basic set of syntax extensions, this is true, for example an - // unclosed code fence is parsed as a code fence. Anything that isn't - // compatible will break very loudly since `Memoizer` never re-joins the - // blocks for rendering. - const newBlocks = extractBlockPositions(content.slice(preservedLength), preservedLength); + // fact that blocks generally do not affect each other. + const processor = UNWRAP(this.#astProcessor); + const parsedTree = processor.parse(content.slice(preservedLength)); + const newBlocks = parsedTree.children.map((child) => UNWRAP(UNWRAP(child.position).start.offset) + preservedLength); blockPositions.splice( blockIndexToReplace, blockPositions.length - blockIndexToReplace, @@ -143,17 +101,26 @@ export class Memoizer { ); this.#previous = content; - // For all touched blocks (newBlocks), find which ones have actually changed - // their contents and run them through the full pipeline. - const processor = UNWRAP(this.#processor); + // For all touched blocks (newBlocks), transform their ASTs. This is done in + // a group to preserve shared contexts, such as reference links. + // TODO: need more effort for reference links without reparsing too much code. const blockSources = this.#blockSources; + const compiler = UNWRAP(this.#compiler); + const renderedBlocks = extractRenderedBlocks( + processor.runSync(parsedTree) as Parent, + newBlocks, + preservedLength, + ); + let reactNodes: ReactNode[] | null = null; - // If items are removed, the array must be resliced. if (this.#reactNodes.length !== blockPositions.length) { + // If items are removed, the array must be resliced. reactNodes = this.#reactNodes.slice(0, blockPositions.length); blockSources.length = blockPositions.length; } for (let i = blockIndexToReplace, len = blockPositions.length; i < len; i += 1) { + // The decision on if to recreate the React element is derived simply from if the + // souce code changes. TODO: perform a light AST-diff? use a mix of approaches? const source = content .slice(UNWRAP(blockPositions[i]), blockPositions[i + 1] ?? content.length) .replace(/\s+$/, ""); @@ -164,8 +131,11 @@ export class Memoizer { } blockSources[i] = source; - // Render the block using all plugins. - let { result } = processor.processSync(source); + // Render the block using the transformed subtree. + let result = compiler.stringify({ + type: "root", + children: renderedBlocks[i - blockIndexToReplace] ?? [], + } as Parent); // Do a little trolling and unwrap the fragment const key = String("m" + i); @@ -186,3 +156,40 @@ export class Memoizer { return this.#reactNodes; } } + +function componentsAreEqual(left: Partial, right: Partial) { + const leftRecord = left as Record; + const rightRecord = right as Record; + const leftKeys = Object.keys(leftRecord); + if (leftKeys.length !== Object.keys(rightRecord).length) return false; + for (const key of leftKeys) { + if (leftRecord[key] !== rightRecord[key]) return false; + } + return true; +} + +function setReactKey(element: JSX.Element, key: Key) { + const { ...props } = element.props; + return jsx(element.type, props, key); +} + +function extractRenderedBlocks(tree: Parent, blockPositions: number[], offset: number) { + const blocks = blockPositions.map(() => [] as Node[]); + + let blockIndex = 0; + for (const child of tree.children) { + const startOffset = child.position?.start.offset; + if (startOffset !== undefined) { + for (; blockIndex + 1 < blockPositions.length; blockIndex += 1) { + if (startOffset + offset < blockPositions[blockIndex + 1]!) break; + } + blocks[blockIndex]!.push(child); + continue; + } + + if (child.type === "text" && /^\s*$/.test(String((child as Literal).value))) continue; + blocks[blockIndex]!.push(child); + } + + return blocks; +} diff --git a/tests/index.test.ts b/tests/index.test.ts index 502cbec01b4f9ad50e049258a6bae9474ff53ce2..48ce17c619eac81e68e866403ebceb43e218c9d0 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -1,9 +1,13 @@ import { act, createElement } from "react"; import { createRoot } from "react-dom/client"; import { expect, test } from "vite-plus/test"; +import remarkGfm from "remark-gfm"; +import remarkParse from "remark-parse"; +import { unified } from "unified"; import { Markdown } from "../src/mod.ts"; Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }); +const gfmProcessor = unified().use(remarkParse).use(remarkGfm); function getParagraphs(container: HTMLElement) { return Array.from(container.querySelectorAll("p")); @@ -74,3 +78,33 @@ test("Markdown reuses rendered nodes across rerenders and preserves them when a container.remove(); } }); + +test("Markdown can render reference definitions and footnotes when they are in the same changed suffix", async () => { + const container = document.createElement("div"); + document.body.append(container); + + const root = createRoot(container); + const content = "[ref][a] and note[^1]\n\n[a]: https://example.com\n[^1]: hi there"; + + try { + await act(async () => { + root.render(createElement(Markdown, { content, processor: gfmProcessor })); + }); + + const link = container.querySelector("a[href='https://example.com']"); + const footnoteRef = container.querySelector("[data-footnote-ref]"); + const footnotes = container.querySelector("section[data-footnotes]"); + const bodyParagraph = container.querySelector("p"); + + expect(bodyParagraph?.textContent).toBe("ref and note1"); + expect(link?.textContent).toBe("ref"); + expect(footnoteRef?.textContent).toBe("1"); + expect(footnotes?.textContent).toContain("hi there"); + } finally { + await act(async () => { + root.unmount(); + }); + + container.remove(); + } +});