| author | |
| committer | |
| log | a5ec1eef9bc66643cc4a597bbcac83d7b713bd3d |
| tree | 2ed7d19e70e110411ecedf1e14dc4055e9bca258 |
| parent | 61a585701f88e36984c72ae977a62a686ad6271d |
| signature |
14 files changed, 935 insertions(+), 65 deletions(-)
README.md+3-5| ... | ... | @@ -12,10 +12,8 @@ such as LLM chat interfaces. |
| 12 | 12 | you can get started with just `<Markdown content="hi" />`. Configuration is |
| 13 | 13 | done either with a provider, or right at the component level. |
| 14 | 14 | - **Handle Partial Markdown**: When the `predict` prop is set, sequences like |
| 15 | `hello **world` will be emitted as `hello <strong>world</strong>`. | |
| 16 | <!--- **Smoothly Animate Streaming Content**: There are separate options for | |
| 17 | controlling how paragraph text animates into the view, as well as control to | |
| 18 | smooth out streamed input text. --> | |
| 15 | `hello **world` will be emitted as `hello <strong>world</strong>`. This | |
| 16 | behavior can be used standlone via the `@clo/react-markdown/Predict` class. | |
| 19 | 17 | |
| 20 | 18 | The motivation for this package is to have an easy to understand version of |
| 21 | 19 | [Streamdown]. With me banning all Vercel software at the company I work at, |
| ... | ... | @@ -29,7 +27,7 @@ to Streamdown: |
| 29 | 27 | - **Headless UI**: No built in styles or components, bring your own CSS to blend |
| 30 | 28 | your markdown with your existing theme. `@clo/react-markdown` simply takes in |
| 31 | 29 | your existing `unified` pipeline and works off of that. |
| 32 | - **Easy to Audit**: Under 450 lines of precise, highly commented TypeScript. | |
| 30 | - **Easy to Audit**: Under 700 lines of precise, highly commented TypeScript. | |
| 33 | 31 | |
| 34 | 32 | [Streamdown]: https://streamdown.ai |
| 35 | 33 | [unified]: https://unifiedjs.com/ |
TODO.md deleted-12| ... | ... | @@ -1,12 +0,0 @@ |
| 1 | - readme claims are not true | |
| 2 | - no `predict` implementation | |
| 3 | - bold `**text` | |
| 4 | - italic `*text` or `_text` | |
| 5 | - bold italic `***text` | |
| 6 | - inline code `` `code`` | |
| 7 | - strike `~~text` | |
| 8 | - link `[hello` or `[hello](world` | |
| 9 | - block katex `$$\nE = m` | |
| 10 | - custom handling maybe | |
| 11 | - animate in paragraph text | |
| 12 | - smoothly interpolate to handle bumpy connections |
example/App.tsx+126-27| ... | ... | @@ -1,32 +1,112 @@ |
| 1 | import { useDeferredValue, useState } from "react"; | |
| 1 | import { type ReactNode, useDeferredValue, useId, useState } from "react"; | |
| 2 | import ReactMarkdown from "react-markdown"; | |
| 2 | 3 | import { Streamdown } from "streamdown"; |
| 3 | 4 | import { Markdown } from "../src/Markdown.ts"; |
| 4 | import { components, streamdownComponents } from "./markdown-components"; | |
| 5 | import { initialMarkdown, processor } from "./markdown-demo"; | |
| 5 | import { | |
| 6 | components, | |
| 7 | reactMarkdownComponents, | |
| 8 | streamdownComponents, | |
| 9 | } from "./markdown-components"; | |
| 10 | import { initialMarkdown, processor, reactMarkdownRemarkPlugins } from "./markdown-demo"; | |
| 11 | ||
| 12 | type Renderer = "memo" | "streamdown" | "react-markdown"; | |
| 6 | 13 | |
| 7 | type Renderer = "memo" | "streamdown"; | |
| 8 | 14 | const rendererOptions: Array<{ label: string; value: Renderer }> = [ |
| 9 | { label: "MemoMarkdown", value: "memo" }, | |
| 10 | { label: "Streamdown", value: "streamdown" }, | |
| 15 | { | |
| 16 | label: "@clo/react-markdown", | |
| 17 | value: "memo", | |
| 18 | }, | |
| 19 | { | |
| 20 | label: "streamdown", | |
| 21 | value: "streamdown", | |
| 22 | }, | |
| 23 | { | |
| 24 | label: "react-markdown", | |
| 25 | value: "react-markdown", | |
| 26 | }, | |
| 11 | 27 | ]; |
| 12 | 28 | |
| 29 | type ToggleSwitchProps = { | |
| 30 | checked: boolean; | |
| 31 | label: string; | |
| 32 | onChange: (checked: boolean) => void; | |
| 33 | }; | |
| 34 | ||
| 35 | function ToggleSwitch({ checked, label, onChange }: ToggleSwitchProps) { | |
| 36 | const id = useId(); | |
| 37 | ||
| 38 | return ( | |
| 39 | <label className="inline-flex items-center gap-3 text-xs font-medium text-stone-600" htmlFor={id}> | |
| 40 | <span>{label}</span> | |
| 41 | <span className="relative inline-flex h-6 w-11 shrink-0"> | |
| 42 | <input | |
| 43 | checked={checked} | |
| 44 | className="peer sr-only" | |
| 45 | id={id} | |
| 46 | onChange={(event) => onChange(event.target.checked)} | |
| 47 | type="checkbox" | |
| 48 | /> | |
| 49 | <span className="absolute inset-0 rounded-full bg-stone-300 transition peer-checked:bg-emerald-500 peer-focus-visible:ring-2 peer-focus-visible:ring-emerald-200 peer-focus-visible:ring-offset-2 peer-focus-visible:ring-offset-white" /> | |
| 50 | <span className="absolute left-0.5 top-0.5 h-5 w-5 rounded-full bg-white shadow-sm transition peer-checked:translate-x-5" /> | |
| 51 | </span> | |
| 52 | </label> | |
| 53 | ); | |
| 54 | } | |
| 55 | ||
| 13 | 56 | function getRendererButtonClass(isActive: boolean) { |
| 14 | 57 | return isActive |
| 15 | 58 | ? "rounded bg-white px-2.5 py-1 text-xs font-medium text-stone-900 shadow-sm" |
| 16 | 59 | : "rounded px-2.5 py-1 text-xs text-stone-600 hover:text-stone-900"; |
| 17 | 60 | } |
| 18 | 61 | |
| 62 | type PreviewPanelProps = { | |
| 63 | children: ReactNode; | |
| 64 | header: ReactNode; | |
| 65 | toolbar: ReactNode; | |
| 66 | }; | |
| 67 | ||
| 68 | function PreviewPanel({ children, header, toolbar }: PreviewPanelProps) { | |
| 69 | return ( | |
| 70 | <article className="flex min-h-[18rem] flex-col overflow-hidden rounded-2xl border border-stone-200 bg-white shadow-sm xl:min-h-0"> | |
| 71 | <header className="flex shrink-0 flex-wrap items-center justify-between gap-3 border-b border-stone-200 px-4 py-3"> | |
| 72 | {header} | |
| 73 | <div className="flex min-h-6 items-center justify-end">{toolbar}</div> | |
| 74 | </header> | |
| 75 | <div className="min-h-0 flex-1 overflow-auto p-4">{children}</div> | |
| 76 | </article> | |
| 77 | ); | |
| 78 | } | |
| 79 | ||
| 19 | 80 | export function App() { |
| 20 | 81 | const [content, setContent] = useState(initialMarkdown); |
| 21 | 82 | const deferredContent = useDeferredValue(content); |
| 83 | const [predictionEnabled, setPredictionEnabled] = useState(true); | |
| 22 | 84 | const [renderer, setRenderer] = useState<Renderer>("memo"); |
| 23 | 85 | |
| 86 | const previewToolbar = | |
| 87 | renderer === "memo" ? ( | |
| 88 | <ToggleSwitch | |
| 89 | checked={predictionEnabled} | |
| 90 | label="Prediction" | |
| 91 | onChange={setPredictionEnabled} | |
| 92 | /> | |
| 93 | ) : renderer === "streamdown" ? ( | |
| 94 | <ToggleSwitch | |
| 95 | checked={predictionEnabled} | |
| 96 | label="Unterminated Block Parsing" | |
| 97 | onChange={setPredictionEnabled} | |
| 98 | /> | |
| 99 | ) : ( | |
| 100 | <span className="rounded-full border border-stone-200 bg-stone-50 px-2.5 py-1 text-xs font-medium text-stone-500"> | |
| 101 | No prediction support | |
| 102 | </span> | |
| 103 | ); | |
| 104 | ||
| 24 | 105 | return ( |
| 25 | <main className="mx-auto flex h-dvh max-w-6xl flex-col overflow-hidden px-4 py-4"> | |
| 26 | <section className="grid min-h-0 flex-1 gap-4 lg:grid-cols-2 lg:grid-rows-1"> | |
| 27 | <div className="flex min-h-0 flex-col overflow-hidden rounded-lg border border-stone-200 bg-white"> | |
| 28 | <div className="flex h-14 shrink-0 items-center justify-between border-b border-stone-200 px-4"> | |
| 29 | <h2 className="text-sm font-medium text-stone-700">Source</h2> | |
| 106 | <main className="flex h-dvh w-full flex-col overflow-hidden bg-stone-100"> | |
| 107 | <section className="grid min-h-0 flex-1 gap-4 overflow-auto p-4 xl:grid-cols-[minmax(22rem,30rem)_minmax(0,1fr)]"> | |
| 108 | <section className="flex min-h-[22rem] flex-col overflow-hidden rounded-2xl border border-stone-200 bg-white shadow-sm xl:min-h-0"> | |
| 109 | <div className="flex shrink-0 items-center justify-end border-b border-stone-200 px-4 py-3"> | |
| 30 | 110 | <button |
| 31 | 111 | className="rounded border border-stone-300 px-2.5 py-1 text-xs text-stone-700 hover:bg-stone-50" |
| 32 | 112 | onClick={() => setContent(initialMarkdown)} |
| ... | ... | @@ -45,11 +125,10 @@ export function App() { |
| 45 | 125 | value={content} |
| 46 | 126 | /> |
| 47 | 127 | </label> |
| 48 | </div> | |
| 128 | </section> | |
| 49 | 129 | |
| 50 | <div className="flex min-h-0 flex-col overflow-hidden rounded-lg border border-stone-200 bg-white"> | |
| 51 | <div className="flex h-14 shrink-0 items-center justify-between gap-3 border-b border-stone-200 px-4"> | |
| 52 | <h2 className="text-sm font-medium text-stone-700">Preview</h2> | |
| 130 | <PreviewPanel | |
| 131 | header={ | |
| 53 | 132 | <div |
| 54 | 133 | aria-label="Preview renderer" |
| 55 | 134 | className="inline-flex rounded-md border border-stone-300 bg-stone-50 p-0.5" |
| ... | ... | @@ -68,19 +147,39 @@ export function App() { |
| 68 | 147 | </button> |
| 69 | 148 | ))} |
| 70 | 149 | </div> |
| 71 | </div> | |
| 72 | <div className="min-h-0 flex-1 overflow-auto p-4"> | |
| 73 | {renderer === "memo" ? ( | |
| 74 | <div className="preview-prose"> | |
| 75 | <Markdown components={components} content={deferredContent} processor={processor} /> | |
| 76 | </div> | |
| 77 | ) : ( | |
| 78 | <Streamdown className="preview-prose" components={streamdownComponents}> | |
| 150 | } | |
| 151 | toolbar={ | |
| 152 | previewToolbar | |
| 153 | } | |
| 154 | > | |
| 155 | {renderer === "memo" ? ( | |
| 156 | <div className="preview-prose"> | |
| 157 | <Markdown | |
| 158 | components={components} | |
| 159 | content={deferredContent} | |
| 160 | predict={predictionEnabled} | |
| 161 | processor={processor} | |
| 162 | /> | |
| 163 | </div> | |
| 164 | ) : renderer === "streamdown" ? ( | |
| 165 | <Streamdown | |
| 166 | className="preview-prose" | |
| 167 | components={streamdownComponents} | |
| 168 | parseIncompleteMarkdown={predictionEnabled} | |
| 169 | > | |
| 170 | {deferredContent} | |
| 171 | </Streamdown> | |
| 172 | ) : ( | |
| 173 | <div className="preview-prose"> | |
| 174 | <ReactMarkdown | |
| 175 | components={reactMarkdownComponents} | |
| 176 | remarkPlugins={reactMarkdownRemarkPlugins} | |
| 177 | > | |
| 79 | 178 | {deferredContent} |
| 80 | </Streamdown> | |
| 81 | )} | |
| 82 | </div> | |
| 83 | </div> | |
| 179 | </ReactMarkdown> | |
| 180 | </div> | |
| 181 | )} | |
| 182 | </PreviewPanel> | |
| 84 | 183 | </section> |
| 85 | 184 | </main> |
| 86 | 185 | ); |
example/index.css+1-1| ... | ... | @@ -20,7 +20,7 @@ body { |
| 20 | 20 | |
| 21 | 21 | .preview-prose { |
| 22 | 22 | @apply prose prose-stone max-w-none; |
| 23 | @apply prose-headings:text-stone-900 prose-p:text-stone-700 | |
| 23 | @apply prose-headings:mt-0 prose-headings:text-stone-900 prose-p:text-stone-700 | |
| 24 | 24 | prose-li:text-stone-700; |
| 25 | 25 | @apply prose-a:text-blue-700; |
| 26 | 26 | @apply prose-pre:overflow-x-auto; |
example/markdown-components.tsx+4-2| ... | ... | @@ -1,5 +1,6 @@ |
| 1 | import { type ComponentPropsWithoutRef, type JSX, memo } from "react"; | |
| 1 | import { type ComponentPropsWithoutRef, type JSX } from "react"; | |
| 2 | 2 | import { type Components as MemoComponents } from "rehype-react"; |
| 3 | import { type Components as ReactMarkdownComponents } from "react-markdown"; | |
| 3 | 4 | import { type Components as StreamdownComponents } from "streamdown"; |
| 4 | 5 | |
| 5 | 6 | type MarkdownProps<Tag extends keyof JSX.IntrinsicElements> = ComponentPropsWithoutRef<Tag> & { |
| ... | ... | @@ -134,7 +135,7 @@ export const components: Partial<MemoComponents> = { |
| 134 | 135 | input: MarkdownInput, |
| 135 | 136 | li: MarkdownListItem, |
| 136 | 137 | ol: MarkdownOrderedList, |
| 137 | p: memo(MarkdownParagraph), | |
| 138 | p: MarkdownParagraph, | |
| 138 | 139 | pre: MarkdownPre, |
| 139 | 140 | strong: MarkdownStrong, |
| 140 | 141 | table: MarkdownTable, |
| ... | ... | @@ -146,4 +147,5 @@ export const components: Partial<MemoComponents> = { |
| 146 | 147 | ul: MarkdownUnorderedList, |
| 147 | 148 | }; |
| 148 | 149 | |
| 150 | export const reactMarkdownComponents = components as ReactMarkdownComponents; | |
| 149 | 151 | export const streamdownComponents = components as StreamdownComponents; |
example/markdown-demo.ts+2-1| ... | ... | @@ -1,7 +1,8 @@ |
| 1 | 1 | import remarkGfm from "remark-gfm"; |
| 2 | 2 | import remarkParse from "remark-parse"; |
| 3 | import { unified } from "unified"; | |
| 3 | import { type PluggableList, unified } from "unified"; | |
| 4 | 4 | import projectReadme from "../README.md?raw"; |
| 5 | 5 | |
| 6 | 6 | export const initialMarkdown = projectReadme; |
| 7 | 7 | export const processor = unified().use(remarkParse).use(remarkGfm); |
| 8 | export const reactMarkdownRemarkPlugins: PluggableList = [remarkGfm]; |
package.json+3-1| ... | ... | @@ -5,7 +5,8 @@ |
| 5 | 5 | ], |
| 6 | 6 | "type": "module", |
| 7 | 7 | "exports": { |
| 8 | ".": "./src/Markdown.ts" | |
| 8 | ".": "./src/Markdown.ts", | |
| 9 | "./Predict": "./src/Predict.ts" | |
| 9 | 10 | }, |
| 10 | 11 | "publishConfig": { |
| 11 | 12 | "access": "public" |
| ... | ... | @@ -35,6 +36,7 @@ |
| 35 | 36 | "hast-util-to-jsx-runtime": "2.3.6", |
| 36 | 37 | "react": "^19.2.4", |
| 37 | 38 | "react-dom": "^19.2.4", |
| 39 | "react-markdown": "^10.1.0", | |
| 38 | 40 | "react-scan": "^0.5.3", |
| 39 | 41 | "rehype-react": "^8.0.0", |
| 40 | 42 | "remark-gfm": "^4.0.1", |
pnpm-lock.yaml+27| ... | ... | @@ -58,6 +58,9 @@ importers: |
| 58 | 58 | react-dom: |
| 59 | 59 | specifier: ^19.2.4 |
| 60 | 60 | version: 19.2.4(react@19.2.4) |
| 61 | react-markdown: | |
| 62 | specifier: ^10.1.0 | |
| 63 | version: 10.1.0(@types/react@19.2.14)(react@19.2.4) | |
| 61 | 64 | react-scan: |
| 62 | 65 | specifier: ^0.5.3 |
| 63 | 66 | version: 0.5.3(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) |
| ... | ... | @@ -2033,6 +2036,12 @@ packages: |
| 2033 | 2036 | react-is@17.0.2: |
| 2034 | 2037 | resolution: {integrity: sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==} |
| 2035 | 2038 | |
| 2039 | react-markdown@10.1.0: | |
| 2040 | resolution: {integrity: sha512-qKxVopLT/TyA6BX3Ue5NwabOsAzm0Q7kAPwq6L+wWDwisYs7R8vZ0nRXqq6rkueboxpkjvLGU9fWifiX/ZZFxQ==} | |
| 2041 | peerDependencies: | |
| 2042 | '@types/react': '>=18' | |
| 2043 | react: '>=18' | |
| 2044 | ||
| 2036 | 2045 | react-scan@0.5.3: |
| 2037 | 2046 | resolution: {integrity: sha512-qde9PupmUf0L3MU1H6bjmoukZNbCXdMyTEwP4Gh8RQ4rZPd2GGNBgEKWszwLm96E8k+sGtMpc0B9P0KyFDP6Bw==} |
| 2038 | 2047 | hasBin: true |
| ... | ... | @@ -4316,6 +4325,24 @@ snapshots: |
| 4316 | 4325 | |
| 4317 | 4326 | react-is@17.0.2: {} |
| 4318 | 4327 | |
| 4328 | react-markdown@10.1.0(@types/react@19.2.14)(react@19.2.4): | |
| 4329 | dependencies: | |
| 4330 | '@types/hast': 3.0.4 | |
| 4331 | '@types/mdast': 4.0.4 | |
| 4332 | '@types/react': 19.2.14 | |
| 4333 | devlop: 1.1.0 | |
| 4334 | hast-util-to-jsx-runtime: 2.3.6 | |
| 4335 | html-url-attributes: 3.0.1 | |
| 4336 | mdast-util-to-hast: 13.2.1 | |
| 4337 | react: 19.2.4 | |
| 4338 | remark-parse: 11.0.0 | |
| 4339 | remark-rehype: 11.1.2 | |
| 4340 | unified: 11.0.5 | |
| 4341 | unist-util-visit: 5.1.0 | |
| 4342 | vfile: 6.0.3 | |
| 4343 | transitivePeerDependencies: | |
| 4344 | - supports-color | |
| 4345 | ||
| 4319 | 4346 | react-scan@0.5.3(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4): |
| 4320 | 4347 | dependencies: |
| 4321 | 4348 | '@babel/core': 7.29.0 |
src/Markdown.ts+19-6| ... | ... | @@ -4,6 +4,7 @@ import { type Processor } from "unified"; |
| 4 | 4 | import { type Components } from "rehype-react"; |
| 5 | 5 | import { jsx } from "react/jsx-runtime"; |
| 6 | 6 | |
| 7 | /** Options for {@linkcode Markdown} */ | |
| 7 | 8 | export interface MarkdownOptions { |
| 8 | 9 | /** Markdown source content. */ |
| 9 | 10 | content: string; |
| ... | ... | @@ -11,6 +12,8 @@ export interface MarkdownOptions { |
| 11 | 12 | processor?: Processor<import("unist").Node> | null | undefined; |
| 12 | 13 | /** Customize component rendering. */ |
| 13 | 14 | components?: Partial<Components> | null | undefined; |
| 15 | /** Automatically predict closing tags, useful for LLM streaming. */ | |
| 16 | predict?: boolean; | |
| 14 | 17 | } |
| 15 | 18 | |
| 16 | 19 | /** |
| ... | ... | @@ -23,14 +26,19 @@ export const Markdown = memo(function Markdown({ |
| 23 | 26 | content, |
| 24 | 27 | processor, |
| 25 | 28 | components, |
| 29 | predict, | |
| 26 | 30 | }: MarkdownOptions) { |
| 27 | 31 | const context = useContext(Context); |
| 28 | 32 | const ref = useRef<Memoizer | null>(null); |
| 29 | 33 | const memoizer = (ref.current ??= new Memoizer()); |
| 30 | memoizer.reconfigure(processor ?? context.processor ?? defaultProcessor, { | |
| 31 | ...components, | |
| 32 | ...context.components, | |
| 33 | }); | |
| 34 | memoizer.reconfigure( | |
| 35 | processor ?? context.processor ?? defaultProcessor, | |
| 36 | predict ?? context.predict ?? false, | |
| 37 | { | |
| 38 | ...components, | |
| 39 | ...context.components, | |
| 40 | }, | |
| 41 | ); | |
| 34 | 42 | return memoizer.update(content); |
| 35 | 43 | }, propsAreEqual); |
| 36 | 44 | |
| ... | ... | @@ -45,17 +53,22 @@ export const MarkdownOptionsProvider = memo(function MarkdownOptionsProvider({ |
| 45 | 53 | processor, |
| 46 | 54 | components, |
| 47 | 55 | children, |
| 56 | predict, | |
| 48 | 57 | }: PropsWithChildren<Context>) { |
| 49 | const value = useMemo(() => ({ processor, components }), [processor, components]); | |
| 58 | const value = useMemo( | |
| 59 | () => ({ processor, components, predict }), | |
| 60 | [processor, components, predict], | |
| 61 | ); | |
| 50 | 62 | return jsx(Context.Provider, { value, children }); |
| 51 | 63 | }, propsAreEqual); |
| 52 | 64 | |
| 53 | 65 | function propsAreEqual<T extends Context & { content?: string }>(prev: T, next: T) { |
| 54 | 66 | if (prev.content !== next.content) return false; |
| 55 | 67 | if (prev.processor !== next.processor) return false; |
| 68 | if (prev.predict !== next.predict) return false; | |
| 56 | 69 | if (prev.components === next.components) return true; |
| 57 | 70 | if (prev.components && next.components) { |
| 58 | 71 | return componentsAreEqual(prev.components, next.components); |
| 59 | 72 | } |
| 60 | return false; | |
| 73 | return !prev.components && !next.components; | |
| 61 | 74 | } |
src/Memoizer.ts+14-3| ... | ... | @@ -1,5 +1,4 @@ |
| 1 | 1 | import { ASSERT, UNWRAP } from "@clo/lib/assert.ts"; |
| 2 | import { indexOfDiff } from "./strings.ts"; | |
| 3 | 2 | import { type JSX, type Key, memo, type ReactNode } from "react"; |
| 4 | 3 | import { type Components } from "rehype-react"; |
| 5 | 4 | import remarkRehype from "remark-rehype"; |
| ... | ... | @@ -9,6 +8,7 @@ import type { Root as HastRoot } from "hast"; |
| 9 | 8 | import type { Literal, Node, Parent } from "unist"; |
| 10 | 9 | import { memoizedHastToReact, type RenderState } from "./hast.ts"; |
| 11 | 10 | import remarkParse from "remark-parse"; |
| 11 | import { Predict } from "./Prediction.ts"; | |
| 12 | 12 | |
| 13 | 13 | export const defaultProcessor = unified().use(remarkParse); |
| 14 | 14 | |
| ... | ... | @@ -35,6 +35,7 @@ export class Memoizer { |
| 35 | 35 | #previousComponents: Partial<Components> = {}; |
| 36 | 36 | |
| 37 | 37 | // incremental parsing graph, structure of arrays |
| 38 | #predict: Predict | null = null; | |
| 38 | 39 | #content: string = ""; |
| 39 | 40 | #positions: number[] = []; |
| 40 | 41 | #parsed: Node[][] = []; |
| ... | ... | @@ -43,12 +44,14 @@ export class Memoizer { |
| 43 | 44 | |
| 44 | 45 | reconfigure( |
| 45 | 46 | processor: Processor<Node, undefined, undefined, undefined, undefined | JSX.Element>, |
| 47 | predict: boolean, | |
| 46 | 48 | components: Partial<Components>, |
| 47 | 49 | ) { |
| 48 | 50 | if ( |
| 49 | 51 | this.#astProcessor !== null && |
| 50 | 52 | this.#baseProcessor === processor && |
| 51 | componentsAreEqual(this.#previousComponents, components) | |
| 53 | componentsAreEqual(this.#previousComponents, components) && | |
| 54 | !!this.#predict === predict | |
| 52 | 55 | ) { |
| 53 | 56 | return; |
| 54 | 57 | } |
| ... | ... | @@ -73,10 +76,12 @@ export class Memoizer { |
| 73 | 76 | this.#parsed = []; |
| 74 | 77 | this.#renderStates = []; |
| 75 | 78 | this.#reactNodes = []; |
| 79 | this.#predict = predict ? new Predict() : null; | |
| 76 | 80 | } |
| 77 | 81 | |
| 78 | 82 | update(content: string): readonly ReactNode[] { |
| 79 | const previous = this.#content; | |
| 83 | if (this.#predict) content = this.#predict.update(content); | |
| 84 | let previous = this.#content; | |
| 80 | 85 | if (content === previous) return this.#reactNodes; |
| 81 | 86 | |
| 82 | 87 | // Locate the last block that has changed content. This is done quickly by finding |
| ... | ... | @@ -166,6 +171,12 @@ export class Memoizer { |
| 166 | 171 | } |
| 167 | 172 | } |
| 168 | 173 | |
| 174 | export function indexOfDiff(a: string, b: string) { | |
| 175 | var i = 0; | |
| 176 | while (a[i] === b[i]) i++; | |
| 177 | return i; | |
| 178 | } | |
| 179 | ||
| 169 | 180 | export function componentsAreEqual(left: Partial<Components>, right: Partial<Components>) { |
| 170 | 181 | const leftRecord = left as Record<string, unknown>; |
| 171 | 182 | const rightRecord = right as Record<string, unknown>; |
src/Prediction.ts created+376| ... | ... | @@ -0,0 +1,376 @@ |
| 1 | /** | |
| 2 | * Incremental parser for predicting closing tags in Markdown text. Works best | |
| 3 | * when continuously appending text to the end of the string; this behavior | |
| 4 | * specifically triggers a fast parsing path. | |
| 5 | */ | |
| 6 | export class Predict { | |
| 7 | #stable = ""; | |
| 8 | ||
| 9 | /** Incrementally reparses the tail. */ | |
| 10 | update(text: string) { | |
| 11 | if (!text.startsWith(this.#stable)) this.#stable = ""; | |
| 12 | let tail = text.slice(this.#stable.length); | |
| 13 | let state = parseTail(tail); | |
| 14 | if (state.commitIndex > 0) { | |
| 15 | this.#stable += tail.slice(0, state.commitIndex); | |
| 16 | tail = tail.slice(state.commitIndex); | |
| 17 | state = parseTail(tail); | |
| 18 | } | |
| 19 | return this.#stable + renderTail(tail, state); | |
| 20 | } | |
| 21 | } | |
| 22 | ||
| 23 | type DelimToken = "***" | "**" | "__" | "~~" | "*" | "_"; | |
| 24 | type DelimState = { start: number; token: DelimToken }; | |
| 25 | type LinkState = | |
| 26 | | { phase: "text"; start: number } | |
| 27 | | { phase: "url_wait"; start: number; textEnd: number } | |
| 28 | | { phase: "url"; start: number; textEnd: number; parenDepth: number }; | |
| 29 | type ExclusiveState = | |
| 30 | | { kind: "code"; start: number; token: string } | |
| 31 | | { kind: "fence"; start: number; token: string } | |
| 32 | | { kind: "math"; start: number; token: "$" | "$$" }; | |
| 33 | interface TailState { | |
| 34 | commitIndex: number; | |
| 35 | delims: DelimState[]; | |
| 36 | exclusive: ExclusiveState | null; | |
| 37 | links: LinkState[]; | |
| 38 | }; | |
| 39 | ||
| 40 | const NESTABLE_DELIMS = new Set<DelimToken>(["*", "**", "_", "__"]); | |
| 41 | ||
| 42 | function parseTail(tail: string): TailState { | |
| 43 | const state: TailState = { | |
| 44 | commitIndex: 0, | |
| 45 | delims: [], | |
| 46 | exclusive: null, | |
| 47 | links: [], | |
| 48 | }; | |
| 49 | ||
| 50 | for (let i = 0; i < tail.length; i++) { | |
| 51 | const char = tail[i]!; | |
| 52 | ||
| 53 | if (char === "\n" && tail[i - 1] === "\n") { | |
| 54 | resetParagraphState(state); | |
| 55 | if (!state.exclusive) state.commitIndex = i + 1; | |
| 56 | continue; | |
| 57 | } | |
| 58 | ||
| 59 | const exclusiveEnd = consumeExclusive(state, tail, i); | |
| 60 | if (exclusiveEnd > i) { | |
| 61 | i = exclusiveEnd - 1; | |
| 62 | continue; | |
| 63 | } | |
| 64 | ||
| 65 | if (state.exclusive) continue; | |
| 66 | ||
| 67 | if (inLinkUrlMode(state)) { | |
| 68 | updateLink(state, i, char); | |
| 69 | continue; | |
| 70 | } | |
| 71 | ||
| 72 | const delimEnd = consumeDelim(state, tail, i); | |
| 73 | if (delimEnd > i) { | |
| 74 | i = delimEnd - 1; | |
| 75 | continue; | |
| 76 | } | |
| 77 | ||
| 78 | if (isEscapedAt(tail, i)) continue; | |
| 79 | ||
| 80 | updateLink(state, i, char); | |
| 81 | } | |
| 82 | ||
| 83 | return state; | |
| 84 | } | |
| 85 | ||
| 86 | function consumeExclusive(state: TailState, tail: string, start: number) { | |
| 87 | const char = tail[start]; | |
| 88 | if (char === "`") return consumeBackticks(state, tail, start); | |
| 89 | if (char === "$") return consumeMath(state, tail, start); | |
| 90 | if (char === "~") return consumeTildes(state, tail, start); | |
| 91 | return start; | |
| 92 | } | |
| 93 | ||
| 94 | function consumeBackticks(state: TailState, tail: string, start: number) { | |
| 95 | const length = runLengthAt(tail, start); | |
| 96 | const token = "`".repeat(length); | |
| 97 | const exclusive = state.exclusive; | |
| 98 | ||
| 99 | if (exclusive?.kind === "fence") { | |
| 100 | if ( | |
| 101 | exclusive.token[0] === "`" && | |
| 102 | isLineStart(tail, start) && | |
| 103 | !isEscapedAt(tail, start) && | |
| 104 | length >= exclusive.token.length | |
| 105 | ) { | |
| 106 | state.exclusive = null; | |
| 107 | } | |
| 108 | return start + length; | |
| 109 | } | |
| 110 | ||
| 111 | if (exclusive?.kind === "code") { | |
| 112 | if (!isEscapedAt(tail, start) && length >= exclusive.token.length) { | |
| 113 | state.exclusive = null; | |
| 114 | } | |
| 115 | return start + length; | |
| 116 | } | |
| 117 | ||
| 118 | if (exclusive) return start + length; | |
| 119 | if (isEscapedAt(tail, start)) return start + length; | |
| 120 | ||
| 121 | if (length >= 3 && isLineStart(tail, start)) { | |
| 122 | state.exclusive = { kind: "fence", start, token }; | |
| 123 | return start + length; | |
| 124 | } | |
| 125 | ||
| 126 | state.exclusive = { kind: "code", start, token }; | |
| 127 | return start + length; | |
| 128 | } | |
| 129 | ||
| 130 | function consumeMath(state: TailState, tail: string, start: number) { | |
| 131 | const length = runLengthAt(tail, start); | |
| 132 | const exclusive = state.exclusive; | |
| 133 | ||
| 134 | if (exclusive?.kind === "math") { | |
| 135 | if (!isEscapedAt(tail, start) && length >= exclusive.token.length) { | |
| 136 | state.exclusive = null; | |
| 137 | } | |
| 138 | return start + length; | |
| 139 | } | |
| 140 | ||
| 141 | if (exclusive) return start + length; | |
| 142 | if (isEscapedAt(tail, start)) return start + length; | |
| 143 | ||
| 144 | state.exclusive = { | |
| 145 | kind: "math", | |
| 146 | start, | |
| 147 | token: length >= 2 ? "$$" : "$", | |
| 148 | }; | |
| 149 | return start + length; | |
| 150 | } | |
| 151 | ||
| 152 | function consumeTildes(state: TailState, tail: string, start: number) { | |
| 153 | const length = runLengthAt(tail, start); | |
| 154 | const exclusive = state.exclusive; | |
| 155 | ||
| 156 | if (exclusive?.kind === "fence" && exclusive.token[0] === "~") { | |
| 157 | if ( | |
| 158 | isLineStart(tail, start) && | |
| 159 | !isEscapedAt(tail, start) && | |
| 160 | length >= exclusive.token.length | |
| 161 | ) { | |
| 162 | state.exclusive = null; | |
| 163 | } | |
| 164 | return start + length; | |
| 165 | } | |
| 166 | ||
| 167 | if (exclusive) return start; | |
| 168 | if (length < 3 || !isLineStart(tail, start) || isEscapedAt(tail, start)) return start; | |
| 169 | ||
| 170 | state.exclusive = { kind: "fence", start, token: "~".repeat(length) }; | |
| 171 | return start + length; | |
| 172 | } | |
| 173 | ||
| 174 | function consumeDelim(state: TailState, tail: string, start: number) { | |
| 175 | const token = matchDelim(tail, start); | |
| 176 | if (!token) return start; | |
| 177 | ||
| 178 | if (isEscapedAt(tail, start)) return start + token.length; | |
| 179 | ||
| 180 | const existingIndex = state.delims.findLastIndex((delim) => delim.token === token); | |
| 181 | if (existingIndex !== -1) { | |
| 182 | state.delims.splice(existingIndex, 1); | |
| 183 | return start + token.length; | |
| 184 | } | |
| 185 | ||
| 186 | if (!canOpenDelim(tail, start, token)) return start + token.length; | |
| 187 | ||
| 188 | state.delims.push({ start, token }); | |
| 189 | return start + token.length; | |
| 190 | } | |
| 191 | ||
| 192 | function matchDelim(tail: string, start: number): DelimToken | undefined { | |
| 193 | const char = tail[start]; | |
| 194 | ||
| 195 | if (char === "*") { | |
| 196 | if (tail.startsWith("***", start)) return "***"; | |
| 197 | if (tail.startsWith("**", start)) return "**"; | |
| 198 | return "*"; | |
| 199 | } | |
| 200 | ||
| 201 | if (char === "_") { | |
| 202 | if (tail.startsWith("__", start)) return "__"; | |
| 203 | return "_"; | |
| 204 | } | |
| 205 | ||
| 206 | if (char === "~" && tail.startsWith("~~", start)) { | |
| 207 | return "~~"; | |
| 208 | } | |
| 209 | } | |
| 210 | ||
| 211 | function canOpenDelim(tail: string, start: number, token: DelimToken) { | |
| 212 | const next = tail[start + token.length]; | |
| 213 | if (!next || /\s/.test(next)) return false; | |
| 214 | ||
| 215 | const prev = tail[start - 1]; | |
| 216 | return !isWordChar(prev) || !isWordChar(next); | |
| 217 | } | |
| 218 | ||
| 219 | function updateLink(state: TailState, index: number, char: string) { | |
| 220 | const top = state.links.at(-1); | |
| 221 | ||
| 222 | if (char === "[") { | |
| 223 | state.links.push({ phase: "text", start: index }); | |
| 224 | return; | |
| 225 | } | |
| 226 | ||
| 227 | if (char === "]" && top?.phase === "text") { | |
| 228 | state.links[state.links.length - 1] = { | |
| 229 | phase: "url_wait", | |
| 230 | start: top.start, | |
| 231 | textEnd: index, | |
| 232 | }; | |
| 233 | return; | |
| 234 | } | |
| 235 | ||
| 236 | if (char === "(" && top?.phase === "url_wait") { | |
| 237 | state.links[state.links.length - 1] = { | |
| 238 | phase: "url", | |
| 239 | start: top.start, | |
| 240 | textEnd: top.textEnd, | |
| 241 | parenDepth: 0, | |
| 242 | }; | |
| 243 | return; | |
| 244 | } | |
| 245 | ||
| 246 | if (char === "(" && top?.phase === "url") { | |
| 247 | top.parenDepth += 1; | |
| 248 | return; | |
| 249 | } | |
| 250 | ||
| 251 | if (char === ")" && top?.phase === "url") { | |
| 252 | if (top.parenDepth > 0) { | |
| 253 | top.parenDepth -= 1; | |
| 254 | return; | |
| 255 | } | |
| 256 | ||
| 257 | state.links.pop(); | |
| 258 | } | |
| 259 | } | |
| 260 | ||
| 261 | function resetParagraphState(state: TailState) { | |
| 262 | state.delims.length = 0; | |
| 263 | state.links.length = 0; | |
| 264 | ||
| 265 | if (state.exclusive?.kind === "fence") return; | |
| 266 | if (state.exclusive?.kind === "math" && state.exclusive.token === "$$") return; | |
| 267 | ||
| 268 | state.exclusive = null; | |
| 269 | } | |
| 270 | ||
| 271 | function inLinkUrlMode(state: TailState) { | |
| 272 | const phase = state.links.at(-1)?.phase; | |
| 273 | return phase === "url_wait" || phase === "url"; | |
| 274 | } | |
| 275 | ||
| 276 | function renderTail(tail: string, state: TailState) { | |
| 277 | const link = state.links.at(-1); | |
| 278 | if (link) return renderOpenLink(tail, link); | |
| 279 | ||
| 280 | const nestedClosers = renderNestedClosers(state); | |
| 281 | if (nestedClosers) return appendCloser(tail, nestedClosers); | |
| 282 | ||
| 283 | const open = findLastOpen(state); | |
| 284 | if (!open) return tail; | |
| 285 | ||
| 286 | if (open.kind === "delim") { | |
| 287 | if (!hasContentAfter(tail, open.start, open.token.length)) return tail; | |
| 288 | if (/\s/.test(tail[tail.length - 1] ?? "")) return tail; | |
| 289 | return appendCloser(tail, open.token); | |
| 290 | } | |
| 291 | ||
| 292 | if (!hasContentAfter(tail, open.start, open.token.length)) return tail; | |
| 293 | if (open.kind === "fence") return tail; | |
| 294 | if (open.kind === "code") return appendCloser(tail, open.token); | |
| 295 | if (open.token === "$$") return appendCloser(tail, (tail.endsWith("\n") ? "" : "\n") + "$$"); | |
| 296 | if (/\s/.test(tail[tail.length - 1] ?? "")) return tail; | |
| 297 | return appendCloser(tail, "$"); | |
| 298 | } | |
| 299 | ||
| 300 | function renderOpenLink(tail: string, state: LinkState) { | |
| 301 | const before = tail.slice(0, state.start); | |
| 302 | ||
| 303 | if (state.phase === "text") { | |
| 304 | if (!hasContentAfter(tail, state.start, 1)) return tail; | |
| 305 | return before + tail.slice(state.start + 1); | |
| 306 | } | |
| 307 | ||
| 308 | const text = tail.slice(state.start + 1, state.textEnd); | |
| 309 | if (state.phase === "url_wait") return before + text + tail.slice(state.textEnd + 1); | |
| 310 | return before + text; | |
| 311 | } | |
| 312 | ||
| 313 | function renderNestedClosers(state: TailState) { | |
| 314 | const closers: string[] = []; | |
| 315 | ||
| 316 | if (state.exclusive) { | |
| 317 | if (state.exclusive.kind !== "code") return undefined; | |
| 318 | closers.push(state.exclusive.token); | |
| 319 | } | |
| 320 | ||
| 321 | for (let i = state.delims.length - 1; i >= 0; i--) { | |
| 322 | const token = state.delims[i]!.token; | |
| 323 | if (!NESTABLE_DELIMS.has(token)) return undefined; | |
| 324 | closers.push(token); | |
| 325 | } | |
| 326 | ||
| 327 | if (closers.length < 2) return undefined; | |
| 328 | return closers.join(""); | |
| 329 | } | |
| 330 | ||
| 331 | function appendCloser(tail: string, closer: string) { | |
| 332 | return tail + closer.slice(overlapLength(tail, closer)); | |
| 333 | } | |
| 334 | ||
| 335 | function overlapLength(tail: string, closer: string) { | |
| 336 | for (let length = Math.min(tail.length, closer.length); length > 0; length -= 1) { | |
| 337 | if (tail.endsWith(closer.slice(0, length))) return length; | |
| 338 | } | |
| 339 | return 0; | |
| 340 | } | |
| 341 | ||
| 342 | function findLastOpen(state: TailState) { | |
| 343 | const delim = state.delims.at(-1); | |
| 344 | if (state.exclusive && (!delim || state.exclusive.start > delim.start)) return state.exclusive; | |
| 345 | if (delim) return { kind: "delim" as const, start: delim.start, token: delim.token }; | |
| 346 | return state.exclusive; | |
| 347 | } | |
| 348 | ||
| 349 | function runLengthAt(text: string, start: number) { | |
| 350 | let end = start + 1; | |
| 351 | while (end < text.length && text[end] === text[start]) end += 1; | |
| 352 | return end - start; | |
| 353 | } | |
| 354 | ||
| 355 | function hasContentAfter(text: string, start: number, tokenLength: number) { | |
| 356 | return text.length > start + tokenLength; | |
| 357 | } | |
| 358 | ||
| 359 | function isLineStart(text: string, index: number) { | |
| 360 | return index === 0 || text[index - 1] === "\n"; | |
| 361 | } | |
| 362 | ||
| 363 | function isEscapedAt(text: string, index: number) { | |
| 364 | let count = 0; | |
| 365 | ||
| 366 | for (let i = index - 1; i >= 0; i--) { | |
| 367 | if (text[i] !== "\\") break; | |
| 368 | count += 1; | |
| 369 | } | |
| 370 | ||
| 371 | return count % 2 === 1; | |
| 372 | } | |
| 373 | ||
| 374 | function isWordChar(char?: string) { | |
| 375 | return !!char && /[A-Za-z0-9]/.test(char); | |
| 376 | } |
src/strings.ts deleted-5| ... | ... | @@ -1,5 +0,0 @@ |
| 1 | export function indexOfDiff(a: string, b: string) { | |
| 2 | var i = 0; | |
| 3 | while (a[i] === b[i]) i++; | |
| 4 | return i; | |
| 5 | } |
tests/Markdown.memoization.test.tsx+17-2| ... | ... | @@ -1,7 +1,7 @@ |
| 1 | 1 | import { cleanup, render, screen } from "@testing-library/react"; |
| 2 | 2 | import type { Components } from "rehype-react"; |
| 3 | 3 | import type { ComponentPropsWithoutRef, JSX } from "react"; |
| 4 | import { afterEach, describe, expect, it, vi } from "vite-plus/test"; | |
| 4 | import { afterEach, expect, it, vi } from "vite-plus/test"; | |
| 5 | 5 | import { Markdown, MarkdownOptionsProvider } from "../src/Markdown.ts"; |
| 6 | 6 | import { Memoizer } from "../src/Memoizer.ts"; |
| 7 | 7 | |
| ... | ... | @@ -107,7 +107,7 @@ it("does not rerender an unchanged inline custom renderer when only adjacent tex |
| 107 | 107 | expect(Link).toHaveBeenCalledTimes(1); |
| 108 | 108 | }); |
| 109 | 109 | |
| 110 | it.only("does not rerender an unchanged inline custom renderer with formatted children when only adjacent text changes", () => { | |
| 110 | it("does not rerender an unchanged inline custom renderer with formatted children when only adjacent text changes", () => { | |
| 111 | 111 | const Link = vi.fn(function Link(props: MarkdownProps<"a">) { |
| 112 | 112 | return <a data-testid="rich-link" {...omitNode(props)} />; |
| 113 | 113 | }); |
| ... | ... | @@ -154,3 +154,18 @@ it("does not rerender Markdown through the provider when the effective component |
| 154 | 154 | expect(Link).toHaveBeenCalledTimes(1); |
| 155 | 155 | expect(screen.getByTestId("provider-link").textContent).toBe("stable"); |
| 156 | 156 | }); |
| 157 | ||
| 158 | it("reprocesses the document when prediction changes", () => { | |
| 159 | const { container, rerender } = render( | |
| 160 | <Markdown content="hello *world" predict={false} />, | |
| 161 | renderOptions, | |
| 162 | ); | |
| 163 | ||
| 164 | expect(container.querySelector("em")).toBeNull(); | |
| 165 | expect(container.textContent).toBe("hello *world"); | |
| 166 | ||
| 167 | rerender(<Markdown content="hello *world" predict />); | |
| 168 | ||
| 169 | expect(container.querySelector("em")?.textContent).toBe("world"); | |
| 170 | expect(container.textContent).toBe("hello world"); | |
| 171 | }); |
tests/prediction.test.ts created+343| ... | ... | @@ -0,0 +1,343 @@ |
| 1 | import { test, describe, expect } from "vite-plus/test"; | |
| 2 | import { Predict } from "../src/Prediction.ts"; | |
| 3 | ||
| 4 | describe.each(["stateless", "stateful"] as const)("%s processing", (mode) => { | |
| 5 | function stateless(text: string) { | |
| 6 | return new Predict().update(text); | |
| 7 | } | |
| 8 | ||
| 9 | function splitBySizes(text: string, sizes: number[]) { | |
| 10 | const chunks: string[] = []; | |
| 11 | let index = 0; | |
| 12 | ||
| 13 | for (const size of sizes) { | |
| 14 | if (index >= text.length) break; | |
| 15 | ||
| 16 | const nextIndex = Math.min(index + size, text.length); | |
| 17 | chunks.push(text.slice(index, nextIndex)); | |
| 18 | index = nextIndex; | |
| 19 | } | |
| 20 | ||
| 21 | if (index < text.length) { | |
| 22 | chunks.push(text.slice(index)); | |
| 23 | } | |
| 24 | ||
| 25 | return chunks; | |
| 26 | } | |
| 27 | ||
| 28 | function addChunking(text: string, sizes: number[], seen: Set<string>, chunkings: string[][]) { | |
| 29 | const chunks = splitBySizes(text, sizes); | |
| 30 | const key = JSON.stringify(chunks); | |
| 31 | ||
| 32 | if (seen.has(key)) return; | |
| 33 | seen.add(key); | |
| 34 | chunkings.push(chunks); | |
| 35 | } | |
| 36 | ||
| 37 | function buildChunkings(text: string) { | |
| 38 | if (text.length === 0) return [[]]; | |
| 39 | ||
| 40 | const seen = new Set<string>(); | |
| 41 | const chunkings: string[][] = []; | |
| 42 | ||
| 43 | if (text.length <= 10) { | |
| 44 | const splitCount = text.length - 1; | |
| 45 | const totalMasks = 1 << splitCount; | |
| 46 | ||
| 47 | for (let mask = 0; mask < totalMasks; mask++) { | |
| 48 | const sizes: number[] = []; | |
| 49 | let last = 0; | |
| 50 | ||
| 51 | for (let bit = 0; bit < splitCount; bit++) { | |
| 52 | if ((mask & (1 << bit)) === 0) continue; | |
| 53 | ||
| 54 | const next = bit + 1; | |
| 55 | sizes.push(next - last); | |
| 56 | last = next; | |
| 57 | } | |
| 58 | ||
| 59 | sizes.push(text.length - last); | |
| 60 | addChunking(text, sizes, seen, chunkings); | |
| 61 | } | |
| 62 | ||
| 63 | return chunkings; | |
| 64 | } | |
| 65 | ||
| 66 | for (let size = 1; size <= text.length; size++) { | |
| 67 | const sizes: number[] = []; | |
| 68 | ||
| 69 | for (let remaining = text.length; remaining > 0; remaining -= size) { | |
| 70 | sizes.push(Math.min(size, remaining)); | |
| 71 | } | |
| 72 | ||
| 73 | addChunking(text, sizes, seen, chunkings); | |
| 74 | } | |
| 75 | ||
| 76 | const maxOffsetSize = Math.min(text.length, 8); | |
| 77 | ||
| 78 | for (let size = 2; size <= maxOffsetSize; size++) { | |
| 79 | for (let head = 1; head < size && head < text.length; head++) { | |
| 80 | const sizes = [head]; | |
| 81 | ||
| 82 | for (let remaining = text.length - head; remaining > 0; remaining -= size) { | |
| 83 | sizes.push(Math.min(size, remaining)); | |
| 84 | } | |
| 85 | ||
| 86 | addChunking(text, sizes, seen, chunkings); | |
| 87 | } | |
| 88 | ||
| 89 | for (let tail = 1; tail < size && tail < text.length; tail++) { | |
| 90 | const sizes: number[] = []; | |
| 91 | ||
| 92 | for (let remaining = text.length - tail; remaining > 0; remaining -= size) { | |
| 93 | sizes.push(Math.min(size, remaining)); | |
| 94 | } | |
| 95 | ||
| 96 | sizes.push(tail); | |
| 97 | addChunking(text, sizes, seen, chunkings); | |
| 98 | } | |
| 99 | } | |
| 100 | ||
| 101 | return chunkings; | |
| 102 | } | |
| 103 | ||
| 104 | function assertStatefulMatchesStateless(text: string) { | |
| 105 | for (const chunks of buildChunkings(text)) { | |
| 106 | const prediction = new Predict(); | |
| 107 | let prefix = ""; | |
| 108 | ||
| 109 | for (const chunk of chunks) { | |
| 110 | prefix += chunk; | |
| 111 | ||
| 112 | const actual = prediction.update(prefix); | |
| 113 | const expected = stateless(prefix); | |
| 114 | ||
| 115 | if (actual !== expected) { | |
| 116 | throw new Error( | |
| 117 | [ | |
| 118 | "stateful prediction diverged from stateless parsing", | |
| 119 | `input: ${JSON.stringify(text)}`, | |
| 120 | `chunks: ${JSON.stringify(chunks)}`, | |
| 121 | `prefix: ${JSON.stringify(prefix)}`, | |
| 122 | `expected: ${JSON.stringify(expected)}`, | |
| 123 | `received: ${JSON.stringify(actual)}`, | |
| 124 | ].join("\n"), | |
| 125 | ); | |
| 126 | } | |
| 127 | } | |
| 128 | } | |
| 129 | } | |
| 130 | ||
| 131 | function check(text: string, expected: string) { | |
| 132 | if (mode === "stateless") { | |
| 133 | expect(stateless(text)).toBe(expected); | |
| 134 | } else { | |
| 135 | assertStatefulMatchesStateless(text); | |
| 136 | } | |
| 137 | } | |
| 138 | ||
| 139 | test.each([ | |
| 140 | ["plain text", "plain text", "plain text"], | |
| 141 | ["italic", "a *hello world", "a *hello world*"], | |
| 142 | ["bold", "a **hello world", "a **hello world**"], | |
| 143 | ["bold italic", "a ***hello world", "a ***hello world***"], | |
| 144 | ["code", "a `hello world", "a `hello world`"], | |
| 145 | ["strike", "a ~~hello world", "a ~~hello world~~"], | |
| 146 | ["katex", "something\n\n$$\nE =", "something\n\n$$\nE =\n$$"], | |
| 147 | ||
| 148 | ||
| 149 | ["bold partial close", "a **hello world*", "a **hello world**"], | |
| 150 | ["bold italic partial close 1", "a ***hello world*", "a ***hello world***"], | |
| 151 | ["bold italic partial close 2", "a ***hello world**", "a ***hello world***"], | |
| 152 | ["code", "a `hello world", "a `hello world`"], | |
| 153 | ["strike", "a ~~hello world", "a ~~hello world~~"], | |
| 154 | ["strike partial close", "a ~~hello world~", "a ~~hello world~~"], | |
| 155 | ["katex", "something\n\n$$\nE =", "something\n\n$$\nE =\n$$"], | |
| 156 | ["katex partial close", "something\n\n$$\nE =\n$", "something\n\n$$\nE =\n$$"], | |
| 157 | ||
| 158 | ["balanced markdown stays unchanged", "a *hello* world", "a *hello* world"], | |
| 159 | ["balanced bold stays unchanged", "a **hello** world", "a **hello** world"], | |
| 160 | ["balanced double underscore stays unchanged", "a __hello__ world", "a __hello__ world"], | |
| 161 | ["balanced strike stays unchanged", "a ~~hello~~ world", "a ~~hello~~ world"], | |
| 162 | ["balanced code stays unchanged", "a `hello` world", "a `hello` world"], | |
| 163 | ["balanced inline math stays unchanged", "a $x$ world", "a $x$ world"], | |
| 164 | [ | |
| 165 | "balanced display math stays unchanged", | |
| 166 | "something\n\n$$\nE = mc^2\n$$", | |
| 167 | "something\n\n$$\nE = mc^2\n$$", | |
| 168 | ], | |
| 169 | ["escaped markdown stays escaped", "a \\*hello", "a \\*hello"], | |
| 170 | ["escaped underscore stays escaped", "a \\_hello", "a \\_hello"], | |
| 171 | ["escaped strike stays escaped", "a \\~~hello", "a \\~~hello"], | |
| 172 | ["escaped code stays escaped", "a \\`hello", "a \\`hello"], | |
| 173 | ["escaped link bracket stays escaped", "a \\[hello", "a \\[hello"], | |
| 174 | ["escaped inline math stays escaped", "a \\$x", "a \\$x"], | |
| 175 | ["code suppresses other inline markdown", "a `code *not emphasis", "a `code *not emphasis`"], | |
| 176 | ["paragraph break resets open formatting", "a **hello\n\nworld", "a **hello\n\nworld"], | |
| 177 | ])("%s", (_name, input, expected) => check(input, expected)); | |
| 178 | ||
| 179 | describe("whitespace around emphasis delimiters", () => { | |
| 180 | test.each([ | |
| 181 | "hello * world", | |
| 182 | "hello* world", | |
| 183 | "hello _ world", | |
| 184 | "hello_ world", | |
| 185 | "hello ** world", | |
| 186 | "hello** world", | |
| 187 | "hello __ world", | |
| 188 | "hello__ world", | |
| 189 | ])("does not predict for %j", (input) => check(input, input)); | |
| 190 | }); | |
| 191 | ||
| 192 | describe("flanking around emphasis delimiters", () => { | |
| 193 | test.each([ | |
| 194 | ["hello *world", "hello *world*"], | |
| 195 | ["hello **world", "hello **world**"], | |
| 196 | ["hello _world", "hello _world_"], | |
| 197 | ["hello __world", "hello __world__"], | |
| 198 | ])("predicts for left-flanking opener %j", (input, expected) => check(input, expected)); | |
| 199 | ||
| 200 | test.each(["snake_case", "foo*bar", "foo**bar", "foo__bar"])( | |
| 201 | "does not predict inside words for %j", | |
| 202 | (input) => check(input, input), | |
| 203 | ); | |
| 204 | ||
| 205 | // TODO: these are dubious | |
| 206 | test.each(["hello *world ", "hello **world ", "hello _world ", "hello __world "])( | |
| 207 | "does not add a closer after trailing whitespace for %j", | |
| 208 | (input) => check(input, input), | |
| 209 | ); | |
| 210 | }); | |
| 211 | ||
| 212 | describe("soft line breaks", () => { | |
| 213 | test.each([ | |
| 214 | ["italic continues across single newline", "a *hello\nworld", "a *hello\nworld*"], | |
| 215 | ["bold continues across single newline", "a **hello\nworld", "a **hello\nworld**"], | |
| 216 | ["strike continues across single newline", "a ~~hello\nworld", "a ~~hello\nworld~~"], | |
| 217 | ["code continues across single newline", "a `hello\nworld", "a `hello\nworld`"], | |
| 218 | ])("%s", (_name, input, expected) => check(input, expected)); | |
| 219 | }); | |
| 220 | ||
| 221 | describe("code span variants", () => { | |
| 222 | test.each([ | |
| 223 | ["double backtick code span", "a ``hello world", "a ``hello world``"], | |
| 224 | ["double backtick can contain a single backtick", "a ``hello ` world", "a ``hello ` world``"], | |
| 225 | ])("%s", (_name, input, expected) => check(input, expected)); | |
| 226 | }); | |
| 227 | ||
| 228 | describe("stuff in code blocks aren't completed", () => { | |
| 229 | test.each([ | |
| 230 | ["backtick code basic", "hello\n\n```tsx\nconst y = '**bold"], | |
| 231 | ["tilde code block", "hello\n\n~~~tsx\nconst y = '**bold"], | |
| 232 | ||
| 233 | ["italic", "hello\n\n```tsx\nconst y = '*italic"], | |
| 234 | ["strike", "hello\n\n```tsx\nconst y = '~~strike"], | |
| 235 | ["link 1", "hello\n\n```tsx\nconst y = '[link"], | |
| 236 | ["link 2", "hello\n\n```tsx\nconst y = '[link]"], | |
| 237 | ["link 3", "hello\n\n```tsx\nconst y = '[link]("], | |
| 238 | ["link 4", "hello\n\n```tsx\nconst y = '[link](meow"], | |
| 239 | ["underscore", "hello\n\n```tsx\nconst y = '__this"], | |
| 240 | ["inline code", "hello\n\n```tsx\nconst y = `template"], | |
| 241 | ||
| 242 | ["backtick code extra line", "hello\n\n```tsx\nmeow\n\nconst y = '**bold"], | |
| 243 | ["tilde code extra line", "hello\n\n~~~tsx\nmeow\n\nconst y = '**bold"], | |
| 244 | ||
| 245 | ["italic", "hello\n\n```tsx\nmeow\n\nconst y = '*italic"], | |
| 246 | ["strike", "hello\n\n```tsx\nmeow\n\nconst y = '~~strike"], | |
| 247 | ["link 1", "hello\n\n```tsx\nmeow\n\nconst y = '[link"], | |
| 248 | ["link 2", "hello\n\n```tsx\nmeow\n\nconst y = '[link]"], | |
| 249 | ["link 3", "hello\n\n```tsx\nmeow\n\nconst y = '[link]("], | |
| 250 | ["link 4", "hello\n\n```tsx\nmeow\n\nconst y = '[link](meow"], | |
| 251 | ["underscore", "hello\n\n```tsx\nmeow\n\nconst y = '__this"], | |
| 252 | ["inline code", "hello\n\n```tsx\nmeow\n\nconst y = `template"], | |
| 253 | ])("%s", (_name, input) => check(input, input)); | |
| 254 | }); | |
| 255 | ||
| 256 | describe("math handling", () => { | |
| 257 | test.each([ | |
| 258 | ["inline math", "a $x + y", "a $x + y$"], | |
| 259 | ["inline math suppresses emphasis markers", "a $x * y", "a $x * y$"], | |
| 260 | [ | |
| 261 | "display math suppresses emphasis markers", | |
| 262 | "something\n\n$$\na * b", | |
| 263 | "something\n\n$$\na * b\n$$", | |
| 264 | ], | |
| 265 | ])("%s", (_name, input, expected) => check(input, expected)); | |
| 266 | }); | |
| 267 | ||
| 268 | describe("nested emphasis handling", () => { | |
| 269 | test.each([ | |
| 270 | ["bold containing italic", "one **two *three", "one **two *three***"], | |
| 271 | ["italic containing bold", "one *two **three", "one *two **three***"], | |
| 272 | ["bold containing code", "**hello `world", "**hello `world`**"], | |
| 273 | ["italic containing code", "*hello `world", "*hello `world`*"], | |
| 274 | ["mixed double underscores inside bold", "one **two __three", "one **two __three__**"], | |
| 275 | ["mixed bold inside double underscores", "one __two **three", "one __two **three**__"], | |
| 276 | ["mixed italic underscore inside bold", "one **two _three", "one **two _three_**"], | |
| 277 | ["mixed bold inside italic underscore", "one _two **three", "one _two **three**_"], | |
| 278 | ["mixed italic markers", "one *two _three", "one *two _three_*"], | |
| 279 | ["mixed italic markers reversed", "one _two *three", "one _two *three*_"], | |
| 280 | ["bold containing code with trailing space", "**hello `world ", "**hello `world `**"], | |
| 281 | ["italic containing code with trailing space", "*hello `world ", "*hello `world `*"], | |
| 282 | ["bold containing code opener", "one **two `three", "one **two `three`**"], | |
| 283 | ["star in code block", "one `two *three", "one `two *three`"], | |
| 284 | ])("%s", (_name, input, expected) => check(input, expected)); | |
| 285 | ||
| 286 | // these cases unsupported | |
| 287 | test.each([ | |
| 288 | ["strike inside bold", "one **two ~~three", "one **two ~~three~~"], | |
| 289 | ["bold inside strike", "one ~~two **three", "one ~~two **three**"], | |
| 290 | // NOTE: if these close just the last one, its ok to edit the test. that behavior is better | |
| 291 | ["double underscore containing double underscore", "one __two __three", "one __two __three"], | |
| 292 | ["bold containing bold", "one **two **three", "one **two **three"], | |
| 293 | ["italic containing italic", "one *two *three", "one *two *three"], | |
| 294 | ["strike containing strike", "one ~~two ~~three", "one ~~two ~~three"], | |
| 295 | ])("%s", (_name, input, expected) => check(input, expected)); | |
| 296 | }); | |
| 297 | ||
| 298 | test.each([ | |
| 299 | ["link 1", "a [hello world", "a hello world"], | |
| 300 | ["link 2", "a [hello world](incomplete", "a hello world"], | |
| 301 | ["link 3", "a [hello] tail", "a hello tail"], | |
| 302 | ["link 4", "a [x](", "a x"], | |
| 303 | ["link full", "a [hello world](complete)", "a [hello world](complete)"], | |
| 304 | ["link full 2", "a [hello world](complete) xyz", "a [hello world](complete) xyz"], | |
| 305 | ["link with parentheses in url", "a [hello](path_(x))", "a [hello](path_(x))"], | |
| 306 | ["link with title", 'a [hello](url "title")', 'a [hello](url "title")'], | |
| 307 | ["link with emphasis in label", "a [*hello*](url)", "a [*hello*](url)"], | |
| 308 | ["link with code in label", "a [`hello`](url)", "a [`hello`](url)"], | |
| 309 | ["link full then open link", "a [link](done) and [open", "a [link](done) and open"], | |
| 310 | ["link full then open emphasis", "a [link](done) and *open", "a [link](done) and *open*"], | |
| 311 | ])("%s", (_name, input, expected) => check(input, expected)); | |
| 312 | ||
| 313 | describe("lossy link edge cases", () => { | |
| 314 | test.each([ | |
| 315 | ["nested open bracket keeps the outer bracket visible", "[a [b", "[a b"], | |
| 316 | [ | |
| 317 | "multiple open links keep the earliest unmatched bracket visible", | |
| 318 | "a [one [two", | |
| 319 | "a [one two", | |
| 320 | ], | |
| 321 | ["unfinished link text can swallow emphasis markers", "[a **b", "a **b"], | |
| 322 | ["unfinished link text can swallow italic markers", "[a *b", "a *b"], | |
| 323 | ["unfinished url drops the url entirely", "hello [label](url and more", "hello label"], | |
| 324 | ["unfinished url with spaces still drops to text", "[x](y z", "x"], | |
| 325 | ["completed label without url degrades to text", "hello [label] tail", "hello label tail"], | |
| 326 | ])("%s", (_name, input, expected) => check(input, expected)); | |
| 327 | }); | |
| 328 | ||
| 329 | describe("opener-only inputs", () => { | |
| 330 | test.each([ | |
| 331 | ["lone italic delimiter", "*", "*"], | |
| 332 | ["lone bold delimiter", "**", "**"], | |
| 333 | ["lone underscore delimiter", "_", "_"], | |
| 334 | ["lone double underscore delimiter", "__", "__"], | |
| 335 | ["lone strike delimiter", "~~", "~~"], | |
| 336 | ["lone code delimiter", "`", "`"], | |
| 337 | ["lone inline math delimiter", "$", "$"], | |
| 338 | ["bare open bracket", "[", "["], | |
| 339 | ["completed label without url at bol", "[x]", "x"], | |
| 340 | ["bare url opener at bol", "[x](", "x"], | |
| 341 | ])("%s", (_name, input, expected) => check(input, expected)); | |
| 342 | }); | |
| 343 | }); |