1// Hard-wrapped markdown, edited as though it were soft-wrapped.
2//
3// A file whose paragraphs are wrapped at a column edits badly under live preview: every newline is
4// a real break, so the caret moves per source line, selection-by-paragraph is impossible, and
5// rewrapping after an edit is manual. This module makes such a paragraph *look* like one line
6// without changing a byte of the document: the line breaks inside a paragraph are hidden behind
7// replace decorations, and the file's own bytes stay in the CodeMirror document.
8//
9// That layering is load-bearing, not a style choice. The workbench builds `ChangeSet`s against the
10// host's text and pushes them into a shared version stream; a transform that rewrote the document
11// on load and un-rewrote it on save would make every offset in every ChangeSet a lie (the same
12// hazard that keeps `createMarkdownEditor`'s mention sentinels out of the file-editing path). So
13// the document is always the file, the join is a decoration, and the only writes are ordinary
14// transactions: `rewrapOnEdit` re-flows the one block the user just typed in, appended to their own
15// transaction, and nothing else in the file is touched.
16//
17// Everything above the CodeMirror section is pure string logic so the properties can run headless.
18import {
19 EditorState,
20 type Extension,
21 type Range,
22 StateField,
23 Transaction,
24} from "@codemirror/state";
25import { Decoration, type DecorationSet, EditorView, WidgetType } from "@codemirror/view";
26
27/** The column the rewrap fills to. No UI surface: consumers pass it, this is the code-level default. */
28export const DEFAULT_WRAP_COLUMN = 80;
29
30export interface HardWrapOptions {
31 /** Fill column for `rewrap*`. Defaults to {@link DEFAULT_WRAP_COLUMN}. Ignored by the scanner. */
32 column?: number;
33}
34
35/** A source range covering one hidden line break: trailing space, the break, and the next line's prefix. */
36export interface JoinRange {
37 from: number;
38 to: number;
39}
40
41/** A run of source lines that renders as a single markdown paragraph (prose, list item, or quoted). */
42export interface WrapBlock {
43 /** Offset of the first line's first character (its indent, not its content). */
44 from: number;
45 /** Offset of the end of the last line's text, excluding its line break. */
46 to: number;
47 lineCount: number;
48 /** The internal breaks, in order. Empty for a single-line block. */
49 joins: JoinRange[];
50 /** Everything before content on the first line: indent, quote markers, and any list marker. */
51 firstPrefix: string;
52 /** Everything before content on continuation lines, taken from the block's second line. */
53 contPrefix: string;
54 /** The block's own line terminator, so a CRLF file is rewrapped with CRLF. */
55 lineBreak: string;
56}
57
58/** A minimal edit produced by the rewrap: always a whitespace run, never a word. */
59export interface TextChange {
60 from: number;
61 to: number;
62 insert: string;
63}
64
65// ---------------------------------------------------------------------------
66// Line scanning
67// ---------------------------------------------------------------------------
68
69interface SourceLine {
70 start: number;
71 /** The line's text without its terminator. */
72 text: string;
73 /** The line's terminator: `\r\n`, `\n`, `\r`, or empty at end of input. */
74 breakText: string;
75 /** Characters consumed by indent plus `>` quote markers plus the indent after them. */
76 prefixLen: number;
77 quote: number;
78 /** Whitespace *before* any quote marker. Four or more of it is an indented code block. */
79 leadIndent: number;
80 /** Whitespace after the quote markers; the column the line's content starts at inside them. */
81 indent: number;
82 body: string;
83 /** True for an empty line and for a bare `>` — both terminate a paragraph. */
84 blank: boolean;
85 /** Two-or-more trailing spaces, or an odd run of trailing backslashes: a markdown line break. */
86 hardBreak: boolean;
87}
88
89const PREFIX = /^([ \t]*)((?:>[ \t]?)*)([ \t]*)/;
90const ATX = /^#{1,6}(?:[ \t]|$)/;
91const FENCE = /^(?:`{3,}|~{3,})/;
92const THEMATIC = /^(?:\*[ \t]*){3,}$|^(?:-[ \t]*){3,}$|^(?:_[ \t]*){3,}$/;
93const SETEXT = /^(?:=+|-+)[ \t]*$/;
94const LIST = /^(?:[-*+]|\d{1,9}[.)])(?:[ \t]|$)/;
95const REF_DEF = /^\[[^\]]*\]:/;
96const HTML_START = /^<[!/A-Za-z]/;
97const TRAILING_BACKSLASHES = /\\*$/;
98
99function hasHardBreak(text: string): boolean {
100 if (/[ \t]{2,}$/.test(text)) return true;
101 const run = TRAILING_BACKSLASHES.exec(text);
102 return (run?.[0].length ?? 0) % 2 === 1;
103}
104
105/** Split into lines, keeping each terminator so CRLF and lone CR survive a round trip. */
106function splitSourceLines(text: string): SourceLine[] {
107 const lines: SourceLine[] = [];
108 let pos = 0;
109 while (pos <= text.length) {
110 let end = text.length;
111 let breakText = "";
112 for (let i = pos; i < text.length; i++) {
113 const ch = text[i];
114 if (ch === "\n") {
115 end = i;
116 breakText = "\n";
117 break;
118 }
119 if (ch === "\r") {
120 end = i;
121 breakText = text[i + 1] === "\n" ? "\r\n" : "\r";
122 break;
123 }
124 }
125 lines.push(describeLine(text.slice(pos, end), pos, breakText));
126 if (breakText === "") break;
127 pos = end + breakText.length;
128 }
129 return lines;
130}
131
132function describeLine(text: string, start: number, breakText: string): SourceLine {
133 const match = PREFIX.exec(text);
134 const prefix = match?.[0] ?? "";
135 const quote = (match?.[2] ?? "").split("").filter((ch) => ch === ">").length;
136 const body = text.slice(prefix.length);
137 return {
138 start,
139 text,
140 breakText,
141 prefixLen: prefix.length,
142 quote,
143 leadIndent: (match?.[1] ?? "").length,
144 indent: quote === 0 ? prefix.length : (match?.[3] ?? "").length,
145 body,
146 blank: body.trim() === "",
147 hardBreak: hasHardBreak(text),
148 };
149}
150
151/**
152 * True when this line's body opens a block of its own. Read in both directions: a paragraph must
153 * not swallow such a line, and a rewrap must never *produce* one — a break that leaves `- x`,
154 * `# x`, `1. x`, `===` or a fence at the head of a line silently turns prose into a new block.
155 */
156export function startsBlockLine(body: string): boolean {
157 return (
158 ATX.test(body) ||
159 FENCE.test(body) ||
160 THEMATIC.test(body) ||
161 SETEXT.test(body) ||
162 LIST.test(body) ||
163 REF_DEF.test(body) ||
164 HTML_START.test(body) ||
165 body.startsWith(">") ||
166 body.includes("|")
167 );
168}
169
170/** Content column of a list marker, or null when the body is not a list item. */
171function listContentOffset(body: string): number | null {
172 const marker = LIST.exec(body);
173 if (!marker) return null;
174 const rest = body.slice(marker[0].length);
175 const extra = /^[ \t]*/.exec(rest)?.[0].length ?? 0;
176 // An empty item ("-" with nothing after) starts no paragraph.
177 if (rest.trim() === "") return null;
178 return marker[0].length + extra;
179}
180
181// ---------------------------------------------------------------------------
182// Block scanning
183// ---------------------------------------------------------------------------
184
185/**
186 * The paragraph-shaped runs of `lines`, each with the internal breaks that may be displayed as a
187 * space. Deliberately conservative: anything the line-level classifier cannot prove is prose
188 * (fences, front matter, tables, setext headings, indented code, HTML, reference definitions, hard
189 * breaks) yields no joins rather than a guess.
190 */
191function scanBlocksFromLines(lines: SourceLine[]): WrapBlock[] {
192 const blocks: WrapBlock[] = [];
193 let i = 0;
194
195 // Front matter: only when the very first line is a `---` fence.
196 if (lines.length > 0 && /^---[ \t]*$/.test(lines[0]?.text ?? "")) {
197 i = 1;
198 while (i < lines.length && !/^(?:---|\.\.\.)[ \t]*$/.test(lines[i]?.text ?? "")) i++;
199 i = Math.min(i + 1, lines.length);
200 }
201
202 while (i < lines.length) {
203 const line = lines[i];
204 if (!line) break;
205 if (line.blank) {
206 i++;
207 continue;
208 }
209 if (FENCE.test(line.body)) {
210 const marker = FENCE.exec(line.body)?.[0] ?? "";
211 const char = marker[0] ?? "`";
212 i++;
213 while (i < lines.length) {
214 const close = lines[i];
215 i++;
216 if (close && new RegExp(`^\\${char}{${marker.length},}[ \t]*$`).test(close.body)) break;
217 }
218 continue;
219 }
220
221 const markerWidth = listContentOffset(line.body);
222 // Four columns of indent is the indented-code floor, and deciding whether a deeper line is code
223 // or a list item's paragraph needs real container parsing. Refuse instead of guessing: a nested
224 // item indented past column 3 simply gets no joins.
225 const startable =
226 line.indent < 4 && line.leadIndent < 4 && !startsBlockLine(line.body.slice(markerWidth ?? 0));
227 if (!startable) {
228 i++;
229 continue;
230 }
231
232 const contentIndent = line.indent + (markerWidth ?? 0);
233
234 let end = i;
235 while (end + 1 < lines.length) {
236 const next = lines[end + 1];
237 const prev = lines[end];
238 if (!next || !prev) break;
239 if (next.blank || prev.hardBreak) break;
240 if (next.quote !== line.quote) break;
241 if (startsBlockLine(next.body)) break;
242 if (next.indent - contentIndent >= 4) break;
243 if (next.leadIndent - line.leadIndent >= 4) break;
244 end++;
245 }
246
247 // A run underlined by `===`/`---` is a setext heading; refuse it rather than reason about it.
248 const after = lines[end + 1];
249 const underlined =
250 after !== undefined && !after.blank && after.quote === line.quote && SETEXT.test(after.body);
251
252 if (end > i && !underlined) blocks.push(buildBlock(lines, i, end, markerWidth ?? 0));
253 i = end + 1;
254 }
255 return blocks;
256}
257
258function contentStart(line: SourceLine, extra = 0): number {
259 return line.start + line.prefixLen + extra;
260}
261
262function buildBlock(
263 lines: SourceLine[],
264 first: number,
265 last: number,
266 markerWidth: number,
267): WrapBlock {
268 const head = lines[first] as SourceLine;
269 const second = lines[first + 1] as SourceLine;
270 const tail = lines[last] as SourceLine;
271 const joins: JoinRange[] = [];
272 for (let k = first; k < last; k++) {
273 const line = lines[k] as SourceLine;
274 const next = lines[k + 1] as SourceLine;
275 joins.push({
276 from: line.start + line.text.replace(/[ \t]+$/, "").length,
277 to: contentStart(next),
278 });
279 }
280 return {
281 from: head.start,
282 to: tail.start + tail.text.length,
283 lineCount: last - first + 1,
284 joins,
285 firstPrefix: head.text.slice(0, head.prefixLen + markerWidth),
286 contPrefix: second.text.slice(0, second.prefixLen),
287 lineBreak: head.breakText || "\n",
288 };
289}
290
291/** The paragraph-shaped runs of `text`. Pure; safe to call on a whole file. */
292export function scanBlocks(text: string): WrapBlock[] {
293 return scanBlocksFromLines(splitSourceLines(text));
294}
295
296/** Every hidden break in `text`, in order. */
297export function scanJoins(text: string): JoinRange[] {
298 return scanBlocks(text).flatMap((block) => block.joins);
299}
300
301// ---------------------------------------------------------------------------
302// The display transform (pure form; the CodeMirror layer below never applies it to a document)
303// ---------------------------------------------------------------------------
304
305/** `text` with every hidden break replaced by one space: what the editor shows. */
306export function unwrapText(text: string, joins = scanJoins(text)): string {
307 let out = "";
308 let pos = 0;
309 for (const join of joins) {
310 out += text.slice(pos, join.from) + " ";
311 pos = join.to;
312 }
313 return out + text.slice(pos);
314}
315
316/** Map a source offset onto the unwrapped view. Offsets inside a join land on its single space. */
317export function toDisplayPos(joins: readonly JoinRange[], pos: number): number {
318 let shift = 0;
319 for (const join of joins) {
320 if (pos <= join.from) break;
321 shift += Math.min(pos, join.to) - join.from - 1;
322 }
323 return pos - shift;
324}
325
326/** Map an offset in the unwrapped view back onto the source. */
327export function fromDisplayPos(joins: readonly JoinRange[], pos: number): number {
328 let shift = 0;
329 for (const join of joins) {
330 const displayFrom = join.from - shift;
331 if (pos <= displayFrom) break;
332 shift += join.to - join.from - 1;
333 }
334 return pos + shift;
335}
336
337// ---------------------------------------------------------------------------
338// The wrap direction
339// ---------------------------------------------------------------------------
340
341interface Gap {
342 from: number;
343 to: number;
344 text: string;
345 /** True when this gap is one of the block's line breaks in the current source. */
346 isBreak: boolean;
347}
348
349interface Token {
350 text: string;
351 from: number;
352}
353
354/**
355 * Rewrap one block to `column`, as the smallest set of whitespace-only edits that gets there.
356 * Words are never moved and never rewritten: every change replaces one inter-word gap. Returns an
357 * empty list when the block already fills correctly, which is what keeps an untouched file
358 * byte-identical.
359 */
360export function rewrapBlockChanges(
361 text: string,
362 block: WrapBlock,
363 column = DEFAULT_WRAP_COLUMN,
364): TextChange[] {
365 if (block.lineCount < 2) return [];
366 const { tokens, gaps } = tokenizeBlock(text, block);
367 if (tokens.length < 2) return [];
368
369 const firstWidth = block.firstPrefix.length;
370 const contWidth = block.contPrefix.length;
371 // Every inter-word gap inside a rewrapped block normalizes to one space. Keeping the original run
372 // is not an option: a gap that lands on a break loses its width, so the next rewrap would measure
373 // the line differently and the fill would never reach a fixed point. The cost is that editing a
374 // paragraph collapses double spaces and tabs between its words — markdown renders them the same.
375 const inlineText = (_gap: Gap) => " ";
376
377 const breakAfter = gaps.map(() => false);
378 let col = firstWidth + (tokens[0]?.text.length ?? 0);
379 for (let i = 0; i < gaps.length; i++) {
380 const gap = gaps[i] as Gap;
381 const next = tokens[i + 1] as Token;
382 const inline = inlineText(gap).length;
383 // Every line already holds at least one token, so breaking here can never leave one empty.
384 if (col + inline + next.text.length > column) {
385 breakAfter[i] = true;
386 col = contWidth + next.text.length;
387 } else {
388 col += inline + next.text.length;
389 }
390 }
391
392 avoidBlockStarts(tokens, gaps, breakAfter, inlineText);
393
394 const changes: TextChange[] = [];
395 for (let i = 0; i < gaps.length; i++) {
396 const gap = gaps[i] as Gap;
397 const insert = breakAfter[i] ? block.lineBreak + block.contPrefix : inlineText(gap);
398 if (insert !== gap.text) changes.push({ from: gap.from, to: gap.to, insert });
399 }
400 return changes;
401}
402
403/**
404 * A break that puts `- x`, `# x`, `> x`, `1. x`, backticks or a setext rule at the start of a
405 * continuation line would turn prose into a new block on the next parse. Pull such a word back onto
406 * the previous line — an over-long line is a cosmetic loss, a silently invented list is corruption.
407 */
408function avoidBlockStarts(
409 tokens: readonly Token[],
410 gaps: readonly Gap[],
411 breakAfter: boolean[],
412 inlineText: (gap: Gap) => string,
413): void {
414 const bodyFrom = (start: number, end: number) => {
415 let out = tokens[start]?.text ?? "";
416 for (let i = start; i < end; i++)
417 out += inlineText(gaps[i] as Gap) + (tokens[i + 1]?.text ?? "");
418 return out;
419 };
420 for (let guard = 0; guard <= tokens.length; guard++) {
421 const starts = [0];
422 for (let i = 0; i < gaps.length; i++) if (breakAfter[i]) starts.push(i + 1);
423 let fixed = false;
424 for (let l = 0; l < starts.length && !fixed; l++) {
425 const start = starts[l] as number;
426 const end = l + 1 < starts.length ? (starts[l + 1] as number) - 1 : tokens.length - 1;
427 if (!startsBlockLine(bodyFrom(start, end))) continue;
428 // A continuation line pulls its first word back; the first line has no line before it, so it
429 // pulls the next word forward instead (`===` alone is a setext rule, `=== word` is prose).
430 if (l > 0) breakAfter[start - 1] = false;
431 else if (end < tokens.length - 1) breakAfter[end] = false;
432 else break;
433 fixed = true;
434 }
435 if (!fixed) return;
436 }
437}
438
439function tokenizeBlock(text: string, block: WrapBlock): { tokens: Token[]; gaps: Gap[] } {
440 const tokens: Token[] = [];
441 const gaps: Gap[] = [];
442 const joins = block.joins;
443 let cursor = block.from + block.firstPrefix.length;
444 for (let k = 0; k < block.lineCount; k++) {
445 const lineEnd = k < joins.length ? (joins[k] as JoinRange).from : block.to;
446 const body = text.slice(cursor, lineEnd);
447 const word = /\S+/g;
448 let match = word.exec(body);
449 while (match) {
450 if (tokens.length > 0 && gaps.length < tokens.length) {
451 const previous = tokens[tokens.length - 1] as Token;
452 const from = previous.from + previous.text.length;
453 const to = cursor + match.index;
454 gaps.push({ from, to, text: text.slice(from, to), isBreak: false });
455 }
456 tokens.push({ text: match[0], from: cursor + match.index });
457 match = word.exec(body);
458 }
459 const join = joins[k];
460 if (join && tokens.length > 0) {
461 gaps.push({
462 from: join.from,
463 to: join.to,
464 text: text.slice(join.from, join.to),
465 isBreak: true,
466 });
467 cursor = join.to;
468 } else if (join) {
469 cursor = join.to;
470 }
471 }
472 // A trailing gap with no token after it cannot exist in a well-formed block; drop it defensively.
473 while (gaps.length >= tokens.length && gaps.length > 0) gaps.pop();
474 return { tokens, gaps };
475}
476
477/**
478 * Rewrap every hard-wrapped block in `text`. Opt-in and destructive by design — it rewrites blocks
479 * nobody edited, so it must never run on save. Exposed for explicit "reflow this file" actions and
480 * as the oracle the properties compare against.
481 */
482export function rewrapText(text: string, options: HardWrapOptions = {}): string {
483 const column = options.column ?? DEFAULT_WRAP_COLUMN;
484 const changes = scanBlocks(text).flatMap((block) => rewrapBlockChanges(text, block, column));
485 let out = "";
486 let pos = 0;
487 for (const change of changes) {
488 out += text.slice(pos, change.from) + change.insert;
489 pos = change.to;
490 }
491 return out + text.slice(pos);
492}
493
494/** The block containing `pos`, or null when `pos` is not inside hard-wrapped prose. */
495export function blockAt(blocks: readonly WrapBlock[], pos: number): WrapBlock | null {
496 return blocks.find((block) => pos >= block.from && pos <= block.to) ?? null;
497}
498
499/**
500 * The wrap column this file appears to already use: the longest line that a rewrap could have
501 * shortened. Null when nothing is hard-wrapped. Never applied automatically — a host that wants to
502 * follow the file instead of a fixed default calls this and passes the result back in.
503 */
504export function detectWrapColumn(text: string): number | null {
505 let widest = 0;
506 for (const block of scanBlocks(text)) {
507 let lineStart = block.from;
508 for (const join of block.joins) {
509 widest = Math.max(widest, join.from - lineStart);
510 lineStart = join.to - block.contPrefix.length;
511 }
512 }
513 return widest === 0 ? null : widest;
514}
515
516// ---------------------------------------------------------------------------
517// The CodeMirror layer
518// ---------------------------------------------------------------------------
519
520export interface HardWrapExtensionOptions extends HardWrapOptions {
521 /**
522 * Re-flow the one block the user just edited, appended to their own transaction (so it is one
523 * undo step and one ChangeSet). Only fires for `input`/`delete`/`move` user events: a snapshot
524 * load, an agent's edit, or any programmatic change is left exactly as it arrived. Default true.
525 */
526 rewrapOnEdit?: boolean;
527 /** Turn on `EditorView.lineWrapping`; without it a joined paragraph just scrolls. Default true. */
528 lineWrapping?: boolean;
529}
530
531/** The space shown in place of a hidden line break. A real space, so the browser may wrap there. */
532class JoinWidget extends WidgetType {
533 override eq(): boolean {
534 return true;
535 }
536
537 override toDOM(): HTMLElement {
538 const span = document.createElement("span");
539 span.className = "cm-hw-join";
540 span.textContent = " ";
541 return span;
542 }
543
544 override ignoreEvent(): boolean {
545 return false;
546 }
547}
548
549const joinWidget = new JoinWidget();
550
551/** The replace decorations that hide a document's paragraph-internal line breaks. */
552export function joinDecorations(state: EditorState): Range<Decoration>[] {
553 const ranges: Range<Decoration>[] = [];
554 for (const join of scanJoins(state.doc.toString())) {
555 if (join.to > join.from) {
556 ranges.push(Decoration.replace({ widget: joinWidget }).range(join.from, join.to));
557 }
558 }
559 return ranges;
560}
561
562const REWRAP_EVENTS = ["input", "delete", "move"] as const;
563
564/**
565 * Show hard-wrapped paragraphs as single soft-wrapped ones, and keep them wrapped as they are
566 * edited. The document is never rewritten behind the host's back: opening a file and saving it
567 * without typing produces the same bytes, because nothing ran.
568 */
569export function hardWrap(options: HardWrapExtensionOptions = {}): Extension {
570 const column = options.column ?? DEFAULT_WRAP_COLUMN;
571 const field = StateField.define<DecorationSet>({
572 create: (state) => Decoration.set(joinDecorations(state), true),
573 update: (previous, transaction) =>
574 transaction.docChanged ? Decoration.set(joinDecorations(transaction.state), true) : previous,
575 provide: (self) => [
576 EditorView.decorations.from(self),
577 EditorView.atomicRanges.of((view) => view.state.field(self)),
578 ],
579 });
580
581 const extensions: Extension[] = [field];
582 if (options.lineWrapping !== false) extensions.push(EditorView.lineWrapping);
583 if (options.rewrapOnEdit !== false) extensions.push(rewrapFilter(column));
584 return extensions;
585}
586
587function rewrapFilter(column: number): Extension {
588 return EditorState.transactionFilter.of((transaction) => {
589 if (!transaction.docChanged) return transaction;
590 if (transaction.annotation(Transaction.remote)) return transaction;
591 if (!REWRAP_EVENTS.some((event) => transaction.isUserEvent(event))) return transaction;
592
593 const text = transaction.newDoc.toString();
594 const blocks = scanBlocks(text);
595 const touched = new Set<WrapBlock>();
596 transaction.changes.iterChangedRanges((_fromA, _toA, fromB, toB) => {
597 for (const block of blocks) {
598 if (block.lineCount >= 2 && toB >= block.from && fromB <= block.to) touched.add(block);
599 }
600 });
601 const changes = [...touched].flatMap((block) => rewrapBlockChanges(text, block, column));
602 if (changes.length === 0) return transaction;
603 changes.sort((a, b) => a.from - b.from);
604 return [transaction, { changes, sequential: true, userEvent: "format.rewrap" }];
605 });
606}