diff --git a/corpus/clipboard/README.md b/corpus/clipboard/README.md index 8ff98c85062e95364b31be341c4cabe37605e033..37aaccdf89b541ab7e79e1539d80e6d8bb627b9d 100644 --- a/corpus/clipboard/README.md +++ b/corpus/clipboard/README.md @@ -9,3 +9,17 @@ built through COM and copied with Ctrl+A, Ctrl+C). Beside it OneNote offered ANSI text, an enhanced metafile and a device-independent bitmap, but no RTF. The tag is absent from the HTML. `crates/canvas/src/editor/html.rs` reads it in its tests. + +`word-2010.html` is the `HTML Format` Word 2010 put on the clipboard for a +document of a bulleted list three levels deep, a plain paragraph, a numbered +list three levels deep (1., a., i.) and a last plain paragraph, typed with +AutoFormat and Tab and copied with Ctrl+A, Ctrl+C (lab, 2026-10-02). Word +writes each item as a paragraph whose `mso-list` style names its list and +level, with its marker in a `mso-list:Ignore` span inside ``. Pasted into OneNote 2010, each level nested under the item +above; bullets kept Word's glyph and font (Symbol `·`, Courier New `o`, +Wingdings `§`) with no gallery index, and numbers took Word's sequence and +punctuation. OneNote also nested the numbered list under the plain paragraph +before it, from Word's indents; Snowbound nests by `mso-list` level alone, as +it reads `ul` and `ol`. `crates/canvas/src/editor/html.rs` and +`crates/canvas/tests/clip_paste.rs` read it. diff --git a/corpus/clipboard/word-2010.html b/corpus/clipboard/word-2010.html new file mode 100644 index 0000000000000000000000000000000000000000..5fbaec2577ea7e54bd7bf9c13db411938fb55165 --- /dev/null +++ b/corpus/clipboard/word-2010.html @@ -0,0 +1,691 @@ +Version:1.0 +StartHTML:0000000105 +EndHTML:0000031043 +StartFragment:0000027392 +EndFragment:0000031003 + + + + + + + + + + + + + + + + + + + + +

·         +Bullet one

+ +

o   +Bullet nested

+ +

§  +Bullet deeper

+ +

·         +Bullet two

+ +

Plain paragraph

+ +

1.       +Number one

+ +

a.       +Number nested

+ +

                                                               +i.      +Number deeper

+ +

2.       +Number two

+ +

Last plain

+ + + + + diff --git a/corpus/word-lists/README.md b/corpus/word-lists/README.md new file mode 100644 index 0000000000000000000000000000000000000000..aee74a045e921e1a075b107415f925f6ec70bcbb --- /dev/null +++ b/corpus/word-lists/README.md @@ -0,0 +1,12 @@ +# Lists pasted from Word + +`candidate/` is `crates/canvas/tests/clip_paste.rs`'s `words_lists_paste_and_store` +section: Word 2010's copy of three-level bulleted and numbered lists +(`corpus/clipboard/word-2010.html`) pasted through the editor at a caret on blank page, +stored as one edit. Bullets keep Word's glyph and font with no gallery index +(`ListMSAAIndex`), as OneNote 2010 stored the same paste (lab, 2026-10-02). +`cold/` is a fresh OneNote 2010 read: the bullets read back as the ones OneNote's own +paste gave (`bullet` 1, 0 and 13), and the numbers as 1., a., i. and 2. +`tools/test_word_lists.py` checks the capture without a VM. Regenerate with +`CANVAS_WORD_LIST_EXPORT` set to a directory while running the test, then +`tools/native_runner.py DIR COLD --expected-pages 1 --collect-notebook --screenshots`. diff --git a/corpus/word-lists/candidate/Open Notebook.onetoc2 b/corpus/word-lists/candidate/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..8ed3775b60c5e3487c3f34466c71cf51a1bdbc1a Binary files /dev/null and b/corpus/word-lists/candidate/Open Notebook.onetoc2 differ diff --git a/corpus/word-lists/candidate/word.one b/corpus/word-lists/candidate/word.one new file mode 100644 index 0000000000000000000000000000000000000000..396a98399847c80fd93c1cd0816ca90f97a93164 Binary files /dev/null and b/corpus/word-lists/candidate/word.one differ diff --git a/corpus/word-lists/cold/commands.jsonl b/corpus/word-lists/cold/commands.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c40af70a6b8a7980cd88ca43c9a1c5e8ecb66ac9 --- /dev/null +++ b/corpus/word-lists/cold/commands.jsonl @@ -0,0 +1,3 @@ +{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-237B5BEB -ExpectedPages 1 -KeepOpen", "exit": 0, "stdout": "Read 1 sections and 1 pages.\r\n", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null} diff --git a/corpus/word-lists/cold/machine.json b/corpus/word-lists/cold/machine.json new file mode 100644 index 0000000000000000000000000000000000000000..9eb1b0030aa2bf61c7cf8dd1fa251862fb259d54 --- /dev/null +++ b/corpus/word-lists/cold/machine.json @@ -0,0 +1 @@ +{"name": "m6-237b5beb", "hostname": "ONE-M6-237B5BEB"} diff --git a/corpus/word-lists/cold/notebook/Open Notebook.onetoc2 b/corpus/word-lists/cold/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..2a29f9ed537f2461256b6afc54d8aa0ca9f81b38 Binary files /dev/null and b/corpus/word-lists/cold/notebook/Open Notebook.onetoc2 differ diff --git a/corpus/word-lists/cold/notebook/word.one b/corpus/word-lists/cold/notebook/word.one new file mode 100644 index 0000000000000000000000000000000000000000..0bacbee6752a0c237de36391e1ce97568cab0bb6 Binary files /dev/null and b/corpus/word-lists/cold/notebook/word.one differ diff --git a/corpus/word-lists/cold/read/environment.json b/corpus/word-lists/cold/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..7748fd7404dbc093964277c576879dd2d3fd88db --- /dev/null +++ b/corpus/word-lists/cold/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-237B5BEB", + "cold": true, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/word-lists/cold/read/hierarchy.xml b/corpus/word-lists/cold/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..ab5197de5ed5f906bace3b42b5e899ceb4feccf7 --- /dev/null +++ b/corpus/word-lists/cold/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/word-lists/cold/read/page-000.navigation.json b/corpus/word-lists/cold/read/page-000.navigation.json new file mode 100644 index 0000000000000000000000000000000000000000..0e1df2f44c8de84709e3a55afc2a0a097b4bc55f --- /dev/null +++ b/corpus/word-lists/cold/read/page-000.navigation.json @@ -0,0 +1,13 @@ +{ + "exit": 0, + "stdout": "", + "stderr": "", + "w": 800, + "h": 600, + "error": null, + "win": { + "title": "Word lists - Microsoft OneNote", + "class": "Framework::CFrame", + "dialog": false + } +} \ No newline at end of file diff --git a/corpus/word-lists/cold/read/page-000.png b/corpus/word-lists/cold/read/page-000.png new file mode 100644 index 0000000000000000000000000000000000000000..e880068e6a526ad23277d35f4e2dcbef1a5887f3 Binary files /dev/null and b/corpus/word-lists/cold/read/page-000.png differ diff --git a/corpus/word-lists/cold/read/page-000.xml b/corpus/word-lists/cold/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..607165b587f4505e4aa1c558c6c25abbbd72a4ea --- /dev/null +++ b/corpus/word-lists/cold/read/page-000.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/word-lists/cold/read/payloads.json b/corpus/word-lists/cold/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..1ce91a413d6fff4d58765187b76756f0e97ae843 --- /dev/null +++ b/corpus/word-lists/cold/read/payloads.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/corpus/word-lists/cold/run.json b/corpus/word-lists/cold/run.json new file mode 100644 index 0000000000000000000000000000000000000000..e7b6452ec0cb991e08280c1d51e959b3a5bc14c7 --- /dev/null +++ b/corpus/word-lists/cold/run.json @@ -0,0 +1,18 @@ +{ + "notebook": "/private/tmp/pastefix/cand61", + "expected_pages": 1, + "author": null, + "author_timeout_seconds": 600, + "inspect": false, + "collect_notebook": true, + "base": { + "file": "win7-office-base.qcow2", + "format": "qcow2", + "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346", + "virtual_size": 68719476736 + }, + "scripts": { + "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331", + "read.ps1": "22bc760f955ef59bb5f0ccfd55271f8d868674084127fbe219e1048bd3777f35" + } +} diff --git a/corpus/word-lists/cold/scripts/cold.ps1 b/corpus/word-lists/cold/scripts/cold.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..a0b99e7aeb86f4dc10032537800501a52ad65b4f --- /dev/null +++ b/corpus/word-lists/cold/scripts/cold.ps1 @@ -0,0 +1,27 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '') +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' } +$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote' +if ($CloneHost) { + if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) { + throw 'The disposable clone hostname does not match this machine.' + } + New-Item "$key\Options\Paths" -Force | Out-Null + New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null +} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or + -not (Test-Path 'C:\one-tests\profile-original-cache')) { + throw 'Park the personal OneNote profile before resetting the test cache.' +} +$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0' +$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root)) +if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' } +New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null +if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked } +if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse } +New-Item "$key\OpenNotebooks" | Out-Null +New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null diff --git a/corpus/word-lists/cold/scripts/read.ps1 b/corpus/word-lists/cold/scripts/read.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..a42366b32e8baaf13dae92e9472cae7a071220cc --- /dev/null +++ b/corpus/word-lists/cold/scripts/read.ps1 @@ -0,0 +1,144 @@ +param( + [Parameter(Mandatory=$true)][string]$Root, + [int]$ExpectedPages = -1, + [switch]$UseCurrentCache, + [switch]$Pdf, + [switch]$KeepOpen, + [string]$CloneHost = '' +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +$notebook = Join-Path $root 'notebook' +$output = Join-Path $root 'read' +if (Test-Path $output) { throw 'Choose a new read destination.' } +if ($UseCurrentCache) { + if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') { + throw 'Park the personal OneNote profile before reading test notebooks.' + } +} else { + & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost +} +New-Item -ItemType Directory -Path $output | Out-Null +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$failure = $null +try { + $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0) + $process = Get-Process ONENOTE + @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion; + powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } | + ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8 + $sections = @() + foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) { + $id = '' + $app.OpenHierarchy($file.FullName, '', [ref]$id, 0) + $sections += $id + } + $deadline = [DateTime]::UtcNow.AddSeconds(300) + $previous = '' + $lastChange = '' + $stableSince = [DateTime]::UtcNow + $settled = $false + do { + $pages = @{} + foreach ($section in $sections) { + $hierarchy = '' + $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) + [xml]$xml = $hierarchy + foreach ($node in $xml.SelectNodes('//*[@path]')) { + if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) { + throw 'OneNote opened a section outside the copied notebook.' + } + } + foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) { + $id = $node.GetAttribute('ID') + $content = '' + $app.GetPageContent($id, [ref]$content, 1, 1) + $pages[$id] = $content + } + } + $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] })) + if ($signature -ne $previous) { + $lastChange = $previous + $previous = $signature + $stableSince = [DateTime]::UtcNow + } + if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or + ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and + ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break } + Start-Sleep -Milliseconds 250 + } while ([DateTime]::UtcNow -lt $deadline) + if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) { + [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8) + $index = 0 + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + $index++ + } + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8) + throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled." + } + $index = 0 + $payloads = @() + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + if ($Pdf) { + $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index) + $app.NavigateTo($id, '', $false) + $app.Publish($id, $pdfPath, 3, '') + if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' } + } + [xml]$page = $pages[$id] + foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) { + $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache')) + $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant() + [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes) + # A paragraph's file is named by its paragraph, one on the page by itself. + $owner = if ($file.ParentNode.LocalName -eq 'Page') { $file } else { $file.ParentNode } + $payloads += @{ page = $id; object = $owner.GetAttribute('objectID'); + kind = $file.LocalName; name = $file.GetAttribute('preferredName'); + sha256 = $hash; bytes = $bytes.Length } + } + $index++ + } + [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8) + $all = '' + $app.GetHierarchy($notebookId, 4, [ref]$all, 1) + [xml]$finalTree = $all + $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique) + if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) { + throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.' + } + [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8) + Write-Output "Read $($sections.Count) sections and $($pages.Count) pages." +} catch { + $failure = $_ + [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + throw +} finally { + try { + try { + if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) } + if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) { + $app.CloseNotebook($notebookId, $false) + } + } catch { + if ($null -eq $failure) { throw } + [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + } + } finally { + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + } + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + if (-not $UseCurrentCache -and -not $CloneHost) { + Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10 + } +} diff --git a/corpus/word-lists/cold/source.json b/corpus/word-lists/cold/source.json new file mode 100644 index 0000000000000000000000000000000000000000..6bc7f85c7bde1d98f26faf621673f308be7694a1 --- /dev/null +++ b/corpus/word-lists/cold/source.json @@ -0,0 +1,14 @@ +[ + { + "path": "Open Notebook.onetoc2", + "bytes": 1596, + "sha256": "2974f1efd4b392b00d5d1a3e2485ad34d049e1ea46c0aceaed754f2ff365bc4e", + "mtime_ns": 1790967579952579436 + }, + { + "path": "word.one", + "bytes": 9632, + "sha256": "144bab7f0967f85881a858efd183c54aa8f334f07043d6959a68569b1fcba664", + "mtime_ns": 1790967579952375355 + } +] diff --git a/corpus/word-lists/cold/teardown.json b/corpus/word-lists/cold/teardown.json new file mode 100644 index 0000000000000000000000000000000000000000..05a47793de40f322e745c4c4183727e3590ecf70 --- /dev/null +++ b/corpus/word-lists/cold/teardown.json @@ -0,0 +1 @@ +{"absent": true} diff --git a/crates/canvas/src/editor/clip.rs b/crates/canvas/src/editor/clip.rs index e750063d6744abb4e01c432f8fb4fae4bc59a2cb..5be34c5aa26cc34c565eaa13bcaf45e8db52d016 100644 --- a/crates/canvas/src/editor/clip.rs +++ b/crates/canvas/src/editor/clip.rs @@ -307,15 +307,13 @@ fn outline( use super::format::AUTOMATIC; for node in nodes { let mut line = format!("{prefix}{}", " ".repeat(node.level as usize - 1)); - match node.lists.last().map(|list| &definitions[list].kind) { - Some(Kind::List { - bullet: Some(_), .. - }) => line.push_str("• "), - Some(kind @ Kind::List { .. }) => { - let (_, kind) = super::html::list_tag(kind); - line.push_str(&format!("{kind}. ")); + if let Some(kind @ Kind::List { .. }) = + node.lists.last().map(|list| &definitions[list].kind) + { + match super::html::list_tag(kind) { + ("ul", _) => line.push_str("• "), + (_, kind) => line.push_str(&format!("{kind}. ")), } - _ => {} } let mut tags = node.tags.clone(); match &node.content { diff --git a/crates/canvas/src/editor/html.rs b/crates/canvas/src/editor/html.rs index 3e0a4f45fe9f00023bc73b47434ff32bf6350181..b6a1894633668e0de5682b8af751914acfcb576c 100644 --- a/crates/canvas/src/editor/html.rs +++ b/crates/canvas/src/editor/html.rs @@ -104,7 +104,10 @@ pub(super) fn list_tag(kind: &Kind) -> (&'static str, &'static str) { let Kind::List { bullet, format, .. } = kind else { return ("ul", "disc"); }; - if bullet.is_some() { + let sequence = format + .as_deref() + .and_then(|format| format.split('\u{fffd}').nth(1)?.chars().next()); + if bullet.is_some() || sequence.is_none() { let kind = match ListStyle::of(kind) { Some(ListStyle::Bullet(3)) => "circle", Some(ListStyle::Bullet(11 | 14)) => "square", @@ -112,9 +115,6 @@ pub(super) fn list_tag(kind: &Kind) -> (&'static str, &'static str) { }; return ("ul", kind); } - let sequence = format - .as_deref() - .and_then(|format| format.split('\u{fffd}').nth(1)?.chars().next()); let kind = match sequence.map(u32::from) { Some(1) => "I", Some(2) => "i", @@ -257,6 +257,8 @@ pub fn html_pieces( tables: Vec::new(), lists: Vec::new(), item: None, + word: None, + sequences: BTreeMap::new(), open: Vec::new(), runs: Vec::new(), shown: false, @@ -267,13 +269,22 @@ pub fn html_pieces( reader.pieces } +/// A paragraph Word marks as a list item, its `mso-list` list and level, and the marker Word +/// wrote before its text, in the marker's font. +struct WordItem { + list: String, + level: usize, + marker: String, + font: Option, +} + /// A paragraph or table read, before it becomes a clip's. enum Block { Text { runs: Vec<(String, Format)>, /// How many lists it sits in, its own not counted. depth: usize, - list: Option, + list: Option>, }, Table { rows: Vec>>, @@ -296,6 +307,8 @@ struct Open { name: String, format: Format, link: Option, + /// Whether it holds a Word list item's marker (`mso-list:Ignore`). + marker: bool, } struct Reader { @@ -307,6 +320,10 @@ struct Reader { lists: Vec, /// The list of the item whose text has yet to come. item: Option, + /// The Word list item being read, as `mso-list` names it. + word: Option, + /// The number sequence each Word list level (`mso-list` list and level) began with. + sequences: BTreeMap<(String, usize), char>, open: Vec, runs: Vec<(String, Format)>, /// Whether the paragraph read so far shows anything. @@ -366,6 +383,10 @@ impl; 2]) -> Option> Reader { let end = tag_end(after); let tag = &after[..end]; rest = after.get(end + 1..).unwrap_or(""); + // Declarations and Word's `` conditions. + if tag.starts_with('!') { + continue; + } let (closing, tag) = match tag.strip_prefix('/') { Some(tag) => (true, tag), None => (false, tag), @@ -394,6 +415,25 @@ impl; 2]) -> Option> Reader { if block || matches!(name, "br" | "table" | "tr" | "td" | "th" | "ul" | "ol") { self.paragraph(); } + let list = style(tag, "mso-list"); + if block && let Some(list) = &list { + let mut words = list.split_whitespace(); + if let (Some(id), Some(Ok(level))) = ( + words.next(), + words + .next() + .and_then(|level| level.strip_prefix("level")) + .map(str::parse::), + ) && level > 0 + { + self.word = Some(WordItem { + list: id.to_owned(), + level, + marker: String::new(), + font: None, + }); + } + } match name { "br" => return, "img" => { @@ -479,6 +519,7 @@ impl; 2]) -> Option> Reader { name: name.to_owned(), format, link, + marker: list.is_some_and(|list| list.eq_ignore_ascii_case("ignore")), }); } @@ -547,6 +588,16 @@ impl; 2]) -> Option> Reader { fn text(&mut self, text: &str) { let format = self.format(); + if self.open.iter().any(|open| open.marker) { + if let Some(item) = &mut self.word { + let shown = text.trim_matches(|c: char| c.is_whitespace() || c == '\u{a0}'); + if !shown.is_empty() && item.marker.is_empty() { + item.font = format.font.clone(); + } + item.marker.push_str(shown); + } + return; + } for c in text.chars() { if c.is_whitespace() && c != '\u{a0}' { if self.shown && self.space.is_none() { @@ -580,15 +631,78 @@ impl; 2]) -> Option> Reader { for open in &mut self.open { open.link = None; } - let list = self.item.take(); - let depth = self.lists.len() - usize::from(list.is_some() && !self.lists.is_empty()); runs.retain(|(text, _)| !text.is_empty()); if runs.is_empty() { runs.push((String::new(), self.format())); } + let (list, depth) = match self.word.take() { + Some(item) => { + let depth = self.lists.len() + item.level - 1; + (Some(Box::new(self.word_list(item, &runs[0].1))), depth) + } + None => { + let list = self.item.take(); + let depth = + self.lists.len() - usize::from(list.is_some() && !self.lists.is_empty()); + ( + list.map(|style| Box::new(list_definition(style, &runs[0].1))), + depth, + ) + } + }; self.sink().push(Block::Text { runs, depth, list }); } + /// The list OneNote 2010 made of a Word list item whose text begins in `format` (lab, + /// 2026-10-02): a marker without a number is a bullet in Word's glyph and font; a number + /// keeps Word's punctuation, in the sequence its list level began with. + fn word_list(&mut self, item: WordItem, format: &Format) -> Definition { + let mut definition = list_definition(ListStyle::NUMBER, format); + let Kind::List { + font, format: list, .. + } = &mut definition.kind + else { + unreachable!("a number is a list") + }; + let marker = item.marker; + let alphanumeric = |c: char| c.is_ascii_alphanumeric(); + let token = marker + .find(alphanumeric) + .zip(marker.rfind(alphanumeric)) + .map(|(start, end)| start..end + 1); + let digits = token + .as_ref() + .is_some_and(|token| marker[token.clone()].bytes().any(|b| b.is_ascii_digit())); + let Some(number) = token.filter(|token| digits || token.len() < marker.len()) else { + *font = item.font; + *list = Some(marker); + return definition; + }; + let token = &marker[number.clone()]; + let sequence = *self + .sequences + .entry((item.list, item.level)) + .or_insert_with(|| { + let roman = token == "i" + || token == "I" + || token.len() > 1 && token.chars().all(|c| "ivxlcdmIVXLCDM".contains(c)); + let upper = token.starts_with(|c: char| c.is_ascii_uppercase()); + match (digits, roman, upper) { + (true, ..) => '\u{0}', + (_, true, true) => '\u{1}', + (_, true, false) => '\u{2}', + (_, false, true) => '\u{3}', + (_, false, false) => '\u{4}', + } + }); + *list = Some(format!( + "{}\u{fffd}{sequence}{}", + &marker[..number.start], + &marker[number.end..] + )); + definition + } + /// Where a block read goes: the open cell, else the clip. fn sink(&mut self) -> &mut Vec { match self.tables.last_mut() { @@ -643,11 +757,10 @@ fn nodes( let level = (depth(&block) - least + 1) as u32; let mut paragraph = match block { Block::Text { runs, list, .. } => { - let first = runs[0].1.clone(); let mut paragraph = node(Paragraph::from_runs(runs), Format::default())?; - if let Some(style) = list { + if let Some(list) = list { let id = new_id()?; - definitions.insert(id, list_definition(style, &first)); + definitions.insert(id, *list); paragraph.lists = vec![id]; } paragraph @@ -1064,6 +1177,83 @@ mod tests { ); } + /// Word 2010's list paragraphs (`corpus/clipboard/word-2010.html`) paste as the lists + /// OneNote 2010 made of them (lab, 2026-10-02): each level nested under the item above, + /// a bullet in Word's own glyph and font, a number in Word's sequence and punctuation. + #[test] + fn words_lists_paste_as_lists() { + let html = include_str!("../../../../corpus/clipboard/word-2010.html"); + let [clip] = &clips(html)[..] else { + panic!("one clip") + }; + assert_eq!( + clip.outline(), + [ + "• Bullet one", + " • Bullet nested", + " • Bullet deeper", + "• Bullet two", + "Plain paragraph", + "1. Number one", + " a. Number nested", + " i. Number deeper", + "1. Number two", + "Last plain", + ] + ); + let lists = crate::document::leaves(&clip.paragraphs, None) + .filter_map( + |(.., node)| match &clip.definitions[node.lists.first()?].kind { + Kind::List { + font, + format, + bullet, + .. + } => Some((font.clone(), format.clone().unwrap(), *bullet)), + _ => None, + }, + ) + .collect::>(); + let bullet = |font: &str, glyph: &str| (Some(font.to_owned()), glyph.to_owned(), None); + let number = |format: &str| (None, format.to_owned(), None); + assert_eq!( + lists, + [ + bullet("Symbol", "\u{b7}"), + bullet("Courier New", "o"), + bullet("Wingdings", "\u{a7}"), + bullet("Symbol", "\u{b7}"), + number("\u{fffd}\u{0}."), + number("\u{fffd}\u{4}."), + number("\u{fffd}\u{2}."), + number("\u{fffd}\u{0}."), + ] + ); + } + + /// LibreOffice writes lists as nested `ul` and `ol` around paragraphs, leaving items + /// unclosed. + #[test] + fn libreoffices_lists_paste_as_lists() { + let clip = &clips( + "
    \n\t

  • Bullet one

    \n\t
      \n\t\t\ +

    • Bullet nested

      \n\t
    \n\t

  • Bullet two

    \n
\n
    \n\t\ +

  1. Number one

    \n\t
      \n\t\t

    1. Number nested

      \n\t\ +
    \n\t

  2. Number two

    \n
", + )[0]; + assert_eq!( + clip.outline(), + [ + "• Bullet one", + " • Bullet nested", + "• Bullet two", + "1. Number one", + " a. Number nested", + "1. Number two", + ] + ); + } + /// Snowbound's HTML reads back as the clip it was written from. #[test] fn snowbound_html_reads_back_as_written() { diff --git a/crates/canvas/tests/clip_paste.rs b/crates/canvas/tests/clip_paste.rs index 632691524ff9f0959e3b651eb0d9dfe76d421501..82defc2b62d8136d571d42d0c084be9be014e358 100644 --- a/crates/canvas/tests/clip_paste.rs +++ b/crates/canvas/tests/clip_paste.rs @@ -16,6 +16,7 @@ use onestore::{ }; const ONENOTE: &str = include_str!("../../../corpus/clipboard/onenote-2010.html"); +const WORD: &str = include_str!("../../../corpus/clipboard/word-2010.html"); /// Each paragraph of a body outline: its level, list kind, text and bold runs. fn outline(page: &Page, outline: &Outline) -> Vec { @@ -23,8 +24,9 @@ fn outline(page: &Page, outline: &Outline) -> Vec { for node in nodes { let list = match node.lists.last().map(|id| &page.definitions[id].kind) { Some(Kind::List { - bullet: Some(_), .. - }) => "• ", + format: Some(format), + .. + }) if !format.contains('\u{fffd}') => "• ", Some(Kind::List { .. }) => "# ", _ => "", }; @@ -310,3 +312,71 @@ fn several_lines_pasted_from_the_title_go_into_the_body() { .unwrap(); } } + +/// Word 2010's lists (`corpus/clipboard/word-2010.html`) paste as lists that the section +/// stores and reads back. `CANVAS_WORD_LIST_EXPORT` names a directory receiving the section +/// as a notebook, for a cold reopen in OneNote 2010. +#[test] +fn words_lists_paste_and_store() { + let source = onestore::create_section("word.one", "Word lists", "Author").unwrap(); + let arena = Arena::default(); + let mut section = Section::open(&arena, source.clone()).unwrap(); + let (space, ..) = section.pages().unwrap()[0].clone(); + let mut engine = TextEngine::default(); + let mut editor = CanvasEditor::from_page(section.page(space).unwrap(), &mut engine).unwrap(); + editor + .place_caret(&mut engine, [72.0, 120.0], 400.0) + .unwrap(); + editor + .paste_pieces(&mut engine, html_pieces(WORD, 0x409, |_, _| None)) + .unwrap(); + let ops = editor.take_ops().unwrap(); + let ops = ops.into_iter().map(|op| Op::Page { space, op }).collect(); + section + .apply( + "Author", + &Edit { + at: 133_000_000_000_000_000, + ops, + }, + ) + .unwrap(); + + let mut image = source; + section.seal().unwrap().unwrap().apply(&mut image).unwrap(); + let arena = Arena::default(); + let reopened = Section::open(&arena, image.clone()).unwrap(); + let page = reopened.page(space).unwrap(); + assert_eq!( + bodies(&page), + [ + vec!["Word lists"], + vec![ + "• Bullet one", + " • Bullet nested", + " • Bullet deeper", + "• Bullet two", + "Plain paragraph", + "# Number one", + " # Number nested", + " # Number deeper", + "# Number two", + "Last plain", + ] + ] + ); + assert_eq!(bodies(&page), bodies(&editor.page().unwrap())); + + if let Some(directory) = std::env::var_os("CANVAS_WORD_LIST_EXPORT") { + let directory = std::path::PathBuf::from(directory); + std::fs::create_dir_all(&directory).unwrap(); + std::fs::write(directory.join("word.one"), &image).unwrap(); + let file_id = Store::parse(&image).unwrap().header.file_id; + std::fs::write( + directory.join("Open Notebook.onetoc2"), + onestore::create_table_of_contents("Open Notebook.onetoc2", &[("word.one", file_id)]) + .unwrap(), + ) + .unwrap(); + } +} diff --git a/tools/test_word_lists.py b/tools/test_word_lists.py new file mode 100644 index 0000000000000000000000000000000000000000..da0b4fe8b2ccb919cf7c074779b4b7af94502e5f --- /dev/null +++ b/tools/test_word_lists.py @@ -0,0 +1,55 @@ +from pathlib import Path +import re +import runpy +import shutil +from tempfile import TemporaryDirectory +import unittest +import xml.etree.ElementTree as ET + +from native_xml import ns + +ROOT = Path(__file__).resolve().parent.parent +FIXTURE = ROOT / 'corpus/word-lists' +compare = runpy.run_path(str(ROOT / 'tools/verify-document.py'))['compare'] + + +class WordListsTest(unittest.TestCase): + def test_lists_pasted_from_word_open_in_onenote_as_its_own_paste_gave_them(self): + read = FIXTURE / 'cold/read' + with TemporaryDirectory() as temporary: + copy = Path(temporary) / 'read' + shutil.copytree(read, copy) + compare(FIXTURE / 'candidate', copy) + page = ET.parse(read / 'page-000.xml').getroot() + lines = [] + + def walk(children, depth): + for element in children.findall('one:OE', ns): + bullet = element.find('one:List/one:Bullet', ns) + number = element.find('one:List/one:Number', ns) + marker = ('bullet %s ' % bullet.get('bullet') if bullet is not None + else number.get('text') + ' ' if number is not None else '') + text = re.sub(r'<[^>]*>', '', element.find('one:T', ns).text or '') + lines.append(' ' * depth + marker + text) + for nested in element.findall('one:OEChildren', ns): + walk(nested, depth + 1) + + outline = page.findall('one:Outline', ns)[-1] + for children in outline.findall('one:OEChildren', ns): + walk(children, 0) + self.assertEqual(lines, [ + 'bullet 1 Bullet one', + ' bullet 0 Bullet nested', + ' bullet 13 Bullet deeper', + 'bullet 1 Bullet two', + 'Plain paragraph', + '1. Number one', + ' a. Number nested', + ' i. Number deeper', + '2. Number two', + 'Last plain', + ]) + + +if __name__ == '__main__': + unittest.main()