diff --git a/corpus/math-edit/README.md b/corpus/math-edit/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41c825e97f632849b7de0c268b6da349d45c3a74 --- /dev/null +++ b/corpus/math-edit/README.md @@ -0,0 +1,35 @@ +# Native equation fixtures for the math model + +`native-editor/` is OneNote 2010 typing five expressions through its equation +editor (Alt+=) on the Rust-authored link page, driven by +`tools/native_math.py` (`equations.ahk` is the AutoHotkey session, +`equations.png` the desktop afterwards): `a_1+b_2`, `x_i^2`, `(a+b)`, +`\int_0^1 x dx` and `\sum_(i=1)^n i`; `\sqrt(x+1)` did not build up and +left an empty paragraph. `before-read/` is the COM read before typing, +`read/` the read afterwards with each expression exported as MathML, and +`notebook/` the section OneNote saved: the linear text with U+FDD0/FDEE/FDEF +object controls and the text-run data naming each object's kind (29 +subscript, 30 sub-superscript, 13 brackets, 21 n-ary with its operator +symbol). `crates/onestore/tests/page_math.rs` renders every expression from +the stored section and compares it with the exported MathML byte for byte; +together with `corpus/m6/native-math-01` (superscripts, a fraction, inline +Greek) that covers the kinds `onestore::page::Math` claims. + +`written/candidate` is the page writer's output for +`equations_are_written_and_read_back`: on a fresh section, every native +equation of both fixtures copied as a new paragraph, then two expressions +built from `Math` trees (`x^2+1` and a fraction with a superscript in its +denominator) through `Math::paragraph`. The writer stores each as OneNote +does: the linear text, one run per span with a Cambria Math italic style +carrying the math flags and the math language, the text-run data array +naming each run's object, and the math language marker on the text object. +`written/cold` is the OneNote 2010 read of that section: its exported MathML +equals `expected-mathml.json` (the native fixtures' exports plus the two +built expressions) and `page-000.png` shows the equations rendered. +`tools/test_math_edit.py` checks both without a VM. + +Regenerate the editor fixture with `tools/native_math.py +corpus/link-edit/candidate OUTPUT` and the written one with +`ONESTORE_MATH_EXPORT` set while running the test, then +`tools/native_runner.py OUTPUT COLD --expected-pages 1 --collect-notebook +--screenshots`. diff --git a/corpus/math-edit/native-drop/README.md b/corpus/math-edit/native-drop/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ade70c8333c4197469b064ba2f7a46a2ef90d85e --- /dev/null +++ b/corpus/math-edit/native-drop/README.md @@ -0,0 +1,23 @@ +# OneNote 2010 drops outline elements written with `{guid},0` identities + +`dropped/candidate/four.one` is one of six identical-content builds of the +ten-equation page (`equations_are_written_and_read_back`); only this build +came back from OneNote 2010 with an empty outline element (`dropped/cold`: +the fifth equation `x_i^2` has no content, alignment or style). The object +bytes, identity tables, run data, styles, revision manifest and transaction +log are identical in shape to the builds that rendered, the drop is +deterministic for a given file, and across thirty builds written with the +model's `{random guid},0` identities seven lost one element each, always an +outline element, at varying positions. OneNote never stores an object as +`{guid},0` (its own objects are `{page guid},n` with `n` from 1); after +`page::text::new_id` moved to `n` 1, eighteen builds and the written +candidate (`../written`) rendered every paragraph. + +`relocated/candidate` is the dropped build with only that element's 56-byte +data chunk copied to the end of the file and the declaration retargeted; it +rendered (`relocated/cold`), so the loss also depends on where the data +sits, which is consistent with the identity being mishandled in a +position-keyed structure rather than being rejected outright. The +transaction writer additionally keeps native files' 1 KiB reservation after +a transaction-log fragment that ends the file, so no object data abuts the +log. diff --git a/corpus/math-edit/native-drop/dropped/candidate/Open Notebook.onetoc2 b/corpus/math-edit/native-drop/dropped/candidate/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..5194ada2c6b37c251317e1f42863c8b3e04af2bb Binary files /dev/null and b/corpus/math-edit/native-drop/dropped/candidate/Open Notebook.onetoc2 differ diff --git a/corpus/math-edit/native-drop/dropped/candidate/four.one b/corpus/math-edit/native-drop/dropped/candidate/four.one new file mode 100644 index 0000000000000000000000000000000000000000..428899c1b6802ee675a9153b221ac4bf51a7ecf4 Binary files /dev/null and b/corpus/math-edit/native-drop/dropped/candidate/four.one differ diff --git a/corpus/math-edit/native-drop/dropped/cold/page-000.xml b/corpus/math-edit/native-drop/dropped/cold/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..ef7f8c5011c96b5d2211b9ba8ad535bfb8db6979 --- /dev/null +++ b/corpus/math-edit/native-drop/dropped/cold/page-000.xml @@ -0,0 +1,3 @@ + +x2+y2=z2]]>a+bc+d]]>Inline ]]>a1+b2]]>a+b]]>∫01x dx ]]>∑i=1ni ]]>x2+1]]>a+bc2]]> diff --git a/corpus/math-edit/native-drop/relocated/candidate/Open Notebook.onetoc2 b/corpus/math-edit/native-drop/relocated/candidate/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..5194ada2c6b37c251317e1f42863c8b3e04af2bb Binary files /dev/null and b/corpus/math-edit/native-drop/relocated/candidate/Open Notebook.onetoc2 differ diff --git a/corpus/math-edit/native-drop/relocated/candidate/four.one b/corpus/math-edit/native-drop/relocated/candidate/four.one new file mode 100644 index 0000000000000000000000000000000000000000..7433711135f00e498c90317f32aa6b1d601588fd Binary files /dev/null and b/corpus/math-edit/native-drop/relocated/candidate/four.one differ diff --git a/corpus/math-edit/native-drop/relocated/cold/commands.jsonl b/corpus/math-edit/native-drop/relocated/cold/commands.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..69850c424cb1b2a1e5df73c83205d4b07edb1240 --- /dev/null +++ b/corpus/math-edit/native-drop/relocated/cold/commands.jsonl @@ -0,0 +1,3 @@ +{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-18994D02 -ExpectedPages 1", "exit": 0, "stdout": "Read 1 sections and 1 pages.\r\n", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null} diff --git a/corpus/math-edit/native-drop/relocated/cold/machine.json b/corpus/math-edit/native-drop/relocated/cold/machine.json new file mode 100644 index 0000000000000000000000000000000000000000..d6aae3c8817206d58cebf3ef9ec46f97e6712199 --- /dev/null +++ b/corpus/math-edit/native-drop/relocated/cold/machine.json @@ -0,0 +1 @@ +{"name": "m6-18994d02", "hostname": "ONE-M6-18994D02"} diff --git a/corpus/math-edit/native-drop/relocated/cold/notebook/Open Notebook.onetoc2 b/corpus/math-edit/native-drop/relocated/cold/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..3a97cd8d0846f19172188a18bd577ce39548178b Binary files /dev/null and b/corpus/math-edit/native-drop/relocated/cold/notebook/Open Notebook.onetoc2 differ diff --git a/corpus/math-edit/native-drop/relocated/cold/notebook/four.one b/corpus/math-edit/native-drop/relocated/cold/notebook/four.one new file mode 100644 index 0000000000000000000000000000000000000000..4cfd0b96d10b7d5ed5f848c747a1874d9f58c821 Binary files /dev/null and b/corpus/math-edit/native-drop/relocated/cold/notebook/four.one differ diff --git a/corpus/math-edit/native-drop/relocated/cold/read/environment.json b/corpus/math-edit/native-drop/relocated/cold/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..fd5f6d54477c7d091c8041477bc82330c935b743 --- /dev/null +++ b/corpus/math-edit/native-drop/relocated/cold/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-18994D02", + "cold": true, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/math-edit/native-drop/relocated/cold/read/hierarchy.xml b/corpus/math-edit/native-drop/relocated/cold/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..6e34c1911acc2e3cdb547d12a452083fe1cf2ceb --- /dev/null +++ b/corpus/math-edit/native-drop/relocated/cold/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/math-edit/native-drop/relocated/cold/read/page-000.xml b/corpus/math-edit/native-drop/relocated/cold/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..3685701532575c53a067a560d91fb8621a89816a --- /dev/null +++ b/corpus/math-edit/native-drop/relocated/cold/read/page-000.xml @@ -0,0 +1,3 @@ + +x2+y2=z2]]>a+bc+d]]>Inline ]]>a1+b2]]>xi2]]>a+b]]>∫01x dx ]]>∑i=1ni ]]>x2+1]]>a+bc2]]> diff --git a/corpus/math-edit/native-drop/relocated/cold/read/payloads.json b/corpus/math-edit/native-drop/relocated/cold/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..1ce91a413d6fff4d58765187b76756f0e97ae843 --- /dev/null +++ b/corpus/math-edit/native-drop/relocated/cold/read/payloads.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/corpus/math-edit/native-drop/relocated/cold/run.json b/corpus/math-edit/native-drop/relocated/cold/run.json new file mode 100644 index 0000000000000000000000000000000000000000..f098e0941df5abfd77ce061acfce6d721e895887 --- /dev/null +++ b/corpus/math-edit/native-drop/relocated/cold/run.json @@ -0,0 +1,18 @@ +{ + "notebook": "/private/tmp/m4/reloc", + "expected_pages": 1, + "author": null, + "author_timeout_seconds": 600, + "inspect": false, + "collect_notebook": true, + "base": { + "file": "win7-office-base.qcow2", + "format": "qcow2", + "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346", + "virtual_size": 68719476736 + }, + "scripts": { + "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331", + "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41" + } +} diff --git a/corpus/math-edit/native-drop/relocated/cold/scripts/cold.ps1 b/corpus/math-edit/native-drop/relocated/cold/scripts/cold.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..a0b99e7aeb86f4dc10032537800501a52ad65b4f --- /dev/null +++ b/corpus/math-edit/native-drop/relocated/cold/scripts/cold.ps1 @@ -0,0 +1,27 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '') +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' } +$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote' +if ($CloneHost) { + if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) { + throw 'The disposable clone hostname does not match this machine.' + } + New-Item "$key\Options\Paths" -Force | Out-Null + New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null +} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or + -not (Test-Path 'C:\one-tests\profile-original-cache')) { + throw 'Park the personal OneNote profile before resetting the test cache.' +} +$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0' +$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root)) +if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' } +New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null +if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked } +if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse } +New-Item "$key\OpenNotebooks" | Out-Null +New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null diff --git a/corpus/math-edit/native-drop/relocated/cold/scripts/read.ps1 b/corpus/math-edit/native-drop/relocated/cold/scripts/read.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..476094ab1e0c47077988a6461db2080cf208642b --- /dev/null +++ b/corpus/math-edit/native-drop/relocated/cold/scripts/read.ps1 @@ -0,0 +1,142 @@ +param( + [Parameter(Mandatory=$true)][string]$Root, + [int]$ExpectedPages = -1, + [switch]$UseCurrentCache, + [switch]$Pdf, + [switch]$KeepOpen, + [string]$CloneHost = '' +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +$notebook = Join-Path $root 'notebook' +$output = Join-Path $root 'read' +if (Test-Path $output) { throw 'Choose a new read destination.' } +if ($UseCurrentCache) { + if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') { + throw 'Park the personal OneNote profile before reading test notebooks.' + } +} else { + & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost +} +New-Item -ItemType Directory -Path $output | Out-Null +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$failure = $null +try { + $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0) + $process = Get-Process ONENOTE + @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion; + powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } | + ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8 + $sections = @() + foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) { + $id = '' + $app.OpenHierarchy($file.FullName, '', [ref]$id, 0) + $sections += $id + } + $deadline = [DateTime]::UtcNow.AddSeconds(300) + $previous = '' + $lastChange = '' + $stableSince = [DateTime]::UtcNow + $settled = $false + do { + $pages = @{} + foreach ($section in $sections) { + $hierarchy = '' + $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) + [xml]$xml = $hierarchy + foreach ($node in $xml.SelectNodes('//*[@path]')) { + if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) { + throw 'OneNote opened a section outside the copied notebook.' + } + } + foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) { + $id = $node.GetAttribute('ID') + $content = '' + $app.GetPageContent($id, [ref]$content, 1, 1) + $pages[$id] = $content + } + } + $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] })) + if ($signature -ne $previous) { + $lastChange = $previous + $previous = $signature + $stableSince = [DateTime]::UtcNow + } + if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or + ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and + ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break } + Start-Sleep -Milliseconds 250 + } while ([DateTime]::UtcNow -lt $deadline) + if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) { + [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8) + $index = 0 + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + $index++ + } + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8) + throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled." + } + $index = 0 + $payloads = @() + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + if ($Pdf) { + $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index) + $app.NavigateTo($id, '', $false) + $app.Publish($id, $pdfPath, 3, '') + if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' } + } + [xml]$page = $pages[$id] + foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) { + $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache')) + $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant() + [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes) + $payloads += @{ page = $id; object = $file.ParentNode.GetAttribute('objectID'); + kind = $file.LocalName; name = $file.GetAttribute('preferredName'); + sha256 = $hash; bytes = $bytes.Length } + } + $index++ + } + [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8) + $all = '' + $app.GetHierarchy($notebookId, 4, [ref]$all, 1) + [xml]$finalTree = $all + $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique) + if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) { + throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.' + } + [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8) + Write-Output "Read $($sections.Count) sections and $($pages.Count) pages." +} catch { + $failure = $_ + [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + throw +} finally { + try { + try { + if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) } + if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) { + $app.CloseNotebook($notebookId, $false) + } + } catch { + if ($null -eq $failure) { throw } + [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + } + } finally { + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + } + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + if (-not $UseCurrentCache -and -not $CloneHost) { + Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10 + } +} diff --git a/corpus/math-edit/native-drop/relocated/cold/source.json b/corpus/math-edit/native-drop/relocated/cold/source.json new file mode 100644 index 0000000000000000000000000000000000000000..53bb12f720a4d76d1e02b4b87f0835318dc0cdbf --- /dev/null +++ b/corpus/math-edit/native-drop/relocated/cold/source.json @@ -0,0 +1,14 @@ +[ + { + "path": "Open Notebook.onetoc2", + "bytes": 1596, + "sha256": "3362ba38101c160a3089a7c35655071f3fefa99af400ee1b93ee9d2a6606382f", + "mtime_ns": 1789266331848783406 + }, + { + "path": "four.one", + "bytes": 14512, + "sha256": "70b21f337a82ff1ac8c6241a7c683f83282a863bdcf2aff70b36be6b9bf279ab", + "mtime_ns": 1789266316867267531 + } +] diff --git a/corpus/math-edit/native-drop/relocated/cold/teardown.json b/corpus/math-edit/native-drop/relocated/cold/teardown.json new file mode 100644 index 0000000000000000000000000000000000000000..05a47793de40f322e745c4c4183727e3590ecf70 --- /dev/null +++ b/corpus/math-edit/native-drop/relocated/cold/teardown.json @@ -0,0 +1 @@ +{"absent": true} diff --git a/corpus/math-edit/native-editor/before-read/environment.json b/corpus/math-edit/native-editor/before-read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..c6f3c040d906633ca3626a4d737023c43ed6bfbf --- /dev/null +++ b/corpus/math-edit/native-editor/before-read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-E6AF28E4", + "cold": true, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/math-edit/native-editor/before-read/hierarchy.xml b/corpus/math-edit/native-editor/before-read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..40f536eca488a69460fc9b98b4ef4b0a534a64c0 --- /dev/null +++ b/corpus/math-edit/native-editor/before-read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/math-edit/native-editor/before-read/page-000.xml b/corpus/math-edit/native-editor/before-read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..671946b6afd0514d154ce0bd177794f7eb93b4e7 --- /dev/null +++ b/corpus/math-edit/native-editor/before-read/page-000.xml @@ -0,0 +1,3 @@ + +the Rust site]]> diff --git a/corpus/math-edit/native-editor/before-read/payloads.json b/corpus/math-edit/native-editor/before-read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..1ce91a413d6fff4d58765187b76756f0e97ae843 --- /dev/null +++ b/corpus/math-edit/native-editor/before-read/payloads.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/corpus/math-edit/native-editor/commands.jsonl b/corpus/math-edit/native-editor/commands.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ccb54ae43192f687407fa302f88a4e7138e4f1d3 --- /dev/null +++ b/corpus/math-edit/native-editor/commands.jsonl @@ -0,0 +1,6 @@ +{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-E6AF28E4 -ExpectedPages 1 -KeepOpen", "exit": 0, "stdout": "Read 1 sections and 1 pages.\r\n", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Rename-Item C:\\one-tests\\runs\\capture\\read before-read\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-E6AF28E4 -UseCurrentCache -ExpectedPages -1", "exit": 0, "stdout": "Read 1 sections and 1 pages.\r\n", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null} diff --git a/corpus/math-edit/native-editor/equations.ahk b/corpus/math-edit/native-editor/equations.ahk new file mode 100644 index 0000000000000000000000000000000000000000..92df26f2f1e8caea89cb1c05977e74189f736940 --- /dev/null +++ b/corpus/math-edit/native-editor/equations.ahk @@ -0,0 +1,68 @@ +ComObject("OneNote.Application").NavigateTo("{1163B9C4-333A-052E-26AB-AA3B02622A56}{1}{B0}", "", false) +WinWait("ahk_exe ONENOTE.EXE", , 30) +WinActivate("ahk_exe ONENOTE.EXE") +WinWaitActive("ahk_exe ONENOTE.EXE", , 30) +WinMaximize("ahk_exe ONENOTE.EXE") +Sleep(3000) +CoordMode("Mouse", "Screen") +Click(300, 140) +Sleep(500) +Send("{End}") +Sleep(300) +Send("{Enter}") +Sleep(500) +Send("!=") +Sleep(600) +SendText("a_1+b_2") +Send(" ") +Sleep(600) +Send("{End}") +Sleep(300) +Send("{Enter}") +Sleep(600) +Send("!=") +Sleep(600) +SendText("\sqrt(x+1)") +Send(" ") +Sleep(600) +Send("{End}") +Sleep(300) +Send("{Enter}") +Sleep(600) +Send("!=") +Sleep(600) +SendText("x_i^2") +Send(" ") +Sleep(600) +Send("{End}") +Sleep(300) +Send("{Enter}") +Sleep(600) +Send("!=") +Sleep(600) +SendText("(a+b)") +Send(" ") +Sleep(600) +Send("{End}") +Sleep(300) +Send("{Enter}") +Sleep(600) +Send("!=") +Sleep(600) +SendText("\int_0^1 x dx") +Send(" ") +Sleep(600) +Send("{End}") +Sleep(300) +Send("{Enter}") +Sleep(600) +Send("!=") +Sleep(600) +SendText("\sum_(i=1)^n i") +Send(" ") +Sleep(600) +Send("{End}") +Sleep(300) +Send("{Enter}") +Sleep(600) +Sleep(2000) diff --git a/corpus/math-edit/native-editor/equations.json b/corpus/math-edit/native-editor/equations.json new file mode 100644 index 0000000000000000000000000000000000000000..83fb4df3362bb2ac7e9fa657fe6db76614b54e31 --- /dev/null +++ b/corpus/math-edit/native-editor/equations.json @@ -0,0 +1,13 @@ +{ + "exit": 0, + "stdout": "", + "stderr": "", + "w": 800, + "h": 600, + "error": null, + "win": { + "title": "Read about Rust the Rust site - Microsoft OneNote", + "class": "Framework::CFrame", + "dialog": false + } +} \ No newline at end of file diff --git a/corpus/math-edit/native-editor/equations.png b/corpus/math-edit/native-editor/equations.png new file mode 100644 index 0000000000000000000000000000000000000000..59b1d11bcc35c962edee976754ef848d6c51c989 Binary files /dev/null and b/corpus/math-edit/native-editor/equations.png differ diff --git a/corpus/math-edit/native-editor/machine.json b/corpus/math-edit/native-editor/machine.json new file mode 100644 index 0000000000000000000000000000000000000000..d77b6d02c2807f7f771543d921df076ba6c6b383 --- /dev/null +++ b/corpus/math-edit/native-editor/machine.json @@ -0,0 +1 @@ +{"name": "m6-e6af28e4", "hostname": "ONE-M6-E6AF28E4"} diff --git a/corpus/math-edit/native-editor/notebook/Open Notebook.onetoc2 b/corpus/math-edit/native-editor/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..adaa534b7ba36343fe368f97feaae448528bbfbc Binary files /dev/null and b/corpus/math-edit/native-editor/notebook/Open Notebook.onetoc2 differ diff --git a/corpus/math-edit/native-editor/notebook/links.one b/corpus/math-edit/native-editor/notebook/links.one new file mode 100644 index 0000000000000000000000000000000000000000..7f26a4000599a0620938af93b9d0b24e8571fdc0 Binary files /dev/null and b/corpus/math-edit/native-editor/notebook/links.one differ diff --git a/corpus/math-edit/native-editor/read/environment.json b/corpus/math-edit/native-editor/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..b4cc83142b43a0c21d9c89c20735fa101afc7717 --- /dev/null +++ b/corpus/math-edit/native-editor/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-E6AF28E4", + "cold": false, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/math-edit/native-editor/read/hierarchy.xml b/corpus/math-edit/native-editor/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..fef7a29b591f027a53d6922d7dead29b56deefc4 --- /dev/null +++ b/corpus/math-edit/native-editor/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/math-edit/native-editor/read/page-000.xml b/corpus/math-edit/native-editor/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..fbdbe2d50c9852606667b37041ad0bc6f5848ea1 --- /dev/null +++ b/corpus/math-edit/native-editor/read/page-000.xml @@ -0,0 +1,3 @@ + +the Rust site]]>a1+b2]]>xi2]]>a+b]]>∫01x dx ]]>∑i=1ni ]]> diff --git a/corpus/math-edit/native-editor/read/payloads.json b/corpus/math-edit/native-editor/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..1ce91a413d6fff4d58765187b76756f0e97ae843 --- /dev/null +++ b/corpus/math-edit/native-editor/read/payloads.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/corpus/math-edit/native-editor/run.json b/corpus/math-edit/native-editor/run.json new file mode 100644 index 0000000000000000000000000000000000000000..9e670c7143179c57e2107d7c15c06d0cb8b81a64 --- /dev/null +++ b/corpus/math-edit/native-editor/run.json @@ -0,0 +1,18 @@ +{ + "notebook": "/Users/clo/dev/one/corpus/link-edit/candidate", + "expected_pages": 1, + "author": null, + "author_timeout_seconds": 600, + "inspect": false, + "collect_notebook": true, + "base": { + "file": "win7-office-base.qcow2", + "format": "qcow2", + "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346", + "virtual_size": 68719476736 + }, + "scripts": { + "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331", + "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41" + } +} diff --git a/corpus/math-edit/native-editor/scripts/cold.ps1 b/corpus/math-edit/native-editor/scripts/cold.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..a0b99e7aeb86f4dc10032537800501a52ad65b4f --- /dev/null +++ b/corpus/math-edit/native-editor/scripts/cold.ps1 @@ -0,0 +1,27 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '') +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' } +$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote' +if ($CloneHost) { + if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) { + throw 'The disposable clone hostname does not match this machine.' + } + New-Item "$key\Options\Paths" -Force | Out-Null + New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null +} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or + -not (Test-Path 'C:\one-tests\profile-original-cache')) { + throw 'Park the personal OneNote profile before resetting the test cache.' +} +$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0' +$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root)) +if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' } +New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null +if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked } +if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse } +New-Item "$key\OpenNotebooks" | Out-Null +New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null diff --git a/corpus/math-edit/native-editor/scripts/read.ps1 b/corpus/math-edit/native-editor/scripts/read.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..476094ab1e0c47077988a6461db2080cf208642b --- /dev/null +++ b/corpus/math-edit/native-editor/scripts/read.ps1 @@ -0,0 +1,142 @@ +param( + [Parameter(Mandatory=$true)][string]$Root, + [int]$ExpectedPages = -1, + [switch]$UseCurrentCache, + [switch]$Pdf, + [switch]$KeepOpen, + [string]$CloneHost = '' +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +$notebook = Join-Path $root 'notebook' +$output = Join-Path $root 'read' +if (Test-Path $output) { throw 'Choose a new read destination.' } +if ($UseCurrentCache) { + if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') { + throw 'Park the personal OneNote profile before reading test notebooks.' + } +} else { + & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost +} +New-Item -ItemType Directory -Path $output | Out-Null +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$failure = $null +try { + $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0) + $process = Get-Process ONENOTE + @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion; + powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } | + ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8 + $sections = @() + foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) { + $id = '' + $app.OpenHierarchy($file.FullName, '', [ref]$id, 0) + $sections += $id + } + $deadline = [DateTime]::UtcNow.AddSeconds(300) + $previous = '' + $lastChange = '' + $stableSince = [DateTime]::UtcNow + $settled = $false + do { + $pages = @{} + foreach ($section in $sections) { + $hierarchy = '' + $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) + [xml]$xml = $hierarchy + foreach ($node in $xml.SelectNodes('//*[@path]')) { + if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) { + throw 'OneNote opened a section outside the copied notebook.' + } + } + foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) { + $id = $node.GetAttribute('ID') + $content = '' + $app.GetPageContent($id, [ref]$content, 1, 1) + $pages[$id] = $content + } + } + $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] })) + if ($signature -ne $previous) { + $lastChange = $previous + $previous = $signature + $stableSince = [DateTime]::UtcNow + } + if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or + ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and + ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break } + Start-Sleep -Milliseconds 250 + } while ([DateTime]::UtcNow -lt $deadline) + if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) { + [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8) + $index = 0 + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + $index++ + } + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8) + throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled." + } + $index = 0 + $payloads = @() + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + if ($Pdf) { + $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index) + $app.NavigateTo($id, '', $false) + $app.Publish($id, $pdfPath, 3, '') + if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' } + } + [xml]$page = $pages[$id] + foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) { + $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache')) + $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant() + [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes) + $payloads += @{ page = $id; object = $file.ParentNode.GetAttribute('objectID'); + kind = $file.LocalName; name = $file.GetAttribute('preferredName'); + sha256 = $hash; bytes = $bytes.Length } + } + $index++ + } + [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8) + $all = '' + $app.GetHierarchy($notebookId, 4, [ref]$all, 1) + [xml]$finalTree = $all + $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique) + if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) { + throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.' + } + [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8) + Write-Output "Read $($sections.Count) sections and $($pages.Count) pages." +} catch { + $failure = $_ + [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + throw +} finally { + try { + try { + if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) } + if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) { + $app.CloseNotebook($notebookId, $false) + } + } catch { + if ($null -eq $failure) { throw } + [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + } + } finally { + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + } + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + if (-not $UseCurrentCache -and -not $CloneHost) { + Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10 + } +} diff --git a/corpus/math-edit/native-editor/source.json b/corpus/math-edit/native-editor/source.json new file mode 100644 index 0000000000000000000000000000000000000000..14786084a474f1cc0a6f522bd49c8104ad3627f0 --- /dev/null +++ b/corpus/math-edit/native-editor/source.json @@ -0,0 +1,14 @@ +[ + { + "path": "Open Notebook.onetoc2", + "bytes": 1604, + "sha256": "6731c4214437fe1cb6aa347b0004542b7f7e4260f47773bf9ec07c4e894de47c", + "mtime_ns": 1789206821144296891 + }, + { + "path": "links.one", + "bytes": 6128, + "sha256": "0cfb356ed374fe919f73fb73fdbe5d66e0baa484d4119f70074d9f869d6db78a", + "mtime_ns": 1789206821116200533 + } +] diff --git a/corpus/math-edit/native-editor/teardown.json b/corpus/math-edit/native-editor/teardown.json new file mode 100644 index 0000000000000000000000000000000000000000..05a47793de40f322e745c4c4183727e3590ecf70 --- /dev/null +++ b/corpus/math-edit/native-editor/teardown.json @@ -0,0 +1 @@ +{"absent": true} diff --git a/corpus/math-edit/written/candidate/Open Notebook.onetoc2 b/corpus/math-edit/written/candidate/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..ed5695c77cfdffc62762c78f37e8fb316a61073c Binary files /dev/null and b/corpus/math-edit/written/candidate/Open Notebook.onetoc2 differ diff --git a/corpus/math-edit/written/candidate/expected-mathml.json b/corpus/math-edit/written/candidate/expected-mathml.json new file mode 100644 index 0000000000000000000000000000000000000000..f3e305976b641ad96017eeb498f4301d01258a78 --- /dev/null +++ b/corpus/math-edit/written/candidate/expected-mathml.json @@ -0,0 +1,12 @@ +[ + "x2+y2=z2", + "a+bc+d", + "𝛼+𝛽", + "a1+b2", + "xi2", + "a+b", + "∫01x dx ", + "∑i=1ni ", + "x2+1", + "a+bc2" +] \ No newline at end of file diff --git a/corpus/math-edit/written/candidate/math.one b/corpus/math-edit/written/candidate/math.one new file mode 100644 index 0000000000000000000000000000000000000000..6cc857f4e22670ede55e5f5f426d0f12fecaf101 Binary files /dev/null and b/corpus/math-edit/written/candidate/math.one differ diff --git a/corpus/math-edit/written/cold/commands.jsonl b/corpus/math-edit/written/cold/commands.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..922076669eed3b079fda589c619d6f68464c8334 --- /dev/null +++ b/corpus/math-edit/written/cold/commands.jsonl @@ -0,0 +1,3 @@ +{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-E32B5D30 -ExpectedPages 1 -KeepOpen", "exit": 0, "stdout": "Read 1 sections and 1 pages.\r\n", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null} diff --git a/corpus/math-edit/written/cold/machine.json b/corpus/math-edit/written/cold/machine.json new file mode 100644 index 0000000000000000000000000000000000000000..1ee1a6d7c811a3e515a7ace85c6a321fa4cdf80b --- /dev/null +++ b/corpus/math-edit/written/cold/machine.json @@ -0,0 +1 @@ +{"name": "m6-e32b5d30", "hostname": "ONE-M6-E32B5D30"} diff --git a/corpus/math-edit/written/cold/notebook/Open Notebook.onetoc2 b/corpus/math-edit/written/cold/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..c4fda78ce4bc1e48d7366fcbe4a575fff08452fa Binary files /dev/null and b/corpus/math-edit/written/cold/notebook/Open Notebook.onetoc2 differ diff --git a/corpus/math-edit/written/cold/notebook/expected-mathml.json b/corpus/math-edit/written/cold/notebook/expected-mathml.json new file mode 100644 index 0000000000000000000000000000000000000000..f3e305976b641ad96017eeb498f4301d01258a78 --- /dev/null +++ b/corpus/math-edit/written/cold/notebook/expected-mathml.json @@ -0,0 +1,12 @@ +[ + "x2+y2=z2", + "a+bc+d", + "𝛼+𝛽", + "a1+b2", + "xi2", + "a+b", + "∫01x dx ", + "∑i=1ni ", + "x2+1", + "a+bc2" +] \ No newline at end of file diff --git a/corpus/math-edit/written/cold/notebook/math.one b/corpus/math-edit/written/cold/notebook/math.one new file mode 100644 index 0000000000000000000000000000000000000000..9cc7e9560eeff1e97754164a05eae7255e13c196 Binary files /dev/null and b/corpus/math-edit/written/cold/notebook/math.one differ diff --git a/corpus/math-edit/written/cold/read/environment.json b/corpus/math-edit/written/cold/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..b75fd21f70b84e9dfdfe623908be0059160d2d53 --- /dev/null +++ b/corpus/math-edit/written/cold/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-E32B5D30", + "cold": true, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/math-edit/written/cold/read/hierarchy.xml b/corpus/math-edit/written/cold/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..657330b7be34cad7b8ccc67edc21563af704ed7e --- /dev/null +++ b/corpus/math-edit/written/cold/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/math-edit/written/cold/read/page-000.navigation.json b/corpus/math-edit/written/cold/read/page-000.navigation.json new file mode 100644 index 0000000000000000000000000000000000000000..af6bcb2175c02024ff0ca87879c85b382404985b --- /dev/null +++ b/corpus/math-edit/written/cold/read/page-000.navigation.json @@ -0,0 +1,13 @@ +{ + "exit": 0, + "stdout": "", + "stderr": "", + "w": 800, + "h": 600, + "error": null, + "win": { + "title": "Equations below - Microsoft OneNote", + "class": "Framework::CFrame", + "dialog": false + } +} \ No newline at end of file diff --git a/corpus/math-edit/written/cold/read/page-000.png b/corpus/math-edit/written/cold/read/page-000.png new file mode 100644 index 0000000000000000000000000000000000000000..0c7fd663288f5eefe1230686a1399567c9d6f8d5 Binary files /dev/null and b/corpus/math-edit/written/cold/read/page-000.png differ diff --git a/corpus/math-edit/written/cold/read/page-000.xml b/corpus/math-edit/written/cold/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..5a2a4051d0554dd8dce44a9ea20aad0d69dd35a1 --- /dev/null +++ b/corpus/math-edit/written/cold/read/page-000.xml @@ -0,0 +1,3 @@ + +x2+y2=z2]]>a+bc+d]]>Inline ]]>a1+b2]]>xi2]]>a+b]]>∫01x dx ]]>∑i=1ni ]]>x2+1]]>a+bc2]]> diff --git a/corpus/math-edit/written/cold/read/payloads.json b/corpus/math-edit/written/cold/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..1ce91a413d6fff4d58765187b76756f0e97ae843 --- /dev/null +++ b/corpus/math-edit/written/cold/read/payloads.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/corpus/math-edit/written/cold/run.json b/corpus/math-edit/written/cold/run.json new file mode 100644 index 0000000000000000000000000000000000000000..068c5a4da94ffd56599463b3d8d1d9d2f744b81a --- /dev/null +++ b/corpus/math-edit/written/cold/run.json @@ -0,0 +1,18 @@ +{ + "notebook": "/private/tmp/m4/written", + "expected_pages": 1, + "author": null, + "author_timeout_seconds": 600, + "inspect": false, + "collect_notebook": true, + "base": { + "file": "win7-office-base.qcow2", + "format": "qcow2", + "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346", + "virtual_size": 68719476736 + }, + "scripts": { + "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331", + "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41" + } +} diff --git a/corpus/math-edit/written/cold/scripts/cold.ps1 b/corpus/math-edit/written/cold/scripts/cold.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..a0b99e7aeb86f4dc10032537800501a52ad65b4f --- /dev/null +++ b/corpus/math-edit/written/cold/scripts/cold.ps1 @@ -0,0 +1,27 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '') +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' } +$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote' +if ($CloneHost) { + if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) { + throw 'The disposable clone hostname does not match this machine.' + } + New-Item "$key\Options\Paths" -Force | Out-Null + New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null +} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or + -not (Test-Path 'C:\one-tests\profile-original-cache')) { + throw 'Park the personal OneNote profile before resetting the test cache.' +} +$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0' +$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root)) +if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' } +New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null +if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked } +if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse } +New-Item "$key\OpenNotebooks" | Out-Null +New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null diff --git a/corpus/math-edit/written/cold/scripts/read.ps1 b/corpus/math-edit/written/cold/scripts/read.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..476094ab1e0c47077988a6461db2080cf208642b --- /dev/null +++ b/corpus/math-edit/written/cold/scripts/read.ps1 @@ -0,0 +1,142 @@ +param( + [Parameter(Mandatory=$true)][string]$Root, + [int]$ExpectedPages = -1, + [switch]$UseCurrentCache, + [switch]$Pdf, + [switch]$KeepOpen, + [string]$CloneHost = '' +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +$notebook = Join-Path $root 'notebook' +$output = Join-Path $root 'read' +if (Test-Path $output) { throw 'Choose a new read destination.' } +if ($UseCurrentCache) { + if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') { + throw 'Park the personal OneNote profile before reading test notebooks.' + } +} else { + & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost +} +New-Item -ItemType Directory -Path $output | Out-Null +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$failure = $null +try { + $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0) + $process = Get-Process ONENOTE + @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion; + powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } | + ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8 + $sections = @() + foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) { + $id = '' + $app.OpenHierarchy($file.FullName, '', [ref]$id, 0) + $sections += $id + } + $deadline = [DateTime]::UtcNow.AddSeconds(300) + $previous = '' + $lastChange = '' + $stableSince = [DateTime]::UtcNow + $settled = $false + do { + $pages = @{} + foreach ($section in $sections) { + $hierarchy = '' + $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) + [xml]$xml = $hierarchy + foreach ($node in $xml.SelectNodes('//*[@path]')) { + if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) { + throw 'OneNote opened a section outside the copied notebook.' + } + } + foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) { + $id = $node.GetAttribute('ID') + $content = '' + $app.GetPageContent($id, [ref]$content, 1, 1) + $pages[$id] = $content + } + } + $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] })) + if ($signature -ne $previous) { + $lastChange = $previous + $previous = $signature + $stableSince = [DateTime]::UtcNow + } + if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or + ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and + ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break } + Start-Sleep -Milliseconds 250 + } while ([DateTime]::UtcNow -lt $deadline) + if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) { + [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8) + $index = 0 + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + $index++ + } + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8) + throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled." + } + $index = 0 + $payloads = @() + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + if ($Pdf) { + $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index) + $app.NavigateTo($id, '', $false) + $app.Publish($id, $pdfPath, 3, '') + if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' } + } + [xml]$page = $pages[$id] + foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) { + $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache')) + $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant() + [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes) + $payloads += @{ page = $id; object = $file.ParentNode.GetAttribute('objectID'); + kind = $file.LocalName; name = $file.GetAttribute('preferredName'); + sha256 = $hash; bytes = $bytes.Length } + } + $index++ + } + [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8) + $all = '' + $app.GetHierarchy($notebookId, 4, [ref]$all, 1) + [xml]$finalTree = $all + $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique) + if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) { + throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.' + } + [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8) + Write-Output "Read $($sections.Count) sections and $($pages.Count) pages." +} catch { + $failure = $_ + [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + throw +} finally { + try { + try { + if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) } + if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) { + $app.CloseNotebook($notebookId, $false) + } + } catch { + if ($null -eq $failure) { throw } + [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + } + } finally { + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + } + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + if (-not $UseCurrentCache -and -not $CloneHost) { + Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10 + } +} diff --git a/corpus/math-edit/written/cold/source.json b/corpus/math-edit/written/cold/source.json new file mode 100644 index 0000000000000000000000000000000000000000..7e5fb30c38c8f532d9a9957daa8499516d02873a --- /dev/null +++ b/corpus/math-edit/written/cold/source.json @@ -0,0 +1,20 @@ +[ + { + "path": "Open Notebook.onetoc2", + "bytes": 1596, + "sha256": "bc31e9f663f68420b4f7684641e2a75abefddd97256074bae432dff0fff5244c", + "mtime_ns": 1789267920902398086 + }, + { + "path": "expected-mathml.json", + "bytes": 1536, + "sha256": "7c0f17965424fbfee0f952b0293a2daf5fcd7040d50b7482e90785cc52192c84", + "mtime_ns": 1789267920927591231 + }, + { + "path": "math.one", + "bytes": 15424, + "sha256": "70b09e61eaca321b6aa555316b59769f282843c7710c4ca311f1b5d2d7ec42bd", + "mtime_ns": 1789267920871282689 + } +] diff --git a/corpus/math-edit/written/cold/teardown.json b/corpus/math-edit/written/cold/teardown.json new file mode 100644 index 0000000000000000000000000000000000000000..05a47793de40f322e745c4c4183727e3590ecf70 --- /dev/null +++ b/corpus/math-edit/written/cold/teardown.json @@ -0,0 +1 @@ +{"absent": true} diff --git a/crates/notebook/README.md b/crates/notebook/README.md index e9b9ee4d9efe9223362d8366610e00664737645e..70ce70ea2b329dd4328c6264bea9cd80cb5160cc 100644 --- a/crates/notebook/README.md +++ b/crates/notebook/README.md @@ -14,7 +14,7 @@ diffs the supplied model against the page stored in the supplied local snapshot, writes the difference into the working image and queues one `Operation::Page(PageIntent { before, after, author })`; `before` is the page the edit started from and is the precondition reconciliation checks. Text, styles, -paragraph formatting, hyperlinks (external and internal), bullets, numbering, note tags, table rows, columns, cell shading, nested tables, pictures (insertion in paragraphs or on the page, position, size, description), attachments, ink strokes, paragraph structure, +paragraph formatting, hyperlinks (external and internal), bullets, numbering, note tags, table rows, columns, cell shading, nested tables, pictures (insertion in paragraphs or on the page, position, size, description), attachments, ink strokes, equations, paragraph structure, outline layout, insertions and deletions are all differences between `before` and `after`; the library never sees editor operations. diff --git a/crates/onestore/README.md b/crates/onestore/README.md index 793babecd34aa7639f4b5435ccea18be46eddaf8..bc267d205c498f34e9ea2a67ccec8b8f6cc6da13 100644 --- a/crates/onestore/README.md +++ b/crates/onestore/README.md @@ -57,7 +57,7 @@ harness also accepts `--client-profile release`. | `ParagraphJoin`, `PreparedEdit::join` | Join adjacent ordinary text while preserving inherited character styles and native text-identity rules | | `TextAttribute`, `PreparedEdit::format` | Change character formatting over a UTF-16 range while sharing immutable styles; preserve unselected runs | | `OutlineEdit`, `PreparedEdit::outline` | Change ordinary outline position/width or a paragraph's saved expansion default, preserving identities and content | -| `PreparedEdit::page` | Publish an edited `page::Page` as one revision per changed space: text, character and paragraph formatting, hyperlinks (external and, via `page::link`, to pages, paragraphs and sections), bullets and numbering, note tags, table rows, columns, cell shading and indents, nested tables, inserted pictures (in paragraphs or on the page) and attachments, picture position, size and description, ink drawings and handwriting (strokes added and erased), paragraph insertion/split/join/move/deletion, outline insertion/deletion/position/width and saved collapse state, lowered onto the typed writers with the model's identities | +| `PreparedEdit::page` | Publish an edited `page::Page` as one revision per changed space: text, character and paragraph formatting, hyperlinks (external and, via `page::link`, to pages, paragraphs and sections), bullets and numbering, note tags, table rows, columns, cell shading and indents, nested tables, inserted pictures (in paragraphs or on the page) and attachments, picture position, size and description, ink drawings and handwriting (strokes added and erased), equations (built from `page::Math` trees), paragraph insertion/split/join/move/deletion, outline insertion/deletion/position/width and saved collapse state, lowered onto the typed writers with the model's identities | | `TreeEdit`, `PreparedEdit::tree` | Move or delete a subtree on one page, normalize surviving containers, and replace an emptied table cell's paragraph atomically | | `PreparedEdit::commit`, `PreparedEdit::commit_file` | Publish the exact prepared image under caller-held exclusion or the conservative filesystem adapter | | `read_file` | Read a snapshot under whole-file exclusion | diff --git a/crates/onestore/src/page/math.rs b/crates/onestore/src/page/math.rs index c1205f2e1ba27412b1b69f5e0245fd5667c6bace..7ae2f63d495c769e352107e0daab7da9f9d5f289 100644 --- a/crates/onestore/src/page/math.rs +++ b/crates/onestore/src/page/math.rs @@ -1,8 +1,9 @@ //! Equations as OneNote stores them: a linear text where U+FDD0 opens an inline object, //! U+FDEE separates its arguments and U+FDEF closes it, with the object's kind on the run //! data of the opening character. `Math::parse` builds the tree and `mathml` renders it the -//! way OneNote's own export does for the kinds a native fixture has verified (superscript -//! and fraction); other kinds follow the text object model's meaning without a native check. +//! way OneNote's own export does for the kinds native fixtures have verified (superscript, +//! subscript, both, fraction, brackets, n-ary operators with limits); radicals, under/over +//! limits and the rest follow the text object model's meaning without a native check. use super::text::Paragraph; use crate::document::MathObject; @@ -96,7 +97,7 @@ impl Math { arguments: frame.arguments, }); } - c if c.is_whitespace() => {} + c if c.is_whitespace() => sequences.last_mut().unwrap().push(Math::Operator(' ')), c if c.is_alphabetic() => sequences .last_mut() .unwrap() @@ -133,11 +134,12 @@ impl Math { symbols, arguments, } => { + // A lone identifier, number or operator stands bare; anything else is a row. let argument = |out: &mut String, index: usize| match arguments .get(index) .map(Vec::as_slice) { - Some([single]) => single.write(out), + Some([single]) if !matches!(single, Math::Object { .. }) => single.write(out), Some(many) => { out.push_str(""); for node in many { @@ -163,16 +165,26 @@ impl Math { (25, _) => wrapped(out, "msqrt", &[arguments.len() - 1]), (19, 2) => wrapped(out, "munder", &[0, 1]), (33, 2) => wrapped(out, "mover", &[0, 1]), - (13, _) => { + // OneNote exports parentheses as default fences, whatever the pair. + (13, _) => wrapped(out, "mfenced", &[0]), + // Lower limit, upper limit, body; integrals take their limits as scripts, + // other operators above and below. + (21, 3) => { + let operator = symbols.first().copied().unwrap_or('∑'); + let element = if ('\u{222b}'..='\u{2233}').contains(&operator) { + "msubsup" + } else { + "munderover" + }; + out.push_str(&format!( + "{operator}" + )); + argument(out, 0); + argument(out, 1); + out.push_str(&format!("")); out.push_str(""); - if let Some(open) = symbols.first() { - tag(out, "mo", &open.to_string()); - } - for index in 0..arguments.len() { - argument(out, index); - } - if let Some(close) = symbols.get(1) { - tag(out, "mo", &close.to_string()); + for node in &arguments[2] { + node.write(out); } out.push_str(""); } @@ -199,6 +211,7 @@ fn tag(out: &mut String, element: &str, content: &str) { '<' => out.push_str("<"), '&' => out.push_str("&"), '>' => out.push_str(">"), + ' ' => out.push_str(" "), c => out.push(c), } } @@ -215,3 +228,96 @@ fn plain(c: char) -> char { _ => c, } } + +impl Math { + /// The paragraph OneNote stores for `nodes`: linear text with object controls, every run + /// formatted the way the equation editor formats it (Cambria Math, italic, the math flags + /// and the math language over `base`), and the run data each object carries. + pub fn paragraph(nodes: &[Math], base: &crate::document::Format) -> Paragraph { + let mut style = base.clone(); + style.italic = Some(true); + style.hidden = Some(false); + style.hyperlink = Some(false); + style.math = Some(true); + style.embedded_object = Some(true); + style.font = Some("Cambria Math".into()); + style.language = Some(0x1007f); + let mut runs: Vec<(String, Option)> = Vec::new(); + write_sequence(nodes, &mut runs); + Paragraph::from_runs(runs.into_iter().map(|(text, object)| { + let mut format = style.clone(); + format.math_object = Some(object.unwrap_or(MathObject { + kind: PLAIN_RUN, + arguments: None, + symbols: Vec::new(), + })); + (text, format) + })) + } +} + +/// The object kind OneNote gives runs between objects. +const PLAIN_RUN: u32 = 0x9000_0000; + +fn write_sequence(nodes: &[Math], runs: &mut Vec<(String, Option)>) { + let mut leaf = String::new(); + for node in nodes { + match node { + Math::Identifier(c) => leaf.push(italic(*c)), + Math::Number(n) => leaf.push_str(n), + Math::Operator(c) => leaf.push(*c), + Math::Object { + kind, + symbols, + arguments, + } => { + if !leaf.is_empty() { + runs.push((std::mem::take(&mut leaf), None)); + } + runs.push(( + OBJECT_START.to_string(), + Some(MathObject { + kind: *kind, + arguments: Some(arguments.len() as u32), + symbols: symbols.clone(), + }), + )); + for (index, argument) in arguments.iter().enumerate() { + let before = runs.len(); + write_sequence(argument, runs); + let control = if index + 1 == arguments.len() { + OBJECT_END + } else { + ARGUMENT_SEPARATOR + }; + let object = MathObject { + kind: *kind, + arguments: (index > 0).then_some(index as u32), + symbols: Vec::new(), + }; + let extended = runs.len() > before; + match runs.last_mut() { + Some((text, slot)) if extended && slot.is_none() => { + text.push(control); + *slot = Some(object); + } + _ => runs.push((control.to_string(), Some(object))), + } + } + } + } + } + if !leaf.is_empty() { + runs.push((leaf, None)); + } +} + +/// The equation editor stores Latin letters as mathematical italics. +fn italic(c: char) -> char { + match c { + 'h' => '\u{210e}', + 'A'..='Z' => char::from_u32(0x1d434 + u32::from(c) - u32::from('A')).unwrap(), + 'a'..='z' => char::from_u32(0x1d44e + u32::from(c) - u32::from('a')).unwrap(), + c => c, + } +} diff --git a/crates/onestore/src/page/text.rs b/crates/onestore/src/page/text.rs index e195c578a4717b0bd436842329d1bb215cb95ebd..8cef50349a53ebdfd299447bf4d170fc50c0ca1a 100644 --- a/crates/onestore/src/page/text.rs +++ b/crates/onestore/src/page/text.rs @@ -154,10 +154,12 @@ impl From for crate::Error { } /// A fresh random identity for a new paragraph or text object. +/// A fresh identity with `n` 1: OneNote never stores an object as `{guid},0`, and OneNote +/// 2010 loses outline elements stored that way (corpus/math-edit/native-drop). pub fn new_id() -> Result { Ok(ExGuid { guid: crate::write::fresh_guid().map_err(|_| EditError::Identity)?, - n: 0, + n: 1, }) } diff --git a/crates/onestore/src/page/write.rs b/crates/onestore/src/page/write.rs index 35abf42da6ecd2792374cb28cac9005215e7d432..8c2285ac616c3957dbf7d5a44981a2e8c51f70b2 100644 --- a/crates/onestore/src/page/write.rs +++ b/crates/onestore/src/page/write.rs @@ -1,7 +1,7 @@ //! Publishes an edited page model by lowering the difference from the stored page onto //! the typed writers, then squashing their transactions into one revision per space. -use super::{Image, Ink, Outline, Page, PageObject, PageParagraph, ParagraphContent, Table}; +use super::{Image, Ink, Math, Outline, Page, PageObject, PageParagraph, ParagraphContent, Table}; use crate::{ Error, ExGuid, Insertion, ObjectData, OutlineEdit, ParagraphJoin, ParagraphSplit, PropertySets, RevisionIndex, Store, TextAttribute, TreeEdit, Value, @@ -184,6 +184,48 @@ fn stroke_values(stroke: &super::InkStroke, index: u32, filetime: u64) -> Result ]) } +/// The character-style properties a span format sets; paragraph-level fields stay on the +/// paragraph style. +fn style_values(format: &crate::document::Format) -> Values { + let mut values = Values::new(); + for (id, flag) in [ + (0x08001c04, format.bold), + (0x08001c05, format.italic), + (0x08001c06, format.underline), + (0x08001c07, format.strike), + (0x08001c08, format.superscript), + (0x08001c09, format.subscript), + (0x08001e16, format.hidden), + (0x08001e14, format.hyperlink), + (0x08001e19, format.hyperlink_label), + (0x08003401, format.math), + (0x08001e22, format.embedded_object), + ] { + if let Some(flag) = flag { + values.push((id | (u32::from(flag) << 31), Vec::new())); + } + } + if let Some(font) = &format.font { + values.push((0x1c001c0a, crate::create::string(font))); + } + if let Some(size) = format.font_size { + values.push(( + 0x10001c0b, + ((size * 2.0).round() as u16).to_le_bytes().to_vec(), + )); + } + if let Some(color) = format.color { + values.push((0x14001c0c, color.to_le_bytes().to_vec())); + } + if let Some(highlight) = format.highlight { + values.push((0x14001c0d, highlight.to_le_bytes().to_vec())); + } + if let Some(language) = format.language { + values.push((0x14001c3b, language.to_le_bytes().to_vec())); + } + values +} + fn picture_fixed_fields(stored: &Image, image: &Image) -> Result<(), Error> { if stored.id != image.id || stored.bytes != image.bytes @@ -494,6 +536,7 @@ impl Lowering<'_> { self.split_and_join(&old, &new, &mut placed, &mut consumed)?; self.place(&old, &new, &placed, &page_order)?; self.delete(&old, &new, &consumed)?; + self.edit_equations(&new)?; self.edit_text(&new)?; self.edit_lists(after, &new)?; self.edit_tags(after, &new)?; @@ -1679,6 +1722,125 @@ impl Lowering<'_> { Ok(()) } + /// Rewrites every equation paragraph whose stored text object differs from the model, the + /// way OneNote stores an equation: the linear text, one run per span with a style + /// carrying the span's format, the run-data array naming each run's inline object, and + /// the math language marker on the text object. + fn edit_equations(&mut self, new: &View<'_>) -> Result<(), Error> { + let current = self.current()?; + let current = View::new(¤t)?; + for (id, paragraph) in &new.paragraphs { + let Some(text) = paragraph.text() else { + continue; + }; + if !Math::is_equation(&text.text) { + continue; + } + let stored = current + .text(self.id(*id)) + .ok_or_else(|| invalid("An equation paragraph is missing after placement"))?; + if stored.text == text.text { + continue; + } + let object = stored.id; + if text.text.text().contains('\u{fffc}') { + return Err(invalid("Equations cannot hold embedded objects")); + } + let encoded: Vec = text + .text + .text() + .encode_utf16() + .chain([0]) + .flat_map(u16::to_le_bytes) + .collect(); + let mut ends = Vec::new(); + let mut styles: Vec = Vec::new(); + let mut sets = Vec::new(); + let all_math = text + .text + .spans() + .iter() + .all(|s| s.format.math == Some(true)); + for span in text.text.spans() { + ends.extend(text.text.utf16_offset(span.end)?.to_le_bytes()); + styles.push(style_values(&span.format)); + sets.push(match &span.format.math_object { + Some(object) => { + let mut set = vec![(0x1400344f, object.kind.to_le_bytes().to_vec())]; + if let Some(count) = object.arguments { + set.push((0x14003450, count.to_le_bytes().to_vec())); + } + for (id, symbol) in [0x10003453, 0x10003454, 0x10003455] + .into_iter() + .zip(&object.symbols) + { + let unit = u16::try_from(u32::from(*symbol)) + .map_err(|_| invalid("Math symbols are single UTF-16 units"))?; + set.push((id, unit.to_le_bytes().to_vec())); + } + set + } + None => Vec::new(), + }); + } + ends.truncate(ends.len() - 4); + let has_objects = sets.iter().any(|set| !set.is_empty()); + let modified = crate::create::current_timestamps()?.0.to_le_bytes(); + let space = self.space; + self.apply(|current| { + crate::write::write_revision(current, space, |raw| { + let mut changed = BTreeMap::new(); + let mut target = PropertyObject::from_object(&raw.objects[&object])?; + let mut references = Vec::new(); + let mut created: Vec<(Values, ExGuid)> = Vec::new(); + for values in &styles { + let id = match created.iter().find(|(known, _)| known == values) { + Some((_, id)) => *id, + None => { + let id = ExGuid { + guid: crate::write::fresh_guid()?, + n: 1, + }; + let mut style = PropertyObject { + jcid: 0x12004d, + bytes: crate::create::properties(values)?, + global_ids: std::sync::Arc::new(BTreeMap::from([(0, id.guid)])), + }; + style.reference(id)?; + changed.insert(id, style); + created.push((values.clone(), id)); + id + } + }; + references.extend(target.reference(id)?); + } + target.remove(&[0x1c003498, 0x40003499])?; + target.set(&[ + (0x1c001c22, &encoded), + (0x1c001e12, &ends), + (0x24001e13, &references), + (0x14001d7a, &modified), + ])?; + if has_objects { + target.set_sets(0x40003499, 0x44000811, &sets)?; + } + if all_math { + // The flags and language marker OneNote's equation editor leaves on + // every equation text object. + target.set(&[ + (0x10001cfe, &0x7f_u16.to_le_bytes()), + (0x14001c3e, &1u32.to_le_bytes()), + (0x14001c84, &1u32.to_le_bytes()), + ])?; + } + changed.insert(object, target); + Ok(changed) + }) + })?; + } + Ok(()) + } + fn edit_text(&mut self, new: &View<'_>) -> Result<(), Error> { let current = self.current()?; let current = View::new(¤t)?; diff --git a/crates/onestore/src/write.rs b/crates/onestore/src/write.rs index b3085fcb2c4719e15eacdebfbab7b3d681cbb83a..a63774c575c95c9d6995134c4c40ffea970897cd 100644 --- a/crates/onestore/src/write.rs +++ b/crates/onestore/src/write.rs @@ -782,6 +782,17 @@ pub(crate) fn write_revisions_with_payloads( index.validate_current()?; let changes = edit(&index)?; let mut output = source.to_vec(); + // Native files reserve 1 KiB per transaction-log fragment; a fragment that ends the file + // keeps that room before the first new chunk. + let tail = store.transaction_fragments.last().unwrap().chunk; + let reserved = usize::try_from(tail.offset + tail.length.max(1024) + 32).unwrap(); + if usize::try_from(tail.offset + tail.length) + .unwrap() + .next_multiple_of(8) + >= output.len() + { + output.resize(reserved, 0); + } let mut maximum = store .transaction_fragments .iter() diff --git a/crates/onestore/tests/page_math.rs b/crates/onestore/tests/page_math.rs index bae4c525ecf5c03c0683dcc5225cf6f7343112aa..c62f12b9b596b23eb78a70f9848833a44647f738 100644 --- a/crates/onestore/tests/page_math.rs +++ b/crates/onestore/tests/page_math.rs @@ -8,10 +8,13 @@ use onestore::{ /// MathML (`corpus/m6/native-math-01`). const NATIVE: &[u8] = include_bytes!("../../../corpus/m6/native-math-01/notebook/synthetic.one"); const NATIVE_READ: &str = include_str!("../../../corpus/m6/native-math-01/read/page-000.xml"); +/// Five more expressions typed through the equation editor by `tools/native_math.py`: +/// subscripts, a sub-superscript, parentheses, an integral and a sum with limits. +const EDITOR: &[u8] = include_bytes!("../../../corpus/math-edit/native-editor/notebook/links.one"); +const EDITOR_READ: &str = include_str!("../../../corpus/math-edit/native-editor/read/page-000.xml"); -fn native_mathml() -> Vec { - NATIVE_READ - .split(" Vec { + read.split("').unwrap() + 1..rest.find("").unwrap()]; @@ -30,9 +33,8 @@ fn native_mathml() -> Vec { .collect() } -#[test] -fn native_equations_render_to_the_mathml_onenote_exports() { - let store = Store::parse(NATIVE).unwrap(); +fn rendered_equations(bytes: &[u8], title: &str) -> Vec { + let store = Store::parse(bytes).unwrap(); let index = RevisionIndex::parse(&store).unwrap(); let document = Document::parse(&index).unwrap(); let page = document @@ -40,7 +42,7 @@ fn native_equations_render_to_the_mathml_onenote_exports() { .unwrap() .into_iter() .map(|(space, _)| Page::from_space(&document, space).unwrap()) - .find(|page| page.title == "Equation controls") + .find(|page| page.title == title) .unwrap(); let mut rendered = Vec::new(); for object in &page.objects { @@ -72,5 +74,236 @@ fn native_equations_render_to_the_mathml_onenote_exports() { rendered.push(Math::mathml(&Math::parse(&equation).unwrap())); } } - assert_eq!(rendered, native_mathml()); + rendered +} + +#[test] +fn native_equations_render_to_the_mathml_onenote_exports() { + assert_eq!( + rendered_equations(NATIVE, "Equation controls"), + native_mathml(NATIVE_READ) + ); +} + +#[test] +fn equation_editor_expressions_render_to_the_mathml_onenote_exports() { + assert_eq!( + rendered_equations(EDITOR, "Read about Rust the Rust site"), + native_mathml(EDITOR_READ) + ); +} + +fn object(kind: u32, symbols: &str, arguments: Vec>) -> Math { + Math::Object { + kind, + symbols: symbols.chars().collect(), + arguments, + } +} + +fn built_equations() -> Vec<(Vec, &'static str)> { + vec![ + ( + vec![ + object( + 31, + "^", + vec![vec![Math::Identifier('x')], vec![Math::Number("2".into())]], + ), + Math::Operator('+'), + Math::Number("1".into()), + ], + "x2+1", + ), + ( + vec![object( + 16, + "/", + vec![ + vec![ + Math::Identifier('a'), + Math::Operator('+'), + Math::Identifier('b'), + ], + vec![object( + 31, + "^", + vec![vec![Math::Identifier('c')], vec![Math::Number("2".into())]], + )], + ], + )], + "a+bc2", + ), + ] +} + +#[test] +fn equations_built_from_the_tree_store_the_native_linear_form_and_parse_back() { + for (nodes, mathml) in built_equations() { + let paragraph = Math::paragraph(&nodes, &onestore::document::Format::default()); + assert!(Math::is_equation(¶graph)); + assert_eq!(Math::parse(¶graph).unwrap(), nodes); + assert_eq!(Math::mathml(&nodes), mathml); + } + let paragraph = Math::paragraph(&built_equations()[0].0, &Default::default()); + assert_eq!(paragraph.text(), "\u{fdd0}\u{1d465}\u{fdee}2\u{fdef}+1"); + let kinds: Vec<(Option, Option)> = paragraph + .spans() + .iter() + .map(|s| { + let object = s.format.math_object.as_ref().unwrap(); + (Some(object.kind), object.arguments) + }) + .collect(); + assert_eq!( + kinds, + [ + (Some(31), Some(2)), + (Some(31), None), + (Some(31), Some(1)), + (Some(0x9000_0000), None) + ] + ); +} + +/// `ONESTORE_MATH_EXPORT` names a new directory receiving the candidate for a cold reopen: +/// every native equation copied onto a fresh page, then two built from the tree. +#[test] +fn equations_are_written_and_read_back() { + use onestore::page::{ParagraphContent, TextObject, text::new_id}; + let source = onestore::create_section("math.one", "Equations below", "Author").unwrap(); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let (space, _) = document.pages().unwrap()[0]; + let before = Page::from_space(&document, space).unwrap(); + let mut after = before.clone(); + let mut expected_mathml = Vec::new(); + let mut paragraphs = Vec::new(); + for (bytes, title, read) in [ + (NATIVE, "Equation controls", NATIVE_READ), + (EDITOR, "Read about Rust the Rust site", EDITOR_READ), + ] { + expected_mathml.extend(native_mathml(read)); + let store = Store::parse(bytes).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let page = document + .pages() + .unwrap() + .into_iter() + .map(|(space, _)| Page::from_space(&document, space).unwrap()) + .find(|page| page.title == title) + .unwrap(); + for object in &page.objects { + let PageObject::Outline(outline) = object else { + continue; + }; + for paragraph in &outline.paragraphs { + if paragraph.text().is_some_and(|t| Math::is_equation(&t.text)) { + paragraphs.push(paragraph.clone()); + } + } + } + } + for (nodes, mathml) in built_equations() { + let mut paragraph = paragraphs[0].clone(); + paragraph.content = ParagraphContent::Text(TextObject { + id: new_id().unwrap(), + date_field: None, + text: Math::paragraph(&nodes, &Default::default()), + tags: Vec::new(), + }); + paragraphs.push(paragraph); + expected_mathml.push(mathml.to_owned()); + } + let outline = after + .objects + .iter_mut() + .find_map(|object| match object { + PageObject::Outline(outline) if !outline.title => Some(outline), + _ => None, + }) + .unwrap(); + for mut paragraph in paragraphs { + paragraph.id = new_id().unwrap(); + paragraph.parent = None; + paragraph.level = 1; + paragraph.lists.clear(); + paragraph.tags.clear(); + paragraph.style = None; + paragraph.collapsed = false; + if let ParagraphContent::Text(text) = &mut paragraph.content { + text.id = new_id().unwrap(); + text.tags.clear(); + // Paragraph-level spacing came from the source page's paragraph style. + let mut at = 0; + text.text = + onestore::page::Paragraph::from_runs(text.text.spans().iter().map(|span| { + let piece = text.text.text()[at..span.end].to_owned(); + at = span.end; + let mut format = span.format.clone(); + format.space_before = None; + format.space_after = None; + format.line_spacing = None; + (piece, format) + })); + } + outline.paragraphs.push(paragraph); + } + let written = onestore::PreparedEdit::page(&source, space, &after, "Math author").unwrap(); + let store = Store::parse(written.as_bytes()).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let stored = Page::from_space(&document, space).unwrap(); + let mut expected = after.clone(); + expected.title = stored.title.clone(); + let body = |page: &Page| { + page.objects + .iter() + .find_map(|object| match object { + PageObject::Outline(outline) if !outline.title => Some(outline.paragraphs.clone()), + _ => None, + }) + .unwrap() + }; + for (index, (a, b)) in body(&stored).iter().zip(body(&expected)).enumerate() { + if a != &b { + let (ta, tb) = (a.text().unwrap(), b.text().unwrap()); + assert_eq!(ta.text.text(), tb.text.text(), "paragraph {index} text"); + for (sa, sb) in ta.text.spans().iter().zip(tb.text.spans()) { + assert_eq!(sa, sb, "paragraph {index} span"); + } + assert_eq!( + ta.text.spans().len(), + tb.text.spans().len(), + "paragraph {index} spans" + ); + assert_eq!(a, &b, "paragraph {index}"); + } + } + assert_eq!(stored, expected); + assert_eq!( + rendered_equations(written.as_bytes(), &stored.title), + expected_mathml + ); + if let Some(directory) = std::env::var_os("ONESTORE_MATH_EXPORT") { + let directory = std::path::PathBuf::from(directory); + std::fs::create_dir(&directory).unwrap(); + std::fs::write(directory.join("math.one"), written.as_bytes()).unwrap(); + std::fs::write( + directory.join("Open Notebook.onetoc2"), + onestore::create_table_of_contents( + "Open Notebook.onetoc2", + &[("math.one", store.header.file_id)], + ) + .unwrap(), + ) + .unwrap(); + std::fs::write( + directory.join("expected-mathml.json"), + serde_json::to_string_pretty(&expected_mathml).unwrap(), + ) + .unwrap(); + } } diff --git a/tools/native_format.py b/tools/native_format.py index 09d090f1ff7a49946197c829c642e0625c71473e..57e19bd1e9f27aa757a76ccf2e8a3eccbbbd1b27 100644 --- a/tools/native_format.py +++ b/tools/native_format.py @@ -97,7 +97,8 @@ def compare_formats(space, nodes, native): for run in kind['runs']: style = {**base, **({k: v for k, v in space['nodes'][run['format']]['format'].items() if v is not None} if run['format'] else {})} text = source[run['start'] * 2:run['end'] * 2].decode('utf-16-le') - if not style.get('hidden'): + # Hidden runs and equation runs (exported as MathML) have no visible native text. + if not style.get('hidden') and not style.get('math'): actual.extend((char, style) for char in text) if actual and actual[-1][0] == '\r': actual.pop() diff --git a/tools/native_math.py b/tools/native_math.py new file mode 100644 index 0000000000000000000000000000000000000000..749c70f1c3b4076e6a41fe9d14fd4769c72424cb --- /dev/null +++ b/tools/native_math.py @@ -0,0 +1,70 @@ +"""Author equations through OneNote 2010's equation editor on a Rust-written page and capture +the COM read, which exports each equation as MathML: the oracle for `onestore::page::Math`. + +Usage: native_math.py NOTEBOOK_DIR OUTPUT_DIR +""" +import base64 +import json +import sys +from pathlib import Path +import xml.etree.ElementTree as ET + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import native_runner as runner + +windows = runner.windows + +# Linear-format input, one equation per paragraph; a space builds each expression up. +EQUATIONS = [ + 'a_1+b_2', + '\\sqrt(x+1)', + 'x_i^2', + '(a+b)', + '\\int_0^1 x dx', + '\\sum_(i=1)^n i', +] + + +def interaction(name, output): + lines = [] + for equation in EQUATIONS: + lines.append('Send("!=")') + lines.append('Sleep(600)') + lines.append('SendText("%s")' % equation.replace('"', '""')) + lines.append('Send(" ")') + lines.append('Sleep(600)') + lines.append('Send("{End}")') + lines.append('Sleep(300)') + lines.append('Send("{Enter}")') + lines.append('Sleep(600)') + page = ET.parse(sorted((output / 'read').glob('page-*.xml'))[0]).getroot().attrib['ID'] + script = '''ComObject("OneNote.Application").NavigateTo("PAGE_ID", "", false) +WinWait("ahk_exe ONENOTE.EXE", , 30) +WinActivate("ahk_exe ONENOTE.EXE") +WinWaitActive("ahk_exe ONENOTE.EXE", , 30) +WinMaximize("ahk_exe ONENOTE.EXE") +Sleep(3000) +CoordMode("Mouse", "Screen") +Click(300, 140) +Sleep(500) +Send("{End}") +Sleep(300) +Send("{Enter}") +Sleep(500) +%s +Sleep(2000) +''' % '\n'.join(lines) + script = script.replace('PAGE_ID', page) + (output / 'equations.ahk').write_text(script) + result = windows.do_exec(script, target=name, timeout_ms=180000, shot_delay_ms=1500) + screenshot = result.pop('png_b64', None) + if screenshot: + (output / 'equations.png').write_bytes(base64.b64decode(screenshot)) + (output / 'equations.json').write_text(json.dumps(result, indent=2)) + if result.get('error') or result.get('exit') != 0: + raise RuntimeError(str(result)) + + +if __name__ == '__main__': + notebook, output = Path(sys.argv[1]), Path(sys.argv[2]) + runner.capture(notebook, output, expected_pages=1, collect_notebook=True, interaction=interaction) diff --git a/tools/test_math_edit.py b/tools/test_math_edit.py new file mode 100644 index 0000000000000000000000000000000000000000..2d6866d8e785b558777dfb321b72a8d4b4a2ffa4 --- /dev/null +++ b/tools/test_math_edit.py @@ -0,0 +1,37 @@ +import json +from pathlib import Path +import re +import runpy +import shutil +from tempfile import TemporaryDirectory +import unittest + +ROOT = Path(__file__).resolve().parent.parent +FIXTURE = ROOT / 'corpus/math-edit' +compare = runpy.run_path(str(ROOT / 'tools/verify-document.py'))['compare'] + + +def exported_mathml(path): + text = path.read_text(encoding='utf-8-sig') + out = [] + for body in re.findall(r']*>(.*?)', text, re.S): + out.append(re.sub(r'&#(\d+);', lambda m: chr(int(m.group(1))), body)) + return out + + +class MathEditTest(unittest.TestCase): + def test_equation_editor_expressions_export_as_mathml(self): + self.assertEqual(len(exported_mathml(FIXTURE / 'native-editor/read/page-000.xml')), 5) + + def test_rust_written_equations_export_the_expected_mathml(self): + with TemporaryDirectory() as temporary: + read = Path(temporary) / 'read' + shutil.copytree(FIXTURE / 'written/cold/read', read) + compare(FIXTURE / 'written/candidate', read) + expected = json.loads((FIXTURE / 'written/candidate/expected-mathml.json').read_text()) + self.assertEqual(exported_mathml(FIXTURE / 'written/cold/read/page-000.xml'), expected) + self.assertTrue((FIXTURE / 'written/cold/read/page-000.png').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/verify-document.py b/tools/verify-document.py index 7b723165a7cda25f6e585f5e54fe5cbf7ab25034..567a87e88b2d5f867562a2b3f78b2a2fb4c9920b 100644 --- a/tools/verify-document.py +++ b/tools/verify-document.py @@ -78,8 +78,10 @@ def verify_pdf_black(path, paragraphs): def visible_text(node, space): kind = node['kind'] data = kind['text'].encode('utf-16-le') + # Hidden runs and equation runs (exported as MathML) have no visible native text. text = ''.join(data[run['start'] * 2:run['end'] * 2].decode('utf-16-le') for run in kind['runs'] - if not run['format'] or not space['nodes'][run['format']]['format']['hidden']) + if not run['format'] or not (space['nodes'][run['format']]['format']['hidden'] + or space['nodes'][run['format']]['format']['math'])) return "" if text == "\u00a0" else project_text(text.removesuffix('\r')) @@ -391,8 +393,9 @@ def compare(notebook, native, versions=None, password_file=None): text_ids = [oid for root in roots for oid, n in walk(space, root) if n['kind']['type'] == 'RichText' and not n['kind']['boilerplate']] for text_id, expected_runs in zip(text_ids, native_runs, strict=True): + # Native XML exports equation runs as MathML, not text (tools/test_math_edit.py compares that). observed_runs = [(char, run['link']) for run in resolved_text[sid][rid][text_id] - if not run['format']['hidden'] for char in run['text']] + if not run['format']['hidden'] and not run['format']['math'] for char in run['text']] if observed_runs and observed_runs[-1][0] == '\r': observed_runs.pop() observed_runs = [(projected, link) for char, link in observed_runs for projected in project_text(char)]