diff --git a/corpus/paragraph-format/README.md b/corpus/paragraph-format/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c7eedf3c43a9fb4a62d850c6e97a79d473f55564 --- /dev/null +++ b/corpus/paragraph-format/README.md @@ -0,0 +1,29 @@ +# Native paragraph formatting controls + +`tools/native/paragraph-formatting.ps1` authored one page, "Paragraph +formatting", with six paragraphs of mixed bold/italic/plain runs: native +defaults, explicit zero values, centred spacing, right-aligned RTL, a named +style (`h1`, Georgia 18 bold with spacing) inherited, and the same style with +paragraph overrides. `before/` holds the authored notebook and its native read +(`before/read`, captured with `--expected-pages 2 --collect-notebook`; +`before-capture/` retains that run's records). OneNote reports the style's +inherited `spaceBefore="8"`/`spaceAfter="4"` as 288/144 in the read, while +explicit paragraph values read back in points. + +`candidate/` is the page writer's output for the model edit in +`crates/onestore/tests/paragraph_formatting.rs`: every paragraph set to centre +alignment, zero space before, 2 pt after and 16 pt line spacing, with runs, +direction and quick styles untouched. `cold/` is its fresh OneNote 2010 read: +each paragraph is centred with `spaceAfter="2.0"` and `spaceBetween="16.0"`, +the zero space before is omitted, `RTL="true"` survives on the right-aligned +case, `quickStyleIndex` and the bold/italic runs match `before/read`, and +`tools/verify-document.py` finds no differences. `tools/test_paragraph_format.py` +checks these without a VM. + +Regenerate the candidate with `ONESTORE_PARAGRAPH_FORMAT_EXPORT` set to a new +absolute directory while running +`paragraph_properties_preserve_native_mixed_runs_and_shared_styles`, then +cold-open it with `tools/native_runner.py OUTPUT COLD --expected-pages 2 +--collect-notebook`. Regenerate `before/` by running the runner on +`corpus/create-notebook` style input with `--author +tools/native/paragraph-formatting.ps1 --expected-pages 2 --collect-notebook`. diff --git a/corpus/paragraph-format/before-capture/machine.json b/corpus/paragraph-format/before-capture/machine.json new file mode 100644 index 0000000000000000000000000000000000000000..07b48c4660662ead1d0344e1e69b5cba0cdba7e7 --- /dev/null +++ b/corpus/paragraph-format/before-capture/machine.json @@ -0,0 +1 @@ +{"name": "m6-b1f8fbb5", "hostname": "ONE-M6-B1F8FBB5"} diff --git a/corpus/paragraph-format/before-capture/run.json b/corpus/paragraph-format/before-capture/run.json new file mode 100644 index 0000000000000000000000000000000000000000..c3b92a543ca6031bd27a5bc9766eb73779656a95 --- /dev/null +++ b/corpus/paragraph-format/before-capture/run.json @@ -0,0 +1,18 @@ +{ + "notebook": "/Users/clo/dev/one/corpus/paragraph-format/before", + "expected_pages": 2, + "author": null, + "author_timeout_seconds": 600, + "inspect": false, + "collect_notebook": true, + "base": { + "file": "win7-office-base.qcow2", + "format": "qcow2", + "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346", + "virtual_size": 68719476736 + }, + "scripts": { + "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331", + "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41" + } +} diff --git a/corpus/paragraph-format/before-capture/source.json b/corpus/paragraph-format/before-capture/source.json new file mode 100644 index 0000000000000000000000000000000000000000..ff53d5ef8f2484eb62e816310fcd4ee28be2b15f --- /dev/null +++ b/corpus/paragraph-format/before-capture/source.json @@ -0,0 +1,14 @@ +[ + { + "path": "Open Notebook.onetoc2", + "bytes": 3780, + "sha256": "bb68c91caf0c082536d5c6ad2cd970a1fe5e68592c528633f97ef8cf406e7915", + "mtime_ns": 1789187746810790205 + }, + { + "path": "synthetic.one", + "bytes": 14196, + "sha256": "780e169db47f5ec5561e7998052763445bca00635174561b76bcb2bb711951b2", + "mtime_ns": 1789187746811006329 + } +] diff --git a/corpus/paragraph-format/before-capture/teardown.json b/corpus/paragraph-format/before-capture/teardown.json new file mode 100644 index 0000000000000000000000000000000000000000..05a47793de40f322e745c4c4183727e3590ecf70 --- /dev/null +++ b/corpus/paragraph-format/before-capture/teardown.json @@ -0,0 +1 @@ +{"absent": true} diff --git a/corpus/paragraph-format/before/Open Notebook.onetoc2 b/corpus/paragraph-format/before/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..7628ea95335c191e02cd7ea2d65ca221f3b6fbf6 Binary files /dev/null and b/corpus/paragraph-format/before/Open Notebook.onetoc2 differ diff --git a/corpus/paragraph-format/before/read/environment.json b/corpus/paragraph-format/before/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..d6e3363bfb7d4607c588c1f5b784323aa6b203f2 --- /dev/null +++ b/corpus/paragraph-format/before/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-B1F8FBB5", + "cold": true, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/paragraph-format/before/read/hierarchy.xml b/corpus/paragraph-format/before/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..d4f0bbb976093360e64c40c63f211fd451a39941 --- /dev/null +++ b/corpus/paragraph-format/before/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/paragraph-format/before/read/page-000.xml b/corpus/paragraph-format/before/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..88db6e9b394996465a4ae23a95e82f87aacc0717 --- /dev/null +++ b/corpus/paragraph-format/before/read/page-000.xml @@ -0,0 +1,10 @@ + +bold italic plain]]>bold italic plain]]>bold italic plain]]>bold italic plain]]>Style inherited: bold italic plain]]>Style overridden: bold italic plain]]> diff --git a/corpus/paragraph-format/before/read/page-001.xml b/corpus/paragraph-format/before/read/page-001.xml new file mode 100644 index 0000000000000000000000000000000000000000..328d4f6fa2ebf439b24f7b46f27ac953de6ae9bb --- /dev/null +++ b/corpus/paragraph-format/before/read/page-001.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/paragraph-format/before/read/payloads.json b/corpus/paragraph-format/before/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..1ce91a413d6fff4d58765187b76756f0e97ae843 --- /dev/null +++ b/corpus/paragraph-format/before/read/payloads.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/corpus/paragraph-format/before/synthetic.one b/corpus/paragraph-format/before/synthetic.one new file mode 100644 index 0000000000000000000000000000000000000000..20f3a15cef7d036acaeeb665961d8e7c90666daa Binary files /dev/null and b/corpus/paragraph-format/before/synthetic.one differ diff --git a/corpus/paragraph-format/candidate/Open Notebook.onetoc2 b/corpus/paragraph-format/candidate/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..d3b4386a0959dba189d929b1c1084c40de097647 Binary files /dev/null and b/corpus/paragraph-format/candidate/Open Notebook.onetoc2 differ diff --git a/corpus/paragraph-format/candidate/synthetic.one b/corpus/paragraph-format/candidate/synthetic.one new file mode 100644 index 0000000000000000000000000000000000000000..39e83362b177fe86bea07c8a07ebdcd965ccf9b4 Binary files /dev/null and b/corpus/paragraph-format/candidate/synthetic.one differ diff --git a/corpus/paragraph-format/cold/commands.jsonl b/corpus/paragraph-format/cold/commands.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0d2c75fe788178a57fff8734dcd1cdb3f817c9df --- /dev/null +++ b/corpus/paragraph-format/cold/commands.jsonl @@ -0,0 +1,3 @@ +{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-6A989DFC -ExpectedPages 2", "exit": 0, "stdout": "Read 1 sections and 2 pages.\r\n", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null} diff --git a/corpus/paragraph-format/cold/machine.json b/corpus/paragraph-format/cold/machine.json new file mode 100644 index 0000000000000000000000000000000000000000..cd43464ea90acb7909b6205b068cffd367395804 --- /dev/null +++ b/corpus/paragraph-format/cold/machine.json @@ -0,0 +1 @@ +{"name": "m6-6a989dfc", "hostname": "ONE-M6-6A989DFC"} diff --git a/corpus/paragraph-format/cold/notebook/Open Notebook.onetoc2 b/corpus/paragraph-format/cold/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..79bfedd2168b054cafb54f21916d067c8c939f5c Binary files /dev/null and b/corpus/paragraph-format/cold/notebook/Open Notebook.onetoc2 differ diff --git a/corpus/paragraph-format/cold/notebook/synthetic.one b/corpus/paragraph-format/cold/notebook/synthetic.one new file mode 100644 index 0000000000000000000000000000000000000000..21ecaf7c03e97db150ffe3c1cd7a38250b55555f Binary files /dev/null and b/corpus/paragraph-format/cold/notebook/synthetic.one differ diff --git a/corpus/paragraph-format/cold/read/environment.json b/corpus/paragraph-format/cold/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..f6485f257ec0987d21ff2ca0267c3fd6d7b77461 --- /dev/null +++ b/corpus/paragraph-format/cold/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-6A989DFC", + "cold": true, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/paragraph-format/cold/read/hierarchy.xml b/corpus/paragraph-format/cold/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..8b191377773a90af25278e848e4b771283032ae4 --- /dev/null +++ b/corpus/paragraph-format/cold/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/paragraph-format/cold/read/page-000.xml b/corpus/paragraph-format/cold/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..eb00e7aa4439f770a60468d3253691090e71eabb --- /dev/null +++ b/corpus/paragraph-format/cold/read/page-000.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/paragraph-format/cold/read/page-001.xml b/corpus/paragraph-format/cold/read/page-001.xml new file mode 100644 index 0000000000000000000000000000000000000000..d959576a83374a5dea55f7dd4415ba060f78ae08 --- /dev/null +++ b/corpus/paragraph-format/cold/read/page-001.xml @@ -0,0 +1,10 @@ + +bold italic plain]]>bold italic plain]]>bold italic plain]]>bold italic plain]]>Style inherited: bold italic plain]]>Style overridden: bold italic plain]]> diff --git a/corpus/paragraph-format/cold/read/payloads.json b/corpus/paragraph-format/cold/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..1ce91a413d6fff4d58765187b76756f0e97ae843 --- /dev/null +++ b/corpus/paragraph-format/cold/read/payloads.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/corpus/paragraph-format/cold/run.json b/corpus/paragraph-format/cold/run.json new file mode 100644 index 0000000000000000000000000000000000000000..e0b4d8979eec0e16cd8d1be47fd5c1fdc56bdddf --- /dev/null +++ b/corpus/paragraph-format/cold/run.json @@ -0,0 +1,18 @@ +{ + "notebook": "/private/tmp/m4/pf", + "expected_pages": 2, + "author": null, + "author_timeout_seconds": 600, + "inspect": false, + "collect_notebook": true, + "base": { + "file": "win7-office-base.qcow2", + "format": "qcow2", + "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346", + "virtual_size": 68719476736 + }, + "scripts": { + "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331", + "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41" + } +} diff --git a/corpus/paragraph-format/cold/scripts/cold.ps1 b/corpus/paragraph-format/cold/scripts/cold.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..a0b99e7aeb86f4dc10032537800501a52ad65b4f --- /dev/null +++ b/corpus/paragraph-format/cold/scripts/cold.ps1 @@ -0,0 +1,27 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '') +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' } +$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote' +if ($CloneHost) { + if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) { + throw 'The disposable clone hostname does not match this machine.' + } + New-Item "$key\Options\Paths" -Force | Out-Null + New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null +} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or + -not (Test-Path 'C:\one-tests\profile-original-cache')) { + throw 'Park the personal OneNote profile before resetting the test cache.' +} +$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0' +$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root)) +if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' } +New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null +if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked } +if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse } +New-Item "$key\OpenNotebooks" | Out-Null +New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null diff --git a/corpus/paragraph-format/cold/scripts/read.ps1 b/corpus/paragraph-format/cold/scripts/read.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..476094ab1e0c47077988a6461db2080cf208642b --- /dev/null +++ b/corpus/paragraph-format/cold/scripts/read.ps1 @@ -0,0 +1,142 @@ +param( + [Parameter(Mandatory=$true)][string]$Root, + [int]$ExpectedPages = -1, + [switch]$UseCurrentCache, + [switch]$Pdf, + [switch]$KeepOpen, + [string]$CloneHost = '' +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +$notebook = Join-Path $root 'notebook' +$output = Join-Path $root 'read' +if (Test-Path $output) { throw 'Choose a new read destination.' } +if ($UseCurrentCache) { + if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') { + throw 'Park the personal OneNote profile before reading test notebooks.' + } +} else { + & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost +} +New-Item -ItemType Directory -Path $output | Out-Null +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$failure = $null +try { + $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0) + $process = Get-Process ONENOTE + @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion; + powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } | + ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8 + $sections = @() + foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) { + $id = '' + $app.OpenHierarchy($file.FullName, '', [ref]$id, 0) + $sections += $id + } + $deadline = [DateTime]::UtcNow.AddSeconds(300) + $previous = '' + $lastChange = '' + $stableSince = [DateTime]::UtcNow + $settled = $false + do { + $pages = @{} + foreach ($section in $sections) { + $hierarchy = '' + $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) + [xml]$xml = $hierarchy + foreach ($node in $xml.SelectNodes('//*[@path]')) { + if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) { + throw 'OneNote opened a section outside the copied notebook.' + } + } + foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) { + $id = $node.GetAttribute('ID') + $content = '' + $app.GetPageContent($id, [ref]$content, 1, 1) + $pages[$id] = $content + } + } + $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] })) + if ($signature -ne $previous) { + $lastChange = $previous + $previous = $signature + $stableSince = [DateTime]::UtcNow + } + if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or + ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and + ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break } + Start-Sleep -Milliseconds 250 + } while ([DateTime]::UtcNow -lt $deadline) + if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) { + [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8) + $index = 0 + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + $index++ + } + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8) + throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled." + } + $index = 0 + $payloads = @() + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + if ($Pdf) { + $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index) + $app.NavigateTo($id, '', $false) + $app.Publish($id, $pdfPath, 3, '') + if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' } + } + [xml]$page = $pages[$id] + foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) { + $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache')) + $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant() + [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes) + $payloads += @{ page = $id; object = $file.ParentNode.GetAttribute('objectID'); + kind = $file.LocalName; name = $file.GetAttribute('preferredName'); + sha256 = $hash; bytes = $bytes.Length } + } + $index++ + } + [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8) + $all = '' + $app.GetHierarchy($notebookId, 4, [ref]$all, 1) + [xml]$finalTree = $all + $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique) + if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) { + throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.' + } + [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8) + Write-Output "Read $($sections.Count) sections and $($pages.Count) pages." +} catch { + $failure = $_ + [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + throw +} finally { + try { + try { + if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) } + if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) { + $app.CloseNotebook($notebookId, $false) + } + } catch { + if ($null -eq $failure) { throw } + [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + } + } finally { + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + } + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + if (-not $UseCurrentCache -and -not $CloneHost) { + Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10 + } +} diff --git a/corpus/paragraph-format/cold/source.json b/corpus/paragraph-format/cold/source.json new file mode 100644 index 0000000000000000000000000000000000000000..159315657a6ba601e29611184f2c11886c677453 --- /dev/null +++ b/corpus/paragraph-format/cold/source.json @@ -0,0 +1,14 @@ +[ + { + "path": "Open Notebook.onetoc2", + "bytes": 1612, + "sha256": "a2c5dbdebb4884003d98144259b2c376ed380fce1b4d3605ce404eb03c2879d4", + "mtime_ns": 1789198428127888130 + }, + { + "path": "synthetic.one", + "bytes": 16488, + "sha256": "dd352fed4d9e4aa02655f6500b4dc921e0c8557287aa32b907e1608894e4caac", + "mtime_ns": 1789198428127605129 + } +] diff --git a/corpus/paragraph-format/cold/teardown.json b/corpus/paragraph-format/cold/teardown.json new file mode 100644 index 0000000000000000000000000000000000000000..05a47793de40f322e745c4c4183727e3590ecf70 --- /dev/null +++ b/corpus/paragraph-format/cold/teardown.json @@ -0,0 +1 @@ +{"absent": true} diff --git a/crates/notebook/tests/sync_formatting.rs b/crates/notebook/tests/sync_formatting.rs index a6699e5801e372ac798687a5892f6096a5d3dfeb..a45e04dceafbdd0ca61bca9600abee508d190558 100644 --- a/crates/notebook/tests/sync_formatting.rs +++ b/crates/notebook/tests/sync_formatting.rs @@ -497,3 +497,44 @@ fn a_formatted_save_of_an_unchanged_model_queues_nothing() { assert_eq!(cache.save(OUTLINES, space, &page, AUTHOR).unwrap(), None); assert!(cache.pending().unwrap().is_empty()); } + +#[test] +fn paragraph_formatting_survives_reopen_rebase_and_a_lost_publication_reply() { + let (space, page) = page_titled(OUTLINES, PAGE); + let ids = plain(&page); + let attributes: &[fn(&mut Format)] = &[ + |f| f.alignment = Some(2), + |f| f.rtl = Some(true), + |f| f.space_before = Some(12.0), + |f| f.space_after = Some(6.0), + |f| f.line_spacing = Some(18.0), + ]; + for apply in attributes { + let directory = tempfile::tempdir().unwrap(); + let cache = cache(&directory); + let id = save(&cache, ids[0], |page| restyle(page, ids[0], 0..6, apply)) + .unwrap() + .unwrap(); + let expected = formats(&page_of(&cache.snapshot().unwrap(), space), ids[0]); + drop(cache); + let cache = Replica::open(directory.path().join("cache.sqlite")).unwrap(); + let mut server = remote_with(space, |page| replace_text(page, ids[3], 0..0, "Remote ")); + server.fault = Fault::UnknownAfter; + assert!( + matches!(cache.sync_once(&mut server), Err(notebook::Error::Remote(error)) if error.state == onestore::CommitState::Unknown) + ); + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::AwaitingConfirmation { .. }) + )); + drop(cache); + let cache = Replica::open(directory.path().join("cache.sqlite")).unwrap(); + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) + ); + assert_eq!(server.publications, 1); + let page = page_of(&server.durable, space); + assert_eq!(formats(&page, ids[0]), expected); + assert!(text_of(&page, ids[3]).starts_with("Remote ")); + } +} diff --git a/crates/onestore/src/formatting.rs b/crates/onestore/src/formatting.rs index 14009351ee40ed34ae53aa12dfbb55fb71792cb8..b6cb67e5f346d7843cb829368c9fa889756d8678 100644 --- a/crates/onestore/src/formatting.rs +++ b/crates/onestore/src/formatting.rs @@ -244,21 +244,32 @@ pub(crate) fn format_text( (0x14001d7a, &modified), ])?; changed.insert(object, target); - let mut pending = parents.get(&object).cloned().unwrap_or_default(); - let mut ancestors = BTreeSet::new(); - while let Some(id) = pending.pop() { - if !ancestors.insert(id) { - continue; - } - let mut ancestor = PropertyObject::from_object(&raw.objects[&id])?; - ancestor.set(&[(0x14001d7a, &modified)])?; - changed.insert(id, ancestor); - pending.extend(parents.get(&id).into_iter().flatten().copied()); - } + touch_ancestors(raw, &parents, object, &modified, &mut changed)?; Ok(changed) }) } +pub(crate) fn touch_ancestors( + raw: &crate::ResolvedRevision<'_>, + parents: &BTreeMap>, + object: ExGuid, + modified: &[u8; 4], + changed: &mut BTreeMap, +) -> Result<(), Error> { + let mut pending = parents.get(&object).cloned().unwrap_or_default(); + let mut ancestors = BTreeSet::new(); + while let Some(id) = pending.pop() { + if !ancestors.insert(id) { + continue; + } + let mut ancestor = PropertyObject::from_object(&raw.objects[&id])?; + ancestor.set(&[(0x14001d7a, modified)])?; + changed.insert(id, ancestor); + pending.extend(parents.get(&id).into_iter().flatten().copied()); + } + Ok(()) +} + pub(crate) fn attribute_values(attributes: &[TextAttribute]) -> Result)>, Error> { if attributes.is_empty() { return Err(invalid("Choose at least one formatting attribute")); diff --git a/crates/onestore/src/page/write.rs b/crates/onestore/src/page/write.rs index 350cb23d36dbd13cd949cff29e7b49e443326ccb..d4b5399efa9a1bde1bcae3c9e2c6993ac9ca3d66 100644 --- a/crates/onestore/src/page/write.rs +++ b/crates/onestore/src/page/write.rs @@ -206,6 +206,7 @@ impl Lowering<'_> { self.place(&old, &new, &placed, &page_order)?; self.delete(&old, &new, &consumed)?; self.edit_text(&new)?; + self.edit_paragraph_formatting(&new)?; self.edit_formatting(&new)?; self.edit_layout(&old, &new)?; Ok(()) @@ -658,6 +659,120 @@ impl Lowering<'_> { Ok(()) } + fn edit_paragraph_formatting(&mut self, new: &View<'_>) -> Result<(), Error> { + let current = self.current()?; + let current = View::new(¤t)?; + for (id, paragraph) in &new.paragraphs { + let Some(text) = paragraph.text() else { + continue; + }; + let stored = current + .text(self.id(*id)) + .ok_or_else(|| invalid("A paragraph is missing after text edits"))?; + let mut values: Vec<(u32, Vec)> = Vec::new(); + macro_rules! field { + ($field:ident, $value:ident, $encode:block) => { + let $value = format_in(&text.text, 0)?.$field.unwrap_or_default(); + if text + .text + .spans() + .iter() + .all(|span| span.format.$field.unwrap_or_default() == $value) + && stored + .text + .spans() + .iter() + .any(|span| span.format.$field.unwrap_or_default() != $value) + { + values.push($encode); + } + }; + } + field!(alignment, value, { + if value > 2 { + return Err(invalid("Paragraph alignment must be left, center or right")); + } + (0x0c003477, vec![value]) + }); + field!(rtl, value, { + (0x08003476 | (u32::from(value) << 31), Vec::new()) + }); + macro_rules! spacing { + ($field:ident, $property:expr) => { + field!($field, value, { + let stored = value / 36.0; + if !stored.is_finite() || !(0.0..=27777.777).contains(&stored) { + return Err(invalid("Paragraph spacing is outside the document range")); + } + ($property, stored.to_le_bytes().to_vec()) + }); + }; + } + spacing!(space_before, 0x1400342e); + spacing!(space_after, 0x1400342f); + spacing!(line_spacing, 0x14003430); + if values.is_empty() { + continue; + } + if text.date_field.is_some() { + return Err(invalid( + "Generated title fields cannot be formatted as ordinary text", + )); + } + let (space, object) = (self.space, stored.id); + self.apply(|image| { + let store = Store::parse(image)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + let parents = crate::edit::editable_parents( + document.active(space)?, + &document.pages_in(space)?, + object, + )?; + let modified = crate::create::current_timestamps()?.0.to_le_bytes(); + crate::write::write_revision(image, space, |raw| { + let mut target = PropertyObject::from_object(&raw.objects[&object])?; + target.set( + &values + .iter() + .map(|(id, bytes)| (*id, bytes.as_slice())) + .collect::>(), + )?; + if let Some((_, alignment)) = values.iter().find(|(id, _)| *id == 0x0c003477) { + for property in [0x14001c3e, 0x14001c84] { + let fields = PropertySets::parse(&target.bytes)?; + let previous = fields.sets[0] + .iter() + .find(|field| field.id == property) + .map(|field| match field.value { + Value::Bytes(bytes) => bytes + .try_into() + .map(u32::from_le_bytes) + .map_err(|_| invalid("Invalid paragraph layout alignment")), + _ => Err(invalid("Invalid paragraph layout alignment")), + }) + .transpose()? + .unwrap_or(0); + let value = (previous & !7) | (u32::from(alignment[0]) + 1); + target.set(&[(property, &value.to_le_bytes())])?; + } + } + target.set(&[(0x14001d7a, &modified)])?; + let mut changed = BTreeMap::from([(object, target)]); + crate::formatting::touch_ancestors( + raw, + &parents, + object, + &modified, + &mut changed, + )?; + Ok(changed) + }) + })?; + } + Ok(()) + } + fn edit_formatting(&mut self, new: &View<'_>) -> Result<(), Error> { let current = self.current()?; let current = View::new(¤t)?; diff --git a/crates/onestore/tests/paragraph_formatting.rs b/crates/onestore/tests/paragraph_formatting.rs new file mode 100644 index 0000000000000000000000000000000000000000..2ef24da7969f4fa40082a3f0734871722c379972 --- /dev/null +++ b/crates/onestore/tests/paragraph_formatting.rs @@ -0,0 +1,125 @@ +use onestore::{ + ExGuid, PreparedEdit, RevisionIndex, Store, + document::{Document, Kind}, + page::{Page, PageObject, Paragraph}, +}; + +const SOURCE: &[u8] = include_bytes!("../../../corpus/paragraph-format/before/synthetic.one"); + +fn page(bytes: &[u8]) -> (ExGuid, Page) { + let store = Store::parse(bytes).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + let document = Document::parse(&index).unwrap(); + document + .pages() + .unwrap() + .into_iter() + .find_map(|(space, _)| { + let page = Page::from_space(&document, space).unwrap(); + (page.title == "Paragraph formatting").then_some((space, page)) + }) + .unwrap() +} + +#[test] +fn paragraph_properties_preserve_native_mixed_runs_and_shared_styles() { + let (space, before) = page(SOURCE); + let mut after = before.clone(); + let mut count = 0; + for object in &mut after.objects { + if let PageObject::Outline(outline) = object { + for paragraph in &mut outline.paragraphs { + let text = paragraph.text_mut().unwrap(); + let mut start = 0; + text.text = Paragraph::from_runs(text.text.spans().iter().map(|span| { + let value = text.text.text()[start..span.end].to_owned(); + start = span.end; + let mut format = span.format.clone(); + format.alignment = Some(1); + format.space_before = Some(0.0); + format.space_after = Some(2.0); + format.line_spacing = Some(16.0); + (value, format) + })); + count += 1; + } + } + } + assert_eq!(count, 6); + let prepared = PreparedEdit::page(SOURCE, space, &after, "Format author").unwrap(); + assert_eq!(page(prepared.as_bytes()).1, after); + let before_store = Store::parse(SOURCE).unwrap(); + let before_index = RevisionIndex::parse(&before_store).unwrap(); + let before_doc = Document::parse(&before_index).unwrap(); + let after_store = Store::parse(prepared.as_bytes()).unwrap(); + let after_index = RevisionIndex::parse(&after_store).unwrap(); + let old = before_index.resolve_active(space).unwrap(); + let new = after_index.resolve_active(space).unwrap(); + for (id, node) in &before_doc.active(space).unwrap().nodes { + if matches!(node.kind, Kind::Style { .. }) { + assert_eq!(old.objects[id].data, new.objects[id].data); + } + } + if let Some(directory) = std::env::var_os("ONESTORE_PARAGRAPH_FORMAT_EXPORT") { + let directory = std::path::PathBuf::from(directory); + std::fs::create_dir(&directory).unwrap(); + std::fs::write(directory.join("synthetic.one"), prepared.as_bytes()).unwrap(); + std::fs::write( + directory.join("Open Notebook.onetoc2"), + onestore::create_table_of_contents( + "Open Notebook.onetoc2", + &[("synthetic.one", after_store.header.file_id)], + ) + .unwrap(), + ) + .unwrap(); + } +} + +#[test] +fn paragraph_formatting_rejects_invalid_values_and_conflicting_run_settings() { + let (space, before) = page(SOURCE); + let invalid: &[fn(&mut onestore::document::Format)] = &[ + |format| format.alignment = Some(3), + |format| format.space_before = Some(-1.0), + |format| format.space_after = Some(f32::NAN), + |format| format.line_spacing = Some(f32::INFINITY), + |format| format.space_after = Some(1_000_100.0), + ]; + for change in invalid { + let mut after = before.clone(); + let text = after + .objects + .iter_mut() + .find_map(|object| match object { + PageObject::Outline(outline) => { + outline.paragraphs.iter_mut().find_map(|p| p.text_mut()) + } + _ => None, + }) + .unwrap(); + let mut format = text.text.format_at(0).unwrap().clone(); + change(&mut format); + text.text = Paragraph::new(text.text.text().to_owned(), format); + assert!(PreparedEdit::page(SOURCE, space, &after, "Author").is_err()); + } + let mut after = before.clone(); + let text = after + .objects + .iter_mut() + .find_map(|object| match object { + PageObject::Outline(outline) => { + outline.paragraphs.iter_mut().find_map(|p| p.text_mut()) + } + _ => None, + }) + .unwrap(); + let mut left = text.text.format_at(0).unwrap().clone(); + let mut right = left.clone(); + left.alignment = Some(1); + right.alignment = Some(2); + text.text = Paragraph::from_runs([("Left".into(), left), ("Right".into(), right)]); + assert!(PreparedEdit::page(SOURCE, space, &after, "Author").is_err()); +} diff --git a/tools/test_paragraph_format.py b/tools/test_paragraph_format.py new file mode 100644 index 0000000000000000000000000000000000000000..e67aac2c995baf7378f3c893dd0f6b6a8623e2ec --- /dev/null +++ b/tools/test_paragraph_format.py @@ -0,0 +1,57 @@ +from pathlib import Path +import runpy +import shutil +from tempfile import TemporaryDirectory +import unittest +import xml.etree.ElementTree as ET + +from native_format import native_characters +from native_xml import ns + +ROOT = Path(__file__).resolve().parent.parent +FIXTURE = ROOT / 'corpus/paragraph-format' +compare = runpy.run_path(str(ROOT / 'tools/verify-document.py'))['compare'] +IGNORED = {'objectID', 'lastModifiedTime', 'creationTime'} + + +def paragraphs(read): + for path in sorted(read.glob('page-*.xml')): + root = ET.parse(path).getroot() + if root.get('name') == 'Paragraph formatting': + outline, = root.findall('one:Outline', ns) + return [({k: v for k, v in oe.attrib.items() if k not in IGNORED}, oe) + for oe in outline.findall('one:OEChildren/one:OE', ns)], root + raise AssertionError('The paragraph formatting page is missing') + + +class ParagraphFormatTest(unittest.TestCase): + def test_native_controls_and_rust_paragraph_properties_reopen_natively(self): + with TemporaryDirectory() as temporary: + for notebook, read in [(FIXTURE / 'before', FIXTURE / 'before/read'), + (FIXTURE / 'candidate', FIXTURE / 'cold/read')]: + folder = Path(temporary) / notebook.name + shutil.copytree(read, folder / 'read') + compare(notebook, folder / 'read') + before, before_page = paragraphs(FIXTURE / 'before/read') + after, after_page = paragraphs(FIXTURE / 'cold/read') + self.assertEqual(len(before), 6) + self.assertEqual([attributes for attributes, _ in before], [ + {'alignment': 'left', 'quickStyleIndex': '0'}, + {'alignment': 'left', 'quickStyleIndex': '0'}, + {'alignment': 'center', 'spaceBefore': '12.0', 'spaceAfter': '6.0', 'spaceBetween': '20.0', 'quickStyleIndex': '0'}, + {'RTL': 'true', 'alignment': 'right', 'spaceBefore': '3.0', 'spaceAfter': '9.0', 'spaceBetween': '18.0', 'quickStyleIndex': '0'}, + {'alignment': 'left', 'spaceBefore': '288.0', 'spaceAfter': '144.0', 'quickStyleIndex': '1'}, + {'alignment': 'center', 'spaceBefore': '288.0', 'spaceAfter': '2.0', 'spaceBetween': '16.0', 'quickStyleIndex': '1'}, + ]) + for index, ((original, _), (attributes, _)) in enumerate(zip(before, after, strict=True)): + expected = {'alignment': 'center', 'spaceAfter': '2.0', 'spaceBetween': '16.0', + 'quickStyleIndex': original['quickStyleIndex']} + if original.get('RTL') == 'true': + expected['RTL'] = 'true' + self.assertEqual(attributes, expected, index) + self.assertEqual(native_characters(after_page, after_page.findall('one:Outline', ns)), + native_characters(before_page, before_page.findall('one:Outline', ns))) + + +if __name__ == '__main__': + unittest.main()