diff --git a/corpus/table-edit/README.md b/corpus/table-edit/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d96fe5fce55b69f51424cb25b55ee36881fb0316 --- /dev/null +++ b/corpus/table-edit/README.md @@ -0,0 +1,28 @@ +# Rust table authoring + +`created/candidate` is the page writer's output for +`a_new_table_is_created_on_a_fresh_page` in +`crates/onestore/tests/page_tables.rs`: on a section created in Rust, a +paragraph between two text paragraphs becomes a two-by-two table with locked +144 pt and 216 pt columns, visible borders and a text paragraph in every +cell. `created/cold` is a fresh OneNote 2010 read showing both columns with +their widths and locks, both rows and every cell text. + +`edited/candidate` is the writer's output for +`a_row_and_a_column_are_added_to_a_native_table_and_removed_again` on +`corpus/outline-edit/tree/before`: the one-row, two-column native table on +"Delete cell subtree" gains an unlocked 120 pt third column and a second row +of three cells. `edited/cold` is its cold read: the native columns keep their +widths and locks, OneNote sizes the unlocked column to its content, and the +new row and cells read back with their text. + +Native rows and cells are plain containers; the writer creates them with the +modification time, the indent array copied from a sibling cell, and the flags +every native cell carries, rebuilds each table's row and cell order to the +model's, and writes the column count, widths, locks and border flag. New +tables start as an empty paragraph whose content the structure pass replaces. +An emptied cell keeps a replacement paragraph, as the tree writer provides. +`tools/test_table_edit.py` checks both captures without a VM. Regenerate with +`ONESTORE_TABLE_EXPORT` and `ONESTORE_TABLE_EDIT_EXPORT` set to new absolute +directories while running the tests, then cold-open them with +`tools/native_runner.py OUTPUT COLD --expected-pages 1|12 --collect-notebook`. diff --git a/corpus/table-edit/created/candidate/Open Notebook.onetoc2 b/corpus/table-edit/created/candidate/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..784f65349498d29d5f5fe5433dd751c740326029 Binary files /dev/null and b/corpus/table-edit/created/candidate/Open Notebook.onetoc2 differ diff --git a/corpus/table-edit/created/candidate/tables.one b/corpus/table-edit/created/candidate/tables.one new file mode 100644 index 0000000000000000000000000000000000000000..75d44cc0da8f1738e0438647e48a8b09cb844710 Binary files /dev/null and b/corpus/table-edit/created/candidate/tables.one differ diff --git a/corpus/table-edit/created/cold/commands.jsonl b/corpus/table-edit/created/cold/commands.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..613db96ba12c5a91daa79bc91aa59630b63ba5bc --- /dev/null +++ b/corpus/table-edit/created/cold/commands.jsonl @@ -0,0 +1,3 @@ +{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-838F23A9 -ExpectedPages 1", "exit": 0, "stdout": "Read 1 sections and 1 pages.\r\n", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null} diff --git a/corpus/table-edit/created/cold/machine.json b/corpus/table-edit/created/cold/machine.json new file mode 100644 index 0000000000000000000000000000000000000000..0a5fbe5a162b6b7a317dc04cb9656215268b42fa --- /dev/null +++ b/corpus/table-edit/created/cold/machine.json @@ -0,0 +1 @@ +{"name": "m6-838f23a9", "hostname": "ONE-M6-838F23A9"} diff --git a/corpus/table-edit/created/cold/notebook/Open Notebook.onetoc2 b/corpus/table-edit/created/cold/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..72c04c527a22a5243a61e843e00244604e967e55 Binary files /dev/null and b/corpus/table-edit/created/cold/notebook/Open Notebook.onetoc2 differ diff --git a/corpus/table-edit/created/cold/notebook/tables.one b/corpus/table-edit/created/cold/notebook/tables.one new file mode 100644 index 0000000000000000000000000000000000000000..7a1d5ea5e66d9b0e24563be6a4d23df29a2a3069 Binary files /dev/null and b/corpus/table-edit/created/cold/notebook/tables.one differ diff --git a/corpus/table-edit/created/cold/read/environment.json b/corpus/table-edit/created/cold/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..9c1eae28f4b03184eee090aa1d307e526b682b83 --- /dev/null +++ b/corpus/table-edit/created/cold/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-838F23A9", + "cold": true, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/table-edit/created/cold/read/hierarchy.xml b/corpus/table-edit/created/cold/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..89b9442957cc33dc9cd148141f94c3deeff7c163 --- /dev/null +++ b/corpus/table-edit/created/cold/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/table-edit/created/cold/read/page-000.xml b/corpus/table-edit/created/cold/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..18e6e85052b8d934f3546717ebb4a168ee13528d --- /dev/null +++ b/corpus/table-edit/created/cold/read/page-000.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/table-edit/created/cold/read/payloads.json b/corpus/table-edit/created/cold/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..1ce91a413d6fff4d58765187b76756f0e97ae843 --- /dev/null +++ b/corpus/table-edit/created/cold/read/payloads.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/corpus/table-edit/created/cold/run.json b/corpus/table-edit/created/cold/run.json new file mode 100644 index 0000000000000000000000000000000000000000..e5de875f30992feac8b4454db76cd812a676a4de --- /dev/null +++ b/corpus/table-edit/created/cold/run.json @@ -0,0 +1,18 @@ +{ + "notebook": "/private/tmp/m4/tables", + "expected_pages": 1, + "author": null, + "author_timeout_seconds": 600, + "inspect": false, + "collect_notebook": true, + "base": { + "file": "win7-office-base.qcow2", + "format": "qcow2", + "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346", + "virtual_size": 68719476736 + }, + "scripts": { + "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331", + "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41" + } +} diff --git a/corpus/table-edit/created/cold/scripts/cold.ps1 b/corpus/table-edit/created/cold/scripts/cold.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..a0b99e7aeb86f4dc10032537800501a52ad65b4f --- /dev/null +++ b/corpus/table-edit/created/cold/scripts/cold.ps1 @@ -0,0 +1,27 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '') +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' } +$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote' +if ($CloneHost) { + if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) { + throw 'The disposable clone hostname does not match this machine.' + } + New-Item "$key\Options\Paths" -Force | Out-Null + New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null +} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or + -not (Test-Path 'C:\one-tests\profile-original-cache')) { + throw 'Park the personal OneNote profile before resetting the test cache.' +} +$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0' +$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root)) +if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' } +New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null +if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked } +if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse } +New-Item "$key\OpenNotebooks" | Out-Null +New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null diff --git a/corpus/table-edit/created/cold/scripts/read.ps1 b/corpus/table-edit/created/cold/scripts/read.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..476094ab1e0c47077988a6461db2080cf208642b --- /dev/null +++ b/corpus/table-edit/created/cold/scripts/read.ps1 @@ -0,0 +1,142 @@ +param( + [Parameter(Mandatory=$true)][string]$Root, + [int]$ExpectedPages = -1, + [switch]$UseCurrentCache, + [switch]$Pdf, + [switch]$KeepOpen, + [string]$CloneHost = '' +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +$notebook = Join-Path $root 'notebook' +$output = Join-Path $root 'read' +if (Test-Path $output) { throw 'Choose a new read destination.' } +if ($UseCurrentCache) { + if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') { + throw 'Park the personal OneNote profile before reading test notebooks.' + } +} else { + & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost +} +New-Item -ItemType Directory -Path $output | Out-Null +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$failure = $null +try { + $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0) + $process = Get-Process ONENOTE + @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion; + powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } | + ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8 + $sections = @() + foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) { + $id = '' + $app.OpenHierarchy($file.FullName, '', [ref]$id, 0) + $sections += $id + } + $deadline = [DateTime]::UtcNow.AddSeconds(300) + $previous = '' + $lastChange = '' + $stableSince = [DateTime]::UtcNow + $settled = $false + do { + $pages = @{} + foreach ($section in $sections) { + $hierarchy = '' + $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) + [xml]$xml = $hierarchy + foreach ($node in $xml.SelectNodes('//*[@path]')) { + if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) { + throw 'OneNote opened a section outside the copied notebook.' + } + } + foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) { + $id = $node.GetAttribute('ID') + $content = '' + $app.GetPageContent($id, [ref]$content, 1, 1) + $pages[$id] = $content + } + } + $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] })) + if ($signature -ne $previous) { + $lastChange = $previous + $previous = $signature + $stableSince = [DateTime]::UtcNow + } + if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or + ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and + ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break } + Start-Sleep -Milliseconds 250 + } while ([DateTime]::UtcNow -lt $deadline) + if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) { + [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8) + $index = 0 + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + $index++ + } + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8) + throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled." + } + $index = 0 + $payloads = @() + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + if ($Pdf) { + $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index) + $app.NavigateTo($id, '', $false) + $app.Publish($id, $pdfPath, 3, '') + if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' } + } + [xml]$page = $pages[$id] + foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) { + $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache')) + $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant() + [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes) + $payloads += @{ page = $id; object = $file.ParentNode.GetAttribute('objectID'); + kind = $file.LocalName; name = $file.GetAttribute('preferredName'); + sha256 = $hash; bytes = $bytes.Length } + } + $index++ + } + [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8) + $all = '' + $app.GetHierarchy($notebookId, 4, [ref]$all, 1) + [xml]$finalTree = $all + $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique) + if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) { + throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.' + } + [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8) + Write-Output "Read $($sections.Count) sections and $($pages.Count) pages." +} catch { + $failure = $_ + [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + throw +} finally { + try { + try { + if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) } + if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) { + $app.CloseNotebook($notebookId, $false) + } + } catch { + if ($null -eq $failure) { throw } + [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + } + } finally { + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + } + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + if (-not $UseCurrentCache -and -not $CloneHost) { + Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10 + } +} diff --git a/corpus/table-edit/created/cold/source.json b/corpus/table-edit/created/cold/source.json new file mode 100644 index 0000000000000000000000000000000000000000..a2b298152ccd8c0d8d5c33b61065d236fe51e883 --- /dev/null +++ b/corpus/table-edit/created/cold/source.json @@ -0,0 +1,14 @@ +[ + { + "path": "Open Notebook.onetoc2", + "bytes": 1604, + "sha256": "b39b2e142e0adce0de37ad336f40c3e9c2ba92ce6927c9d109139e74c1fb4a6e", + "mtime_ns": 1789239342755220465 + }, + { + "path": "tables.one", + "bytes": 11016, + "sha256": "e1c014220e05352a01c98752031b32c17512407295aa812ff8e9c776e48eef6e", + "mtime_ns": 1789239342754903465 + } +] diff --git a/corpus/table-edit/created/cold/teardown.json b/corpus/table-edit/created/cold/teardown.json new file mode 100644 index 0000000000000000000000000000000000000000..05a47793de40f322e745c4c4183727e3590ecf70 --- /dev/null +++ b/corpus/table-edit/created/cold/teardown.json @@ -0,0 +1 @@ +{"absent": true} diff --git a/corpus/table-edit/edited/candidate/Open Notebook.onetoc2 b/corpus/table-edit/edited/candidate/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..7baa1ed5468231cbec163f9cbdee4219dac644d7 Binary files /dev/null and b/corpus/table-edit/edited/candidate/Open Notebook.onetoc2 differ diff --git a/corpus/table-edit/edited/candidate/synthetic.one b/corpus/table-edit/edited/candidate/synthetic.one new file mode 100644 index 0000000000000000000000000000000000000000..be94b7b23efb111c839fb6184ead00a3fd31f2f7 Binary files /dev/null and b/corpus/table-edit/edited/candidate/synthetic.one differ diff --git a/corpus/table-edit/edited/cold/commands.jsonl b/corpus/table-edit/edited/cold/commands.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c96d0f091be95e979b634be55d13b6018adfdf29 --- /dev/null +++ b/corpus/table-edit/edited/cold/commands.jsonl @@ -0,0 +1,3 @@ +{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-C8AB6EC4 -ExpectedPages 12", "exit": 0, "stdout": "Read 1 sections and 12 pages.\r\n", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null} diff --git a/corpus/table-edit/edited/cold/machine.json b/corpus/table-edit/edited/cold/machine.json new file mode 100644 index 0000000000000000000000000000000000000000..366a1bac6efdfeffcb5c436ce8337ca91f48f7a4 --- /dev/null +++ b/corpus/table-edit/edited/cold/machine.json @@ -0,0 +1 @@ +{"name": "m6-c8ab6ec4", "hostname": "ONE-M6-C8AB6EC4"} diff --git a/corpus/table-edit/edited/cold/notebook/Open Notebook.onetoc2 b/corpus/table-edit/edited/cold/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..5b242159bcdd17896cdd58cc0860ae14ab0a2a32 Binary files /dev/null and b/corpus/table-edit/edited/cold/notebook/Open Notebook.onetoc2 differ diff --git a/corpus/table-edit/edited/cold/notebook/synthetic.one b/corpus/table-edit/edited/cold/notebook/synthetic.one new file mode 100644 index 0000000000000000000000000000000000000000..abc6b7d2afe3336b3dfc818271a373bf10afc416 Binary files /dev/null and b/corpus/table-edit/edited/cold/notebook/synthetic.one differ diff --git a/corpus/table-edit/edited/cold/read/environment.json b/corpus/table-edit/edited/cold/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..55f0d8d4eaf69da26d29cd1a5ffdb5832cd4431d --- /dev/null +++ b/corpus/table-edit/edited/cold/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-C8AB6EC4", + "cold": true, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/table-edit/edited/cold/read/hierarchy.xml b/corpus/table-edit/edited/cold/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..14baf191414f72df567a651ea2366eabdb127935 --- /dev/null +++ b/corpus/table-edit/edited/cold/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/table-edit/edited/cold/read/page-000.xml b/corpus/table-edit/edited/cold/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..7e01ef5eccb26f3b93e8aa93527be85d602efd1a --- /dev/null +++ b/corpus/table-edit/edited/cold/read/page-000.xml @@ -0,0 +1,5 @@ + +Target 🦀 é Link label]]>Child]]> diff --git a/corpus/table-edit/edited/cold/read/page-001.xml b/corpus/table-edit/edited/cold/read/page-001.xml new file mode 100644 index 0000000000000000000000000000000000000000..ccd8cbe4adb0497888439ed7329e3d0591d28d13 --- /dev/null +++ b/corpus/table-edit/edited/cold/read/page-001.xml @@ -0,0 +1,5 @@ + +Target 🦀 é Link label]]>Child]]> diff --git a/corpus/table-edit/edited/cold/read/page-002.xml b/corpus/table-edit/edited/cold/read/page-002.xml new file mode 100644 index 0000000000000000000000000000000000000000..a194a97ac55a36f0fd1b2fbea947eac751bb6732 --- /dev/null +++ b/corpus/table-edit/edited/cold/read/page-002.xml @@ -0,0 +1,5 @@ + +Target 🦀 é Link label]]>Child]]> diff --git a/corpus/table-edit/edited/cold/read/page-003.xml b/corpus/table-edit/edited/cold/read/page-003.xml new file mode 100644 index 0000000000000000000000000000000000000000..3b054566709aa4f70f762969e387c38a9dfe81f6 --- /dev/null +++ b/corpus/table-edit/edited/cold/read/page-003.xml @@ -0,0 +1,5 @@ + +Target 🦀 é Link label]]>Child]]> diff --git a/corpus/table-edit/edited/cold/read/page-004.xml b/corpus/table-edit/edited/cold/read/page-004.xml new file mode 100644 index 0000000000000000000000000000000000000000..c63730b827fb33abf24bcd20c120100faa5fc2e4 --- /dev/null +++ b/corpus/table-edit/edited/cold/read/page-004.xml @@ -0,0 +1,5 @@ + +Target 🦀 é Link label]]>Child]]> diff --git a/corpus/table-edit/edited/cold/read/page-005.xml b/corpus/table-edit/edited/cold/read/page-005.xml new file mode 100644 index 0000000000000000000000000000000000000000..8a87fcf7d3d1bad53e7b9c077ede4c5b74bda386 --- /dev/null +++ b/corpus/table-edit/edited/cold/read/page-005.xml @@ -0,0 +1,20 @@ + +Bold 🦀 italic é color 東京
+End
]]>
Fictitious: café, 東京, مرحبا]]>
diff --git a/corpus/table-edit/edited/cold/read/page-006.xml b/corpus/table-edit/edited/cold/read/page-006.xml new file mode 100644 index 0000000000000000000000000000000000000000..674f336f3956bf269378bf7ef9bdb89c8709c804 --- /dev/null +++ b/corpus/table-edit/edited/cold/read/page-006.xml @@ -0,0 +1,4 @@ + +Target 🦀 é Link label]]> diff --git a/corpus/table-edit/edited/cold/read/page-007.xml b/corpus/table-edit/edited/cold/read/page-007.xml new file mode 100644 index 0000000000000000000000000000000000000000..51370350597d3babe05014eab64f81eb727bb9c4 --- /dev/null +++ b/corpus/table-edit/edited/cold/read/page-007.xml @@ -0,0 +1,5 @@ + +Target 🦀 é Link label]]>Child]]> diff --git a/corpus/table-edit/edited/cold/read/page-008.xml b/corpus/table-edit/edited/cold/read/page-008.xml new file mode 100644 index 0000000000000000000000000000000000000000..ea632e15e318bae07fe9b8d0dc3917e67b0cf274 --- /dev/null +++ b/corpus/table-edit/edited/cold/read/page-008.xml @@ -0,0 +1,4 @@ + +Target 🦀 é Link label]]> diff --git a/corpus/table-edit/edited/cold/read/page-009.xml b/corpus/table-edit/edited/cold/read/page-009.xml new file mode 100644 index 0000000000000000000000000000000000000000..33525ee6752bd754e338a24907f11d49a23c2f45 --- /dev/null +++ b/corpus/table-edit/edited/cold/read/page-009.xml @@ -0,0 +1,5 @@ + +Target 🦀 é Link label]]>Child]]> diff --git a/corpus/table-edit/edited/cold/read/page-010.xml b/corpus/table-edit/edited/cold/read/page-010.xml new file mode 100644 index 0000000000000000000000000000000000000000..460ca52aa917dcf42070a98cc36eadc974d9cb8d --- /dev/null +++ b/corpus/table-edit/edited/cold/read/page-010.xml @@ -0,0 +1,5 @@ + +Target 🦀 é Link label]]>Child]]> diff --git a/corpus/table-edit/edited/cold/read/page-011.xml b/corpus/table-edit/edited/cold/read/page-011.xml new file mode 100644 index 0000000000000000000000000000000000000000..95dbb17b8e9fac2d4fbfe20cefd2ac2bc9c3c7d6 --- /dev/null +++ b/corpus/table-edit/edited/cold/read/page-011.xml @@ -0,0 +1,5 @@ + +Target 🦀 é Link label]]>Child]]> diff --git a/corpus/table-edit/edited/cold/read/payloads.json b/corpus/table-edit/edited/cold/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..1ce91a413d6fff4d58765187b76756f0e97ae843 --- /dev/null +++ b/corpus/table-edit/edited/cold/read/payloads.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/corpus/table-edit/edited/cold/run.json b/corpus/table-edit/edited/cold/run.json new file mode 100644 index 0000000000000000000000000000000000000000..25b225e714721fa0f1e8ca0f82c9dcf4f988b9ad --- /dev/null +++ b/corpus/table-edit/edited/cold/run.json @@ -0,0 +1,18 @@ +{ + "notebook": "/private/tmp/m4/table-edit", + "expected_pages": 12, + "author": null, + "author_timeout_seconds": 600, + "inspect": false, + "collect_notebook": true, + "base": { + "file": "win7-office-base.qcow2", + "format": "qcow2", + "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346", + "virtual_size": 68719476736 + }, + "scripts": { + "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331", + "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41" + } +} diff --git a/corpus/table-edit/edited/cold/scripts/cold.ps1 b/corpus/table-edit/edited/cold/scripts/cold.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..a0b99e7aeb86f4dc10032537800501a52ad65b4f --- /dev/null +++ b/corpus/table-edit/edited/cold/scripts/cold.ps1 @@ -0,0 +1,27 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '') +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' } +$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote' +if ($CloneHost) { + if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) { + throw 'The disposable clone hostname does not match this machine.' + } + New-Item "$key\Options\Paths" -Force | Out-Null + New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null +} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or + -not (Test-Path 'C:\one-tests\profile-original-cache')) { + throw 'Park the personal OneNote profile before resetting the test cache.' +} +$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0' +$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root)) +if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' } +New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null +if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked } +if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse } +New-Item "$key\OpenNotebooks" | Out-Null +New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null diff --git a/corpus/table-edit/edited/cold/scripts/read.ps1 b/corpus/table-edit/edited/cold/scripts/read.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..476094ab1e0c47077988a6461db2080cf208642b --- /dev/null +++ b/corpus/table-edit/edited/cold/scripts/read.ps1 @@ -0,0 +1,142 @@ +param( + [Parameter(Mandatory=$true)][string]$Root, + [int]$ExpectedPages = -1, + [switch]$UseCurrentCache, + [switch]$Pdf, + [switch]$KeepOpen, + [string]$CloneHost = '' +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +$notebook = Join-Path $root 'notebook' +$output = Join-Path $root 'read' +if (Test-Path $output) { throw 'Choose a new read destination.' } +if ($UseCurrentCache) { + if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') { + throw 'Park the personal OneNote profile before reading test notebooks.' + } +} else { + & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost +} +New-Item -ItemType Directory -Path $output | Out-Null +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$failure = $null +try { + $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0) + $process = Get-Process ONENOTE + @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion; + powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } | + ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8 + $sections = @() + foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) { + $id = '' + $app.OpenHierarchy($file.FullName, '', [ref]$id, 0) + $sections += $id + } + $deadline = [DateTime]::UtcNow.AddSeconds(300) + $previous = '' + $lastChange = '' + $stableSince = [DateTime]::UtcNow + $settled = $false + do { + $pages = @{} + foreach ($section in $sections) { + $hierarchy = '' + $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) + [xml]$xml = $hierarchy + foreach ($node in $xml.SelectNodes('//*[@path]')) { + if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) { + throw 'OneNote opened a section outside the copied notebook.' + } + } + foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) { + $id = $node.GetAttribute('ID') + $content = '' + $app.GetPageContent($id, [ref]$content, 1, 1) + $pages[$id] = $content + } + } + $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] })) + if ($signature -ne $previous) { + $lastChange = $previous + $previous = $signature + $stableSince = [DateTime]::UtcNow + } + if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or + ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and + ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break } + Start-Sleep -Milliseconds 250 + } while ([DateTime]::UtcNow -lt $deadline) + if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) { + [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8) + $index = 0 + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + $index++ + } + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8) + throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled." + } + $index = 0 + $payloads = @() + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + if ($Pdf) { + $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index) + $app.NavigateTo($id, '', $false) + $app.Publish($id, $pdfPath, 3, '') + if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' } + } + [xml]$page = $pages[$id] + foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) { + $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache')) + $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant() + [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes) + $payloads += @{ page = $id; object = $file.ParentNode.GetAttribute('objectID'); + kind = $file.LocalName; name = $file.GetAttribute('preferredName'); + sha256 = $hash; bytes = $bytes.Length } + } + $index++ + } + [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8) + $all = '' + $app.GetHierarchy($notebookId, 4, [ref]$all, 1) + [xml]$finalTree = $all + $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique) + if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) { + throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.' + } + [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8) + Write-Output "Read $($sections.Count) sections and $($pages.Count) pages." +} catch { + $failure = $_ + [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + throw +} finally { + try { + try { + if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) } + if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) { + $app.CloseNotebook($notebookId, $false) + } + } catch { + if ($null -eq $failure) { throw } + [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + } + } finally { + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + } + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + if (-not $UseCurrentCache -and -not $CloneHost) { + Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10 + } +} diff --git a/corpus/table-edit/edited/cold/source.json b/corpus/table-edit/edited/cold/source.json new file mode 100644 index 0000000000000000000000000000000000000000..b8b914af1aabb8a82f77a0a2f06c0af0a523f348 --- /dev/null +++ b/corpus/table-edit/edited/cold/source.json @@ -0,0 +1,14 @@ +[ + { + "path": "Open Notebook.onetoc2", + "bytes": 1612, + "sha256": "f49128a77aa74d62e6a1b1281abf6fd76f0789bf90cf47d812162404995c4276", + "mtime_ns": 1789239343363554023 + }, + { + "path": "synthetic.one", + "bytes": 92288, + "sha256": "1d0632852ecbf66bff16e9b36e2dfa3df059bdfd15807f4943bfeabda0e920ce", + "mtime_ns": 1789239343362662188 + } +] diff --git a/corpus/table-edit/edited/cold/teardown.json b/corpus/table-edit/edited/cold/teardown.json new file mode 100644 index 0000000000000000000000000000000000000000..05a47793de40f322e745c4c4183727e3590ecf70 --- /dev/null +++ b/corpus/table-edit/edited/cold/teardown.json @@ -0,0 +1 @@ +{"absent": true} diff --git a/crates/notebook/README.md b/crates/notebook/README.md index 1d1ae9d4f51e9c682863df7666f5493b7cd69975..3d4db5159b4d900edd52c2e189a7583132f6cb7f 100644 --- a/crates/notebook/README.md +++ b/crates/notebook/README.md @@ -14,7 +14,7 @@ diffs the supplied model against the page stored in the supplied local snapshot, writes the difference into the working image and queues one `Operation::Page(PageIntent { before, after, author })`; `before` is the page the edit started from and is the precondition reconciliation checks. Text, styles, -paragraph formatting, hyperlinks, bullets, numbering and note tags, paragraph structure, +paragraph formatting, hyperlinks, bullets, numbering, note tags, table rows and columns, paragraph structure, outline layout, insertions and deletions are all differences between `before` and `after`; the library never sees editor operations. diff --git a/crates/onestore/README.md b/crates/onestore/README.md index 8c58b5d6df89604fd4ee49f3d91bb06f82c6980e..ee8a2d030c56a902c6e51c434e892dfd289f815e 100644 --- a/crates/onestore/README.md +++ b/crates/onestore/README.md @@ -57,7 +57,7 @@ harness also accepts `--client-profile release`. | `ParagraphJoin`, `PreparedEdit::join` | Join adjacent ordinary text while preserving inherited character styles and native text-identity rules | | `TextAttribute`, `PreparedEdit::format` | Change character formatting over a UTF-16 range while sharing immutable styles; preserve unselected runs | | `OutlineEdit`, `PreparedEdit::outline` | Change ordinary outline position/width or a paragraph's saved expansion default, preserving identities and content | -| `PreparedEdit::page` | Publish an edited `page::Page` as one revision per changed space: text, character and paragraph formatting, hyperlinks, bullets and numbering, note tags, paragraph insertion/split/join/move/deletion, outline insertion/deletion/position/width and saved collapse state, lowered onto the typed writers with the model's identities | +| `PreparedEdit::page` | Publish an edited `page::Page` as one revision per changed space: text, character and paragraph formatting, hyperlinks, bullets and numbering, note tags, table rows and columns, paragraph insertion/split/join/move/deletion, outline insertion/deletion/position/width and saved collapse state, lowered onto the typed writers with the model's identities | | `TreeEdit`, `PreparedEdit::tree` | Move or delete a subtree on one page, normalize surviving containers, and replace an emptied table cell's paragraph atomically | | `PreparedEdit::commit`, `PreparedEdit::commit_file` | Publish the exact prepared image under caller-held exclusion or the conservative filesystem adapter | | `read_file` | Read a snapshot under whole-file exclusion | diff --git a/crates/onestore/src/page/write.rs b/crates/onestore/src/page/write.rs index c69cc31bf9a1d6b74f23d6306950cb9ff58d826c..099509b4a23b96415abd83b85dfb730c1263a620 100644 --- a/crates/onestore/src/page/write.rs +++ b/crates/onestore/src/page/write.rs @@ -1,7 +1,7 @@ //! Publishes an edited page model by lowering the difference from the stored page onto //! the typed writers, then squashing their transactions into one revision per space. -use super::{Outline, Page, PageObject, PageParagraph, ParagraphContent}; +use super::{Outline, Page, PageObject, PageParagraph, ParagraphContent, Table}; use crate::{ Error, ExGuid, Insertion, ObjectData, OutlineEdit, ParagraphJoin, ParagraphSplit, PropertySets, RevisionIndex, Store, TextAttribute, TreeEdit, Value, @@ -20,6 +20,50 @@ fn invalid(message: &'static str) -> Error { Error { offset: 0, message } } +/// A native measurement array: a count byte, `header - 1` reserved bytes, then each value +/// in inches. +fn measurement_bytes(values: &[f32], header: usize) -> Result, Error> { + let count = u8::try_from(values.len()) + .map_err(|_| invalid("A measurement array exceeds the document range"))?; + let mut bytes = vec![0u8; header]; + bytes[0] = count; + for value in values { + if !value.is_finite() { + return Err(invalid("A measurement must be finite")); + } + bytes.extend_from_slice(&(value / 36.0).to_le_bytes()); + } + Ok(bytes) +} + +/// Every row carries one cell per column and widths are usable. +fn validate_table(table: &Table) -> Result<(), Error> { + if table.rows.is_empty() || table.columns.is_empty() { + return Err(invalid("A table needs at least one row and one column")); + } + if table + .columns + .iter() + .any(|c| !c.width.is_finite() || c.width < 36.0) + { + return Err(invalid("Table columns are at least 36 points wide")); + } + for row in &table.rows { + if row.cells.len() != table.columns.len() { + return Err(invalid("Every table row has one cell per column")); + } + for cell in &row.cells { + // An emptied cell keeps a replacement paragraph, as the tree writer provides. + for paragraph in &cell.paragraphs { + if let ParagraphContent::Table(nested) = ¶graph.content { + validate_table(nested)?; + } + } + } + } + Ok(()) +} + pub(crate) fn write_page( source: &[u8], space: ExGuid, @@ -154,6 +198,16 @@ impl Lowering<'_> { self.alias.get(&model).copied().unwrap_or(model) } + /// A writer identity for a new object; squash renames it to the model's. + fn allocate(&mut self, model: ExGuid) -> Result { + let written = ExGuid { + guid: crate::write::fresh_guid()?, + n: 1, + }; + self.alias.insert(model, written); + Ok(written) + } + fn apply(&mut self, edit: impl FnOnce(&[u8]) -> Result, Error>) -> Result<(), Error> { self.image = edit(&self.image)?; Ok(()) @@ -187,6 +241,11 @@ impl Lowering<'_> { } let old = View::new(before)?; let new = View::new(after)?; + for paragraph in new.paragraphs.values() { + if let ParagraphContent::Table(table) = ¶graph.content { + validate_table(table)?; + } + } self.check_fixed_objects(before, after, &old, &new)?; for id in new.outlines.keys().chain(new.paragraphs.keys()) { if !old.outlines.contains_key(id) @@ -295,31 +354,28 @@ impl Lowering<'_> { } } (ParagraphContent::Table(table), ParagraphContent::Table(previous)) => { - let same = table.id == previous.id - && table.columns == previous.columns - && table.borders == previous.borders - && table.layout == previous.layout - && table.tags == previous.tags - && table.rows.len() == previous.rows.len() - && table - .rows - .iter() - .zip(&previous.rows) - .all(|(row, previous)| { - row.id == previous.id - && row.cells.len() == previous.cells.len() - && row.cells.iter().zip(&previous.cells).all( - |(cell, previous)| { - cell.id == previous.id - && cell.layout == previous.layout - && cell.indents == previous.indents - && cell.shading == previous.shading - && cell.unsupported == previous.unsupported - }, - ) - }); - if !same { - return Err(invalid("Table structure cannot be edited")); + if table.id != previous.id + || table.layout != previous.layout + || table.tags != previous.tags + { + return Err(invalid("Table identity, layout and tags cannot be edited")); + } + let unchanged_cells = + table.rows.iter().flat_map(|row| &row.cells).all(|cell| { + previous + .rows + .iter() + .flat_map(|row| &row.cells) + .find(|before| before.id == cell.id) + .is_none_or(|before| { + cell.layout == before.layout + && cell.indents == before.indents + && cell.shading == before.shading + && cell.unsupported == before.unsupported + }) + }); + if !unchanged_cells { + return Err(invalid("Cell layout, indents and shading cannot be edited")); } } (ParagraphContent::Unsupported(a), ParagraphContent::Unsupported(b)) if a == b => {} @@ -544,10 +600,30 @@ impl Lowering<'_> { collect_containers(&outline.paragraphs, &mut containers); } } + // Cells that do not exist yet receive their paragraphs once the table structure does. + let (existing, deferred): (Vec, Vec) = + containers.into_iter().partition(|container| { + old.children.contains_key(container) + || new.paragraphs.contains_key(container) + || new.outlines.contains_key(container) + }); + self.place_containers(old, new, placed, &existing)?; + self.edit_table_structure(old, new)?; + self.place_containers(old, new, placed, &deferred)?; + Ok(()) + } + + fn place_containers( + &mut self, + old: &View<'_>, + new: &View<'_>, + placed: &BTreeMap>, + containers: &[ExGuid], + ) -> Result<(), Error> { for container in containers { - let after = &new.children[&container]; + let after = &new.children[container]; let current: Vec = placed - .get(&container) + .get(container) .map(|list| { list.iter() .copied() @@ -561,26 +637,35 @@ impl Lowering<'_> { let anchor = next.map(|n| self.id(n)); if !old.paragraphs.contains_key(id) && !self.alias.contains_key(id) { let paragraph = new.paragraphs[id]; - let text = paragraph - .text() - .filter(|text| text.date_field.is_none()) - .filter(|_| paragraph.style.is_none()) - .ok_or_else(|| { - invalid("New paragraphs contain plain text without fields or styles") - })?; - let insertion = Insertion::paragraph( - self.id(container), - anchor, - text.text.text(), - self.author, - )?; + if paragraph.style.is_some() { + return Err(invalid( + "New paragraphs contain plain text without fields or styles", + )); + } + // A new table starts as an empty text paragraph whose content the + // structure pass replaces with the table. + let (text, text_id) = match ¶graph.content { + ParagraphContent::Text(text) if text.date_field.is_none() => { + (text.text.text(), Some(text.id)) + } + ParagraphContent::Table(_) => ("", None), + _ => { + return Err(invalid( + "New paragraphs contain plain text without fields or styles", + )); + } + }; + let insertion = + Insertion::paragraph(self.id(*container), anchor, text, self.author)?; let space = self.space; self.apply(|image| insertion.apply(image, space))?; self.alias.insert(*id, insertion.object()); - self.alias.insert(text.id, insertion.text_object()); + if let Some(text_id) = text_id { + self.alias.insert(text_id, insertion.text_object()); + } } else if !kept.contains(id) { let edit = - TreeEdit::move_to(self.id(*id), self.id(container), anchor, self.author)?; + TreeEdit::move_to(self.id(*id), self.id(*container), anchor, self.author)?; let space = self.space; self.apply(|image| edit.apply(image, space))?; } @@ -590,6 +675,187 @@ impl Lowering<'_> { Ok(()) } + /// Creates tables, rows and cells the model added, rebuilds every table's row and cell + /// order to the model's, and writes the column widths, locks and border flag. Native + /// rows and cells are plain containers; a cell carries its indent array and the flags + /// every native cell has. + fn edit_table_structure(&mut self, old: &View<'_>, new: &View<'_>) -> Result<(), Error> { + let old_tables: BTreeMap = old + .paragraphs + .values() + .filter_map(|p| match &p.content { + ParagraphContent::Table(table) => Some((table.id, table)), + _ => None, + }) + .collect(); + for (paragraph_id, paragraph) in &new.paragraphs { + let ParagraphContent::Table(table) = ¶graph.content else { + continue; + }; + let previous = old_tables.get(&table.id).copied(); + let unchanged = previous.is_some_and(|previous| { + previous.columns == table.columns + && previous.borders == table.borders + && previous.rows.len() == table.rows.len() + && previous.rows.iter().zip(&table.rows).all(|(a, b)| { + a.id == b.id + && a.cells.len() == b.cells.len() + && a.cells.iter().zip(&b.cells).all(|(x, y)| x.id == y.id) + }) + }); + if unchanged { + continue; + } + let modified = crate::create::current_timestamps()?.0.to_le_bytes(); + let mut created: Vec<(ExGuid, u32, Values)> = Vec::new(); + let table_id = if previous.is_some() { + self.id(table.id) + } else { + self.allocate(table.id)? + }; + let existing_rows: BTreeSet = previous + .map(|p| p.rows.iter().map(|r| r.id).collect()) + .unwrap_or_default(); + let existing_cells: BTreeSet = previous + .map(|p| { + p.rows + .iter() + .flat_map(|r| r.cells.iter().map(|c| c.id)) + .collect() + }) + .unwrap_or_default(); + let template = previous + .and_then(|p| p.rows.first()) + .and_then(|r| r.cells.first()); + let mut rows = Vec::new(); + for row in &table.rows { + let row_id = if existing_rows.contains(&row.id) { + self.id(row.id) + } else { + self.allocate(row.id)? + }; + let mut cells = Vec::new(); + for cell in &row.cells { + let cell_id = if existing_cells.contains(&cell.id) { + self.id(cell.id) + } else { + let written = self.allocate(cell.id)?; + let indents = if cell.indents.is_empty() { + template.map(|t| t.indents.clone()).unwrap_or_default() + } else { + cell.indents.clone() + }; + let mut values: Values = vec![ + (0x14001d7a, modified.to_vec()), + (0x0c001c13, vec![0]), + (0x0c001c03, vec![1]), + (0x88001c91, Vec::new()), + ]; + if !indents.is_empty() { + values.push((0x1c001c12, measurement_bytes(&indents, 4)?)); + } + if let Some(shading) = cell.shading { + values.push((0x14001e26, shading.to_le_bytes().to_vec())); + } + created.push((written, 0x60024, values)); + written + }; + cells.push(cell_id); + } + if !existing_rows.contains(&row.id) { + created.push((row_id, 0x60023, vec![(0x14001d7a, modified.to_vec())])); + } + rows.push((row_id, cells)); + } + let mut table_values: Values = vec![ + (0x14001d7a, modified.to_vec()), + (0x14001d57, (table.rows.len() as u32).to_le_bytes().to_vec()), + ( + 0x14001d58, + (table.columns.len() as u32).to_le_bytes().to_vec(), + ), + ( + 0x1c001d66, + measurement_bytes( + &table.columns.iter().map(|c| c.width).collect::>(), + 1, + )?, + ), + ]; + let mut locks = vec![table.columns.len() as u8]; + locks.extend(vec![0u8; table.columns.len().div_ceil(8)]); + for (i, column) in table.columns.iter().enumerate() { + if column.locked { + locks[1 + i / 8] |= 1 << (i % 8); + } + } + table_values.push((0x1c001d7d, locks)); + table_values.push(( + 0x08001d5e | (u32::from(table.borders.unwrap_or(true)) << 31), + Vec::new(), + )); + if previous.is_none() { + table_values.push((0x14001c3e, 1u32.to_le_bytes().to_vec())); + table_values.push((0x14001c84, 1u32.to_le_bytes().to_vec())); + created.push((table_id, 0x60022, Vec::new())); + } + let holder = self.id(*paragraph_id); + let is_new = previous.is_none(); + let space = self.space; + self.apply(|image| { + crate::write::write_revision(image, space, |raw| { + let mut changed: BTreeMap = BTreeMap::new(); + for (id, jcid, values) in &created { + let mut node = PropertyObject { + jcid: *jcid, + bytes: crate::create::properties(values)?, + global_ids: std::sync::Arc::new(BTreeMap::from([(0, id.guid)])), + }; + node.reference(*id)?; + changed.insert(*id, node); + } + let mut row_refs = Vec::new(); + for (row_id, cells) in &rows { + let mut row = match changed.remove(row_id) { + Some(row) => row, + None => PropertyObject::from_object(&raw.objects[row_id])?, + }; + let mut cell_refs = Vec::new(); + for cell in cells { + cell_refs.extend_from_slice(&row.reference(*cell)?); + } + row.set(&[(0x24001c20, &cell_refs), (0x14001d7a, &modified)])?; + changed.insert(*row_id, row); + } + let mut table_object = match changed.remove(&table_id) { + Some(object) => object, + None => PropertyObject::from_object(&raw.objects[&table_id])?, + }; + for (row_id, _) in &rows { + row_refs.extend_from_slice(&table_object.reference(*row_id)?); + } + table_object.remove(&[0x1c001d7d, 0x08001d5e])?; + table_object.set( + &table_values + .iter() + .map(|(id, bytes)| (*id, bytes.as_slice())) + .collect::>(), + )?; + table_object.set(&[(0x24001c20, &row_refs)])?; + changed.insert(table_id, table_object); + if is_new { + let mut object = PropertyObject::from_object(&raw.objects[&holder])?; + let reference = object.reference(table_id)?; + object.set(&[(0x24001c1f, &reference), (0x14001d7a, &modified)])?; + changed.insert(holder, object); + } + Ok(changed) + }) + })?; + } + Ok(()) + } + fn delete( &mut self, old: &View<'_>, @@ -609,6 +875,10 @@ impl Lowering<'_> { || (old.paragraphs.contains_key(&ancestor) && !new.paragraphs.contains_key(&ancestor) && !consumed.contains(&ancestor)) + || (old.children.contains_key(&ancestor) + && !old.paragraphs.contains_key(&ancestor) + && !old.outlines.contains_key(&ancestor) + && !new.children.contains_key(&ancestor)) { covered = true; break; diff --git a/crates/onestore/tests/page_tables.rs b/crates/onestore/tests/page_tables.rs new file mode 100644 index 0000000000000000000000000000000000000000..cf91ec2dbf79281159c6cf07428341f9483d4dbf --- /dev/null +++ b/crates/onestore/tests/page_tables.rs @@ -0,0 +1,306 @@ +use onestore::{ + ExGuid, PreparedEdit, RevisionIndex, Store, + document::{Document, Format}, + page::{ + Page, PageObject, PageParagraph, ParagraphContent, Table, TableCell, TableColumn, TableRow, + TextObject, text::new_id, + }, +}; + +const TREES: &[u8] = + include_bytes!("../../../corpus/outline-edit/tree/before/notebook/synthetic.one"); +const AUTHOR: &str = "Table author"; + +fn page_by_title(bytes: &[u8], title: &str) -> (ExGuid, Page) { + let store = Store::parse(bytes).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + document + .pages() + .unwrap() + .into_iter() + .find_map(|(space, _)| { + let page = Page::from_space(&document, space).unwrap(); + (page.title == title).then_some((space, page)) + }) + .unwrap() +} + +fn page_in(bytes: &[u8], space: ExGuid) -> Page { + let store = Store::parse(bytes).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + Page::from_space(&Document::parse(&index).unwrap(), space).unwrap() +} + +fn body_paragraphs(page: &mut Page) -> &mut Vec { + page.objects + .iter_mut() + .find_map(|object| match object { + PageObject::Outline(outline) if !outline.title => Some(&mut outline.paragraphs), + _ => None, + }) + .unwrap() +} + +fn table_mut(page: &mut Page) -> &mut Table { + body_paragraphs(page) + .iter_mut() + .find_map(|p| match &mut p.content { + ParagraphContent::Table(table) => Some(table), + _ => None, + }) + .unwrap() +} + +/// Definitions are read back only while a paragraph references them. +fn referenced(page: &Page) -> std::collections::BTreeSet { + fn walk(paragraphs: &[PageParagraph], out: &mut std::collections::BTreeSet) { + for paragraph in paragraphs { + out.extend(paragraph.lists.iter().copied()); + out.extend(paragraph.style); + out.extend(paragraph.tags.iter().filter_map(|t| t.definition)); + match ¶graph.content { + ParagraphContent::Text(text) => { + out.extend(text.tags.iter().filter_map(|t| t.definition)); + } + ParagraphContent::Table(table) => { + for row in &table.rows { + for cell in &row.cells { + walk(&cell.paragraphs, out); + } + } + } + ParagraphContent::Unsupported(_) => {} + } + } + } + let mut out = std::collections::BTreeSet::new(); + for object in &page.objects { + match object { + PageObject::Outline(outline) => walk(&outline.paragraphs, &mut out), + PageObject::Title(title) => { + for outline in &title.outlines { + walk(&outline.paragraphs, &mut out); + } + } + _ => {} + } + } + out +} + +fn assert_same(written: &[u8], space: ExGuid, expected: &Page) -> Page { + let stored = page_in(written, space); + let mut expected = expected.clone(); + expected.title = stored.title.clone(); + let live = referenced(&expected); + expected.definitions.retain(|id, _| live.contains(id)); + assert_eq!(stored, expected); + stored +} + +fn plain_text(text: &str) -> TextObject { + TextObject { + id: new_id().unwrap(), + date_field: None, + text: onestore::page::Paragraph::new( + text.into(), + Format { + font: Some("Calibri".into()), + font_size: Some(11.0), + language: Some(1033), + ..Default::default() + }, + ), + tags: Vec::new(), + } +} + +fn cell_paragraph(template: &PageParagraph, text: &str) -> PageParagraph { + let mut paragraph = template.clone(); + paragraph.id = new_id().unwrap(); + paragraph.parent = None; + paragraph.level = 1; + paragraph.lists.clear(); + paragraph.tags.clear(); + paragraph.style = None; + paragraph.collapsed = false; + paragraph.content = ParagraphContent::Text(plain_text(text)); + paragraph +} + +fn new_cell(template: &TableCell, text: &str) -> TableCell { + TableCell { + id: new_id().unwrap(), + layout: Default::default(), + indents: template.indents.clone(), + shading: None, + paragraphs: vec![cell_paragraph(&template.paragraphs[0], text)], + unsupported: Vec::new(), + } +} + +#[test] +fn a_row_and_a_column_are_added_to_a_native_table_and_removed_again() { + let (space, before) = page_by_title(TREES, "Delete cell subtree"); + let mut after = before.clone(); + let table = table_mut(&mut after); + assert_eq!((table.rows.len(), table.columns.len()), (1, 2)); + let template = table.rows[0].cells[0].clone(); + table.columns.push(TableColumn { + width: 120.0, + locked: false, + }); + table.rows[0] + .cells + .push(new_cell(&template, "Third column")); + table.rows.push(TableRow { + id: new_id().unwrap(), + cells: vec![ + new_cell(&template, "Second row"), + new_cell(&template, "Second row, second column"), + new_cell(&template, "Second row, third column"), + ], + }); + let written = PreparedEdit::page(TREES, space, &after, AUTHOR).unwrap(); + let stored = assert_same(written.as_bytes(), space, &after); + assert_eq!( + PreparedEdit::page(written.as_bytes(), space, &stored, AUTHOR) + .unwrap() + .as_bytes(), + written.as_bytes() + ); + if let Some(directory) = std::env::var_os("ONESTORE_TABLE_EDIT_EXPORT") { + let directory = std::path::PathBuf::from(directory); + std::fs::create_dir(&directory).unwrap(); + std::fs::write(directory.join("synthetic.one"), written.as_bytes()).unwrap(); + let written_store = Store::parse(written.as_bytes()).unwrap(); + std::fs::write( + directory.join("Open Notebook.onetoc2"), + onestore::create_table_of_contents( + "Open Notebook.onetoc2", + &[("synthetic.one", written_store.header.file_id)], + ) + .unwrap(), + ) + .unwrap(); + } + let mut smaller = stored.clone(); + let table = table_mut(&mut smaller); + table.columns.remove(1); + for row in &mut table.rows { + row.cells.remove(1); + } + table.rows.remove(0); + let again = PreparedEdit::page(written.as_bytes(), space, &smaller, AUTHOR).unwrap(); + let mut stored = assert_same(again.as_bytes(), space, &smaller); + let table = table_mut(&mut stored); + assert_eq!((table.rows.len(), table.columns.len()), (1, 2)); +} + +#[test] +fn column_widths_locks_and_borders_change_in_place() { + let (space, before) = page_by_title(TREES, "Delete cell subtree"); + let mut after = before.clone(); + let table = table_mut(&mut after); + table.columns[0].width = 300.0; + table.columns[1].locked = false; + table.borders = Some(false); + let written = PreparedEdit::page(TREES, space, &after, AUTHOR).unwrap(); + assert_same(written.as_bytes(), space, &after); +} + +#[test] +fn inconsistent_tables_are_refused() { + let (space, before) = page_by_title(TREES, "Delete cell subtree"); + let mut ragged = before.clone(); + let template = table_mut(&mut ragged).rows[0].cells[0].clone(); + table_mut(&mut ragged).rows[0] + .cells + .push(new_cell(&template, "Extra")); + assert!(PreparedEdit::page(TREES, space, &ragged, AUTHOR).is_err()); + let mut empty = before.clone(); + table_mut(&mut empty).rows[0].cells[0].paragraphs.clear(); + let written = PreparedEdit::page(TREES, space, &empty, AUTHOR).unwrap(); + let mut stored = page_in(written.as_bytes(), space); + let cell = &table_mut(&mut stored).rows[0].cells[0]; + assert_eq!(cell.paragraphs.len(), 1); + assert_eq!(cell.paragraphs[0].text().unwrap().text.text(), ""); + let mut narrow = before.clone(); + table_mut(&mut narrow).columns[0].width = 10.0; + assert!(PreparedEdit::page(TREES, space, &narrow, AUTHOR).is_err()); + let mut rowless = before.clone(); + table_mut(&mut rowless).rows.clear(); + assert!(PreparedEdit::page(TREES, space, &rowless, AUTHOR).is_err()); +} + +/// `ONESTORE_TABLE_EXPORT` names a new directory receiving the candidate for a cold reopen. +#[test] +fn a_new_table_is_created_on_a_fresh_page() { + let source = onestore::create_section("tables.one", "Before the table", "Author").unwrap(); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let (space, _) = document.pages().unwrap()[0]; + let before = Page::from_space(&document, space).unwrap(); + let mut after = before.clone(); + let template = body_paragraphs(&mut after)[0].clone(); + let cell = |text: &str| TableCell { + id: new_id().unwrap(), + layout: Default::default(), + indents: Vec::new(), + shading: None, + paragraphs: vec![cell_paragraph(&template, text)], + unsupported: Vec::new(), + }; + let table = Table { + id: new_id().unwrap(), + columns: vec![ + TableColumn { + width: 144.0, + locked: true, + }, + TableColumn { + width: 216.0, + locked: true, + }, + ], + rows: vec![ + TableRow { + id: new_id().unwrap(), + cells: vec![cell("Name"), cell("Value")], + }, + TableRow { + id: new_id().unwrap(), + cells: vec![cell("Rust 🦀"), cell("Written natively")], + }, + ], + borders: Some(true), + layout: Default::default(), + tags: Vec::new(), + }; + let mut holder = cell_paragraph(&template, ""); + holder.content = ParagraphContent::Table(table); + holder.format = Format::default(); + body_paragraphs(&mut after).push(holder); + body_paragraphs(&mut after).push(cell_paragraph(&template, "After the table")); + let written = PreparedEdit::page(&source, space, &after, AUTHOR).unwrap(); + assert_same(written.as_bytes(), space, &after); + if let Some(directory) = std::env::var_os("ONESTORE_TABLE_EXPORT") { + let directory = std::path::PathBuf::from(directory); + std::fs::create_dir(&directory).unwrap(); + std::fs::write(directory.join("tables.one"), written.as_bytes()).unwrap(); + let written_store = Store::parse(written.as_bytes()).unwrap(); + std::fs::write( + directory.join("Open Notebook.onetoc2"), + onestore::create_table_of_contents( + "Open Notebook.onetoc2", + &[("tables.one", written_store.header.file_id)], + ) + .unwrap(), + ) + .unwrap(); + } +} diff --git a/crates/onestore/tests/page_writer.rs b/crates/onestore/tests/page_writer.rs index f842bd485bb4d470740e955ba93fe34e3ed405c1..b7efc8395d6c50d52f9df30c10cc0705733d36c3 100644 --- a/crates/onestore/tests/page_writer.rs +++ b/crates/onestore/tests/page_writer.rs @@ -604,7 +604,7 @@ fn unsupported_model_edits_are_rejected_before_writing() { } } } - assert!(PreparedEdit::page(TREES, space, &widened, AUTHOR).is_err()); + assert!(PreparedEdit::page(TREES, space, &widened, AUTHOR).is_ok()); let mut relisted = page_in(TREES, space); let outline = body(&page)[0]; for object in &mut relisted.objects { diff --git a/tools/test_table_edit.py b/tools/test_table_edit.py new file mode 100644 index 0000000000000000000000000000000000000000..12305d45cde9b4fa729a8e4cbe9b154042d685b7 --- /dev/null +++ b/tools/test_table_edit.py @@ -0,0 +1,58 @@ +import html +from pathlib import Path +import re +import runpy +import shutil +from tempfile import TemporaryDirectory +import unittest +import xml.etree.ElementTree as ET + +from native_xml import ns + +ROOT = Path(__file__).resolve().parent.parent +FIXTURE = ROOT / 'corpus/table-edit' +compare = runpy.run_path(str(ROOT / 'tools/verify-document.py'))['compare'] + + +def tables(read, title): + for path in sorted(read.glob('page-*.xml')): + page = ET.parse(path).getroot() + if page.get('name') != title: + continue + out = [] + for table in page.iter('{%s}Table' % ns['one']): + columns = [(column.get('width'), column.get('isLocked')) for column in table.find('one:Columns', ns)] + rows = [[[html.unescape(re.sub(r'<[^>]*>', '', text.text or '')) for text in cell.findall('.//one:T', ns)] + for cell in row.findall('one:Cell', ns)] for row in table.findall('one:Row', ns)] + out.append((table.get('bordersVisible'), columns, rows)) + return out + raise AssertionError(title) + + +class TableEditTest(unittest.TestCase): + def check(self, name): + with TemporaryDirectory() as temporary: + read = Path(temporary) / 'read' + shutil.copytree(FIXTURE / name / 'cold/read', read) + compare(FIXTURE / name / 'candidate', read) + + def test_a_rust_table_renders_natively(self): + self.check('created') + self.assertEqual(tables(FIXTURE / 'created/cold/read', 'Before the table'), [ + ('true', [('144.0', 'true'), ('216.0', 'true')], + [[['Name'], ['Value']], [['Rust 🦀'], ['Written natively']]]), + ]) + + def test_rows_and_columns_added_to_a_native_table_render_natively(self): + self.check('edited') + (borders, columns, rows), = tables(FIXTURE / 'edited/cold/read', 'Delete cell subtree') + self.assertEqual(borders, 'true') + self.assertEqual(columns[:2], [('240.0', 'true'), ('180.0', 'true')]) + self.assertEqual(len(columns), 3) + self.assertIsNone(columns[2][1]) + self.assertEqual(rows[0][2], ['Third column']) + self.assertEqual(rows[1], [['Second row'], ['Second row, second column'], ['Second row, third column']]) + + +if __name__ == '__main__': + unittest.main()