diff --git a/corpus/attachment-edit/README.md b/corpus/attachment-edit/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c6a769a2c5b96c0d89d5c8a0ac19476a575f437d --- /dev/null +++ b/corpus/attachment-edit/README.md @@ -0,0 +1,23 @@ +# Rust attachment authoring + +`plain/candidate` and `icon/candidate` are the page writer's output for +`an_attachment_inserted_on_a_fresh_page_reads_back_and_can_be_removed` in +`crates/onestore/tests/page_attachments.rs`: on a section created in Rust, a +paragraph between two text paragraphs becomes an inserted file +(`notes 🦀.txt`, 28 bytes), stored the way OneNote stores one +(`corpus/native/cold-05-06-attachment`): the payload embedded in the +section's file-data store list, an embedded-file object declaring it by +identity and extension, and an attachment object with the file name, the +24 pt icon size and the flags every native attachment carries. `icon/` also +embeds the native fixture's 724-byte icon PNG as the preview picture; +`plain/` omits it, which OneNote accepts (it renders its own icon). +OneNote's integrity check refuses an attachment object without +`0x1c001d61` (twenty bytes: 16, 1, zeros), so the writer always stores it. + +`*/cold` are fresh OneNote 2010 reads: the file reads back as an +`InsertedFile` with the preferred name and the exact payload bytes, between +the two text paragraphs. `tools/test_attachment_edit.py` checks both captures +without a VM. Regenerate with `ONESTORE_ATTACHMENT_EXPORT` set to a directory +while running the test (it writes `plain/` and `icon/`), then cold-open each +with `tools/native_runner.py OUTPUT/plain COLD --expected-pages 1 +--collect-notebook`. diff --git a/corpus/attachment-edit/icon/candidate/Open Notebook.onetoc2 b/corpus/attachment-edit/icon/candidate/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..b1e75d18a4b4afe2e1578f6984ce26245e06e601 Binary files /dev/null and b/corpus/attachment-edit/icon/candidate/Open Notebook.onetoc2 differ diff --git a/corpus/attachment-edit/icon/candidate/files.one b/corpus/attachment-edit/icon/candidate/files.one new file mode 100644 index 0000000000000000000000000000000000000000..f2a7bc271a6845ace199cadb3a064020c8629d30 Binary files /dev/null and b/corpus/attachment-edit/icon/candidate/files.one differ diff --git a/corpus/attachment-edit/icon/cold/commands.jsonl b/corpus/attachment-edit/icon/cold/commands.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..9def35e2e711d7d73fc98cc696645b981257650c --- /dev/null +++ b/corpus/attachment-edit/icon/cold/commands.jsonl @@ -0,0 +1,3 @@ +{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-5534DCBB -ExpectedPages 1", "exit": 0, "stdout": "Read 1 sections and 1 pages.\r\n", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null} diff --git a/corpus/attachment-edit/icon/cold/machine.json b/corpus/attachment-edit/icon/cold/machine.json new file mode 100644 index 0000000000000000000000000000000000000000..37d696a7cd52402b04269e5713066036b300a606 --- /dev/null +++ b/corpus/attachment-edit/icon/cold/machine.json @@ -0,0 +1 @@ +{"name": "m6-5534dcbb", "hostname": "ONE-M6-5534DCBB"} diff --git a/corpus/attachment-edit/icon/cold/notebook/Open Notebook.onetoc2 b/corpus/attachment-edit/icon/cold/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..1257f97ca743e3779ae9999ec188b339781cdd82 Binary files /dev/null and b/corpus/attachment-edit/icon/cold/notebook/Open Notebook.onetoc2 differ diff --git a/corpus/attachment-edit/icon/cold/notebook/files.one b/corpus/attachment-edit/icon/cold/notebook/files.one new file mode 100644 index 0000000000000000000000000000000000000000..59abe526cb0c8505a61bd396e7d02205f436c264 Binary files /dev/null and b/corpus/attachment-edit/icon/cold/notebook/files.one differ diff --git a/corpus/attachment-edit/icon/cold/read/b20a14e044e33eac39b4665e7d79813b8a3672dbbd27ed9d2e4ac866c13ec4b6.attachment b/corpus/attachment-edit/icon/cold/read/b20a14e044e33eac39b4665e7d79813b8a3672dbbd27ed9d2e4ac866c13ec4b6.attachment new file mode 100644 index 0000000000000000000000000000000000000000..ae0d23c6d54266cd83b053571137ccdab0adb80a --- /dev/null +++ b/corpus/attachment-edit/icon/cold/read/b20a14e044e33eac39b4665e7d79813b8a3672dbbd27ed9d2e4ac866c13ec4b6.attachment @@ -0,0 +1 @@ +Rust wrote this attachment. diff --git a/corpus/attachment-edit/icon/cold/read/environment.json b/corpus/attachment-edit/icon/cold/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..9949345eafe711d01410767db91d603343d39ad9 --- /dev/null +++ b/corpus/attachment-edit/icon/cold/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-5534DCBB", + "cold": true, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/attachment-edit/icon/cold/read/hierarchy.xml b/corpus/attachment-edit/icon/cold/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..06673c4211d0b0a40ffe481d1618adc9e451a38c --- /dev/null +++ b/corpus/attachment-edit/icon/cold/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/attachment-edit/icon/cold/read/page-000.xml b/corpus/attachment-edit/icon/cold/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..74db9382bd8c0e7dc06d47345b1bb207d22dc70a --- /dev/null +++ b/corpus/attachment-edit/icon/cold/read/page-000.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/attachment-edit/icon/cold/read/payloads.json b/corpus/attachment-edit/icon/cold/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..e23523cda593af0e8951098f8ad12dae2c5a28b5 --- /dev/null +++ b/corpus/attachment-edit/icon/cold/read/payloads.json @@ -0,0 +1,10 @@ +[ + { + "sha256": "b20a14e044e33eac39b4665e7d79813b8a3672dbbd27ed9d2e4ac866c13ec4b6", + "name": "notes 🦀.txt", + "object": "{7FE89054-6937-03E5-03EB-E0B7A346CCCA}{0}{B0}", + "kind": "InsertedFile", + "page": "{650CBE9C-812A-0909-10A6-1E17B31C633C}{1}{B0}", + "bytes": 28 + } +] \ No newline at end of file diff --git a/corpus/attachment-edit/icon/cold/run.json b/corpus/attachment-edit/icon/cold/run.json new file mode 100644 index 0000000000000000000000000000000000000000..32b105282d2e3956b9ec59b399b5e5eae8fe081d --- /dev/null +++ b/corpus/attachment-edit/icon/cold/run.json @@ -0,0 +1,18 @@ +{ + "notebook": "/private/tmp/m4/files/icon", + "expected_pages": 1, + "author": null, + "author_timeout_seconds": 600, + "inspect": false, + "collect_notebook": true, + "base": { + "file": "win7-office-base.qcow2", + "format": "qcow2", + "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346", + "virtual_size": 68719476736 + }, + "scripts": { + "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331", + "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41" + } +} diff --git a/corpus/attachment-edit/icon/cold/scripts/cold.ps1 b/corpus/attachment-edit/icon/cold/scripts/cold.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..a0b99e7aeb86f4dc10032537800501a52ad65b4f --- /dev/null +++ b/corpus/attachment-edit/icon/cold/scripts/cold.ps1 @@ -0,0 +1,27 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '') +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' } +$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote' +if ($CloneHost) { + if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) { + throw 'The disposable clone hostname does not match this machine.' + } + New-Item "$key\Options\Paths" -Force | Out-Null + New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null +} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or + -not (Test-Path 'C:\one-tests\profile-original-cache')) { + throw 'Park the personal OneNote profile before resetting the test cache.' +} +$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0' +$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root)) +if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' } +New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null +if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked } +if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse } +New-Item "$key\OpenNotebooks" | Out-Null +New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null diff --git a/corpus/attachment-edit/icon/cold/scripts/read.ps1 b/corpus/attachment-edit/icon/cold/scripts/read.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..476094ab1e0c47077988a6461db2080cf208642b --- /dev/null +++ b/corpus/attachment-edit/icon/cold/scripts/read.ps1 @@ -0,0 +1,142 @@ +param( + [Parameter(Mandatory=$true)][string]$Root, + [int]$ExpectedPages = -1, + [switch]$UseCurrentCache, + [switch]$Pdf, + [switch]$KeepOpen, + [string]$CloneHost = '' +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +$notebook = Join-Path $root 'notebook' +$output = Join-Path $root 'read' +if (Test-Path $output) { throw 'Choose a new read destination.' } +if ($UseCurrentCache) { + if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') { + throw 'Park the personal OneNote profile before reading test notebooks.' + } +} else { + & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost +} +New-Item -ItemType Directory -Path $output | Out-Null +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$failure = $null +try { + $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0) + $process = Get-Process ONENOTE + @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion; + powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } | + ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8 + $sections = @() + foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) { + $id = '' + $app.OpenHierarchy($file.FullName, '', [ref]$id, 0) + $sections += $id + } + $deadline = [DateTime]::UtcNow.AddSeconds(300) + $previous = '' + $lastChange = '' + $stableSince = [DateTime]::UtcNow + $settled = $false + do { + $pages = @{} + foreach ($section in $sections) { + $hierarchy = '' + $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) + [xml]$xml = $hierarchy + foreach ($node in $xml.SelectNodes('//*[@path]')) { + if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) { + throw 'OneNote opened a section outside the copied notebook.' + } + } + foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) { + $id = $node.GetAttribute('ID') + $content = '' + $app.GetPageContent($id, [ref]$content, 1, 1) + $pages[$id] = $content + } + } + $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] })) + if ($signature -ne $previous) { + $lastChange = $previous + $previous = $signature + $stableSince = [DateTime]::UtcNow + } + if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or + ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and + ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break } + Start-Sleep -Milliseconds 250 + } while ([DateTime]::UtcNow -lt $deadline) + if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) { + [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8) + $index = 0 + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + $index++ + } + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8) + throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled." + } + $index = 0 + $payloads = @() + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + if ($Pdf) { + $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index) + $app.NavigateTo($id, '', $false) + $app.Publish($id, $pdfPath, 3, '') + if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' } + } + [xml]$page = $pages[$id] + foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) { + $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache')) + $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant() + [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes) + $payloads += @{ page = $id; object = $file.ParentNode.GetAttribute('objectID'); + kind = $file.LocalName; name = $file.GetAttribute('preferredName'); + sha256 = $hash; bytes = $bytes.Length } + } + $index++ + } + [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8) + $all = '' + $app.GetHierarchy($notebookId, 4, [ref]$all, 1) + [xml]$finalTree = $all + $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique) + if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) { + throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.' + } + [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8) + Write-Output "Read $($sections.Count) sections and $($pages.Count) pages." +} catch { + $failure = $_ + [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + throw +} finally { + try { + try { + if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) } + if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) { + $app.CloseNotebook($notebookId, $false) + } + } catch { + if ($null -eq $failure) { throw } + [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + } + } finally { + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + } + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + if (-not $UseCurrentCache -and -not $CloneHost) { + Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10 + } +} diff --git a/corpus/attachment-edit/icon/cold/source.json b/corpus/attachment-edit/icon/cold/source.json new file mode 100644 index 0000000000000000000000000000000000000000..96577bee44aed9761dff291cf70c0c23e2df240b --- /dev/null +++ b/corpus/attachment-edit/icon/cold/source.json @@ -0,0 +1,14 @@ +[ + { + "path": "Open Notebook.onetoc2", + "bytes": 1604, + "sha256": "98f31fe15497a403a6e2896c8ae2e6566824c0971e541187046a1581c4612721", + "mtime_ns": 1789246495204529692 + }, + { + "path": "files.one", + "bytes": 8288, + "sha256": "473e6527a585dc98d178343cf53ba9e08354b3d4d53a4f5ce984c0036ecfd76a", + "mtime_ns": 1789246495204285026 + } +] diff --git a/corpus/attachment-edit/icon/cold/teardown.json b/corpus/attachment-edit/icon/cold/teardown.json new file mode 100644 index 0000000000000000000000000000000000000000..05a47793de40f322e745c4c4183727e3590ecf70 --- /dev/null +++ b/corpus/attachment-edit/icon/cold/teardown.json @@ -0,0 +1 @@ +{"absent": true} diff --git a/corpus/attachment-edit/plain/candidate/Open Notebook.onetoc2 b/corpus/attachment-edit/plain/candidate/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..55dbe4e743b89d2e8f6dca8226453e4eaaeab88a Binary files /dev/null and b/corpus/attachment-edit/plain/candidate/Open Notebook.onetoc2 differ diff --git a/corpus/attachment-edit/plain/candidate/files.one b/corpus/attachment-edit/plain/candidate/files.one new file mode 100644 index 0000000000000000000000000000000000000000..f03ffb8cb2275239d0303743363eb292ff26f8da Binary files /dev/null and b/corpus/attachment-edit/plain/candidate/files.one differ diff --git a/corpus/attachment-edit/plain/cold/commands.jsonl b/corpus/attachment-edit/plain/cold/commands.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5c48c3229dfca20efedbed19f4ab4d103d7b830c --- /dev/null +++ b/corpus/attachment-edit/plain/cold/commands.jsonl @@ -0,0 +1,3 @@ +{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-B7D04704 -ExpectedPages 1", "exit": 0, "stdout": "Read 1 sections and 1 pages.\r\n", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null} diff --git a/corpus/attachment-edit/plain/cold/machine.json b/corpus/attachment-edit/plain/cold/machine.json new file mode 100644 index 0000000000000000000000000000000000000000..60691db38b35f7f5adab927b7fec5830c7ccab7c --- /dev/null +++ b/corpus/attachment-edit/plain/cold/machine.json @@ -0,0 +1 @@ +{"name": "m6-b7d04704", "hostname": "ONE-M6-B7D04704"} diff --git a/corpus/attachment-edit/plain/cold/notebook/Open Notebook.onetoc2 b/corpus/attachment-edit/plain/cold/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..4a5546bae6531fe237ee901e9dd7dacb7796e1e5 Binary files /dev/null and b/corpus/attachment-edit/plain/cold/notebook/Open Notebook.onetoc2 differ diff --git a/corpus/attachment-edit/plain/cold/notebook/files.one b/corpus/attachment-edit/plain/cold/notebook/files.one new file mode 100644 index 0000000000000000000000000000000000000000..7a04015d191c758afaef2704218fd28f00b7ddeb Binary files /dev/null and b/corpus/attachment-edit/plain/cold/notebook/files.one differ diff --git a/corpus/attachment-edit/plain/cold/read/b20a14e044e33eac39b4665e7d79813b8a3672dbbd27ed9d2e4ac866c13ec4b6.attachment b/corpus/attachment-edit/plain/cold/read/b20a14e044e33eac39b4665e7d79813b8a3672dbbd27ed9d2e4ac866c13ec4b6.attachment new file mode 100644 index 0000000000000000000000000000000000000000..ae0d23c6d54266cd83b053571137ccdab0adb80a --- /dev/null +++ b/corpus/attachment-edit/plain/cold/read/b20a14e044e33eac39b4665e7d79813b8a3672dbbd27ed9d2e4ac866c13ec4b6.attachment @@ -0,0 +1 @@ +Rust wrote this attachment. diff --git a/corpus/attachment-edit/plain/cold/read/environment.json b/corpus/attachment-edit/plain/cold/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..62364ddd944733ee96ed6015a8b9dcc18a160a55 --- /dev/null +++ b/corpus/attachment-edit/plain/cold/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-B7D04704", + "cold": true, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/attachment-edit/plain/cold/read/hierarchy.xml b/corpus/attachment-edit/plain/cold/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..5a6ff174d2ecb518f85080680083a89384fab38b --- /dev/null +++ b/corpus/attachment-edit/plain/cold/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/attachment-edit/plain/cold/read/page-000.xml b/corpus/attachment-edit/plain/cold/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..566f4cd5092d9427a2b0dad6d075f474df0328fb --- /dev/null +++ b/corpus/attachment-edit/plain/cold/read/page-000.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/attachment-edit/plain/cold/read/payloads.json b/corpus/attachment-edit/plain/cold/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..ce18cfb2962b0c7f8313c6ce191010486f5a00b0 --- /dev/null +++ b/corpus/attachment-edit/plain/cold/read/payloads.json @@ -0,0 +1,10 @@ +[ + { + "sha256": "b20a14e044e33eac39b4665e7d79813b8a3672dbbd27ed9d2e4ac866c13ec4b6", + "name": "notes 🦀.txt", + "object": "{6DE415DD-4090-0E54-15BB-41A8635DE4EB}{0}{B0}", + "kind": "InsertedFile", + "page": "{62772D92-5840-014B-3B34-46BA7FAC9472}{1}{B0}", + "bytes": 28 + } +] \ No newline at end of file diff --git a/corpus/attachment-edit/plain/cold/run.json b/corpus/attachment-edit/plain/cold/run.json new file mode 100644 index 0000000000000000000000000000000000000000..b95bfc1392795e5d9175e140691e211da4b0151f --- /dev/null +++ b/corpus/attachment-edit/plain/cold/run.json @@ -0,0 +1,18 @@ +{ + "notebook": "/private/tmp/m4/files/plain", + "expected_pages": 1, + "author": null, + "author_timeout_seconds": 600, + "inspect": false, + "collect_notebook": true, + "base": { + "file": "win7-office-base.qcow2", + "format": "qcow2", + "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346", + "virtual_size": 68719476736 + }, + "scripts": { + "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331", + "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41" + } +} diff --git a/corpus/attachment-edit/plain/cold/scripts/cold.ps1 b/corpus/attachment-edit/plain/cold/scripts/cold.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..a0b99e7aeb86f4dc10032537800501a52ad65b4f --- /dev/null +++ b/corpus/attachment-edit/plain/cold/scripts/cold.ps1 @@ -0,0 +1,27 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '') +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' } +$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote' +if ($CloneHost) { + if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) { + throw 'The disposable clone hostname does not match this machine.' + } + New-Item "$key\Options\Paths" -Force | Out-Null + New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null +} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or + -not (Test-Path 'C:\one-tests\profile-original-cache')) { + throw 'Park the personal OneNote profile before resetting the test cache.' +} +$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0' +$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root)) +if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' } +New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null +if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked } +if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse } +New-Item "$key\OpenNotebooks" | Out-Null +New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null diff --git a/corpus/attachment-edit/plain/cold/scripts/read.ps1 b/corpus/attachment-edit/plain/cold/scripts/read.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..476094ab1e0c47077988a6461db2080cf208642b --- /dev/null +++ b/corpus/attachment-edit/plain/cold/scripts/read.ps1 @@ -0,0 +1,142 @@ +param( + [Parameter(Mandatory=$true)][string]$Root, + [int]$ExpectedPages = -1, + [switch]$UseCurrentCache, + [switch]$Pdf, + [switch]$KeepOpen, + [string]$CloneHost = '' +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +$notebook = Join-Path $root 'notebook' +$output = Join-Path $root 'read' +if (Test-Path $output) { throw 'Choose a new read destination.' } +if ($UseCurrentCache) { + if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') { + throw 'Park the personal OneNote profile before reading test notebooks.' + } +} else { + & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost +} +New-Item -ItemType Directory -Path $output | Out-Null +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$failure = $null +try { + $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0) + $process = Get-Process ONENOTE + @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion; + powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } | + ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8 + $sections = @() + foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) { + $id = '' + $app.OpenHierarchy($file.FullName, '', [ref]$id, 0) + $sections += $id + } + $deadline = [DateTime]::UtcNow.AddSeconds(300) + $previous = '' + $lastChange = '' + $stableSince = [DateTime]::UtcNow + $settled = $false + do { + $pages = @{} + foreach ($section in $sections) { + $hierarchy = '' + $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) + [xml]$xml = $hierarchy + foreach ($node in $xml.SelectNodes('//*[@path]')) { + if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) { + throw 'OneNote opened a section outside the copied notebook.' + } + } + foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) { + $id = $node.GetAttribute('ID') + $content = '' + $app.GetPageContent($id, [ref]$content, 1, 1) + $pages[$id] = $content + } + } + $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] })) + if ($signature -ne $previous) { + $lastChange = $previous + $previous = $signature + $stableSince = [DateTime]::UtcNow + } + if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or + ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and + ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break } + Start-Sleep -Milliseconds 250 + } while ([DateTime]::UtcNow -lt $deadline) + if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) { + [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8) + $index = 0 + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + $index++ + } + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8) + throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled." + } + $index = 0 + $payloads = @() + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + if ($Pdf) { + $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index) + $app.NavigateTo($id, '', $false) + $app.Publish($id, $pdfPath, 3, '') + if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' } + } + [xml]$page = $pages[$id] + foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) { + $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache')) + $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant() + [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes) + $payloads += @{ page = $id; object = $file.ParentNode.GetAttribute('objectID'); + kind = $file.LocalName; name = $file.GetAttribute('preferredName'); + sha256 = $hash; bytes = $bytes.Length } + } + $index++ + } + [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8) + $all = '' + $app.GetHierarchy($notebookId, 4, [ref]$all, 1) + [xml]$finalTree = $all + $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique) + if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) { + throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.' + } + [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8) + Write-Output "Read $($sections.Count) sections and $($pages.Count) pages." +} catch { + $failure = $_ + [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + throw +} finally { + try { + try { + if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) } + if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) { + $app.CloseNotebook($notebookId, $false) + } + } catch { + if ($null -eq $failure) { throw } + [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + } + } finally { + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + } + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + if (-not $UseCurrentCache -and -not $CloneHost) { + Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10 + } +} diff --git a/corpus/attachment-edit/plain/cold/source.json b/corpus/attachment-edit/plain/cold/source.json new file mode 100644 index 0000000000000000000000000000000000000000..42d2b6dca1bd65182f68a4f513f9e57b21d7cb89 --- /dev/null +++ b/corpus/attachment-edit/plain/cold/source.json @@ -0,0 +1,14 @@ +[ + { + "path": "Open Notebook.onetoc2", + "bytes": 1604, + "sha256": "a0aa6bb1b80a7e9dd300ffef5d547ea08b685b53cbc91abde9829ea8b6054489", + "mtime_ns": 1789246495204137318 + }, + { + "path": "files.one", + "bytes": 7160, + "sha256": "cdb9b28f8bd716d612675321ad8bb752349c118735331e9d323c9b8e876aea73", + "mtime_ns": 1789246495203833985 + } +] diff --git a/corpus/attachment-edit/plain/cold/teardown.json b/corpus/attachment-edit/plain/cold/teardown.json new file mode 100644 index 0000000000000000000000000000000000000000..05a47793de40f322e745c4c4183727e3590ecf70 --- /dev/null +++ b/corpus/attachment-edit/plain/cold/teardown.json @@ -0,0 +1 @@ +{"absent": true} diff --git a/corpus/picture-edit/README.md b/corpus/picture-edit/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d4793d321000015fda144dd5745d11cbf10c2985 --- /dev/null +++ b/corpus/picture-edit/README.md @@ -0,0 +1,20 @@ +# Rust picture authoring + +`candidate/` is the page writer's output for +`a_picture_inserted_on_a_fresh_page_reads_back_and_can_be_removed` in +`crates/onestore/tests/page_images.rs`: on a section created in Rust, a +paragraph between two text paragraphs becomes a picture, stored the way +OneNote stores an inserted picture (`corpus/native/cold-05-05-image`): the +PNG embedded in the section's file-data store list as a file-data store +object, a picture-container file object declaring it by identity and +extension, and a picture object with the displayed size that the paragraph +holds as content. OneNote's own picture objects also carry a DPAPI-protected +blob (`0x1c001dfb`) that only the authoring Windows user can decrypt; the +writer omits it. + +`cold/` is a fresh OneNote 2010 read: the image reads back as `format="png"` +with the exact payload bytes, between the two text paragraphs. +`tools/test_picture_edit.py` checks this without a VM. Regenerate with +`ONESTORE_IMAGE_EXPORT` set to a new absolute directory while running the +test, then cold-open it with `tools/native_runner.py OUTPUT COLD +--expected-pages 1 --collect-notebook`. diff --git a/corpus/picture-edit/candidate/Open Notebook.onetoc2 b/corpus/picture-edit/candidate/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..79f8f955db3b58cd2ae471c00ae2b66c95f81f62 Binary files /dev/null and b/corpus/picture-edit/candidate/Open Notebook.onetoc2 differ diff --git a/corpus/picture-edit/candidate/pictures.one b/corpus/picture-edit/candidate/pictures.one new file mode 100644 index 0000000000000000000000000000000000000000..af4169c68f7c76ead1f32f25685131438a9fc7a1 Binary files /dev/null and b/corpus/picture-edit/candidate/pictures.one differ diff --git a/corpus/picture-edit/cold/commands.jsonl b/corpus/picture-edit/cold/commands.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f077978e124b796017ac989959a7b5e48d27684c --- /dev/null +++ b/corpus/picture-edit/cold/commands.jsonl @@ -0,0 +1,3 @@ +{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-A5A8705B -ExpectedPages 1", "exit": 0, "stdout": "Read 1 sections and 1 pages.\r\n", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null} diff --git a/corpus/picture-edit/cold/machine.json b/corpus/picture-edit/cold/machine.json new file mode 100644 index 0000000000000000000000000000000000000000..80a47e2a0dd156babc8d6e6c895da3a3144fd28a --- /dev/null +++ b/corpus/picture-edit/cold/machine.json @@ -0,0 +1 @@ +{"name": "m6-a5a8705b", "hostname": "ONE-M6-A5A8705B"} diff --git a/corpus/picture-edit/cold/notebook/Open Notebook.onetoc2 b/corpus/picture-edit/cold/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..836282818b6fe532b91bf32edf78fba4fe160b7a Binary files /dev/null and b/corpus/picture-edit/cold/notebook/Open Notebook.onetoc2 differ diff --git a/corpus/picture-edit/cold/notebook/pictures.one b/corpus/picture-edit/cold/notebook/pictures.one new file mode 100644 index 0000000000000000000000000000000000000000..0dd87dcf4b41be4ca1b5b4b921d9462358aae3b1 Binary files /dev/null and b/corpus/picture-edit/cold/notebook/pictures.one differ diff --git a/corpus/picture-edit/cold/read/environment.json b/corpus/picture-edit/cold/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..f1032fa5ff0c8dddcee05b60f81366ec2671b1f9 --- /dev/null +++ b/corpus/picture-edit/cold/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-A5A8705B", + "cold": true, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/picture-edit/cold/read/hierarchy.xml b/corpus/picture-edit/cold/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..dce86fad6744777127a6c19a6146cadb358f799f --- /dev/null +++ b/corpus/picture-edit/cold/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/picture-edit/cold/read/page-000.xml b/corpus/picture-edit/cold/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..aa8840aa5f2fb3e14fe2cb7223dc8638d73f1a92 --- /dev/null +++ b/corpus/picture-edit/cold/read/page-000.xml @@ -0,0 +1,4 @@ + +iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4z8AAAAMBAQAY3Y2w +AAAAAElFTkSuQmCC + diff --git a/corpus/picture-edit/cold/read/payloads.json b/corpus/picture-edit/cold/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..1ce91a413d6fff4d58765187b76756f0e97ae843 --- /dev/null +++ b/corpus/picture-edit/cold/read/payloads.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/corpus/picture-edit/cold/run.json b/corpus/picture-edit/cold/run.json new file mode 100644 index 0000000000000000000000000000000000000000..6e98c47fb0e8f3f15888fbc77851de46e6db6301 --- /dev/null +++ b/corpus/picture-edit/cold/run.json @@ -0,0 +1,18 @@ +{ + "notebook": "/private/tmp/m4/pictures", + "expected_pages": 1, + "author": null, + "author_timeout_seconds": 600, + "inspect": false, + "collect_notebook": true, + "base": { + "file": "win7-office-base.qcow2", + "format": "qcow2", + "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346", + "virtual_size": 68719476736 + }, + "scripts": { + "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331", + "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41" + } +} diff --git a/corpus/picture-edit/cold/scripts/cold.ps1 b/corpus/picture-edit/cold/scripts/cold.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..a0b99e7aeb86f4dc10032537800501a52ad65b4f --- /dev/null +++ b/corpus/picture-edit/cold/scripts/cold.ps1 @@ -0,0 +1,27 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '') +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' } +$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote' +if ($CloneHost) { + if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) { + throw 'The disposable clone hostname does not match this machine.' + } + New-Item "$key\Options\Paths" -Force | Out-Null + New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null +} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or + -not (Test-Path 'C:\one-tests\profile-original-cache')) { + throw 'Park the personal OneNote profile before resetting the test cache.' +} +$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0' +$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root)) +if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' } +New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null +if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked } +if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse } +New-Item "$key\OpenNotebooks" | Out-Null +New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null diff --git a/corpus/picture-edit/cold/scripts/read.ps1 b/corpus/picture-edit/cold/scripts/read.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..476094ab1e0c47077988a6461db2080cf208642b --- /dev/null +++ b/corpus/picture-edit/cold/scripts/read.ps1 @@ -0,0 +1,142 @@ +param( + [Parameter(Mandatory=$true)][string]$Root, + [int]$ExpectedPages = -1, + [switch]$UseCurrentCache, + [switch]$Pdf, + [switch]$KeepOpen, + [string]$CloneHost = '' +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +$notebook = Join-Path $root 'notebook' +$output = Join-Path $root 'read' +if (Test-Path $output) { throw 'Choose a new read destination.' } +if ($UseCurrentCache) { + if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') { + throw 'Park the personal OneNote profile before reading test notebooks.' + } +} else { + & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost +} +New-Item -ItemType Directory -Path $output | Out-Null +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$failure = $null +try { + $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0) + $process = Get-Process ONENOTE + @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion; + powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } | + ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8 + $sections = @() + foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) { + $id = '' + $app.OpenHierarchy($file.FullName, '', [ref]$id, 0) + $sections += $id + } + $deadline = [DateTime]::UtcNow.AddSeconds(300) + $previous = '' + $lastChange = '' + $stableSince = [DateTime]::UtcNow + $settled = $false + do { + $pages = @{} + foreach ($section in $sections) { + $hierarchy = '' + $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) + [xml]$xml = $hierarchy + foreach ($node in $xml.SelectNodes('//*[@path]')) { + if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) { + throw 'OneNote opened a section outside the copied notebook.' + } + } + foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) { + $id = $node.GetAttribute('ID') + $content = '' + $app.GetPageContent($id, [ref]$content, 1, 1) + $pages[$id] = $content + } + } + $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] })) + if ($signature -ne $previous) { + $lastChange = $previous + $previous = $signature + $stableSince = [DateTime]::UtcNow + } + if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or + ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and + ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break } + Start-Sleep -Milliseconds 250 + } while ([DateTime]::UtcNow -lt $deadline) + if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) { + [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8) + $index = 0 + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + $index++ + } + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8) + throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled." + } + $index = 0 + $payloads = @() + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + if ($Pdf) { + $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index) + $app.NavigateTo($id, '', $false) + $app.Publish($id, $pdfPath, 3, '') + if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' } + } + [xml]$page = $pages[$id] + foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) { + $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache')) + $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant() + [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes) + $payloads += @{ page = $id; object = $file.ParentNode.GetAttribute('objectID'); + kind = $file.LocalName; name = $file.GetAttribute('preferredName'); + sha256 = $hash; bytes = $bytes.Length } + } + $index++ + } + [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8) + $all = '' + $app.GetHierarchy($notebookId, 4, [ref]$all, 1) + [xml]$finalTree = $all + $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique) + if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) { + throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.' + } + [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8) + Write-Output "Read $($sections.Count) sections and $($pages.Count) pages." +} catch { + $failure = $_ + [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + throw +} finally { + try { + try { + if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) } + if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) { + $app.CloseNotebook($notebookId, $false) + } + } catch { + if ($null -eq $failure) { throw } + [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + } + } finally { + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + } + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + if (-not $UseCurrentCache -and -not $CloneHost) { + Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10 + } +} diff --git a/corpus/picture-edit/cold/source.json b/corpus/picture-edit/cold/source.json new file mode 100644 index 0000000000000000000000000000000000000000..006d1122e0e6b26381b4b3c32834b6beb5866625 --- /dev/null +++ b/corpus/picture-edit/cold/source.json @@ -0,0 +1,14 @@ +[ + { + "path": "Open Notebook.onetoc2", + "bytes": 1604, + "sha256": "0eddd691b52d26bac222cf7e52add3c9909446647b452fb193f03d49e273426a", + "mtime_ns": 1789243635388246438 + }, + { + "path": "pictures.one", + "bytes": 7064, + "sha256": "3076880e02cf74eb85400852c133060ab8ed50383c6b9df8a07bde8389891a70", + "mtime_ns": 1789243635387879438 + } +] diff --git a/corpus/picture-edit/cold/teardown.json b/corpus/picture-edit/cold/teardown.json new file mode 100644 index 0000000000000000000000000000000000000000..05a47793de40f322e745c4c4183727e3590ecf70 --- /dev/null +++ b/corpus/picture-edit/cold/teardown.json @@ -0,0 +1 @@ +{"absent": true} diff --git a/crates/canvas/src/document.rs b/crates/canvas/src/document.rs index e5bb2edb513e7607dabd6e5c527cfd4a81550c28..54682ff839128bd574e3f89169fe638450fda89b 100644 --- a/crates/canvas/src/document.rs +++ b/crates/canvas/src/document.rs @@ -153,7 +153,11 @@ pub(crate) fn validate_nodes( return Err(EditError::InvalidStructure); } } - ParagraphContent::Unsupported(_) => return Err(EditError::UnsupportedContent), + ParagraphContent::Image(_) + | ParagraphContent::Attachment(_) + | ParagraphContent::Unsupported(_) => { + return Err(EditError::UnsupportedContent); + } ParagraphContent::Table(table) => { if !ids.insert(table.id) || table.rows.is_empty() diff --git a/crates/canvas/src/editor.rs b/crates/canvas/src/editor.rs index 3d1a811ea53147e1a523b61498127267119c2447..d783b2c35fd8d4f7daa9b53025f73e5b2774cd3c 100644 --- a/crates/canvas/src/editor.rs +++ b/crates/canvas/src/editor.rs @@ -455,7 +455,9 @@ impl TextOutline { pending.push(visible_paragraphs(cell.paragraphs.iter())); } } - onestore::page::ParagraphContent::Unsupported(_) => { + onestore::page::ParagraphContent::Image(_) + | onestore::page::ParagraphContent::Attachment(_) + | onestore::page::ParagraphContent::Unsupported(_) => { return Err(EditError::UnsupportedContent); } } @@ -504,7 +506,9 @@ impl TextOutline { .rect[1] - 1.86 } - onestore::page::ParagraphContent::Unsupported(_) => { + onestore::page::ParagraphContent::Image(_) + | onestore::page::ParagraphContent::Attachment(_) + | onestore::page::ParagraphContent::Unsupported(_) => { return Err(EditError::UnsupportedContent); } }; @@ -1690,7 +1694,9 @@ impl CanvasEditor { .ok_or(EditError::InvalidStructure)?; f64::from(cell.rect[3]) + 1.68 } - onestore::page::ParagraphContent::Unsupported(_) => { + onestore::page::ParagraphContent::Image(_) + | onestore::page::ParagraphContent::Attachment(_) + | onestore::page::ParagraphContent::Unsupported(_) => { return Err(EditError::UnsupportedContent.into()); } } + f64::from(outline.origin()[1]); @@ -4348,6 +4354,7 @@ mod tests { (120.0, 80.0, true), ] .map(|(x, y, background)| Image { + size: None, id: onestore::page::text::new_id().unwrap(), layout: onestore::document::Layout { x: Some(x), diff --git a/crates/canvas/src/gpu/page.rs b/crates/canvas/src/gpu/page.rs index 12d41d8c471aec7b13c03692db6d08b709690127..3067f71a5fe3bc961d2ca6f91f66c745203a5733 100644 --- a/crates/canvas/src/gpu/page.rs +++ b/crates/canvas/src/gpu/page.rs @@ -907,6 +907,7 @@ mod tests { (120.0, 60.0, false), ] { page.objects.push(PageObject::Image(Image { + size: None, id: onestore::page::text::new_id().unwrap(), bytes: Some(image_bytes.clone()), layout: Layout { @@ -1230,6 +1231,7 @@ mod tests { unsupported: vec![unknown.clone()], }), PageObject::Image(Image { + size: None, id: ExGuid::default(), layout: Layout { y: Some(150.0), @@ -1388,6 +1390,7 @@ mod tests { objects: vec![ outline(2.0, "first"), PageObject::Image(Image { + size: None, id: ExGuid::default(), layout: Layout { x: Some(5.0), @@ -1506,6 +1509,7 @@ mod tests { margin_origin: [0.0; 2], definitions: BTreeMap::new(), objects: vec![PageObject::Image(Image { + size: None, id: ExGuid::default(), layout: Layout { x: Some(2.0), diff --git a/crates/canvas/src/outline.rs b/crates/canvas/src/outline.rs index 55eabfb43753088f994672cc08a8c27597b69b7f..c7a989f8c440673f9a6c9ed0d2246b0303c405cf 100644 --- a/crates/canvas/src/outline.rs +++ b/crates/canvas/src/outline.rs @@ -374,7 +374,11 @@ impl OutlineLayout { bottom += f64::from(child.size[1]); result.append(child, [x, y]); } - ParagraphContent::Unsupported(_) => return Err(LayoutError::UnsupportedContent), + ParagraphContent::Image(_) + | ParagraphContent::Attachment(_) + | ParagraphContent::Unsupported(_) => { + return Err(LayoutError::UnsupportedContent); + } } if !(bottom as f32).is_finite() || !result.size[0].is_finite() { return Err(LayoutError::InvalidSpacing); diff --git a/crates/notebook/README.md b/crates/notebook/README.md index 3d4db5159b4d900edd52c2e189a7583132f6cb7f..d2fb85623a5db93250338f7956b8880ced069b87 100644 --- a/crates/notebook/README.md +++ b/crates/notebook/README.md @@ -14,7 +14,7 @@ diffs the supplied model against the page stored in the supplied local snapshot, writes the difference into the working image and queues one `Operation::Page(PageIntent { before, after, author })`; `before` is the page the edit started from and is the precondition reconciliation checks. Text, styles, -paragraph formatting, hyperlinks, bullets, numbering, note tags, table rows and columns, paragraph structure, +paragraph formatting, hyperlinks, bullets, numbering, note tags, table rows and columns, pictures and attachments, paragraph structure, outline layout, insertions and deletions are all differences between `before` and `after`; the library never sees editor operations. diff --git a/crates/onestore/README.md b/crates/onestore/README.md index ee8a2d030c56a902c6e51c434e892dfd289f815e..784e3a7dee3818756ef49d8d618e946c16045bd1 100644 --- a/crates/onestore/README.md +++ b/crates/onestore/README.md @@ -57,7 +57,7 @@ harness also accepts `--client-profile release`. | `ParagraphJoin`, `PreparedEdit::join` | Join adjacent ordinary text while preserving inherited character styles and native text-identity rules | | `TextAttribute`, `PreparedEdit::format` | Change character formatting over a UTF-16 range while sharing immutable styles; preserve unselected runs | | `OutlineEdit`, `PreparedEdit::outline` | Change ordinary outline position/width or a paragraph's saved expansion default, preserving identities and content | -| `PreparedEdit::page` | Publish an edited `page::Page` as one revision per changed space: text, character and paragraph formatting, hyperlinks, bullets and numbering, note tags, table rows and columns, paragraph insertion/split/join/move/deletion, outline insertion/deletion/position/width and saved collapse state, lowered onto the typed writers with the model's identities | +| `PreparedEdit::page` | Publish an edited `page::Page` as one revision per changed space: text, character and paragraph formatting, hyperlinks, bullets and numbering, note tags, table rows and columns, inserted pictures and attachments, paragraph insertion/split/join/move/deletion, outline insertion/deletion/position/width and saved collapse state, lowered onto the typed writers with the model's identities | | `TreeEdit`, `PreparedEdit::tree` | Move or delete a subtree on one page, normalize surviving containers, and replace an emptied table cell's paragraph atomically | | `PreparedEdit::commit`, `PreparedEdit::commit_file` | Publish the exact prepared image under caller-held exclusion or the conservative filesystem adapter | | `read_file` | Read a snapshot under whole-file exclusion | diff --git a/crates/onestore/src/document.rs b/crates/onestore/src/document.rs index a640f59089200170ccbb8c09eaa1f35dcfbf9356..9f1c193fbed44e228e23bc2e5fe466369f610d3a 100644 --- a/crates/onestore/src/document.rs +++ b/crates/onestore/src/document.rs @@ -374,6 +374,9 @@ pub enum Kind<'a> { source_path: Option, recording_id: Option<[u8; 16]>, recording_type: Option, + /// Displayed icon width and height in points. + icon_width: Option, + icon_height: Option, }, File { reference: FileDataReference, @@ -1161,6 +1164,8 @@ impl<'a> Element<'a> { source_path: f.text(0x1c001d9d)?, recording_id: f.fixed(0x1c001c97)?, recording_type: f.u32(0x14001d24)?, + icon_width: f.float(0x140034cd, 36.0)?, + icon_height: f.float(0x140034ce, 36.0)?, }, 0x120001 => Kind::Author { name: f.text(0x1c001d75)?, diff --git a/crates/onestore/src/page/mod.rs b/crates/onestore/src/page/mod.rs index ec2d744bbc0825eb4b8e498357f3e99610d48bb9..8322927c7389448b2edc12a9272822548c5a7fef 100644 --- a/crates/onestore/src/page/mod.rs +++ b/crates/onestore/src/page/mod.rs @@ -109,9 +109,81 @@ pub struct PageParagraph { pub enum ParagraphContent { Text(TextObject), Table(Table), + /// A picture inside the paragraph, as OneNote inserts pictures into outlines. + Image(Image), + /// An embedded file, as OneNote inserts attachments into outlines. + Attachment(Attachment), Unsupported(Unsupported), } +/// Payload bytes are identified by `id`, so equality and serialization leave them out. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub struct Attachment { + pub id: ExGuid, + /// The name OneNote shows and saves the file under. + pub filename: String, + /// Where the file was inserted from, when recorded. + pub source_path: Option, + /// The icon OneNote rendered for the file, when present. + pub size: Option<[f32; 2]>, + #[serde(skip)] + pub bytes: Option>, + #[serde(skip)] + pub preview: Option>, +} + +impl PartialEq for Attachment { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + && self.filename == other.filename + && self.source_path == other.source_path + && self.size == other.size + } +} + +impl Attachment { + fn read( + revision: &Revision<'_>, + id: ExGuid, + node: &crate::document::Element<'_>, + ) -> Result { + let invalid = |message| Error { offset: 0, message }; + let Kind::Attachment { + container, + preview, + filename, + source_path, + icon_width, + icon_height, + .. + } = &node.kind + else { + unreachable!() + }; + let payload = |target: &Option| -> Result>, Error> { + let Some(target) = target else { + return Ok(None); + }; + let data = revision + .nodes + .get(target) + .ok_or_else(|| invalid("Missing canvas attachment data"))?; + match &data.kind { + Kind::File { payload, .. } => Ok(payload.map(Arc::from)), + _ => Err(invalid("Canvas attachment data has the wrong type")), + } + }; + Ok(Self { + id, + filename: filename.clone().unwrap_or_default(), + source_path: source_path.clone(), + size: icon_width.zip(*icon_height).map(|(w, h)| [w, h]), + bytes: payload(container)?, + preview: payload(preview)?, + }) + } +} + #[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] pub struct Table { pub id: ExGuid, @@ -157,6 +229,8 @@ pub struct TextObject { pub struct Image { pub id: ExGuid, pub layout: Layout, + /// Displayed picture width and height in points. + pub size: Option<[f32; 2]>, #[serde(skip)] pub bytes: Option>, pub alt: Option, @@ -167,11 +241,53 @@ impl PartialEq for Image { fn eq(&self, other: &Self) -> bool { self.id == other.id && self.layout == other.layout + && self.size == other.size && self.alt == other.alt && self.background == other.background } } +impl Image { + fn read( + revision: &Revision<'_>, + id: ExGuid, + node: &crate::document::Element<'_>, + ) -> Result { + let invalid = |message| Error { offset: 0, message }; + let Kind::Image { + container, + alt, + background, + picture_width, + picture_height, + .. + } = &node.kind + else { + unreachable!() + }; + let bytes = if let Some(container) = container { + let data = revision + .nodes + .get(container) + .ok_or_else(|| invalid("Missing canvas image data"))?; + match &data.kind { + Kind::File { payload, .. } => payload.map(Arc::from), + _ => return Err(invalid("Canvas image data has the wrong type")), + } + } else { + None + }; + Ok(Self { + id, + layout: node.layout.clone(), + size: picture_width.zip(*picture_height).map(|(w, h)| [w, h]), + bytes, + alt: alt.clone(), + background: background.unwrap_or(false), + }) + } +} + #[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] pub struct Unsupported { pub id: ExGuid, @@ -341,31 +457,9 @@ impl Page { page.objects.push(PageObject::Outline(outline)); } } - Kind::Image { - container, - alt, - background, - .. - } => { - let bytes = if let Some(container) = container { - let data = revision - .nodes - .get(container) - .ok_or_else(|| invalid("Missing canvas image data"))?; - match &data.kind { - Kind::File { payload, .. } => payload.map(Arc::from), - _ => return Err(invalid("Canvas image data has the wrong type")), - } - } else { - None - }; - page.objects.push(PageObject::Image(Image { - id, - layout: node.layout.clone(), - bytes, - alt: alt.clone(), - background: background.unwrap_or(false), - })); + Kind::Image { .. } => { + page.objects + .push(PageObject::Image(Image::read(revision, id, node)?)); } _ => page.objects.push(PageObject::Unsupported(Unsupported { id, @@ -493,6 +587,10 @@ fn read_paragraphs( text: text_content, tags: content.tags.clone(), }) + } else if matches!(content.kind, Kind::Image { .. }) { + ParagraphContent::Image(Image::read(revision, *content_id, content)?) + } else if matches!(content.kind, Kind::Attachment { .. }) { + ParagraphContent::Attachment(Attachment::read(revision, *content_id, content)?) } else if matches!(content.kind, Kind::Table { .. }) { ParagraphContent::Table(read_table( revision, @@ -515,7 +613,9 @@ fn read_paragraphs( .chain(match &content { ParagraphContent::Text(text) => text.tags.as_slice(), ParagraphContent::Table(table) => table.tags.as_slice(), - ParagraphContent::Unsupported(_) => &[], + ParagraphContent::Image(_) + | ParagraphContent::Attachment(_) + | ParagraphContent::Unsupported(_) => &[], }) .filter_map(|tag| tag.definition.as_ref()) .map(|id| (id, false)), diff --git a/crates/onestore/src/page/write.rs b/crates/onestore/src/page/write.rs index 099509b4a23b96415abd83b85dfb730c1263a620..b7654c07f92eb4706936f8dd6b1da8cdf1907a7c 100644 --- a/crates/onestore/src/page/write.rs +++ b/crates/onestore/src/page/write.rs @@ -6,7 +6,7 @@ use crate::{ Error, ExGuid, Insertion, ObjectData, OutlineEdit, ParagraphJoin, ParagraphSplit, PropertySets, RevisionIndex, Store, TextAttribute, TreeEdit, Value, document::{Document, Format, Kind, Tag}, - write::{PropertyObject, RevisionEdit, write_revisions}, + write::{PropertyObject, RevisionEdit}, }; use std::{ collections::{BTreeMap, BTreeSet}, @@ -22,6 +22,12 @@ fn invalid(message: &'static str) -> Error { /// A native measurement array: a count byte, `header - 1` reserved bytes, then each value /// in inches. +/// The `{GUID}` form a file-data declaration uses to name a payload in the file-data store. +fn payload_reference(guid: [u8; 16]) -> String { + let id = ExGuid { guid, n: 0 }.to_string(); + format!("{}", id.split(',').next().unwrap()) +} + fn measurement_bytes(values: &[f32], header: usize) -> Result, Error> { let count = u8::try_from(values.len()) .map_err(|_| invalid("A measurement array exceeds the document range"))?; @@ -379,6 +385,16 @@ impl Lowering<'_> { } } (ParagraphContent::Unsupported(a), ParagraphContent::Unsupported(b)) if a == b => {} + (ParagraphContent::Image(a), ParagraphContent::Image(b)) => { + if a != b { + return Err(invalid("A stored picture cannot be edited")); + } + } + (ParagraphContent::Attachment(a), ParagraphContent::Attachment(b)) => { + if a != b { + return Err(invalid("A stored attachment cannot be edited")); + } + } _ => return Err(invalid("Paragraph content type cannot change")), } } @@ -609,10 +625,205 @@ impl Lowering<'_> { }); self.place_containers(old, new, placed, &existing)?; self.edit_table_structure(old, new)?; + self.edit_images(old, new)?; + self.edit_attachments(old, new)?; self.place_containers(old, new, placed, &deferred)?; Ok(()) } + /// Gives each new attachment paragraph what OneNote stores for an inserted file: the + /// payload embedded in the file-data store, an embedded-file container declaring it, + /// and an attachment object naming the file that the paragraph holds as content. + fn edit_attachments(&mut self, old: &View<'_>, new: &View<'_>) -> Result<(), Error> { + for (paragraph_id, paragraph) in &new.paragraphs { + let ParagraphContent::Attachment(attachment) = ¶graph.content else { + continue; + }; + if let Some(previous) = old.paragraphs.get(paragraph_id) { + if previous.content != paragraph.content { + return Err(invalid("A stored attachment cannot be edited")); + } + continue; + } + let Some(bytes) = &attachment.bytes else { + return Err(invalid("A new attachment needs its payload")); + }; + let name = attachment.filename.as_str(); + if name.is_empty() || name.contains(['\0', '/', '\\']) { + return Err(invalid( + "An attachment needs a file name without path separators", + )); + } + let extension = name + .rfind('.') + .filter(|dot| *dot > 0) + .map(|dot| &name[dot..]) + .unwrap_or(""); + let attachment_id = self.allocate(attachment.id)?; + let file_id = ExGuid { + guid: crate::write::fresh_guid()?, + n: 1, + }; + let payload_guid = crate::write::fresh_guid()?; + let reference = payload_reference(payload_guid); + let modified = crate::create::current_timestamps()?.0.to_le_bytes(); + let mut values: Values = vec![(0x14001d7a, modified.to_vec())]; + if let Some([width, height]) = attachment.size { + if !(width.is_finite() && height.is_finite() && width > 0.0 && height > 0.0) { + return Err(invalid("Attachment icon size must be positive")); + } + values.push((0x140034cd, (width / 36.0).to_le_bytes().to_vec())); + values.push((0x140034ce, (height / 36.0).to_le_bytes().to_vec())); + } + values.push((0x14001c3b, 0x409_u32.to_le_bytes().to_vec())); + values.push((0x10001cfe, 0x409_u16.to_le_bytes().to_vec())); + values.push((0x1c001dcf, vec![0; 32])); + values.push(( + 0x1c001d61, + [16u32, 1, 0, 0, 0] + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect(), + )); + values.push((0x14001c3e, 1u32.to_le_bytes().to_vec())); + values.push((0x14001c84, 1u32.to_le_bytes().to_vec())); + values.push((0x1c001c22, crate::create::string(name))); + values.push((0x1c001d9c, crate::create::string(name))); + if let Some(path) = &attachment.source_path { + values.push((0x1c001d9d, crate::create::string(path))); + } + let holder = self.id(*paragraph_id); + let space = self.space; + let mut payloads: Vec<([u8; 16], &[u8])> = vec![(payload_guid, bytes)]; + let mut preview = None; + if let Some(icon) = &attachment.preview { + if !icon.starts_with(&[0x89, b'P', b'N', b'G']) { + return Err(invalid("An attachment preview is a PNG icon")); + } + let guid = crate::write::fresh_guid()?; + payloads.push((guid, icon)); + preview = Some(( + ExGuid { + guid: crate::write::fresh_guid()?, + n: 1, + }, + payload_reference(guid), + )); + } + self.apply(|current| { + crate::write::write_revision_with_payloads(current, space, &payloads, |raw| { + let mut changed = BTreeMap::new(); + let mut file = PropertyObject::file(file_id, &reference, extension)?; + file.jcid = 0x80036; + changed.insert(file_id, file); + let mut node = PropertyObject { + jcid: 0x60035, + bytes: crate::create::properties(&values)?, + global_ids: std::sync::Arc::new(BTreeMap::from([(0, attachment_id.guid)])), + }; + node.reference(attachment_id)?; + let container = node.reference(file_id)?; + node.set(&[(0x20001d9b, &container)])?; + if let Some((icon_id, icon_reference)) = &preview { + changed.insert( + *icon_id, + PropertyObject::file(*icon_id, icon_reference, ".png")?, + ); + let icon = node.reference(*icon_id)?; + node.set(&[(0x20001c3f, &icon)])?; + } + changed.insert(attachment_id, node); + let mut object = PropertyObject::from_object(&raw.objects[&holder])?; + let content = object.reference(attachment_id)?; + object.set(&[(0x24001c1f, &content), (0x14001d7a, &modified)])?; + changed.insert(holder, object); + Ok(changed) + }) + })?; + } + Ok(()) + } + + /// Gives each new picture paragraph what OneNote stores for an inserted picture: the + /// payload embedded in the section's file-data store, a file-data object declaring it + /// by identity and extension, and a picture object the paragraph holds as content. + fn edit_images(&mut self, old: &View<'_>, new: &View<'_>) -> Result<(), Error> { + for (paragraph_id, paragraph) in &new.paragraphs { + let ParagraphContent::Image(image) = ¶graph.content else { + continue; + }; + if let Some(previous) = old.paragraphs.get(paragraph_id) { + if previous.content != paragraph.content { + return Err(invalid("A stored picture cannot be edited")); + } + continue; + } + let Some(bytes) = &image.bytes else { + return Err(invalid("A new picture needs its payload")); + }; + let extension = match bytes.as_ref() { + [0x89, b'P', b'N', b'G', ..] => ".png", + [0xff, 0xd8, 0xff, ..] => ".jpg", + [b'G', b'I', b'F', b'8', ..] => ".gif", + [b'B', b'M', ..] => ".bmp", + _ => return Err(invalid("Choose a PNG, JPEG, GIF or BMP picture")), + }; + let image_id = self.allocate(image.id)?; + let file_id = ExGuid { + guid: crate::write::fresh_guid()?, + n: 1, + }; + let payload_guid = crate::write::fresh_guid()?; + let reference = payload_reference(payload_guid); + let modified = crate::create::current_timestamps()?.0.to_le_bytes(); + let mut values: Values = vec![(0x14001d7a, modified.to_vec())]; + if let Some([width, height]) = image.size { + if !(width.is_finite() && height.is_finite() && width > 0.0 && height > 0.0) { + return Err(invalid("Picture size must be positive")); + } + values.push((0x140034cd, (width / 36.0).to_le_bytes().to_vec())); + values.push((0x140034ce, (height / 36.0).to_le_bytes().to_vec())); + } + if let Some(alt) = &image.alt { + values.push((0x1c001e58, crate::create::string(alt))); + } + if image.background { + values.push((0x08001d13 | (1 << 31), Vec::new())); + } + values.push((0x08001d85, Vec::new())); + let holder = self.id(*paragraph_id); + let space = self.space; + let payload: &[u8] = bytes; + self.apply(|current| { + crate::write::write_revision_with_payloads( + current, + space, + &[(payload_guid, payload)], + |raw| { + let mut changed = BTreeMap::new(); + let file = PropertyObject::file(file_id, &reference, extension)?; + changed.insert(file_id, file); + let mut picture = PropertyObject { + jcid: 0x60011, + bytes: crate::create::properties(&values)?, + global_ids: std::sync::Arc::new(BTreeMap::from([(0, image_id.guid)])), + }; + picture.reference(image_id)?; + let container = picture.reference(file_id)?; + picture.set(&[(0x20001c3f, &container)])?; + changed.insert(image_id, picture); + let mut object = PropertyObject::from_object(&raw.objects[&holder])?; + let content = object.reference(image_id)?; + object.set(&[(0x24001c1f, &content), (0x14001d7a, &modified)])?; + changed.insert(holder, object); + Ok(changed) + }, + ) + })?; + } + Ok(()) + } + fn place_containers( &mut self, old: &View<'_>, @@ -648,7 +859,9 @@ impl Lowering<'_> { ParagraphContent::Text(text) if text.date_field.is_none() => { (text.text.text(), Some(text.id)) } - ParagraphContent::Table(_) => ("", None), + ParagraphContent::Table(_) + | ParagraphContent::Image(_) + | ParagraphContent::Attachment(_) => ("", None), _ => { return Err(invalid( "New paragraphs contain plain text without fields or styles", @@ -1737,7 +1950,25 @@ fn squash( .collect(); let applied_store = Store::parse(applied)?; let applied_index = RevisionIndex::parse(&applied_store)?; - write_revisions(source, |index| { + // Payloads the typed edits embedded travel into the squashed transaction as well. + let source_store = Store::parse(source)?; + let declared = |store: &Store<'_>| -> Vec<[u8; 16]> { + store + .lists + .values() + .flat_map(|list| &list.nodes) + .filter(|node| node.id == 0x94) + .filter_map(|node| node.payload.get(..16).and_then(|g| g.try_into().ok())) + .collect() + }; + let existing = declared(&source_store); + let mut payloads = Vec::new(); + for guid in declared(&applied_store) { + if !existing.contains(&guid) { + payloads.push((guid, applied_store.file_data(guid)?)); + } + } + crate::write::write_revisions_with_payloads(source, &payloads, |index| { let mut changes = BTreeMap::new(); for sid in applied_index.spaces.keys() { let Some(space) = index.spaces.get(sid) else { @@ -1766,12 +1997,35 @@ fn squash( }) { continue; } - let ObjectData::Properties(_) = object.data else { - return Err(invalid("Page edits only produce property objects")); - }; - let mut replacement = PropertyObject::from_object(object)?; - remap(&mut replacement, &rename)?; let id = rename.get(id).copied().unwrap_or(*id); + let mut replacement = match object.data { + ObjectData::Properties(_) => { + let mut replacement = PropertyObject::from_object(object)?; + remap(&mut replacement, &rename)?; + replacement + } + ObjectData::File { + reference, + extension, + } => { + let text = |bytes: &[u8]| { + String::from_utf16( + &bytes + .chunks_exact(2) + .map(|pair| u16::from_le_bytes([pair[0], pair[1]])) + .collect::>(), + ) + .map_err(|_| invalid("Invalid UTF-16 file-data declaration")) + }; + let mut replacement = + PropertyObject::file(id, &text(reference)?, &text(extension)?)?; + replacement.jcid = object.jcid; + replacement + } + ObjectData::Encrypted(_) => { + return Err(invalid("Page edits only produce property objects")); + } + }; replacement.reference(id)?; if before.objects.contains_key(&id) && rename.values().any(|model| *model == id) { return Err(invalid( diff --git a/crates/onestore/src/write.rs b/crates/onestore/src/write.rs index 72662adc7232ab16d41f1de295fe9b1fd0f49727..b3085fcb2c4719e15eacdebfbab7b3d681cbb83a 100644 --- a/crates/onestore/src/write.rs +++ b/crates/onestore/src/write.rs @@ -353,6 +353,15 @@ pub(crate) fn replace_objects( }) } +/// The object type OneNote gives embedded picture payload declarations; embedded files +/// use `EMBEDDED_FILE_JCID` with the same declaration shape. +pub(crate) const FILE_DATA_JCID: u32 = 0x80039; +pub(crate) const EMBEDDED_FILE_JCID: u32 = 0x80036; + +fn is_file_declaration(jcid: u32) -> bool { + jcid == FILE_DATA_JCID || jcid == EMBEDDED_FILE_JCID +} + pub(crate) struct PropertyObject { pub jcid: u32, pub bytes: Vec, @@ -368,6 +377,51 @@ pub(crate) enum RevisionEdit { } impl PropertyObject { + /// A file-data object declaring an embedded payload by its store identity, or an + /// external payload by name; `extension` includes its leading dot. + pub fn file(id: ExGuid, reference: &str, extension: &str) -> Result { + if reference.is_empty() || reference.contains('\0') || extension.contains('\0') { + return Err(Error { + offset: 0, + message: "File-data references and extensions are nonempty and contain no NUL", + }); + } + let mut bytes = Vec::new(); + for text in [reference, extension] { + let encoded: Vec = text.encode_utf16().flat_map(u16::to_le_bytes).collect(); + bytes.extend_from_slice(&u32::try_from(encoded.len()).unwrap().to_le_bytes()); + bytes.extend_from_slice(&encoded); + } + Ok(Self { + jcid: FILE_DATA_JCID, + bytes, + global_ids: Arc::new(BTreeMap::from([(0, id.guid)])), + }) + } + + /// The reference and extension of a file-data declaration built by `file`. + fn file_declaration(&self) -> Result<(&[u8], &[u8])> { + let malformed = || Error { + offset: 0, + message: "Malformed file-data declaration", + }; + let mut rest = self.bytes.as_slice(); + let mut parts = Vec::new(); + for _ in 0..2 { + let length = usize::try_from(u32::from_le_bytes( + rest.get(..4).ok_or_else(malformed)?.try_into().unwrap(), + )) + .map_err(|_| malformed())?; + let bytes = rest.get(4..4 + length).ok_or_else(malformed)?; + parts.push(bytes); + rest = &rest[4 + length..]; + } + if !rest.is_empty() { + return Err(malformed()); + } + Ok((parts[0], parts[1])) + } + pub fn from_object(object: &crate::Object<'_>) -> Result { let ObjectData::Properties(bytes) = object.data else { return Err(Error { @@ -649,7 +703,16 @@ pub(crate) fn write_revision( space: ExGuid, edit: impl FnOnce(&crate::ResolvedRevision<'_>) -> Result>, ) -> Result> { - write_revisions(source, |index| { + write_revision_with_payloads(source, space, &[], edit) +} + +pub(crate) fn write_revision_with_payloads( + source: &[u8], + space: ExGuid, + payloads: &[([u8; 16], &[u8])], + edit: impl FnOnce(&crate::ResolvedRevision<'_>) -> Result>, +) -> Result> { + write_revisions_with_payloads(source, payloads, |index| { let rid = index.active(space)?; let revision = index.resolve(space, rid)?; Ok(BTreeMap::from([( @@ -695,6 +758,17 @@ fn append_fragment( pub(crate) fn write_revisions( source: &[u8], edit: impl FnOnce(&RevisionIndex<'_>) -> Result>, +) -> Result> { + write_revisions_with_payloads(source, &[], edit) +} + +/// `write_revisions` that also stores embedded payloads: each becomes a file-data store +/// object referenced from the root file node list under its identity, as OneNote embeds +/// pictures and attachments. +pub(crate) fn write_revisions_with_payloads( + source: &[u8], + payloads: &[([u8; 16], &[u8])], + edit: impl FnOnce(&RevisionIndex<'_>) -> Result>, ) -> Result> { let store = Store::parse(source)?; let is_section = store.header.file_type == FileType::Section; @@ -780,16 +854,29 @@ pub(crate) fn write_revisions( message: "New objects require a section file", }); } - if replacement.jcid & 0x20000 == 0 - || replacement.global_ids.keys().any(|i| *i > 0xffffff) - { + if replacement.global_ids.keys().any(|i| *i > 0xffffff) { return Err(Error { offset: 0, message: "Invalid property object declaration", }); } compact(*id, &replacement.global_ids)?; - PropertySets::parse(&replacement.bytes)?; + if is_file_declaration(replacement.jcid) { + if !is_section { + return Err(Error { + offset: 0, + message: "File-data objects require a section file", + }); + } + replacement.file_declaration()?; + } else if replacement.jcid & 0x20000 == 0 { + return Err(Error { + offset: 0, + message: "Invalid property object declaration", + }); + } else { + PropertySets::parse(&replacement.bytes)?; + } } // Native coalescing of duplicate readonly styles can leave dangling references. let mut aliases = BTreeMap::new(); @@ -824,6 +911,9 @@ pub(crate) fn write_revisions( replacements.remove(id); } for object in replacements.values_mut() { + if is_file_declaration(object.jcid) { + continue; + } let mut remapped = Vec::new(); for property in PropertySets::parse(&object.bytes)?.sets.iter().flatten() { if let Value::References { @@ -859,12 +949,21 @@ pub(crate) fn write_revisions( } let mut changed: BTreeSet<_> = replacements.keys().copied().collect(); for (id, replacement) in &replacements { + let data = if is_file_declaration(replacement.jcid) { + let (reference, extension) = replacement.file_declaration()?; + ObjectData::File { + reference, + extension, + } + } else { + ObjectData::Properties(&replacement.bytes) + }; revision.objects.insert( *id, crate::Object { jcid: replacement.jcid, reference_count: 0, - data: ObjectData::Properties(&replacement.bytes), + data, global_ids: Arc::clone(&replacement.global_ids), }, ); @@ -1139,6 +1238,55 @@ pub(crate) fn write_revisions( )?); } } + let mut data_nodes = Vec::new(); + for (guid, payload) in payloads { + if !is_section { + return Err(Error { + offset: 0, + message: "Embedded payloads require a section file", + }); + } + let mut blob = vec![ + 0xe7, 0x16, 0xe3, 0xbd, 0x65, 0x26, 0x11, 0x45, 0xa4, 0xc4, 0x8d, 0x4d, 0x0b, 0x7a, + 0x9e, 0xac, + ]; + blob.extend_from_slice(&(payload.len() as u64).to_le_bytes()); + blob.extend_from_slice(&[0; 12]); + blob.extend_from_slice(payload); + blob.resize(blob.len().next_multiple_of(8), 0); + blob.extend_from_slice(&[ + 0x22, 0xa7, 0xfb, 0x71, 0x79, 0x0f, 0x0b, 0x4a, 0xbb, 0x13, 0x89, 0x92, 0x56, 0x42, + 0x6b, 0x24, + ]); + let chunk = append(&mut output, &blob)?; + data_nodes.push(node(0x94, Some(Reference::Data(chunk)), guid)?); + } + if !data_nodes.is_empty() { + // Payload declarations live in the file-data store list the root list references. + let root = store.list(store.header.root)?; + match root.nodes.iter().find(|node| node.id == 0x90) { + Some(reference) => { + let Some(Reference::NodeList(chunk)) = reference.reference else { + return Err(Error { + offset: reference.offset, + message: "File-data store reference lacks a list", + }); + }; + counts.push(append_fragment( + source, + &mut output, + store.list(chunk)?, + &data_nodes, + )?); + } + None => { + let list_id = allocate_list()?; + let chunk = append_list(&mut output, list_id, &data_nodes)?; + counts.push((list_id, data_nodes.len())); + root_nodes.push(node(0x90, Some(Reference::NodeList(chunk)), &[])?); + } + } + } if !root_nodes.is_empty() { counts.push(append_fragment( source, diff --git a/crates/onestore/tests/page_attachments.rs b/crates/onestore/tests/page_attachments.rs new file mode 100644 index 0000000000000000000000000000000000000000..5feeaa00824a57094e105c775e36467ed1be38dd --- /dev/null +++ b/crates/onestore/tests/page_attachments.rs @@ -0,0 +1,203 @@ +use onestore::{ + ExGuid, PreparedEdit, RevisionIndex, Store, + document::{Document, Format}, + page::{ + Attachment, Page, PageObject, PageParagraph, ParagraphContent, TextObject, text::new_id, + }, +}; +use std::sync::Arc; + +const NATIVE: &[u8] = + include_bytes!("../../../corpus/native/cold-05-06-attachment/notebook/synthetic.one"); +const AUTHOR: &str = "Attachment author"; +const PAYLOAD: &[u8] = b"Rust wrote this attachment.\n"; + +fn first_page(bytes: &[u8]) -> (ExGuid, Page) { + let store = Store::parse(bytes).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let (space, _) = document.pages().unwrap()[0]; + (space, Page::from_space(&document, space).unwrap()) +} + +fn page_in(bytes: &[u8], space: ExGuid) -> Page { + let store = Store::parse(bytes).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + Page::from_space(&Document::parse(&index).unwrap(), space).unwrap() +} + +fn body_paragraphs(page: &mut Page) -> &mut Vec { + page.objects + .iter_mut() + .find_map(|object| match object { + PageObject::Outline(outline) if !outline.title => Some(&mut outline.paragraphs), + _ => None, + }) + .unwrap() +} + +fn attachments(page: &Page) -> Vec<&Attachment> { + page.objects + .iter() + .filter_map(|object| match object { + PageObject::Outline(outline) => Some(outline), + _ => None, + }) + .flat_map(|outline| outline.paragraphs.iter()) + .filter_map(|p| match &p.content { + ParagraphContent::Attachment(attachment) => Some(attachment), + _ => None, + }) + .collect() +} + +fn plain_paragraph(template: &PageParagraph, text: &str) -> PageParagraph { + let mut paragraph = template.clone(); + paragraph.id = new_id().unwrap(); + paragraph.parent = None; + paragraph.level = 1; + paragraph.lists.clear(); + paragraph.tags.clear(); + paragraph.style = None; + paragraph.collapsed = false; + paragraph.content = ParagraphContent::Text(TextObject { + id: new_id().unwrap(), + date_field: None, + text: onestore::page::Paragraph::new( + text.into(), + Format { + font: Some("Calibri".into()), + font_size: Some(11.0), + language: Some(1033), + ..Default::default() + }, + ), + tags: Vec::new(), + }); + paragraph +} + +#[test] +fn a_native_attachment_reads_as_paragraph_content_with_payload_and_preview() { + let (_, page) = first_page(NATIVE); + let found = attachments(&page); + let [attachment] = found.as_slice() else { + panic!("{page:?}"); + }; + assert_eq!(attachment.filename, "fictitious-attachment.txt"); + assert!( + attachment + .source_path + .as_deref() + .unwrap() + .ends_with("fictitious-attachment.txt") + ); + assert_eq!(attachment.bytes.as_ref().map(|b| b.len()), Some(43)); + assert_eq!(attachment.preview.as_ref().map(|b| b.len()), Some(724)); + assert_eq!(attachment.size, Some([24.0, 24.0])); +} + +fn insert_attachment(section: &str, preview: Option>) -> Vec { + let source = onestore::create_section(section, "Before the file", "Author").unwrap(); + let (space, before) = first_page(&source); + let mut after = before.clone(); + let template = body_paragraphs(&mut after)[0].clone(); + let mut holder = plain_paragraph(&template, ""); + holder.format = Format::default(); + holder.content = ParagraphContent::Attachment(Attachment { + id: new_id().unwrap(), + filename: "notes 🦀.txt".into(), + source_path: None, + size: Some([24.0, 24.0]), + bytes: Some(Arc::from(PAYLOAD)), + preview: preview.clone(), + }); + body_paragraphs(&mut after).push(holder.clone()); + body_paragraphs(&mut after).push(plain_paragraph(&template, "After the file")); + let written = PreparedEdit::page(&source, space, &after, AUTHOR).unwrap(); + let stored = page_in(written.as_bytes(), space); + let mut expected = after.clone(); + expected.title = stored.title.clone(); + assert_eq!(stored, expected); + let found = attachments(&stored); + let [stored_attachment] = found.as_slice() else { + panic!() + }; + assert_eq!(stored_attachment.bytes.as_deref(), Some(PAYLOAD)); + assert_eq!(stored_attachment.preview, preview); + assert_eq!( + PreparedEdit::page(written.as_bytes(), space, &stored, AUTHOR) + .unwrap() + .as_bytes(), + written.as_bytes() + ); + let mut removed = stored.clone(); + body_paragraphs(&mut removed).retain(|p| p.id != holder.id); + let again = PreparedEdit::page(written.as_bytes(), space, &removed, AUTHOR).unwrap(); + assert!(attachments(&page_in(again.as_bytes(), space)).is_empty()); + written.as_bytes().to_vec() +} + +/// `ONESTORE_ATTACHMENT_EXPORT` names a directory receiving both candidates for a cold reopen: +/// `plain/` without an icon preview and `icon/` with the native fixture's icon. +#[test] +fn an_attachment_inserted_on_a_fresh_page_reads_back_and_can_be_removed() { + let (_, native) = first_page(NATIVE); + let icon = attachments(&native)[0].preview.clone().unwrap(); + let plain = insert_attachment("files.one", None); + let with_icon = insert_attachment("icon.one", Some(icon)); + if let Some(directory) = std::env::var_os("ONESTORE_ATTACHMENT_EXPORT") { + let directory = std::path::PathBuf::from(directory); + for (name, written) in [("plain", &plain), ("icon", &with_icon)] { + let directory = directory.join(name); + std::fs::create_dir_all(&directory).unwrap(); + std::fs::write(directory.join("files.one"), written).unwrap(); + let file_id = Store::parse(written).unwrap().header.file_id; + std::fs::write( + directory.join("Open Notebook.onetoc2"), + onestore::create_table_of_contents( + "Open Notebook.onetoc2", + &[("files.one", file_id)], + ) + .unwrap(), + ) + .unwrap(); + } + } +} + +#[test] +fn attachments_need_a_file_name_and_stored_ones_stay_fixed() { + let source = onestore::create_section("files.one", "Text", "Author").unwrap(); + let (space, before) = first_page(&source); + let mut after = before.clone(); + let template = body_paragraphs(&mut after)[0].clone(); + let mut holder = plain_paragraph(&template, ""); + holder.content = ParagraphContent::Attachment(Attachment { + id: new_id().unwrap(), + filename: "sub/dir.txt".into(), + source_path: None, + size: None, + bytes: Some(Arc::from(PAYLOAD)), + preview: None, + }); + body_paragraphs(&mut after).push(holder); + assert!(PreparedEdit::page(&source, space, &after, AUTHOR).is_err()); + let (space, native) = first_page(NATIVE); + let mut renamed = native.clone(); + let mut found = 0; + for object in &mut renamed.objects { + if let PageObject::Outline(outline) = object { + for paragraph in &mut outline.paragraphs { + if let ParagraphContent::Attachment(attachment) = &mut paragraph.content { + attachment.filename = "renamed.txt".into(); + found += 1; + } + } + } + } + assert_eq!(found, 1); + assert!(PreparedEdit::page(NATIVE, space, &renamed, AUTHOR).is_err()); +} diff --git a/crates/onestore/tests/page_images.rs b/crates/onestore/tests/page_images.rs new file mode 100644 index 0000000000000000000000000000000000000000..44e7ea1a603e2741f8c0b45145c39f4bfb13a848 --- /dev/null +++ b/crates/onestore/tests/page_images.rs @@ -0,0 +1,190 @@ +use onestore::{ + ExGuid, PreparedEdit, RevisionIndex, Store, + document::{Document, Format}, + page::{Image, Page, PageObject, PageParagraph, ParagraphContent, TextObject, text::new_id}, +}; +use std::sync::Arc; + +const NATIVE: &[u8] = + include_bytes!("../../../corpus/native/cold-05-05-image/notebook/synthetic.one"); +const AUTHOR: &str = "Picture author"; +/// A one-pixel PNG. +const PNG: &[u8] = &[ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53, + 0xde, 0x00, 0x00, 0x00, 0x0c, 0x49, 0x44, 0x41, 0x54, 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0x00, + 0x00, 0x03, 0x01, 0x01, 0x00, 0x18, 0xdd, 0x8d, 0xb0, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4e, + 0x44, 0xae, 0x42, 0x60, 0x82, +]; + +fn first_page(bytes: &[u8]) -> (ExGuid, Page) { + let store = Store::parse(bytes).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let (space, _) = document.pages().unwrap()[0]; + (space, Page::from_space(&document, space).unwrap()) +} + +fn page_in(bytes: &[u8], space: ExGuid) -> Page { + let store = Store::parse(bytes).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + Page::from_space(&Document::parse(&index).unwrap(), space).unwrap() +} + +fn body_paragraphs(page: &mut Page) -> &mut Vec { + page.objects + .iter_mut() + .find_map(|object| match object { + PageObject::Outline(outline) if !outline.title => Some(&mut outline.paragraphs), + _ => None, + }) + .unwrap() +} + +fn pictures(page: &Page) -> Vec<&Image> { + page.objects + .iter() + .filter_map(|object| match object { + PageObject::Outline(outline) => Some(outline), + _ => None, + }) + .flat_map(|outline| outline.paragraphs.iter()) + .filter_map(|p| match &p.content { + ParagraphContent::Image(image) => Some(image), + _ => None, + }) + .collect() +} + +fn plain_paragraph(template: &PageParagraph, text: &str) -> PageParagraph { + let mut paragraph = template.clone(); + paragraph.id = new_id().unwrap(); + paragraph.parent = None; + paragraph.level = 1; + paragraph.lists.clear(); + paragraph.tags.clear(); + paragraph.style = None; + paragraph.collapsed = false; + paragraph.content = ParagraphContent::Text(TextObject { + id: new_id().unwrap(), + date_field: None, + text: onestore::page::Paragraph::new( + text.into(), + Format { + font: Some("Calibri".into()), + font_size: Some(11.0), + language: Some(1033), + ..Default::default() + }, + ), + tags: Vec::new(), + }); + paragraph +} + +#[test] +fn a_native_picture_reads_as_paragraph_content_with_its_payload() { + let (_, page) = first_page(NATIVE); + let found = pictures(&page); + let [picture] = found.as_slice() else { + panic!("{page:?}"); + }; + assert_eq!(picture.bytes.as_ref().map(|b| b.len()), Some(68)); + assert!(picture.bytes.as_ref().unwrap().starts_with(&PNG[..8])); + assert_eq!(picture.size, Some([0.75, 0.75])); +} + +/// `ONESTORE_IMAGE_EXPORT` names a new directory receiving the candidate for a cold reopen. +#[test] +fn a_picture_inserted_on_a_fresh_page_reads_back_and_can_be_removed() { + let source = onestore::create_section("pictures.one", "Before the picture", "Author").unwrap(); + let (space, before) = first_page(&source); + let mut after = before.clone(); + let template = body_paragraphs(&mut after)[0].clone(); + let mut holder = plain_paragraph(&template, ""); + holder.format = Format::default(); + let image = Image { + id: new_id().unwrap(), + layout: Default::default(), + size: Some([0.75, 0.75]), + bytes: Some(Arc::from(PNG)), + alt: None, + background: false, + }; + holder.content = ParagraphContent::Image(image.clone()); + body_paragraphs(&mut after).push(holder.clone()); + body_paragraphs(&mut after).push(plain_paragraph(&template, "After the picture")); + let written = PreparedEdit::page(&source, space, &after, AUTHOR).unwrap(); + let stored = page_in(written.as_bytes(), space); + let mut expected = after.clone(); + expected.title = stored.title.clone(); + assert_eq!(stored, expected); + let found = pictures(&stored); + let [picture] = found.as_slice() else { + panic!() + }; + assert_eq!(picture.bytes.as_deref(), Some(PNG)); + assert_eq!( + PreparedEdit::page(written.as_bytes(), space, &stored, AUTHOR) + .unwrap() + .as_bytes(), + written.as_bytes() + ); + if let Some(directory) = std::env::var_os("ONESTORE_IMAGE_EXPORT") { + let directory = std::path::PathBuf::from(directory); + std::fs::create_dir(&directory).unwrap(); + std::fs::write(directory.join("pictures.one"), written.as_bytes()).unwrap(); + let written_store = Store::parse(written.as_bytes()).unwrap(); + std::fs::write( + directory.join("Open Notebook.onetoc2"), + onestore::create_table_of_contents( + "Open Notebook.onetoc2", + &[("pictures.one", written_store.header.file_id)], + ) + .unwrap(), + ) + .unwrap(); + } + let mut removed = stored.clone(); + body_paragraphs(&mut removed).retain(|p| p.id != holder.id); + let again = PreparedEdit::page(written.as_bytes(), space, &removed, AUTHOR).unwrap(); + let stored = page_in(again.as_bytes(), space); + assert!(pictures(&stored).is_empty()); + assert_eq!(body_paragraphs(&mut stored.clone()).len(), 2); +} + +#[test] +fn pictures_need_a_recognised_payload_and_stored_ones_stay_fixed() { + let source = onestore::create_section("pictures.one", "Text", "Author").unwrap(); + let (space, before) = first_page(&source); + let mut after = before.clone(); + let template = body_paragraphs(&mut after)[0].clone(); + let mut holder = plain_paragraph(&template, ""); + holder.content = ParagraphContent::Image(Image { + id: new_id().unwrap(), + layout: Default::default(), + size: None, + bytes: Some(Arc::from(b"not a picture".as_slice())), + alt: None, + background: false, + }); + body_paragraphs(&mut after).push(holder); + assert!(PreparedEdit::page(&source, space, &after, AUTHOR).is_err()); + let (space, native) = first_page(NATIVE); + let mut resized = native.clone(); + let mut found = 0; + for object in &mut resized.objects { + if let PageObject::Outline(outline) = object { + for paragraph in &mut outline.paragraphs { + if let ParagraphContent::Image(image) = &mut paragraph.content { + image.size = Some([10.0, 10.0]); + found += 1; + } + } + } + } + assert_eq!(found, 1); + assert!(PreparedEdit::page(NATIVE, space, &resized, AUTHOR).is_err()); +} diff --git a/crates/onestore/tests/page_model.rs b/crates/onestore/tests/page_model.rs index 2541b51c07785ad12012d5f874e5182cb66242a6..527b69bbdf60b17f68bb8235afd3d8db5632dca5 100644 --- a/crates/onestore/tests/page_model.rs +++ b/crates/onestore/tests/page_model.rs @@ -38,6 +38,8 @@ fn identities(page: &Page) -> Vec { } } } + ParagraphContent::Image(image) => ids.push(image.id), + ParagraphContent::Attachment(attachment) => ids.push(attachment.id), ParagraphContent::Unsupported(unsupported) => ids.push(unsupported.id), } } diff --git a/crates/onestore/tests/page_tables.rs b/crates/onestore/tests/page_tables.rs index cf91ec2dbf79281159c6cf07428341f9483d4dbf..5c9e8207172193b26225f7724a457c13662d878b 100644 --- a/crates/onestore/tests/page_tables.rs +++ b/crates/onestore/tests/page_tables.rs @@ -72,7 +72,9 @@ fn referenced(page: &Page) -> std::collections::BTreeSet { } } } - ParagraphContent::Unsupported(_) => {} + ParagraphContent::Image(_) + | ParagraphContent::Attachment(_) + | ParagraphContent::Unsupported(_) => {} } } } diff --git a/crates/onestore/tests/support/page_edits.rs b/crates/onestore/tests/support/page_edits.rs index f9d7ef96669d76957ae403c9bba307f0e6a9da78..831af3719600e739f6ea25d014bb697d7607dfd9 100644 --- a/crates/onestore/tests/support/page_edits.rs +++ b/crates/onestore/tests/support/page_edits.rs @@ -123,6 +123,12 @@ fn projection(page: &Page) -> String { ParagraphContent::Table(table) => { out.push_str(&format!(" table {}", table.id)) } + ParagraphContent::Image(image) => { + out.push_str(&format!(" image {}", image.id)) + } + ParagraphContent::Attachment(attachment) => { + out.push_str(&format!(" attachment {}", attachment.id)) + } ParagraphContent::Unsupported(u) => { out.push_str(&format!(" unsupported {}", u.id)) } diff --git a/tools/test_attachment_edit.py b/tools/test_attachment_edit.py new file mode 100644 index 0000000000000000000000000000000000000000..e257c7611eb8941b6a056d4a47ae6e5576e4cec1 --- /dev/null +++ b/tools/test_attachment_edit.py @@ -0,0 +1,36 @@ +from pathlib import Path +import re +import runpy +import shutil +from tempfile import TemporaryDirectory +import unittest +import xml.etree.ElementTree as ET + +from native_xml import ns + +ROOT = Path(__file__).resolve().parent.parent +FIXTURE = ROOT / 'corpus/attachment-edit' +compare = runpy.run_path(str(ROOT / 'tools/verify-document.py'))['compare'] + + +class AttachmentEditTest(unittest.TestCase): + def check(self, variant): + with TemporaryDirectory() as temporary: + read = Path(temporary) / 'read' + shutil.copytree(FIXTURE / variant / 'cold/read', read) + compare(FIXTURE / variant / 'candidate', read) + page, = (ET.parse(path).getroot() for path in sorted((FIXTURE / variant / 'cold/read').glob('page-*.xml'))) + inserted, = page.iter('{%s}InsertedFile' % ns['one']) + self.assertEqual(inserted.get('preferredName'), 'notes 🦀.txt') + texts = [re.sub(r'<[^>]*>', '', t.text or '') for t in page.findall('.//one:OE/one:T', ns)] + self.assertEqual(texts, ['Before the file', 'After the file']) + + def test_a_rust_inserted_attachment_renders_natively_with_its_payload(self): + self.check('plain') + + def test_a_rust_inserted_attachment_with_an_icon_preview_renders_natively(self): + self.check('icon') + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/test_picture_edit.py b/tools/test_picture_edit.py new file mode 100644 index 0000000000000000000000000000000000000000..f2227113405fec3023555b78d436dd16fef85ede --- /dev/null +++ b/tools/test_picture_edit.py @@ -0,0 +1,34 @@ +import base64 +from pathlib import Path +import re +import runpy +import shutil +from tempfile import TemporaryDirectory +import unittest +import xml.etree.ElementTree as ET + +from native_xml import ns + +ROOT = Path(__file__).resolve().parent.parent +FIXTURE = ROOT / 'corpus/picture-edit' +compare = runpy.run_path(str(ROOT / 'tools/verify-document.py'))['compare'] + + +class PictureEditTest(unittest.TestCase): + def test_a_rust_inserted_picture_renders_natively_with_its_payload(self): + with TemporaryDirectory() as temporary: + read = Path(temporary) / 'read' + shutil.copytree(FIXTURE / 'cold/read', read) + compare(FIXTURE / 'candidate', read) + page, = (ET.parse(path).getroot() for path in sorted((FIXTURE / 'cold/read').glob('page-*.xml'))) + image, = page.iter('{%s}Image' % ns['one']) + self.assertEqual(image.get('format'), 'png') + payload = base64.b64decode(image.find('one:Data', ns).text) + self.assertEqual(payload[:8], b'\x89PNG\r\n\x1a\n') + self.assertEqual(len(payload), 69) + texts = [re.sub(r'<[^>]*>', '', t.text or '') for t in page.findall('.//one:OE/one:T', ns)] + self.assertEqual(texts, ['Before the picture', 'After the picture']) + + +if __name__ == '__main__': + unittest.main()