diff --git a/corpus/attachment-edit/README.md b/corpus/attachment-edit/README.md index c6a769a2c5b96c0d89d5c8a0ac19476a575f437d..cb168e20325c7e9341e9e18f62d3776e6e12d523 100644 --- a/corpus/attachment-edit/README.md +++ b/corpus/attachment-edit/README.md @@ -21,3 +21,15 @@ without a VM. Regenerate with `ONESTORE_ATTACHMENT_EXPORT` set to a directory while running the test (it writes `plain/` and `icon/`), then cold-open each with `tools/native_runner.py OUTPUT/plain COLD --expected-pages 1 --collect-notebook`. + +## Renaming a stored attachment + +`renamed/candidate` is the writer's output for +`attachments_need_a_file_name_and_stored_ones_are_renamed_in_place` in +`crates/onestore/tests/page_attachments.rs`: the native attachment of +`corpus/native/cold-05-06-attachment` with its shown name and recorded source +path changed in place (properties `0x1c001c22`, `0x1c001d9c` and +`0x1c001d9d` on the attachment node), payload and preview untouched. +`renamed/cold` is its cold read: `preferredName` and `pathSource` carry the +new values. Regenerate with `ONESTORE_ATTACHMENT_RENAME_EXPORT` and cold-open +with `--expected-pages 1`. diff --git a/corpus/attachment-edit/renamed/candidate/Open Notebook.onetoc2 b/corpus/attachment-edit/renamed/candidate/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..96d7cf292d064465a5f0c577bfb2f67eec013527 Binary files /dev/null and b/corpus/attachment-edit/renamed/candidate/Open Notebook.onetoc2 differ diff --git a/corpus/attachment-edit/renamed/candidate/synthetic.one b/corpus/attachment-edit/renamed/candidate/synthetic.one new file mode 100644 index 0000000000000000000000000000000000000000..c0cc0a52087bcc02f3c098b72f5b72069dfd67a0 Binary files /dev/null and b/corpus/attachment-edit/renamed/candidate/synthetic.one differ diff --git a/corpus/attachment-edit/renamed/cold/commands.jsonl b/corpus/attachment-edit/renamed/cold/commands.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e1f9f4e16cb3b0c0f8a00fafe9f0470de3bc08be --- /dev/null +++ b/corpus/attachment-edit/renamed/cold/commands.jsonl @@ -0,0 +1,3 @@ +{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-0A713CC8 -ExpectedPages 1", "exit": 0, "stdout": "Read 1 sections and 1 pages.\r\n", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null} diff --git a/corpus/attachment-edit/renamed/cold/machine.json b/corpus/attachment-edit/renamed/cold/machine.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1bab0ce7d6c1cf413b1ec8932880549d8fa5e4 --- /dev/null +++ b/corpus/attachment-edit/renamed/cold/machine.json @@ -0,0 +1 @@ +{"name": "m6-0a713cc8", "hostname": "ONE-M6-0A713CC8"} diff --git a/corpus/attachment-edit/renamed/cold/notebook/Open Notebook.onetoc2 b/corpus/attachment-edit/renamed/cold/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..180ba6770401181374cac8454d062d1a295728e3 Binary files /dev/null and b/corpus/attachment-edit/renamed/cold/notebook/Open Notebook.onetoc2 differ diff --git a/corpus/attachment-edit/renamed/cold/notebook/synthetic.one b/corpus/attachment-edit/renamed/cold/notebook/synthetic.one new file mode 100644 index 0000000000000000000000000000000000000000..f36451aa996243bb0cba272c4e45a7be8187c24c Binary files /dev/null and b/corpus/attachment-edit/renamed/cold/notebook/synthetic.one differ diff --git a/corpus/attachment-edit/renamed/cold/read/af1af4b8987109ce1e335e538910fb3860b7c57eb3b6110a06aa2114e06d69a7.attachment b/corpus/attachment-edit/renamed/cold/read/af1af4b8987109ce1e335e538910fb3860b7c57eb3b6110a06aa2114e06d69a7.attachment new file mode 100644 index 0000000000000000000000000000000000000000..66b6151e91f90df6a61a19ef2f4447da278e07b2 --- /dev/null +++ b/corpus/attachment-edit/renamed/cold/read/af1af4b8987109ce1e335e538910fb3860b7c57eb3b6110a06aa2114e06d69a7.attachment @@ -0,0 +1 @@ +Fictitious attachment for native corpus. \ No newline at end of file diff --git a/corpus/attachment-edit/renamed/cold/read/environment.json b/corpus/attachment-edit/renamed/cold/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..0c16a3301cf381b1a9601bce5d23f4fc1d74fe4d --- /dev/null +++ b/corpus/attachment-edit/renamed/cold/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-0A713CC8", + "cold": true, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/attachment-edit/renamed/cold/read/hierarchy.xml b/corpus/attachment-edit/renamed/cold/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..07947089e33a06eb8138a88efc4f064b7df9dcb8 --- /dev/null +++ b/corpus/attachment-edit/renamed/cold/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/attachment-edit/renamed/cold/read/page-000.xml b/corpus/attachment-edit/renamed/cold/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..d24863d517c8e1def63026be0bb4c21aa08fd864 --- /dev/null +++ b/corpus/attachment-edit/renamed/cold/read/page-000.xml @@ -0,0 +1,8 @@ + +Fictitious: café, 東京, مرحبا]]>iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9J1uoA +AAAASUVORK5CYII= diff --git a/corpus/attachment-edit/renamed/cold/read/payloads.json b/corpus/attachment-edit/renamed/cold/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..ba0dea930c14a7ec44406df59e15ed3618073fbc --- /dev/null +++ b/corpus/attachment-edit/renamed/cold/read/payloads.json @@ -0,0 +1,10 @@ +[ + { + "sha256": "af1af4b8987109ce1e335e538910fb3860b7c57eb3b6110a06aa2114e06d69a7", + "name": "renamed 🦀.txt", + "object": "{4E773A8B-F556-4F2B-AA89-7C345FD361D2}{11}{B0}", + "kind": "InsertedFile", + "page": "{C7383ECD-2D94-4CA8-AC10-BE6DDCFFE747}{1}{B0}", + "bytes": 43 + } +] \ No newline at end of file diff --git a/corpus/attachment-edit/renamed/cold/run.json b/corpus/attachment-edit/renamed/cold/run.json new file mode 100644 index 0000000000000000000000000000000000000000..57488db26ebe5c285f3aa17e24a6b8c9f092f6ff --- /dev/null +++ b/corpus/attachment-edit/renamed/cold/run.json @@ -0,0 +1,18 @@ +{ + "notebook": "/private/tmp/m4/attachment-rename", + "expected_pages": 1, + "author": null, + "author_timeout_seconds": 600, + "inspect": false, + "collect_notebook": true, + "base": { + "file": "win7-office-base.qcow2", + "format": "qcow2", + "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346", + "virtual_size": 68719476736 + }, + "scripts": { + "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331", + "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41" + } +} diff --git a/corpus/attachment-edit/renamed/cold/scripts/cold.ps1 b/corpus/attachment-edit/renamed/cold/scripts/cold.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..a0b99e7aeb86f4dc10032537800501a52ad65b4f --- /dev/null +++ b/corpus/attachment-edit/renamed/cold/scripts/cold.ps1 @@ -0,0 +1,27 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '') +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' } +$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote' +if ($CloneHost) { + if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) { + throw 'The disposable clone hostname does not match this machine.' + } + New-Item "$key\Options\Paths" -Force | Out-Null + New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null +} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or + -not (Test-Path 'C:\one-tests\profile-original-cache')) { + throw 'Park the personal OneNote profile before resetting the test cache.' +} +$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0' +$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root)) +if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' } +New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null +if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked } +if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse } +New-Item "$key\OpenNotebooks" | Out-Null +New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null diff --git a/corpus/attachment-edit/renamed/cold/scripts/read.ps1 b/corpus/attachment-edit/renamed/cold/scripts/read.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..476094ab1e0c47077988a6461db2080cf208642b --- /dev/null +++ b/corpus/attachment-edit/renamed/cold/scripts/read.ps1 @@ -0,0 +1,142 @@ +param( + [Parameter(Mandatory=$true)][string]$Root, + [int]$ExpectedPages = -1, + [switch]$UseCurrentCache, + [switch]$Pdf, + [switch]$KeepOpen, + [string]$CloneHost = '' +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +$notebook = Join-Path $root 'notebook' +$output = Join-Path $root 'read' +if (Test-Path $output) { throw 'Choose a new read destination.' } +if ($UseCurrentCache) { + if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') { + throw 'Park the personal OneNote profile before reading test notebooks.' + } +} else { + & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost +} +New-Item -ItemType Directory -Path $output | Out-Null +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$failure = $null +try { + $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0) + $process = Get-Process ONENOTE + @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion; + powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } | + ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8 + $sections = @() + foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) { + $id = '' + $app.OpenHierarchy($file.FullName, '', [ref]$id, 0) + $sections += $id + } + $deadline = [DateTime]::UtcNow.AddSeconds(300) + $previous = '' + $lastChange = '' + $stableSince = [DateTime]::UtcNow + $settled = $false + do { + $pages = @{} + foreach ($section in $sections) { + $hierarchy = '' + $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) + [xml]$xml = $hierarchy + foreach ($node in $xml.SelectNodes('//*[@path]')) { + if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) { + throw 'OneNote opened a section outside the copied notebook.' + } + } + foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) { + $id = $node.GetAttribute('ID') + $content = '' + $app.GetPageContent($id, [ref]$content, 1, 1) + $pages[$id] = $content + } + } + $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] })) + if ($signature -ne $previous) { + $lastChange = $previous + $previous = $signature + $stableSince = [DateTime]::UtcNow + } + if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or + ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and + ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break } + Start-Sleep -Milliseconds 250 + } while ([DateTime]::UtcNow -lt $deadline) + if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) { + [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8) + $index = 0 + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + $index++ + } + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8) + throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled." + } + $index = 0 + $payloads = @() + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + if ($Pdf) { + $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index) + $app.NavigateTo($id, '', $false) + $app.Publish($id, $pdfPath, 3, '') + if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' } + } + [xml]$page = $pages[$id] + foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) { + $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache')) + $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant() + [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes) + $payloads += @{ page = $id; object = $file.ParentNode.GetAttribute('objectID'); + kind = $file.LocalName; name = $file.GetAttribute('preferredName'); + sha256 = $hash; bytes = $bytes.Length } + } + $index++ + } + [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8) + $all = '' + $app.GetHierarchy($notebookId, 4, [ref]$all, 1) + [xml]$finalTree = $all + $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique) + if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) { + throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.' + } + [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8) + Write-Output "Read $($sections.Count) sections and $($pages.Count) pages." +} catch { + $failure = $_ + [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + throw +} finally { + try { + try { + if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) } + if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) { + $app.CloseNotebook($notebookId, $false) + } + } catch { + if ($null -eq $failure) { throw } + [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + } + } finally { + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + } + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + if (-not $UseCurrentCache -and -not $CloneHost) { + Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10 + } +} diff --git a/corpus/attachment-edit/renamed/cold/source.json b/corpus/attachment-edit/renamed/cold/source.json new file mode 100644 index 0000000000000000000000000000000000000000..45256bf6c7e5488a2f4c623994986571a369a437 --- /dev/null +++ b/corpus/attachment-edit/renamed/cold/source.json @@ -0,0 +1,14 @@ +[ + { + "path": "Open Notebook.onetoc2", + "bytes": 1612, + "sha256": "e95a685134505ff745de97cb2795f289c23329966ed49eead8a2dd4f98b91571", + "mtime_ns": 1789278022646769677 + }, + { + "path": "synthetic.one", + "bytes": 21896, + "sha256": "ff9cba8456136873d2929bba7f2fa01931eb9aa69a1d5ebbaa61a669724e10e5", + "mtime_ns": 1789278022626746201 + } +] diff --git a/corpus/attachment-edit/renamed/cold/teardown.json b/corpus/attachment-edit/renamed/cold/teardown.json new file mode 100644 index 0000000000000000000000000000000000000000..05a47793de40f322e745c4c4183727e3590ecf70 --- /dev/null +++ b/corpus/attachment-edit/renamed/cold/teardown.json @@ -0,0 +1 @@ +{"absent": true} diff --git a/corpus/notebook-edit/README.md b/corpus/notebook-edit/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f73cf66b688ad12b7ccc64f4927e9391b8793ccb --- /dev/null +++ b/corpus/notebook-edit/README.md @@ -0,0 +1,40 @@ +# Notebook structure + +`native-structure/` is OneNote 2010 building structure through the COM API on +a Rust-created notebook (`tools/native/notebook-structure.ps1`): it created +two sections and a section group with a section inside, renamed the notebook, +reopened it, and deleted a section; `structure-*.xml` are the hierarchy reads +after each step, `notebook-renamed/` the files before the delete. OneNote +keeps the documented table of contents (MS-ONE 2.2.14): one root object with +an entry array whose entries carry the file identity, order, filename and +colour (`0xffffffff` for sections, absent for groups); a section's colour +lives in its own metadata. Deleting moves the file into `OneNote_RecycleBin`, +a group with its own TOC that the root lists. OneNote also names every file +for its place in the header: `guidAncestor` is the parent TOC's file +identity and `crcName` the CRC of the section file name or the group folder +name. The Rust section `links.one`, created with a zero ancestor, was +re-identified on open and listed a second time under its new identity. + +`native-reorder/` is the COM API asked to move the last root section first +through `UpdateHierarchy` (`tools/native/notebook-reorder.ps1`): the +in-session read shows the new order, but after `SyncHierarchy`, close and +reopen the order is back and the TOC unchanged, so section order cannot be +authored through COM; the order rows below are proved by cold reads of Rust +candidates alone. Section display order is the entry's ordering number +ascending, sections before groups, which the owner's own notebook confirms +(array order there differs from the numbers, and OneNote lists by number). + +`structured/` is the notebook `crates/notebook/tests/structure.rs` builds +through `notebook::session::Notebook` (create sections and a group, rename +both, colour a section, order the root as group, renamed, first), placed the +same way; `deleted/` is the same notebook after deleting a section. Each +`cold/` is a fresh OneNote 2010 read: the hierarchy lists the entries in the +written order with the written colours, the recycle bin carries +`isRecycleBin`, and every file keeps the identity Rust wrote. +Every TOC revision the writer appends carries one global id table; with one +table per object group, as the section writer emits, OneNote applied a +reordered entry's value to the wrong entry. `tools/test_notebook_edit.py` +checks this without a VM. Regenerate with +`NOTEBOOK_STRUCTURE_EXPORT` and `NOTEBOOK_STRUCTURE_EXPORT_DELETED` set to +new directories while running the test, then cold-open each with +`tools/native_runner.py OUTPUT COLD --expected-pages -1 --collect-notebook`. diff --git a/corpus/notebook-edit/deleted/candidate/Kept/First.one b/corpus/notebook-edit/deleted/candidate/Kept/First.one new file mode 100644 index 0000000000000000000000000000000000000000..7c767bf992a6d9a522fb8556f13b14160842e9cf Binary files /dev/null and b/corpus/notebook-edit/deleted/candidate/Kept/First.one differ diff --git a/corpus/notebook-edit/deleted/candidate/Kept/Open Notebook.onetoc2 b/corpus/notebook-edit/deleted/candidate/Kept/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..bd1cabf50275bdbe34b3579bba78a17f3e61dad5 Binary files /dev/null and b/corpus/notebook-edit/deleted/candidate/Kept/Open Notebook.onetoc2 differ diff --git a/corpus/notebook-edit/deleted/candidate/OneNote_RecycleBin/First.one b/corpus/notebook-edit/deleted/candidate/OneNote_RecycleBin/First.one new file mode 100644 index 0000000000000000000000000000000000000000..65b5f1f600f5810df63e0cbca775851580602c07 Binary files /dev/null and b/corpus/notebook-edit/deleted/candidate/OneNote_RecycleBin/First.one differ diff --git a/corpus/notebook-edit/deleted/candidate/OneNote_RecycleBin/Open Notebook.onetoc2 b/corpus/notebook-edit/deleted/candidate/OneNote_RecycleBin/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..9960294c46ef3dcf7aa082edaf66f88376cc1d6a Binary files /dev/null and b/corpus/notebook-edit/deleted/candidate/OneNote_RecycleBin/Open Notebook.onetoc2 differ diff --git a/corpus/notebook-edit/deleted/candidate/Open Notebook.onetoc2 b/corpus/notebook-edit/deleted/candidate/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..f5f3c075ffed0e96eed2e99ee11517b704b4de48 Binary files /dev/null and b/corpus/notebook-edit/deleted/candidate/Open Notebook.onetoc2 differ diff --git a/corpus/notebook-edit/deleted/candidate/Renamed.one b/corpus/notebook-edit/deleted/candidate/Renamed.one new file mode 100644 index 0000000000000000000000000000000000000000..31fba4276c85f0e413b6cb036992a8102c31d014 Binary files /dev/null and b/corpus/notebook-edit/deleted/candidate/Renamed.one differ diff --git a/corpus/notebook-edit/deleted/cold/commands.jsonl b/corpus/notebook-edit/deleted/cold/commands.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..250d639f4fa7b6432ed08c3040cc5a6007f54902 --- /dev/null +++ b/corpus/notebook-edit/deleted/cold/commands.jsonl @@ -0,0 +1,3 @@ +{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-3574EFF5 -ExpectedPages -1", "exit": 0, "stdout": "Read 3 sections and 3 pages.\r\n", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null} diff --git a/corpus/notebook-edit/deleted/cold/machine.json b/corpus/notebook-edit/deleted/cold/machine.json new file mode 100644 index 0000000000000000000000000000000000000000..b5dfe53f9cdcc6be1289b2ef5f323a3939ee0cfc --- /dev/null +++ b/corpus/notebook-edit/deleted/cold/machine.json @@ -0,0 +1 @@ +{"name": "m6-3574eff5", "hostname": "ONE-M6-3574EFF5"} diff --git a/corpus/notebook-edit/deleted/cold/notebook/Kept/First.one b/corpus/notebook-edit/deleted/cold/notebook/Kept/First.one new file mode 100644 index 0000000000000000000000000000000000000000..7c767bf992a6d9a522fb8556f13b14160842e9cf Binary files /dev/null and b/corpus/notebook-edit/deleted/cold/notebook/Kept/First.one differ diff --git a/corpus/notebook-edit/deleted/cold/notebook/Kept/Open Notebook.onetoc2 b/corpus/notebook-edit/deleted/cold/notebook/Kept/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..bd1cabf50275bdbe34b3579bba78a17f3e61dad5 Binary files /dev/null and b/corpus/notebook-edit/deleted/cold/notebook/Kept/Open Notebook.onetoc2 differ diff --git a/corpus/notebook-edit/deleted/cold/notebook/OneNote_RecycleBin/First.one b/corpus/notebook-edit/deleted/cold/notebook/OneNote_RecycleBin/First.one new file mode 100644 index 0000000000000000000000000000000000000000..65b5f1f600f5810df63e0cbca775851580602c07 Binary files /dev/null and b/corpus/notebook-edit/deleted/cold/notebook/OneNote_RecycleBin/First.one differ diff --git a/corpus/notebook-edit/deleted/cold/notebook/OneNote_RecycleBin/Open Notebook.onetoc2 b/corpus/notebook-edit/deleted/cold/notebook/OneNote_RecycleBin/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..9960294c46ef3dcf7aa082edaf66f88376cc1d6a Binary files /dev/null and b/corpus/notebook-edit/deleted/cold/notebook/OneNote_RecycleBin/Open Notebook.onetoc2 differ diff --git a/corpus/notebook-edit/deleted/cold/notebook/Open Notebook.onetoc2 b/corpus/notebook-edit/deleted/cold/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..7547b2573857a848c268836f1800122ef540e851 Binary files /dev/null and b/corpus/notebook-edit/deleted/cold/notebook/Open Notebook.onetoc2 differ diff --git a/corpus/notebook-edit/deleted/cold/notebook/Renamed.one b/corpus/notebook-edit/deleted/cold/notebook/Renamed.one new file mode 100644 index 0000000000000000000000000000000000000000..31fba4276c85f0e413b6cb036992a8102c31d014 Binary files /dev/null and b/corpus/notebook-edit/deleted/cold/notebook/Renamed.one differ diff --git a/corpus/notebook-edit/deleted/cold/read/environment.json b/corpus/notebook-edit/deleted/cold/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..7f5a3869aca0881ba74087b19fc76d989b96dd9f --- /dev/null +++ b/corpus/notebook-edit/deleted/cold/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-3574EFF5", + "cold": true, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/notebook-edit/deleted/cold/read/hierarchy.xml b/corpus/notebook-edit/deleted/cold/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..11c04ae171ce950913442b9c554b0669c7ea2229 --- /dev/null +++ b/corpus/notebook-edit/deleted/cold/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/notebook-edit/deleted/cold/read/page-000.xml b/corpus/notebook-edit/deleted/cold/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..a4a0c21aafb018bd9e7d8d975fbfd86a5060fafd --- /dev/null +++ b/corpus/notebook-edit/deleted/cold/read/page-000.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/notebook-edit/deleted/cold/read/page-001.xml b/corpus/notebook-edit/deleted/cold/read/page-001.xml new file mode 100644 index 0000000000000000000000000000000000000000..fd6f5d67aa0aca95b864e297b61ac902b272d001 --- /dev/null +++ b/corpus/notebook-edit/deleted/cold/read/page-001.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/notebook-edit/deleted/cold/read/page-002.xml b/corpus/notebook-edit/deleted/cold/read/page-002.xml new file mode 100644 index 0000000000000000000000000000000000000000..25998a0313e6f184bda856249a5d39e6470d903a --- /dev/null +++ b/corpus/notebook-edit/deleted/cold/read/page-002.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/notebook-edit/deleted/cold/read/payloads.json b/corpus/notebook-edit/deleted/cold/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..1ce91a413d6fff4d58765187b76756f0e97ae843 --- /dev/null +++ b/corpus/notebook-edit/deleted/cold/read/payloads.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/corpus/notebook-edit/deleted/cold/run.json b/corpus/notebook-edit/deleted/cold/run.json new file mode 100644 index 0000000000000000000000000000000000000000..e02a3e8b7cdff223ff00ea4337a5512be238a916 --- /dev/null +++ b/corpus/notebook-edit/deleted/cold/run.json @@ -0,0 +1,18 @@ +{ + "notebook": "/private/tmp/m4/structure3-deleted", + "expected_pages": -1, + "author": null, + "author_timeout_seconds": 600, + "inspect": false, + "collect_notebook": true, + "base": { + "file": "win7-office-base.qcow2", + "format": "qcow2", + "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346", + "virtual_size": 68719476736 + }, + "scripts": { + "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331", + "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41" + } +} diff --git a/corpus/notebook-edit/deleted/cold/scripts/cold.ps1 b/corpus/notebook-edit/deleted/cold/scripts/cold.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..a0b99e7aeb86f4dc10032537800501a52ad65b4f --- /dev/null +++ b/corpus/notebook-edit/deleted/cold/scripts/cold.ps1 @@ -0,0 +1,27 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '') +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' } +$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote' +if ($CloneHost) { + if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) { + throw 'The disposable clone hostname does not match this machine.' + } + New-Item "$key\Options\Paths" -Force | Out-Null + New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null +} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or + -not (Test-Path 'C:\one-tests\profile-original-cache')) { + throw 'Park the personal OneNote profile before resetting the test cache.' +} +$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0' +$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root)) +if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' } +New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null +if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked } +if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse } +New-Item "$key\OpenNotebooks" | Out-Null +New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null diff --git a/corpus/notebook-edit/deleted/cold/scripts/read.ps1 b/corpus/notebook-edit/deleted/cold/scripts/read.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..476094ab1e0c47077988a6461db2080cf208642b --- /dev/null +++ b/corpus/notebook-edit/deleted/cold/scripts/read.ps1 @@ -0,0 +1,142 @@ +param( + [Parameter(Mandatory=$true)][string]$Root, + [int]$ExpectedPages = -1, + [switch]$UseCurrentCache, + [switch]$Pdf, + [switch]$KeepOpen, + [string]$CloneHost = '' +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +$notebook = Join-Path $root 'notebook' +$output = Join-Path $root 'read' +if (Test-Path $output) { throw 'Choose a new read destination.' } +if ($UseCurrentCache) { + if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') { + throw 'Park the personal OneNote profile before reading test notebooks.' + } +} else { + & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost +} +New-Item -ItemType Directory -Path $output | Out-Null +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$failure = $null +try { + $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0) + $process = Get-Process ONENOTE + @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion; + powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } | + ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8 + $sections = @() + foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) { + $id = '' + $app.OpenHierarchy($file.FullName, '', [ref]$id, 0) + $sections += $id + } + $deadline = [DateTime]::UtcNow.AddSeconds(300) + $previous = '' + $lastChange = '' + $stableSince = [DateTime]::UtcNow + $settled = $false + do { + $pages = @{} + foreach ($section in $sections) { + $hierarchy = '' + $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) + [xml]$xml = $hierarchy + foreach ($node in $xml.SelectNodes('//*[@path]')) { + if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) { + throw 'OneNote opened a section outside the copied notebook.' + } + } + foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) { + $id = $node.GetAttribute('ID') + $content = '' + $app.GetPageContent($id, [ref]$content, 1, 1) + $pages[$id] = $content + } + } + $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] })) + if ($signature -ne $previous) { + $lastChange = $previous + $previous = $signature + $stableSince = [DateTime]::UtcNow + } + if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or + ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and + ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break } + Start-Sleep -Milliseconds 250 + } while ([DateTime]::UtcNow -lt $deadline) + if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) { + [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8) + $index = 0 + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + $index++ + } + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8) + throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled." + } + $index = 0 + $payloads = @() + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + if ($Pdf) { + $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index) + $app.NavigateTo($id, '', $false) + $app.Publish($id, $pdfPath, 3, '') + if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' } + } + [xml]$page = $pages[$id] + foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) { + $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache')) + $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant() + [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes) + $payloads += @{ page = $id; object = $file.ParentNode.GetAttribute('objectID'); + kind = $file.LocalName; name = $file.GetAttribute('preferredName'); + sha256 = $hash; bytes = $bytes.Length } + } + $index++ + } + [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8) + $all = '' + $app.GetHierarchy($notebookId, 4, [ref]$all, 1) + [xml]$finalTree = $all + $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique) + if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) { + throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.' + } + [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8) + Write-Output "Read $($sections.Count) sections and $($pages.Count) pages." +} catch { + $failure = $_ + [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + throw +} finally { + try { + try { + if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) } + if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) { + $app.CloseNotebook($notebookId, $false) + } + } catch { + if ($null -eq $failure) { throw } + [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + } + } finally { + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + } + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + if (-not $UseCurrentCache -and -not $CloneHost) { + Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10 + } +} diff --git a/corpus/notebook-edit/deleted/cold/source.json b/corpus/notebook-edit/deleted/cold/source.json new file mode 100644 index 0000000000000000000000000000000000000000..b6176cbeb96c675874da6fb07b38478798a8a585 --- /dev/null +++ b/corpus/notebook-edit/deleted/cold/source.json @@ -0,0 +1,38 @@ +[ + { + "path": "Kept/First.one", + "bytes": 3164, + "sha256": "61be453f86b7791923497293f74db57bc7f33b6e8d5fd38ad57631571475a1cf", + "mtime_ns": 1789279283845197267 + }, + { + "path": "Kept/Open Notebook.onetoc2", + "bytes": 4120, + "sha256": "7d734133a1019a00260296bfd143282a7411b667f68956874f7e7ffaa97518b8", + "mtime_ns": 1789279283861857142 + }, + { + "path": "OneNote_RecycleBin/First.one", + "bytes": 3212, + "sha256": "25d2946bbef7c546dc7391f848d43b7b7a4270cf8e0c35264e6939a1a083821c", + "mtime_ns": 1789279283804243010 + }, + { + "path": "OneNote_RecycleBin/Open Notebook.onetoc2", + "bytes": 3824, + "sha256": "8e3ad16bb0ed85c55577d0996a6a3f98df2537f82e3d1d48a35be40283cf4a40", + "mtime_ns": 1789279283820890719 + }, + { + "path": "Open Notebook.onetoc2", + "bytes": 6328, + "sha256": "0c10b7470bb74f2e17b3319707f2f422370fcbde05f09bd7c0e9dd088483c97e", + "mtime_ns": 1789279283837910881 + }, + { + "path": "Renamed.one", + "bytes": 5504, + "sha256": "9ee7bd23c1679ce5035c92adca0416730b149d11bedd1b4a7c181a0186df7007", + "mtime_ns": 1789279283726949659 + } +] diff --git a/corpus/notebook-edit/deleted/cold/teardown.json b/corpus/notebook-edit/deleted/cold/teardown.json new file mode 100644 index 0000000000000000000000000000000000000000..05a47793de40f322e745c4c4183727e3590ecf70 --- /dev/null +++ b/corpus/notebook-edit/deleted/cold/teardown.json @@ -0,0 +1 @@ +{"absent": true} diff --git a/corpus/notebook-edit/native-reorder/commands.jsonl b/corpus/notebook-edit/native-reorder/commands.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..21415205c388ac9c9609bd6d17bffb3e9e713ac6 --- /dev/null +++ b/corpus/notebook-edit/native-reorder/commands.jsonl @@ -0,0 +1,4 @@ +{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\author.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-42A0DA0E", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-42A0DA0E -ExpectedPages -1 -UseCurrentCache", "exit": 0, "stdout": "Read 4 sections and 1 pages.\r\n", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null} diff --git a/corpus/notebook-edit/native-reorder/machine.json b/corpus/notebook-edit/native-reorder/machine.json new file mode 100644 index 0000000000000000000000000000000000000000..93855edd8d6c6a5596b870f0dad0465d440d3a09 --- /dev/null +++ b/corpus/notebook-edit/native-reorder/machine.json @@ -0,0 +1 @@ +{"name": "m6-42a0da0e", "hostname": "ONE-M6-42A0DA0E"} diff --git a/corpus/notebook-edit/native-reorder/notebook-reordered/Group/Inner.one b/corpus/notebook-edit/native-reorder/notebook-reordered/Group/Inner.one new file mode 100644 index 0000000000000000000000000000000000000000..ba9b977cddfbdba559da17d844cf9339f4d0983a Binary files /dev/null and b/corpus/notebook-edit/native-reorder/notebook-reordered/Group/Inner.one differ diff --git a/corpus/notebook-edit/native-reorder/notebook-reordered/Group/Open Notebook.onetoc2 b/corpus/notebook-edit/native-reorder/notebook-reordered/Group/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..8b7a7eca827559a46b8bc3d21a19e7bd5da6c092 Binary files /dev/null and b/corpus/notebook-edit/native-reorder/notebook-reordered/Group/Open Notebook.onetoc2 differ diff --git a/corpus/notebook-edit/native-reorder/notebook-reordered/Open Notebook.onetoc2 b/corpus/notebook-edit/native-reorder/notebook-reordered/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..491ef31bae9ed88e2453db2e4c6bbcbd248cd2e2 Binary files /dev/null and b/corpus/notebook-edit/native-reorder/notebook-reordered/Open Notebook.onetoc2 differ diff --git a/corpus/notebook-edit/native-reorder/notebook-reordered/Second.one b/corpus/notebook-edit/native-reorder/notebook-reordered/Second.one new file mode 100644 index 0000000000000000000000000000000000000000..ff01d734a4719f80b339c3d449f371081d56f5c9 Binary files /dev/null and b/corpus/notebook-edit/native-reorder/notebook-reordered/Second.one differ diff --git a/corpus/notebook-edit/native-reorder/notebook-reordered/Third.one b/corpus/notebook-edit/native-reorder/notebook-reordered/Third.one new file mode 100644 index 0000000000000000000000000000000000000000..0fbb322cfd0d38596cc6b0d2c0d8bbfd39ff0c53 Binary files /dev/null and b/corpus/notebook-edit/native-reorder/notebook-reordered/Third.one differ diff --git a/corpus/notebook-edit/native-reorder/notebook-reordered/links.one b/corpus/notebook-edit/native-reorder/notebook-reordered/links.one new file mode 100644 index 0000000000000000000000000000000000000000..c53bf910edd84ce5721c52e18ea5022d415c730c Binary files /dev/null and b/corpus/notebook-edit/native-reorder/notebook-reordered/links.one differ diff --git a/corpus/notebook-edit/native-reorder/notebook/Group/Inner.one b/corpus/notebook-edit/native-reorder/notebook/Group/Inner.one new file mode 100644 index 0000000000000000000000000000000000000000..ba9b977cddfbdba559da17d844cf9339f4d0983a Binary files /dev/null and b/corpus/notebook-edit/native-reorder/notebook/Group/Inner.one differ diff --git a/corpus/notebook-edit/native-reorder/notebook/Group/Open Notebook.onetoc2 b/corpus/notebook-edit/native-reorder/notebook/Group/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..8b7a7eca827559a46b8bc3d21a19e7bd5da6c092 Binary files /dev/null and b/corpus/notebook-edit/native-reorder/notebook/Group/Open Notebook.onetoc2 differ diff --git a/corpus/notebook-edit/native-reorder/notebook/Open Notebook.onetoc2 b/corpus/notebook-edit/native-reorder/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..491ef31bae9ed88e2453db2e4c6bbcbd248cd2e2 Binary files /dev/null and b/corpus/notebook-edit/native-reorder/notebook/Open Notebook.onetoc2 differ diff --git a/corpus/notebook-edit/native-reorder/notebook/Second.one b/corpus/notebook-edit/native-reorder/notebook/Second.one new file mode 100644 index 0000000000000000000000000000000000000000..ff01d734a4719f80b339c3d449f371081d56f5c9 Binary files /dev/null and b/corpus/notebook-edit/native-reorder/notebook/Second.one differ diff --git a/corpus/notebook-edit/native-reorder/notebook/Third.one b/corpus/notebook-edit/native-reorder/notebook/Third.one new file mode 100644 index 0000000000000000000000000000000000000000..0fbb322cfd0d38596cc6b0d2c0d8bbfd39ff0c53 Binary files /dev/null and b/corpus/notebook-edit/native-reorder/notebook/Third.one differ diff --git a/corpus/notebook-edit/native-reorder/notebook/links.one b/corpus/notebook-edit/native-reorder/notebook/links.one new file mode 100644 index 0000000000000000000000000000000000000000..c53bf910edd84ce5721c52e18ea5022d415c730c Binary files /dev/null and b/corpus/notebook-edit/native-reorder/notebook/links.one differ diff --git a/corpus/notebook-edit/native-reorder/read/environment.json b/corpus/notebook-edit/native-reorder/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..c45b314aec57ef1a2274499d9d6f4b9395296b1b --- /dev/null +++ b/corpus/notebook-edit/native-reorder/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-42A0DA0E", + "cold": false, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/notebook-edit/native-reorder/read/hierarchy.xml b/corpus/notebook-edit/native-reorder/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..7600b2ee2751be04ef7af8bf4080fc28ebbf11f9 --- /dev/null +++ b/corpus/notebook-edit/native-reorder/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/notebook-edit/native-reorder/read/page-000.xml b/corpus/notebook-edit/native-reorder/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..09bbc41f9b5eb40793162004d7004c8b20bac55a --- /dev/null +++ b/corpus/notebook-edit/native-reorder/read/page-000.xml @@ -0,0 +1,3 @@ + +the Rust site]]> diff --git a/corpus/notebook-edit/native-reorder/read/payloads.json b/corpus/notebook-edit/native-reorder/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..1ce91a413d6fff4d58765187b76756f0e97ae843 --- /dev/null +++ b/corpus/notebook-edit/native-reorder/read/payloads.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/corpus/notebook-edit/native-reorder/reorder-00-opened.xml b/corpus/notebook-edit/native-reorder/reorder-00-opened.xml new file mode 100644 index 0000000000000000000000000000000000000000..536899dbd29217eeaa3d7064eeeb878b081248f2 --- /dev/null +++ b/corpus/notebook-edit/native-reorder/reorder-00-opened.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/notebook-edit/native-reorder/reorder-01-update.xml b/corpus/notebook-edit/native-reorder/reorder-01-update.xml new file mode 100644 index 0000000000000000000000000000000000000000..3f97f191e282bee7f8224345c15f50a5ac45b9b3 --- /dev/null +++ b/corpus/notebook-edit/native-reorder/reorder-01-update.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/corpus/notebook-edit/native-reorder/reorder-02-reordered.xml b/corpus/notebook-edit/native-reorder/reorder-02-reordered.xml new file mode 100644 index 0000000000000000000000000000000000000000..c6e94bc84fd5026bdb3b36f8eac780ad7c8d30aa --- /dev/null +++ b/corpus/notebook-edit/native-reorder/reorder-02-reordered.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/notebook-edit/native-reorder/reorder-03-reopened.xml b/corpus/notebook-edit/native-reorder/reorder-03-reopened.xml new file mode 100644 index 0000000000000000000000000000000000000000..e261b606fccf929e5292fc54e08312ae7124a7d5 --- /dev/null +++ b/corpus/notebook-edit/native-reorder/reorder-03-reopened.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/notebook-edit/native-reorder/run.json b/corpus/notebook-edit/native-reorder/run.json new file mode 100644 index 0000000000000000000000000000000000000000..b06e8ae926ac3e811a3f7207e7b2837a7d1166df --- /dev/null +++ b/corpus/notebook-edit/native-reorder/run.json @@ -0,0 +1,19 @@ +{ + "notebook": "/Users/clo/dev/one/corpus/notebook-edit/native-structure/notebook-renamed", + "expected_pages": -1, + "author": "tools/native/notebook-reorder.ps1", + "author_timeout_seconds": 600, + "inspect": false, + "collect_notebook": true, + "base": { + "file": "win7-office-base.qcow2", + "format": "qcow2", + "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346", + "virtual_size": 68719476736 + }, + "scripts": { + "author.ps1": "61594f0aec7a19bf3c5b2bdc478a98aed9d59637f33f7fde074cdc4567716e03", + "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331", + "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41" + } +} diff --git a/corpus/notebook-edit/native-reorder/scripts/author.ps1 b/corpus/notebook-edit/native-reorder/scripts/author.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..b672af93e4abe3eeeaa403c5ff4ed48c775238ac --- /dev/null +++ b/corpus/notebook-edit/native-reorder/scripts/author.ps1 @@ -0,0 +1,50 @@ +param([Parameter(Mandatory=$true)][string]$Root, [Parameter(Mandatory=$true)][string]$CloneHost) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +& "$PSScriptRoot\cold-current.ps1" -Root $Root -CloneHost $CloneHost +$namespace = 'http://schemas.microsoft.com/office/onenote/2010/onenote' +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +function Read-Tree { + $deadline = [DateTime]::UtcNow.AddSeconds(60) + do { + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [xml]$tree = $hierarchy + if (@($tree.SelectNodes('//*[local-name()="Section"]')).Count -ge 4) { return $hierarchy } + Start-Sleep -Milliseconds 500 + } while ([DateTime]::UtcNow -lt $deadline) + throw 'The notebook did not finish loading.' +} +try { + $app.OpenHierarchy((Join-Path $Root 'notebook'), '', [ref]$notebookId, 0) + $hierarchy = Read-Tree + [IO.File]::WriteAllText((Join-Path $Root 'reorder-00-opened.xml'), $hierarchy, [Text.Encoding]::UTF8) + [xml]$tree = $hierarchy + $manager = New-Object Xml.XmlNamespaceManager($tree.NameTable) + $manager.AddNamespace('one', $namespace) + $notebookNode = $tree.DocumentElement + $sections = @($notebookNode.SelectNodes('one:Section', $manager)) + if ($sections.Count -lt 3) { throw "Expected three root sections, found $($sections.Count)." } + $last = $sections[$sections.Count - 1] + [void]$notebookNode.RemoveChild($last) + [void]$notebookNode.InsertBefore($last, $sections[0]) + [IO.File]::WriteAllText((Join-Path $Root 'reorder-01-update.xml'), $tree.OuterXml, [Text.Encoding]::UTF8) + $app.UpdateHierarchy($tree.OuterXml, 1) + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $Root 'reorder-02-reordered.xml'), $hierarchy, [Text.Encoding]::UTF8) + $app.SyncHierarchy($notebookId) + $app.CloseNotebook($notebookId, $false) + $notebookId = '' + Copy-Item -Recurse (Join-Path $Root 'notebook') (Join-Path $Root 'notebook-reordered') + $app.OpenHierarchy((Join-Path $Root 'notebook'), '', [ref]$notebookId, 0) + $hierarchy = Read-Tree + [IO.File]::WriteAllText((Join-Path $Root 'reorder-03-reopened.xml'), $hierarchy, [Text.Encoding]::UTF8) + $app.CloseNotebook($notebookId, $false) + $notebookId = '' +} finally { + if ($notebookId) { $app.CloseNotebook($notebookId, $false) } + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + [GC]::Collect() + [GC]::WaitForPendingFinalizers() +} diff --git a/corpus/notebook-edit/native-reorder/scripts/cold.ps1 b/corpus/notebook-edit/native-reorder/scripts/cold.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..a0b99e7aeb86f4dc10032537800501a52ad65b4f --- /dev/null +++ b/corpus/notebook-edit/native-reorder/scripts/cold.ps1 @@ -0,0 +1,27 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '') +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' } +$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote' +if ($CloneHost) { + if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) { + throw 'The disposable clone hostname does not match this machine.' + } + New-Item "$key\Options\Paths" -Force | Out-Null + New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null +} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or + -not (Test-Path 'C:\one-tests\profile-original-cache')) { + throw 'Park the personal OneNote profile before resetting the test cache.' +} +$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0' +$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root)) +if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' } +New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null +if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked } +if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse } +New-Item "$key\OpenNotebooks" | Out-Null +New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null diff --git a/corpus/notebook-edit/native-reorder/scripts/read.ps1 b/corpus/notebook-edit/native-reorder/scripts/read.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..476094ab1e0c47077988a6461db2080cf208642b --- /dev/null +++ b/corpus/notebook-edit/native-reorder/scripts/read.ps1 @@ -0,0 +1,142 @@ +param( + [Parameter(Mandatory=$true)][string]$Root, + [int]$ExpectedPages = -1, + [switch]$UseCurrentCache, + [switch]$Pdf, + [switch]$KeepOpen, + [string]$CloneHost = '' +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +$notebook = Join-Path $root 'notebook' +$output = Join-Path $root 'read' +if (Test-Path $output) { throw 'Choose a new read destination.' } +if ($UseCurrentCache) { + if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') { + throw 'Park the personal OneNote profile before reading test notebooks.' + } +} else { + & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost +} +New-Item -ItemType Directory -Path $output | Out-Null +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$failure = $null +try { + $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0) + $process = Get-Process ONENOTE + @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion; + powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } | + ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8 + $sections = @() + foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) { + $id = '' + $app.OpenHierarchy($file.FullName, '', [ref]$id, 0) + $sections += $id + } + $deadline = [DateTime]::UtcNow.AddSeconds(300) + $previous = '' + $lastChange = '' + $stableSince = [DateTime]::UtcNow + $settled = $false + do { + $pages = @{} + foreach ($section in $sections) { + $hierarchy = '' + $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) + [xml]$xml = $hierarchy + foreach ($node in $xml.SelectNodes('//*[@path]')) { + if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) { + throw 'OneNote opened a section outside the copied notebook.' + } + } + foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) { + $id = $node.GetAttribute('ID') + $content = '' + $app.GetPageContent($id, [ref]$content, 1, 1) + $pages[$id] = $content + } + } + $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] })) + if ($signature -ne $previous) { + $lastChange = $previous + $previous = $signature + $stableSince = [DateTime]::UtcNow + } + if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or + ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and + ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break } + Start-Sleep -Milliseconds 250 + } while ([DateTime]::UtcNow -lt $deadline) + if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) { + [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8) + $index = 0 + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + $index++ + } + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8) + throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled." + } + $index = 0 + $payloads = @() + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + if ($Pdf) { + $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index) + $app.NavigateTo($id, '', $false) + $app.Publish($id, $pdfPath, 3, '') + if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' } + } + [xml]$page = $pages[$id] + foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) { + $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache')) + $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant() + [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes) + $payloads += @{ page = $id; object = $file.ParentNode.GetAttribute('objectID'); + kind = $file.LocalName; name = $file.GetAttribute('preferredName'); + sha256 = $hash; bytes = $bytes.Length } + } + $index++ + } + [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8) + $all = '' + $app.GetHierarchy($notebookId, 4, [ref]$all, 1) + [xml]$finalTree = $all + $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique) + if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) { + throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.' + } + [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8) + Write-Output "Read $($sections.Count) sections and $($pages.Count) pages." +} catch { + $failure = $_ + [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + throw +} finally { + try { + try { + if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) } + if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) { + $app.CloseNotebook($notebookId, $false) + } + } catch { + if ($null -eq $failure) { throw } + [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + } + } finally { + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + } + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + if (-not $UseCurrentCache -and -not $CloneHost) { + Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10 + } +} diff --git a/corpus/notebook-edit/native-reorder/source.json b/corpus/notebook-edit/native-reorder/source.json new file mode 100644 index 0000000000000000000000000000000000000000..06d1e5fafe5381cd296e513e70b0c7f59285f6ea --- /dev/null +++ b/corpus/notebook-edit/native-reorder/source.json @@ -0,0 +1,38 @@ +[ + { + "path": "Group/Inner.one", + "bytes": 4016, + "sha256": "7a482b9feafd8220bc9f8c60aca78b4710d64cdc233d7e4aff2d43214f30d26d", + "mtime_ns": 1789276348740792164 + }, + { + "path": "Group/Open Notebook.onetoc2", + "bytes": 3752, + "sha256": "5a81be6da6b559291303c74fede90e0d6f7f78b640c3bce9c1365edb0d99d1b4", + "mtime_ns": 1789276348740542872 + }, + { + "path": "Open Notebook.onetoc2", + "bytes": 4116, + "sha256": "bb8943cc70108ff95a393122f53fd06864db347aafe865f2cf4094ff202c3bbc", + "mtime_ns": 1789276348741057831 + }, + { + "path": "Second.one", + "bytes": 4016, + "sha256": "b1c37af371bb1d7b93950b6b73920d5080178156b7448f3aa30f45472f441ad5", + "mtime_ns": 1789276348741246832 + }, + { + "path": "Third.one", + "bytes": 4016, + "sha256": "6654509286c8c342844a6b716407cfb3dc1303aa8b9c7a0cfff606cc900aea33", + "mtime_ns": 1789276348739887203 + }, + { + "path": "links.one", + "bytes": 6128, + "sha256": "85d17a08e48b0fc0c6d49c4389c80c58910ff64ed2ffd464522523256d2db5dc", + "mtime_ns": 1789276348740094245 + } +] diff --git a/corpus/notebook-edit/native-reorder/teardown.json b/corpus/notebook-edit/native-reorder/teardown.json new file mode 100644 index 0000000000000000000000000000000000000000..05a47793de40f322e745c4c4183727e3590ecf70 --- /dev/null +++ b/corpus/notebook-edit/native-reorder/teardown.json @@ -0,0 +1 @@ +{"absent": true} diff --git a/corpus/notebook-edit/native-structure/commands.jsonl b/corpus/notebook-edit/native-structure/commands.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..9dafabf3699e65d78f3b303f159cb7f367ed0017 --- /dev/null +++ b/corpus/notebook-edit/native-structure/commands.jsonl @@ -0,0 +1,4 @@ +{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\author.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-266B13AF", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-266B13AF -ExpectedPages -1 -UseCurrentCache", "exit": 0, "stdout": "Read 5 sections and 1 pages.\r\n", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null} diff --git a/corpus/notebook-edit/native-structure/machine.json b/corpus/notebook-edit/native-structure/machine.json new file mode 100644 index 0000000000000000000000000000000000000000..987ff337497fcaf05840f5e21dd78e6bea7b389a --- /dev/null +++ b/corpus/notebook-edit/native-structure/machine.json @@ -0,0 +1 @@ +{"name": "m6-266b13af", "hostname": "ONE-M6-266B13AF"} diff --git a/corpus/notebook-edit/native-structure/notebook-renamed/Group/Inner.one b/corpus/notebook-edit/native-structure/notebook-renamed/Group/Inner.one new file mode 100644 index 0000000000000000000000000000000000000000..ba9b977cddfbdba559da17d844cf9339f4d0983a Binary files /dev/null and b/corpus/notebook-edit/native-structure/notebook-renamed/Group/Inner.one differ diff --git a/corpus/notebook-edit/native-structure/notebook-renamed/Group/Open Notebook.onetoc2 b/corpus/notebook-edit/native-structure/notebook-renamed/Group/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..8b7a7eca827559a46b8bc3d21a19e7bd5da6c092 Binary files /dev/null and b/corpus/notebook-edit/native-structure/notebook-renamed/Group/Open Notebook.onetoc2 differ diff --git a/corpus/notebook-edit/native-structure/notebook-renamed/Open Notebook.onetoc2 b/corpus/notebook-edit/native-structure/notebook-renamed/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..491ef31bae9ed88e2453db2e4c6bbcbd248cd2e2 Binary files /dev/null and b/corpus/notebook-edit/native-structure/notebook-renamed/Open Notebook.onetoc2 differ diff --git a/corpus/notebook-edit/native-structure/notebook-renamed/Second.one b/corpus/notebook-edit/native-structure/notebook-renamed/Second.one new file mode 100644 index 0000000000000000000000000000000000000000..ff01d734a4719f80b339c3d449f371081d56f5c9 Binary files /dev/null and b/corpus/notebook-edit/native-structure/notebook-renamed/Second.one differ diff --git a/corpus/notebook-edit/native-structure/notebook-renamed/Third.one b/corpus/notebook-edit/native-structure/notebook-renamed/Third.one new file mode 100644 index 0000000000000000000000000000000000000000..0fbb322cfd0d38596cc6b0d2c0d8bbfd39ff0c53 Binary files /dev/null and b/corpus/notebook-edit/native-structure/notebook-renamed/Third.one differ diff --git a/corpus/notebook-edit/native-structure/notebook-renamed/links.one b/corpus/notebook-edit/native-structure/notebook-renamed/links.one new file mode 100644 index 0000000000000000000000000000000000000000..c53bf910edd84ce5721c52e18ea5022d415c730c Binary files /dev/null and b/corpus/notebook-edit/native-structure/notebook-renamed/links.one differ diff --git a/corpus/notebook-edit/native-structure/notebook/Group/Inner.one b/corpus/notebook-edit/native-structure/notebook/Group/Inner.one new file mode 100644 index 0000000000000000000000000000000000000000..ba9b977cddfbdba559da17d844cf9339f4d0983a Binary files /dev/null and b/corpus/notebook-edit/native-structure/notebook/Group/Inner.one differ diff --git a/corpus/notebook-edit/native-structure/notebook/Group/Open Notebook.onetoc2 b/corpus/notebook-edit/native-structure/notebook/Group/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..8b7a7eca827559a46b8bc3d21a19e7bd5da6c092 Binary files /dev/null and b/corpus/notebook-edit/native-structure/notebook/Group/Open Notebook.onetoc2 differ diff --git a/corpus/notebook-edit/native-structure/notebook/OneNote_RecycleBin/OneNote_DeletedPages.one b/corpus/notebook-edit/native-structure/notebook/OneNote_RecycleBin/OneNote_DeletedPages.one new file mode 100644 index 0000000000000000000000000000000000000000..0b35e4075f90bb3ef9acad76415f0cbcf3bd9d06 Binary files /dev/null and b/corpus/notebook-edit/native-structure/notebook/OneNote_RecycleBin/OneNote_DeletedPages.one differ diff --git a/corpus/notebook-edit/native-structure/notebook/OneNote_RecycleBin/Open Notebook.onetoc2 b/corpus/notebook-edit/native-structure/notebook/OneNote_RecycleBin/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..d5cd409141148e98332aa0f4404563392a4a5f56 Binary files /dev/null and b/corpus/notebook-edit/native-structure/notebook/OneNote_RecycleBin/Open Notebook.onetoc2 differ diff --git a/corpus/notebook-edit/native-structure/notebook/OneNote_RecycleBin/Third.one b/corpus/notebook-edit/native-structure/notebook/OneNote_RecycleBin/Third.one new file mode 100644 index 0000000000000000000000000000000000000000..b1b6224c9ebacb66370578d83d9d275e0616fbb4 Binary files /dev/null and b/corpus/notebook-edit/native-structure/notebook/OneNote_RecycleBin/Third.one differ diff --git a/corpus/notebook-edit/native-structure/notebook/Open Notebook.onetoc2 b/corpus/notebook-edit/native-structure/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..88430cfcbf5f58d02bffc29907e5e23c55cc44ad Binary files /dev/null and b/corpus/notebook-edit/native-structure/notebook/Open Notebook.onetoc2 differ diff --git a/corpus/notebook-edit/native-structure/notebook/Second.one b/corpus/notebook-edit/native-structure/notebook/Second.one new file mode 100644 index 0000000000000000000000000000000000000000..ff01d734a4719f80b339c3d449f371081d56f5c9 Binary files /dev/null and b/corpus/notebook-edit/native-structure/notebook/Second.one differ diff --git a/corpus/notebook-edit/native-structure/notebook/links.one b/corpus/notebook-edit/native-structure/notebook/links.one new file mode 100644 index 0000000000000000000000000000000000000000..c53bf910edd84ce5721c52e18ea5022d415c730c Binary files /dev/null and b/corpus/notebook-edit/native-structure/notebook/links.one differ diff --git a/corpus/notebook-edit/native-structure/read/environment.json b/corpus/notebook-edit/native-structure/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..3a6eeb9e7081153938f3dde5e315a1d4722edc94 --- /dev/null +++ b/corpus/notebook-edit/native-structure/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-266B13AF", + "cold": false, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/notebook-edit/native-structure/read/hierarchy.xml b/corpus/notebook-edit/native-structure/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..e57b3c07b38e4cf2fef1c55fb358abba994eee5c --- /dev/null +++ b/corpus/notebook-edit/native-structure/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/notebook-edit/native-structure/read/page-000.xml b/corpus/notebook-edit/native-structure/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..aae586285f92029feab8401da5465764fba2406f --- /dev/null +++ b/corpus/notebook-edit/native-structure/read/page-000.xml @@ -0,0 +1,3 @@ + +the Rust site]]> diff --git a/corpus/notebook-edit/native-structure/read/payloads.json b/corpus/notebook-edit/native-structure/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..1ce91a413d6fff4d58765187b76756f0e97ae843 --- /dev/null +++ b/corpus/notebook-edit/native-structure/read/payloads.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/corpus/notebook-edit/native-structure/run.json b/corpus/notebook-edit/native-structure/run.json new file mode 100644 index 0000000000000000000000000000000000000000..6c3a356ea771b65ae32e22c7df0dcdbbc9e9988b --- /dev/null +++ b/corpus/notebook-edit/native-structure/run.json @@ -0,0 +1,19 @@ +{ + "notebook": "/Users/clo/dev/one/corpus/link-edit/candidate", + "expected_pages": -1, + "author": "tools/native/notebook-structure.ps1", + "author_timeout_seconds": 600, + "inspect": false, + "collect_notebook": true, + "base": { + "file": "win7-office-base.qcow2", + "format": "qcow2", + "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346", + "virtual_size": 68719476736 + }, + "scripts": { + "author.ps1": "1369d3e7ae7205370d11e9c6abc3fcf4dd9931e7c0da50b3b86f1e2945e1d4a6", + "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331", + "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41" + } +} diff --git a/corpus/notebook-edit/native-structure/scripts/author.ps1 b/corpus/notebook-edit/native-structure/scripts/author.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..caa6737aea0e40f0695880dee51d166868e8ce3e --- /dev/null +++ b/corpus/notebook-edit/native-structure/scripts/author.ps1 @@ -0,0 +1,71 @@ +param([Parameter(Mandatory=$true)][string]$Root, [Parameter(Mandatory=$true)][string]$CloneHost) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +& "$PSScriptRoot\cold-current.ps1" -Root $Root -CloneHost $CloneHost +$namespace = 'http://schemas.microsoft.com/office/onenote/2010/onenote' +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$log = @() +try { + $app.OpenHierarchy((Join-Path $Root 'notebook'), '', [ref]$notebookId, 0) + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $Root 'structure-00-opened.xml'), $hierarchy, [Text.Encoding]::UTF8) + $second = '' + $app.OpenHierarchy('Second.one', $notebookId, [ref]$second, 3) + $third = '' + $app.OpenHierarchy('Third.one', $notebookId, [ref]$third, 3) + $group = '' + $app.OpenHierarchy('Group', $notebookId, [ref]$group, 2) + $inner = '' + $app.OpenHierarchy('Inner.one', $group, [ref]$inner, 3) + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $Root 'structure-01-created.xml'), $hierarchy, [Text.Encoding]::UTF8) + $app.SyncHierarchy($notebookId) + $app.CloseNotebook($notebookId, $false) + $notebookId = '' + $app.OpenHierarchy((Join-Path $Root 'notebook'), '', [ref]$notebookId, 0) + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [xml]$tree = $hierarchy + $manager = New-Object Xml.XmlNamespaceManager($tree.NameTable) + $manager.AddNamespace('one', $namespace) + $sections = @($tree.SelectNodes('//one:Section', $manager)) + foreach ($section in $sections) { + switch ($section.GetAttribute('name')) { + 'Second' { $section.SetAttribute('name', 'Renamed'); $section.SetAttribute('color', '#FFD75E') } + 'Third' { $section.SetAttribute('color', '#B7C997') } + } + } + [IO.File]::WriteAllText((Join-Path $Root 'structure-02-update.xml'), $tree.OuterXml, [Text.Encoding]::UTF8) + $app.UpdateHierarchy($tree.OuterXml, 1) + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $Root 'structure-03-renamed.xml'), $hierarchy, [Text.Encoding]::UTF8) + $app.SyncHierarchy($notebookId) + $app.CloseNotebook($notebookId, $false) + $notebookId = '' + Copy-Item -Recurse (Join-Path $Root 'notebook') (Join-Path $Root 'notebook-renamed') + $app.OpenHierarchy((Join-Path $Root 'notebook'), '', [ref]$notebookId, 0) + $deadline = [DateTime]::UtcNow.AddSeconds(60) + do { + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [xml]$tree = $hierarchy + if (@($tree.SelectNodes('//*[local-name()="Section"]')).Count -ge 4) { break } + Start-Sleep -Milliseconds 500 + } while ([DateTime]::UtcNow -lt $deadline) + $manager = New-Object Xml.XmlNamespaceManager($tree.NameTable) + $manager.AddNamespace('one', $namespace) + [IO.File]::WriteAllText((Join-Path $Root 'structure-03b-reopened.xml'), $hierarchy, [Text.Encoding]::UTF8) + $doomed = @($tree.SelectNodes('//*[local-name()="Section"]') | Where-Object { $_.GetAttribute('name') -eq 'Third' }) + if ($doomed.Count -ne 1) { throw "Expected one section named Third, found $($doomed.Count)." } + $app.DeleteHierarchy($doomed[0].GetAttribute('ID'), [DateTime]::MinValue, $false) + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $Root 'structure-04-deleted.xml'), $hierarchy, [Text.Encoding]::UTF8) + $app.SyncHierarchy($notebookId) + $app.CloseNotebook($notebookId, $false) + $notebookId = '' +} finally { + if ($notebookId) { $app.CloseNotebook($notebookId, $false) } + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + [GC]::Collect() + [GC]::WaitForPendingFinalizers() +} diff --git a/corpus/notebook-edit/native-structure/scripts/cold.ps1 b/corpus/notebook-edit/native-structure/scripts/cold.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..a0b99e7aeb86f4dc10032537800501a52ad65b4f --- /dev/null +++ b/corpus/notebook-edit/native-structure/scripts/cold.ps1 @@ -0,0 +1,27 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '') +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' } +$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote' +if ($CloneHost) { + if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) { + throw 'The disposable clone hostname does not match this machine.' + } + New-Item "$key\Options\Paths" -Force | Out-Null + New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null +} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or + -not (Test-Path 'C:\one-tests\profile-original-cache')) { + throw 'Park the personal OneNote profile before resetting the test cache.' +} +$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0' +$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root)) +if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' } +New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null +if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked } +if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse } +New-Item "$key\OpenNotebooks" | Out-Null +New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null diff --git a/corpus/notebook-edit/native-structure/scripts/read.ps1 b/corpus/notebook-edit/native-structure/scripts/read.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..476094ab1e0c47077988a6461db2080cf208642b --- /dev/null +++ b/corpus/notebook-edit/native-structure/scripts/read.ps1 @@ -0,0 +1,142 @@ +param( + [Parameter(Mandatory=$true)][string]$Root, + [int]$ExpectedPages = -1, + [switch]$UseCurrentCache, + [switch]$Pdf, + [switch]$KeepOpen, + [string]$CloneHost = '' +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +$notebook = Join-Path $root 'notebook' +$output = Join-Path $root 'read' +if (Test-Path $output) { throw 'Choose a new read destination.' } +if ($UseCurrentCache) { + if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') { + throw 'Park the personal OneNote profile before reading test notebooks.' + } +} else { + & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost +} +New-Item -ItemType Directory -Path $output | Out-Null +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$failure = $null +try { + $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0) + $process = Get-Process ONENOTE + @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion; + powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } | + ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8 + $sections = @() + foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) { + $id = '' + $app.OpenHierarchy($file.FullName, '', [ref]$id, 0) + $sections += $id + } + $deadline = [DateTime]::UtcNow.AddSeconds(300) + $previous = '' + $lastChange = '' + $stableSince = [DateTime]::UtcNow + $settled = $false + do { + $pages = @{} + foreach ($section in $sections) { + $hierarchy = '' + $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) + [xml]$xml = $hierarchy + foreach ($node in $xml.SelectNodes('//*[@path]')) { + if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) { + throw 'OneNote opened a section outside the copied notebook.' + } + } + foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) { + $id = $node.GetAttribute('ID') + $content = '' + $app.GetPageContent($id, [ref]$content, 1, 1) + $pages[$id] = $content + } + } + $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] })) + if ($signature -ne $previous) { + $lastChange = $previous + $previous = $signature + $stableSince = [DateTime]::UtcNow + } + if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or + ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and + ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break } + Start-Sleep -Milliseconds 250 + } while ([DateTime]::UtcNow -lt $deadline) + if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) { + [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8) + $index = 0 + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + $index++ + } + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8) + throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled." + } + $index = 0 + $payloads = @() + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + if ($Pdf) { + $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index) + $app.NavigateTo($id, '', $false) + $app.Publish($id, $pdfPath, 3, '') + if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' } + } + [xml]$page = $pages[$id] + foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) { + $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache')) + $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant() + [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes) + $payloads += @{ page = $id; object = $file.ParentNode.GetAttribute('objectID'); + kind = $file.LocalName; name = $file.GetAttribute('preferredName'); + sha256 = $hash; bytes = $bytes.Length } + } + $index++ + } + [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8) + $all = '' + $app.GetHierarchy($notebookId, 4, [ref]$all, 1) + [xml]$finalTree = $all + $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique) + if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) { + throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.' + } + [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8) + Write-Output "Read $($sections.Count) sections and $($pages.Count) pages." +} catch { + $failure = $_ + [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + throw +} finally { + try { + try { + if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) } + if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) { + $app.CloseNotebook($notebookId, $false) + } + } catch { + if ($null -eq $failure) { throw } + [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + } + } finally { + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + } + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + if (-not $UseCurrentCache -and -not $CloneHost) { + Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10 + } +} diff --git a/corpus/notebook-edit/native-structure/source.json b/corpus/notebook-edit/native-structure/source.json new file mode 100644 index 0000000000000000000000000000000000000000..14786084a474f1cc0a6f522bd49c8104ad3627f0 --- /dev/null +++ b/corpus/notebook-edit/native-structure/source.json @@ -0,0 +1,14 @@ +[ + { + "path": "Open Notebook.onetoc2", + "bytes": 1604, + "sha256": "6731c4214437fe1cb6aa347b0004542b7f7e4260f47773bf9ec07c4e894de47c", + "mtime_ns": 1789206821144296891 + }, + { + "path": "links.one", + "bytes": 6128, + "sha256": "0cfb356ed374fe919f73fb73fdbe5d66e0baa484d4119f70074d9f869d6db78a", + "mtime_ns": 1789206821116200533 + } +] diff --git a/corpus/notebook-edit/native-structure/structure-00-opened.xml b/corpus/notebook-edit/native-structure/structure-00-opened.xml new file mode 100644 index 0000000000000000000000000000000000000000..4e4c71ddc2d95b643864e00e4ca9b8442721ad31 --- /dev/null +++ b/corpus/notebook-edit/native-structure/structure-00-opened.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/notebook-edit/native-structure/structure-01-created.xml b/corpus/notebook-edit/native-structure/structure-01-created.xml new file mode 100644 index 0000000000000000000000000000000000000000..d5aecd6a994608493b531c2376be05a83a289140 --- /dev/null +++ b/corpus/notebook-edit/native-structure/structure-01-created.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/notebook-edit/native-structure/structure-02-update.xml b/corpus/notebook-edit/native-structure/structure-02-update.xml new file mode 100644 index 0000000000000000000000000000000000000000..0ba2ecae962b71c5557241f6dcce09f5f5c1b83c --- /dev/null +++ b/corpus/notebook-edit/native-structure/structure-02-update.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/corpus/notebook-edit/native-structure/structure-03-renamed.xml b/corpus/notebook-edit/native-structure/structure-03-renamed.xml new file mode 100644 index 0000000000000000000000000000000000000000..e6bbf68ba1b4aa95710683a7c7d85b9b51b3a468 --- /dev/null +++ b/corpus/notebook-edit/native-structure/structure-03-renamed.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/notebook-edit/native-structure/structure-03b-reopened.xml b/corpus/notebook-edit/native-structure/structure-03b-reopened.xml new file mode 100644 index 0000000000000000000000000000000000000000..19831bb3f4ffdcffb15dd6cda847ff5c35e13e0e --- /dev/null +++ b/corpus/notebook-edit/native-structure/structure-03b-reopened.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/notebook-edit/native-structure/structure-04-deleted.xml b/corpus/notebook-edit/native-structure/structure-04-deleted.xml new file mode 100644 index 0000000000000000000000000000000000000000..0d58e35673332aebe601d62d209fbe6c3b78026c --- /dev/null +++ b/corpus/notebook-edit/native-structure/structure-04-deleted.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/notebook-edit/native-structure/teardown.json b/corpus/notebook-edit/native-structure/teardown.json new file mode 100644 index 0000000000000000000000000000000000000000..05a47793de40f322e745c4c4183727e3590ecf70 --- /dev/null +++ b/corpus/notebook-edit/native-structure/teardown.json @@ -0,0 +1 @@ +{"absent": true} diff --git a/corpus/notebook-edit/structured/candidate/First.one b/corpus/notebook-edit/structured/candidate/First.one new file mode 100644 index 0000000000000000000000000000000000000000..73c4e8efe60b799c1de48850f501516327e165bc Binary files /dev/null and b/corpus/notebook-edit/structured/candidate/First.one differ diff --git a/corpus/notebook-edit/structured/candidate/Kept/Inner.one b/corpus/notebook-edit/structured/candidate/Kept/Inner.one new file mode 100644 index 0000000000000000000000000000000000000000..52c134aeb17662b915978ff668e8fa5aed1a79b9 Binary files /dev/null and b/corpus/notebook-edit/structured/candidate/Kept/Inner.one differ diff --git a/corpus/notebook-edit/structured/candidate/Kept/Open Notebook.onetoc2 b/corpus/notebook-edit/structured/candidate/Kept/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..5991fddfa49d5235a429f41fee6aefa7ae655195 Binary files /dev/null and b/corpus/notebook-edit/structured/candidate/Kept/Open Notebook.onetoc2 differ diff --git a/corpus/notebook-edit/structured/candidate/Open Notebook.onetoc2 b/corpus/notebook-edit/structured/candidate/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..0f0cfa53f1ab928351bb9ff5b1836d37a422da03 Binary files /dev/null and b/corpus/notebook-edit/structured/candidate/Open Notebook.onetoc2 differ diff --git a/corpus/notebook-edit/structured/candidate/Renamed.one b/corpus/notebook-edit/structured/candidate/Renamed.one new file mode 100644 index 0000000000000000000000000000000000000000..31fba4276c85f0e413b6cb036992a8102c31d014 Binary files /dev/null and b/corpus/notebook-edit/structured/candidate/Renamed.one differ diff --git a/corpus/notebook-edit/structured/cold/commands.jsonl b/corpus/notebook-edit/structured/cold/commands.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..97e2d1eb8ce74259020f9c18671dc67067b8f5da --- /dev/null +++ b/corpus/notebook-edit/structured/cold/commands.jsonl @@ -0,0 +1,3 @@ +{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-A31627CC -ExpectedPages -1", "exit": 0, "stdout": "Read 3 sections and 3 pages.\r\n", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null} diff --git a/corpus/notebook-edit/structured/cold/machine.json b/corpus/notebook-edit/structured/cold/machine.json new file mode 100644 index 0000000000000000000000000000000000000000..362a2d1d14429c44e98dbab0ef63ce8c0d9cfe2b --- /dev/null +++ b/corpus/notebook-edit/structured/cold/machine.json @@ -0,0 +1 @@ +{"name": "m6-a31627cc", "hostname": "ONE-M6-A31627CC"} diff --git a/corpus/notebook-edit/structured/cold/notebook/First.one b/corpus/notebook-edit/structured/cold/notebook/First.one new file mode 100644 index 0000000000000000000000000000000000000000..73c4e8efe60b799c1de48850f501516327e165bc Binary files /dev/null and b/corpus/notebook-edit/structured/cold/notebook/First.one differ diff --git a/corpus/notebook-edit/structured/cold/notebook/Kept/Inner.one b/corpus/notebook-edit/structured/cold/notebook/Kept/Inner.one new file mode 100644 index 0000000000000000000000000000000000000000..52c134aeb17662b915978ff668e8fa5aed1a79b9 Binary files /dev/null and b/corpus/notebook-edit/structured/cold/notebook/Kept/Inner.one differ diff --git a/corpus/notebook-edit/structured/cold/notebook/Kept/Open Notebook.onetoc2 b/corpus/notebook-edit/structured/cold/notebook/Kept/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..5991fddfa49d5235a429f41fee6aefa7ae655195 Binary files /dev/null and b/corpus/notebook-edit/structured/cold/notebook/Kept/Open Notebook.onetoc2 differ diff --git a/corpus/notebook-edit/structured/cold/notebook/Open Notebook.onetoc2 b/corpus/notebook-edit/structured/cold/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..aca80d29b252adba4ae4fbd7b553da11023ed3d9 Binary files /dev/null and b/corpus/notebook-edit/structured/cold/notebook/Open Notebook.onetoc2 differ diff --git a/corpus/notebook-edit/structured/cold/notebook/Renamed.one b/corpus/notebook-edit/structured/cold/notebook/Renamed.one new file mode 100644 index 0000000000000000000000000000000000000000..31fba4276c85f0e413b6cb036992a8102c31d014 Binary files /dev/null and b/corpus/notebook-edit/structured/cold/notebook/Renamed.one differ diff --git a/corpus/notebook-edit/structured/cold/read/environment.json b/corpus/notebook-edit/structured/cold/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..b119e4fcd06ca1a80360de9c9c571427c3a3cf9a --- /dev/null +++ b/corpus/notebook-edit/structured/cold/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-A31627CC", + "cold": true, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/notebook-edit/structured/cold/read/hierarchy.xml b/corpus/notebook-edit/structured/cold/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..37e4cd0165325041c867d2439844ea14c068758f --- /dev/null +++ b/corpus/notebook-edit/structured/cold/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/notebook-edit/structured/cold/read/page-000.xml b/corpus/notebook-edit/structured/cold/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..52ece39370fe0bbef1d7fb32c6801cf76803a930 --- /dev/null +++ b/corpus/notebook-edit/structured/cold/read/page-000.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/notebook-edit/structured/cold/read/page-001.xml b/corpus/notebook-edit/structured/cold/read/page-001.xml new file mode 100644 index 0000000000000000000000000000000000000000..803fd8b035cd8815f744d0825401b690b29d9305 --- /dev/null +++ b/corpus/notebook-edit/structured/cold/read/page-001.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/notebook-edit/structured/cold/read/page-002.xml b/corpus/notebook-edit/structured/cold/read/page-002.xml new file mode 100644 index 0000000000000000000000000000000000000000..8c42c95e23447652243442f606cbc2614694a2b1 --- /dev/null +++ b/corpus/notebook-edit/structured/cold/read/page-002.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/notebook-edit/structured/cold/read/payloads.json b/corpus/notebook-edit/structured/cold/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..1ce91a413d6fff4d58765187b76756f0e97ae843 --- /dev/null +++ b/corpus/notebook-edit/structured/cold/read/payloads.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/corpus/notebook-edit/structured/cold/run.json b/corpus/notebook-edit/structured/cold/run.json new file mode 100644 index 0000000000000000000000000000000000000000..efc1a27f8ac2fb96ec2c151500177de83d5852f8 --- /dev/null +++ b/corpus/notebook-edit/structured/cold/run.json @@ -0,0 +1,18 @@ +{ + "notebook": "/private/tmp/m4/structure3", + "expected_pages": -1, + "author": null, + "author_timeout_seconds": 600, + "inspect": false, + "collect_notebook": true, + "base": { + "file": "win7-office-base.qcow2", + "format": "qcow2", + "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346", + "virtual_size": 68719476736 + }, + "scripts": { + "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331", + "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41" + } +} diff --git a/corpus/notebook-edit/structured/cold/scripts/cold.ps1 b/corpus/notebook-edit/structured/cold/scripts/cold.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..a0b99e7aeb86f4dc10032537800501a52ad65b4f --- /dev/null +++ b/corpus/notebook-edit/structured/cold/scripts/cold.ps1 @@ -0,0 +1,27 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '') +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' } +$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote' +if ($CloneHost) { + if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) { + throw 'The disposable clone hostname does not match this machine.' + } + New-Item "$key\Options\Paths" -Force | Out-Null + New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null +} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or + -not (Test-Path 'C:\one-tests\profile-original-cache')) { + throw 'Park the personal OneNote profile before resetting the test cache.' +} +$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0' +$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root)) +if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' } +New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null +if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked } +if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse } +New-Item "$key\OpenNotebooks" | Out-Null +New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null diff --git a/corpus/notebook-edit/structured/cold/scripts/read.ps1 b/corpus/notebook-edit/structured/cold/scripts/read.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..476094ab1e0c47077988a6461db2080cf208642b --- /dev/null +++ b/corpus/notebook-edit/structured/cold/scripts/read.ps1 @@ -0,0 +1,142 @@ +param( + [Parameter(Mandatory=$true)][string]$Root, + [int]$ExpectedPages = -1, + [switch]$UseCurrentCache, + [switch]$Pdf, + [switch]$KeepOpen, + [string]$CloneHost = '' +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +$notebook = Join-Path $root 'notebook' +$output = Join-Path $root 'read' +if (Test-Path $output) { throw 'Choose a new read destination.' } +if ($UseCurrentCache) { + if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') { + throw 'Park the personal OneNote profile before reading test notebooks.' + } +} else { + & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost +} +New-Item -ItemType Directory -Path $output | Out-Null +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$failure = $null +try { + $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0) + $process = Get-Process ONENOTE + @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion; + powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } | + ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8 + $sections = @() + foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) { + $id = '' + $app.OpenHierarchy($file.FullName, '', [ref]$id, 0) + $sections += $id + } + $deadline = [DateTime]::UtcNow.AddSeconds(300) + $previous = '' + $lastChange = '' + $stableSince = [DateTime]::UtcNow + $settled = $false + do { + $pages = @{} + foreach ($section in $sections) { + $hierarchy = '' + $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) + [xml]$xml = $hierarchy + foreach ($node in $xml.SelectNodes('//*[@path]')) { + if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) { + throw 'OneNote opened a section outside the copied notebook.' + } + } + foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) { + $id = $node.GetAttribute('ID') + $content = '' + $app.GetPageContent($id, [ref]$content, 1, 1) + $pages[$id] = $content + } + } + $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] })) + if ($signature -ne $previous) { + $lastChange = $previous + $previous = $signature + $stableSince = [DateTime]::UtcNow + } + if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or + ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and + ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break } + Start-Sleep -Milliseconds 250 + } while ([DateTime]::UtcNow -lt $deadline) + if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) { + [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8) + $index = 0 + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + $index++ + } + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8) + throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled." + } + $index = 0 + $payloads = @() + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + if ($Pdf) { + $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index) + $app.NavigateTo($id, '', $false) + $app.Publish($id, $pdfPath, 3, '') + if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' } + } + [xml]$page = $pages[$id] + foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) { + $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache')) + $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant() + [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes) + $payloads += @{ page = $id; object = $file.ParentNode.GetAttribute('objectID'); + kind = $file.LocalName; name = $file.GetAttribute('preferredName'); + sha256 = $hash; bytes = $bytes.Length } + } + $index++ + } + [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8) + $all = '' + $app.GetHierarchy($notebookId, 4, [ref]$all, 1) + [xml]$finalTree = $all + $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique) + if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) { + throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.' + } + [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8) + Write-Output "Read $($sections.Count) sections and $($pages.Count) pages." +} catch { + $failure = $_ + [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + throw +} finally { + try { + try { + if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) } + if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) { + $app.CloseNotebook($notebookId, $false) + } + } catch { + if ($null -eq $failure) { throw } + [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + } + } finally { + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + } + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + if (-not $UseCurrentCache -and -not $CloneHost) { + Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10 + } +} diff --git a/corpus/notebook-edit/structured/cold/source.json b/corpus/notebook-edit/structured/cold/source.json new file mode 100644 index 0000000000000000000000000000000000000000..b364775d9806efe99f98c992ccb7c2faf10f66d8 --- /dev/null +++ b/corpus/notebook-edit/structured/cold/source.json @@ -0,0 +1,32 @@ +[ + { + "path": "First.one", + "bytes": 3212, + "sha256": "d926b2f5502dd1bb2768928e44020497248299b9e6c1c8ddcec999117279c177", + "mtime_ns": 1789279283377754822 + }, + { + "path": "Kept/Inner.one", + "bytes": 3164, + "sha256": "33e43aef96619ba34e4499b605e21540ac25475542ae8817e37ed10382ac25ca", + "mtime_ns": 1789279283538323423 + }, + { + "path": "Kept/Open Notebook.onetoc2", + "bytes": 3824, + "sha256": "432b3413622bb8eabacc640ffc8c6ad80dee94cd45bed6154f5afbc2ae443965", + "mtime_ns": 1789279283654960588 + }, + { + "path": "Open Notebook.onetoc2", + "bytes": 5408, + "sha256": "e713f4d9cca21552e7fd650c54b6072fc2d8875e48b738b00887324f82331728", + "mtime_ns": 1789279283758856923 + }, + { + "path": "Renamed.one", + "bytes": 5504, + "sha256": "9ee7bd23c1679ce5035c92adca0416730b149d11bedd1b4a7c181a0186df7007", + "mtime_ns": 1789279283726949659 + } +] diff --git a/corpus/notebook-edit/structured/cold/teardown.json b/corpus/notebook-edit/structured/cold/teardown.json new file mode 100644 index 0000000000000000000000000000000000000000..05a47793de40f322e745c4c4183727e3590ecf70 --- /dev/null +++ b/corpus/notebook-edit/structured/cold/teardown.json @@ -0,0 +1 @@ +{"absent": true} diff --git a/crates/notebook/README.md b/crates/notebook/README.md index 70ce70ea2b329dd4328c6264bea9cd80cb5160cc..8d854ac6801ad9acfc7a82075f3c5ed11cc84a35 100644 --- a/crates/notebook/README.md +++ b/crates/notebook/README.md @@ -160,6 +160,17 @@ in one local transaction. The supplied images must still match `snapshot()` and so another overlapping remote edit can produce a new conflict. Pending intents and uncertain attempts cannot be reviewed. +An attempt that stays `AwaitingConfirmation` is released by review, never by +replay: `release_attempt(id, local, remote, archive, after)` on the replica +(`release` on a session) exports the branch to a new archive, then retires +the oldest edit's attempt. `Some(page)` continues from a page reviewed +against the current remote image as a fresh intent under the same id, +keeping later edits; `None` abandons the local branch, whose ids report +`EditStatus::Archived { archive }` from then on, and the working image +returns to the remote image. Neither writes a receipt: the uncertain +publication may or may not have landed, and the archive is the record of +what was attempted. + Opening recognizes the application identity and the current schema version only; caches and archives written by other versions are refused unchanged. Until the application is usable end to end there are no migrations. @@ -295,7 +306,8 @@ Discovery returns ordered sections and nested groups with file identities and share-relative paths. Section display-name overrides remain distinct from file names. TOC references whose identities are absent from the directory remain inspectable; a cached filename never substitutes for an identity match. -Reserved `_onefiles` directories are excluded from section-group traversal. +Reserved `_onefiles` directories are excluded from section-group traversal; +OneNote's `OneNote_RecycleBin` is listed as the section group OneNote shows. Encrypted sections and valid storage with an unreadable document graph retain their identity as `Locked` or `Unreadable`, without being presented as empty pages. Malformed storage, failed reads and ambiguous identities reject the discovery. @@ -312,6 +324,37 @@ Missing files, permissions and size failures retain their I/O error kinds; an empty payload is a successful empty buffer. Embedded payloads remain available directly from the core document model. +## Notebook structure + +`Notebook::create_section`, `create_group`, `rename`, `set_section_color`, +`reorder` and `delete` change a notebook the way OneNote does: the table of +contents (`Open Notebook.onetoc2`, created when a folder has none) gains, +renames, reorders or loses entries; a section's colour lives in its own +metadata; a deleted section moves into `OneNote_RecycleBin`, a group with its +own TOC. Every created, renamed or moved file is placed with +`onestore::place_file`, which sets the header's ancestor to the parent TOC's +identity and the name CRC OneNote checks on open; a file without them is +re-identified and listed anew. These operate on the mounted directory; +the SMB transport has no create, rename or delete. + +`Notebook::find_page(url)` resolves a stored internal link to a section path +and page space by identity: the linked section first, then every readable +section, so a link follows its page when the section is renamed or moved and +when the page itself was moved to another section. Other URLs and unknown +targets are `None`. + +With the optional `protected` feature, `Notebook::unlock(path, password)` +reads the pages of a `Locked` section for display; nothing is cached or +written, and a wrong password is `Error::Protected(PasswordMismatch)`. + +`Notebook::refresh` rereads the directory on the caller's schedule and reports +what another client changed as `Change`s keyed by file identity: a renamed or +moved section is `Moved`, not removed and added, and a deleted section moves +into `OneNote_RecycleBin`; a folder whose surviving entries changed sequence +is `Reordered`. A failed read returns the error and keeps +the previous catalog, so an unreachable share never reads as an emptied +notebook. + ## SMB (feature `smb`) `notebook::smb` provides blocking SMB access for OneNote sections and diff --git a/crates/notebook/examples/recovery_probe.rs b/crates/notebook/examples/recovery_probe.rs index b69d3c3e351fbc46501a7b8d6b9fcf1b50861781..841d92c2aaa8a7b6dd47213c635bb2999d4f765e 100644 --- a/crates/notebook/examples/recovery_probe.rs +++ b/crates/notebook/examples/recovery_probe.rs @@ -91,6 +91,7 @@ fn report(cache: &Replica, root: &Path) -> Result<(), Box } Some(EditStatus::Published { revision }) => ("published", Some(revision.to_string())), Some(EditStatus::Conflict(_)) => ("conflict", None), + Some(EditStatus::Archived { archive }) => ("archived", Some(archive)), None => ("missing", None), }; println!( diff --git a/crates/notebook/src/lib.rs b/crates/notebook/src/lib.rs index 0e4cd47db025b66e4136562154dd7dfe011acbae..e35e6a7e4c8c52a22a195225f4bb0480bfec3ab6 100644 --- a/crates/notebook/src/lib.rs +++ b/crates/notebook/src/lib.rs @@ -51,12 +51,15 @@ pub enum Error { Discovery(#[from] discover::Error), #[error("External payload identity now refers to different bytes")] AssetChanged, + #[cfg(feature = "protected")] + #[error(transparent)] + Protected(#[from] onestore::protected::Error), } type Result = std::result::Result; const APPLICATION_ID: u32 = 0x4f4e454f; -const SCHEMA_VERSION: u32 = 12; +const SCHEMA_VERSION: u32 = 13; /// An edited page model together with the stored model it was edited from. /// `before` is the precondition reconciliation checks against the remote page. diff --git a/crates/notebook/src/schema.rs b/crates/notebook/src/schema.rs index ef12c0c62a5e7d1e06e4a0a08ef12a53857f3a06..4c163f268b3b3164cc6083a714c6729213884641 100644 --- a/crates/notebook/src/schema.rs +++ b/crates/notebook/src/schema.rs @@ -27,6 +27,10 @@ pub(crate) fn create(transaction: &Transaction<'_>) -> Result<()> { CREATE TABLE receipts ( edit_id INTEGER PRIMARY KEY CHECK(edit_id>0), revision TEXT NOT NULL + ) STRICT; + CREATE TABLE archived ( + edit_id INTEGER PRIMARY KEY CHECK(edit_id>0), + archive TEXT NOT NULL ) STRICT;", )?; transaction.execute_batch(CONFLICTS)?; diff --git a/crates/notebook/src/session.rs b/crates/notebook/src/session.rs index 9bc1b894349395de053b30128038e3487378717f..e117740c1587bbb0aa10d6531f7f7c23c5fc42df 100644 --- a/crates/notebook/src/session.rs +++ b/crates/notebook/src/session.rs @@ -8,7 +8,9 @@ use onestore::{ CommitError, ExGuid, PreparedEdit, RevisionIndex, Store, document::Document, page::Page, }; use std::{ + collections::BTreeMap, io, + io::Write, path::{Path, PathBuf}, sync::{ Arc, @@ -17,6 +19,52 @@ use std::{ time::Duration, }; +/// A difference between two catalog reads of a notebook, in catalog paths. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Change { + /// A section or group appeared, including one replacing a file under an existing path. + Added(String), + /// A section or group is no longer in the notebook. + Removed(String), + /// The same file now lives at another path: a rename or a move between groups. + Moved { from: String, to: String }, + /// A folder's surviving sections and groups changed order. + Reordered(String), +} + +/// Sections and groups are followed by file identity; a group without a TOC only by path. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +enum Key { + File([u8; 16]), + Folder(String), +} + +type Entries = (BTreeMap, BTreeMap>); + +fn entries(folder: &discover::Folder) -> Entries { + fn walk(folder: &discover::Folder, out: &mut Entries) { + let mut order = Vec::new(); + for section in &folder.sections { + let key = Key::File(section.file_id); + out.0.insert(key.clone(), section.path.clone()); + order.push(key); + } + for group in &folder.groups { + let key = match &group.toc { + Some(toc) => Key::File(toc.file_id), + None => Key::Folder(group.path.clone()), + }; + out.0.insert(key.clone(), group.path.clone()); + order.push(key); + walk(group, out); + } + out.1.insert(folder.path.clone(), order); + } + let mut out = Entries::default(); + walk(folder, &mut out); + out +} + /// A notebook directory and the cache directory holding its section replicas. pub struct Notebook { root: PathBuf, @@ -48,6 +96,365 @@ impl Notebook { &self.catalog } + /// Rereads the notebook directory and reports what changed since the last catalog, + /// keyed by file identity so a renamed or moved section stays the same section. A + /// failed read keeps the previous catalog: an unreachable notebook is not an empty one. + pub fn refresh(&mut self) -> Result> { + let catalog = discover::discover( + &mut discover::Local::open(&self.root)?, + discover::Limits { + entries: 100_000, + bytes_per_file: 256 * 1024 * 1024, + depth: 64, + }, + )?; + let (before, before_orders) = entries(&self.catalog); + let (after, after_orders) = entries(&catalog); + let mut changes = Vec::new(); + for key in before_orders.values().flatten() { + let from = &before[key]; + match after.get(key) { + None => changes.push(Change::Removed(from.clone())), + Some(to) if to != from => changes.push(Change::Moved { + from: from.clone(), + to: to.clone(), + }), + Some(_) => {} + } + } + for key in after_orders.values().flatten() { + if !before.contains_key(key) { + changes.push(Change::Added(after[key].clone())); + } + } + for (folder, order) in &after_orders { + let Some(previous) = before_orders.get(folder) else { + continue; + }; + let kept = |sequence: &[Key], other: &[Key]| -> Vec { + sequence + .iter() + .filter(|key| other.contains(key)) + .cloned() + .collect() + }; + if kept(previous, order) != kept(order, previous) { + changes.push(Change::Reordered(folder.clone())); + } + } + self.catalog = catalog; + Ok(changes) + } + + fn folder(&self, path: &str) -> Result<&discover::Folder> { + let mut folders = vec![&self.catalog]; + while let Some(folder) = folders.pop() { + if folder.path == path { + return Ok(folder); + } + folders.extend(&folder.groups); + } + Err(io::Error::from(io::ErrorKind::NotFound).into()) + } + + fn directory(&self, path: &str) -> Result { + let directory = if path.is_empty() { + self.root.clone() + } else { + self.root.join(path).canonicalize()? + }; + if !directory.starts_with(&self.root) || !directory.is_dir() { + return Err(io::Error::from(io::ErrorKind::PermissionDenied).into()); + } + Ok(directory) + } + + /// A folder's TOC path and file identity, creating the TOC when the folder has none + /// (OneNote names it `Open Notebook.onetoc2`). + fn toc(&self, folder: &str) -> Result<(PathBuf, [u8; 16])> { + let directory = self.directory(folder)?; + match &self.folder(folder)?.toc { + Some(toc) => Ok((directory.join(&toc.filename), toc.file_id)), + None => { + let path = directory.join("Open Notebook.onetoc2"); + let bytes = onestore::create_table_of_contents("Open Notebook.onetoc2", &[])?; + std::fs::File::create_new(&path)?.write_all(&bytes)?; + Ok((path, onestore::Store::parse(&bytes)?.header.file_id)) + } + } + } + + fn edit_toc(&self, folder: &str, edits: &[onestore::TocEdit]) -> Result<()> { + let (toc, _) = self.toc(folder)?; + let source = onestore::read_file(&toc)?; + PreparedEdit::table_of_contents(&source, edits)? + .commit_file(&toc) + .map_err(|error| error.error)?; + Ok(()) + } + + /// Creates `name.one` in `folder` with one empty page and lists it last in the folder's + /// TOC, as OneNote creates a section. Returns the new catalog path. + pub fn create_section(&mut self, folder: &str, name: &str, author: &str) -> Result { + let filename = format!("{name}.one"); + let directory = self.directory(folder)?; + let (_, ancestor) = self.toc(folder)?; + let bytes = onestore::create_section(&filename, "", author)?; + let path = directory.join(&filename); + std::fs::File::create_new(&path)?.write_all(&bytes)?; + onestore::place_file(&path, ancestor, &filename)?; + let identity = onestore::Store::parse(&bytes)?.header.file_id; + self.edit_toc( + folder, + &[onestore::TocEdit::Add { + filename: filename.clone(), + identity, + group: false, + }], + )?; + self.refresh()?; + Ok(catalog_path(folder, &filename)) + } + + /// Creates a section group: a folder with its own TOC, listed last in the parent's TOC. + pub fn create_group(&mut self, folder: &str, name: &str) -> Result { + let directory = self.directory(folder)?; + if !component(name) { + return Err(io::Error::from(io::ErrorKind::InvalidInput).into()); + } + let group = directory.join(name); + let (_, ancestor) = self.toc(folder)?; + std::fs::create_dir(&group)?; + let bytes = onestore::create_table_of_contents("Open Notebook.onetoc2", &[])?; + let path = group.join("Open Notebook.onetoc2"); + std::fs::File::create_new(&path)?.write_all(&bytes)?; + onestore::place_file(&path, ancestor, name)?; + let identity = onestore::Store::parse(&bytes)?.header.file_id; + self.edit_toc( + folder, + &[onestore::TocEdit::Add { + filename: name.to_owned(), + identity, + group: true, + }], + )?; + self.refresh()?; + Ok(catalog_path(folder, name)) + } + + /// Renames a section or section group: the file or folder on disk and its TOC entry. + pub fn rename(&mut self, path: &str, name: &str) -> Result { + let (folder, entry) = split(path); + let directory = self.directory(folder)?; + let (filename, identity) = self.entry(folder, entry)?; + let target = if filename.to_ascii_lowercase().ends_with(".one") { + format!("{name}.one") + } else { + name.to_owned() + }; + if !component(&target) || directory.join(&target).exists() { + return Err(io::Error::from(io::ErrorKind::AlreadyExists).into()); + } + std::fs::rename(directory.join(&filename), directory.join(&target))?; + let (_, ancestor) = self.toc(folder)?; + let placed = if target.ends_with(".one") { + directory.join(&target) + } else { + directory.join(&target).join("Open Notebook.onetoc2") + }; + onestore::place_file(placed, ancestor, &target)?; + self.edit_toc( + folder, + &[onestore::TocEdit::Rename { + identity, + filename: target.clone(), + }], + )?; + self.refresh()?; + Ok(catalog_path(folder, &target)) + } + + /// Sets a section's colour (COLORREF) in its own metadata, where OneNote keeps it. + pub fn set_section_color(&mut self, path: &str, color: Option) -> Result<()> { + let file = self.root.join(path).canonicalize()?; + if !file.starts_with(&self.root) { + return Err(io::Error::from(io::ErrorKind::PermissionDenied).into()); + } + let source = onestore::read_file(&file)?; + let store = onestore::Store::parse(&source)?; + let index = onestore::RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + let revision = document.active(document.root)?; + let metadata = *revision + .roots + .get(&2) + .ok_or_else(|| io::Error::from(io::ErrorKind::InvalidData))?; + onestore::commit_file_property( + &file, + &source, + document.root, + metadata, + 0x14001cbe, + &color.unwrap_or(0xffff_ffff).to_le_bytes(), + ) + .map_err(|error| error.error)?; + self.refresh()?; + Ok(()) + } + + /// Orders a folder's sections and groups; entries left out follow in their current order. + pub fn reorder(&mut self, folder: &str, paths: &[&str]) -> Result<()> { + let mut identities = Vec::new(); + for path in paths { + let (parent, entry) = split(path); + if parent != folder { + return Err(io::Error::from(io::ErrorKind::InvalidInput).into()); + } + identities.push(self.entry(folder, entry)?.1); + } + self.edit_toc(folder, &[onestore::TocEdit::Order(identities)])?; + self.refresh().map(drop) + } + + /// Deletes a section the way OneNote does: the file moves into the notebook's + /// `OneNote_RecycleBin` folder, a section group with its own TOC that lists it (and that + /// the root TOC lists), and its own folder's TOC entry goes. + pub fn delete(&mut self, path: &str) -> Result<()> { + let (folder, entry) = split(path); + let directory = self.directory(folder)?; + let (filename, identity) = self.entry(folder, entry)?; + if !filename.to_ascii_lowercase().ends_with(".one") { + return Err(io::Error::from(io::ErrorKind::InvalidInput).into()); + } + let bin = self.root.join("OneNote_RecycleBin"); + let bin_toc = bin.join("Open Notebook.onetoc2"); + if !bin_toc.exists() { + std::fs::create_dir_all(&bin)?; + let bytes = onestore::create_table_of_contents("Open Notebook.onetoc2", &[])?; + std::fs::File::create_new(&bin_toc)?.write_all(&bytes)?; + onestore::place_file(&bin_toc, self.toc("")?.1, "OneNote_RecycleBin")?; + let bin_identity = onestore::Store::parse(&bytes)?.header.file_id; + self.edit_toc( + "", + &[onestore::TocEdit::Add { + filename: "OneNote_RecycleBin".into(), + identity: bin_identity, + group: true, + }], + )?; + } + let mut target = filename.clone(); + let mut attempt = 1; + while bin.join(&target).exists() { + attempt += 1; + let (stem, extension) = filename.rsplit_once('.').unwrap_or((&filename, "")); + target = format!("{stem} ({attempt}).{extension}"); + } + std::fs::rename(directory.join(&filename), bin.join(&target))?; + let source = onestore::read_file(&bin_toc)?; + onestore::place_file( + bin.join(&target), + onestore::Store::parse(&source)?.header.file_id, + &target, + )?; + PreparedEdit::table_of_contents( + &source, + &[onestore::TocEdit::Add { + filename: target, + identity, + group: false, + }], + )? + .commit_file(&bin_toc) + .map_err(|error| error.error)?; + self.edit_toc(folder, &[onestore::TocEdit::Remove { identity }])?; + self.refresh().map(drop) + } + + /// The stored filename and TOC identity of a folder's section or group. + fn entry(&self, folder: &str, name: &str) -> Result<(String, [u8; 16])> { + let parent = self.folder(folder)?; + if let Some(section) = parent + .sections + .iter() + .find(|section| split(§ion.path).1 == name) + { + return Ok((name.to_owned(), section.file_id)); + } + if let Some(group) = parent + .groups + .iter() + .find(|group| split(&group.path).1 == name) + { + let toc = group + .toc + .as_ref() + .ok_or_else(|| io::Error::from(io::ErrorKind::NotFound))?; + return Ok((name.to_owned(), toc.file_id)); + } + Err(io::Error::from(io::ErrorKind::NotFound).into()) + } + + /// The section path and page space a stored internal link opens, found by identity: + /// in the linked section first, then in every other readable section, so a link + /// follows its page across sections. `None` for other URLs and unknown targets. + pub fn find_page(&self, url: &str) -> Result)>> { + let Some(link) = onestore::page::link::parse_internal_link(url) else { + return Ok(None); + }; + let mut sections = Vec::new(); + let mut folders = vec![&self.catalog]; + while let Some(folder) = folders.pop() { + sections.extend(folder.sections.iter().filter(|section| { + matches!(section.state, discover::SectionState::Readable { .. }) + })); + folders.extend(&folder.groups); + } + sections.sort_by_key(|section| section.file_id != link.section); + let Some(page) = link.page else { + return Ok(sections + .first() + .filter(|section| section.file_id == link.section) + .map(|section| (section.path.clone(), None))); + }; + for section in sections { + let bytes = onestore::read_file(self.root.join(§ion.path))?; + let store = Store::parse(&bytes)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + for (space, _) in document.pages()? { + if Page::identity_of(document.active(space)?) == Some(page) { + return Ok(Some((section.path.clone(), Some(space)))); + } + } + } + Ok(None) + } + + /// The pages of a password-protected section, for reading: nothing is cached or + /// written, and the decoded buffers go when the pages have been built. + #[cfg(feature = "protected")] + pub fn unlock(&self, path: &str, password: &str) -> Result> { + let file = self.root.join(path).canonicalize()?; + if !file.starts_with(&self.root) { + return Err(io::Error::from(io::ErrorKind::PermissionDenied).into()); + } + let bytes = onestore::read_file(&file)?; + let store = Store::parse(&bytes)?; + let index = RevisionIndex::parse(&store)?; + let unlocked = onestore::protected::UnlockedSection::open( + &index, + password, + onestore::protected::Limits::default(), + )?; + let document = unlocked.document()?; + document + .pages()? + .into_iter() + .map(|(space, _)| Ok((space, Page::from_space(&document, space)?))) + .collect() + } + /// Opens a section by its catalog path. pub fn section(&self, path: &str, notify: impl Fn() + Send + 'static) -> Result
{ let mut folders = vec![&self.catalog]; @@ -65,6 +472,25 @@ impl Notebook { } } +fn component(name: &str) -> bool { + !name.is_empty() && !name.contains(['/', '\\', '\0']) && name != "." && name != ".." +} + +fn split(path: &str) -> (&str, &str) { + match path.rsplit_once('/') { + Some((folder, name)) => (folder, name), + None => ("", path), + } +} + +fn catalog_path(folder: &str, name: &str) -> String { + if folder.is_empty() { + name.to_owned() + } else { + format!("{folder}/{name}") + } +} + /// What happened on the synchronization thread since the last poll. #[derive(Debug)] pub enum Event { @@ -181,7 +607,7 @@ impl Section { Ok(None) => Event::Refreshed, Ok(Some((id, status))) => Event::Attempt { id: *id, - status: *status, + status: status.clone(), }, Err(Error::RemoteIo(error)) => Event::Unreachable(match error.raw_os_error() { Some(code) => io::Error::from_raw_os_error(code), @@ -308,6 +734,18 @@ impl Section { Ok(()) } + /// Retires an uncertain attempt after review, exporting the branch to `archive` + /// first: `Some(page)` continues from the reviewed page against `remote_page`, + /// `None` abandons the local branch. Neither claims the attempt was acknowledged. + pub fn release(&self, id: u64, archive: impl AsRef, after: Option<&Page>) -> Result<()> { + let local = self.replica.snapshot()?; + let remote = self.replica.remote_snapshot()?; + self.replica + .release_attempt(id, &local, &remote, archive.as_ref(), after)?; + self.wake(); + Ok(()) + } + /// Captures both images, the queue and its states in a read-only archive. pub fn export_recovery(&self, path: impl AsRef) -> Result<()> { self.replica.export_recovery(path) diff --git a/crates/notebook/src/sync.rs b/crates/notebook/src/sync.rs index 56b71f181a3cf4db24010ae7a15e3a8cbe09367b..5093fbe14797971599f6262ef65685d9bc65ca27 100644 --- a/crates/notebook/src/sync.rs +++ b/crates/notebook/src/sync.rs @@ -27,7 +27,7 @@ pub enum ConflictKind { ContentChanged = 3, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum EditStatus { Pending, /// Retained publication attempt; this revision alone may be insufficient to confirm it. @@ -39,6 +39,11 @@ pub enum EditStatus { Published { revision: ExGuid, }, + /// Retired unpublished by a reviewed release; the archive at this path holds the + /// intent, its attempt evidence and both images. + Archived { + archive: String, + }, } impl Replica { @@ -355,6 +360,120 @@ impl Replica { }) } + /// Retires the oldest edit's uncertain attempt after review. The branch is first + /// exported to `archive` (a new file), which is the record: no receipt is written. + /// `Some(after)` continues from the reviewed page as a fresh intent against the + /// current remote page, keeping the edit's id and its dependents; `None` abandons + /// the whole local branch, whose edits become `Archived`, and the working image + /// returns to the remote image. Both need the reviewed images to be current. + pub fn release_attempt( + &self, + id: u64, + local: &[u8], + remote: &[u8], + archive: &Path, + after: Option<&Page>, + ) -> Result<()> { + let owner = self.sync_owner()?; + let intent = self + .pending()? + .into_iter() + .next() + .filter(|intent| intent.id == id) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "Only the oldest edit can hold an attempt", + ) + })?; + let continued = match after { + Some(after) => { + let Operation::Page(edit) = intent.operation else { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Select a page save to continue from", + ) + .into()); + }; + let before = page_of(remote, intent.space)?.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "The page is no longer in the remote section", + ) + })?; + PreparedEdit::page(remote, intent.space, after, &edit.author)?; + Some(Operation::Page(PageIntent { + before, + after: after.clone(), + author: edit.author, + })) + } + None => None, + }; + let mut connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let (base, working): (Vec, Vec) = + transaction.query_row("SELECT base, working FROM replica WHERE id=1", [], |row| { + Ok((row.get(0)?, row.get(1)?)) + })?; + if working != local || base != remote { + return Err(io::Error::new( + io::ErrorKind::ResourceBusy, + "The reviewed cache images changed", + ) + .into()); + } + let id = i64::try_from(id).map_err(io::Error::other)?; + let attempted: bool = transaction.query_row( + "SELECT EXISTS(SELECT 1 FROM attempt WHERE edit_id=?1)", + [id], + |row| row.get(0), + )?; + if !attempted { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "The edit has no uncertain attempt", + ) + .into()); + } + drop(transaction); + drop(connection); + self.export_recovery(archive)?; + let mut connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + transaction.execute("DELETE FROM attempt WHERE edit_id=?1", [id])?; + match continued { + Some(operation) => { + transaction.execute( + "UPDATE edits SET operation=?1 WHERE id=?2", + params![ + serde_json::to_string(&operation).map_err(io::Error::other)?, + id + ], + )?; + } + None => { + transaction.execute( + "INSERT INTO archived(edit_id, archive) SELECT id, ?1 FROM edits", + [archive.to_string_lossy().into_owned()], + )?; + transaction.execute("DELETE FROM edits", [])?; + transaction.execute("UPDATE replica SET working=base WHERE id=1", [])?; + } + } + transaction.commit()?; + drop(connection); + drop(owner); + self.wake_sync(); + Ok(()) + } + fn resolve_conflict( &self, id: u64, @@ -462,6 +581,16 @@ pub(crate) fn status(connection: &Connection, id: u64) -> Result(0), + ) + .optional()? + { + return Ok(Some(EditStatus::Archived { archive })); + } let record: Option<(Option, Option, String)> = connection.query_row( "SELECT attempt.revisions, conflicts.kind, edits.space FROM edits LEFT JOIN attempt ON attempt.edit_id=edits.id LEFT JOIN conflicts ON conflicts.edit_id=edits.id WHERE edits.id=?1", [id], |row| Ok((row.get(0)?,row.get(1)?, row.get(2)?))).optional()?; Ok(match record { diff --git a/crates/notebook/tests/changes.rs b/crates/notebook/tests/changes.rs new file mode 100644 index 0000000000000000000000000000000000000000..1131d39958c722393ffcedb30c3b651d8d264c9b --- /dev/null +++ b/crates/notebook/tests/changes.rs @@ -0,0 +1,78 @@ +use notebook::session::{Change, Notebook}; +use std::io::Write; + +/// One client watches a notebook while another changes it on disk; changes follow file +/// identity, and an unreachable notebook keeps the last catalog. +#[test] +fn external_structure_changes_are_reported_by_identity() { + let temporary = tempfile::tempdir().unwrap(); + let root = temporary.path().join("notebook"); + std::fs::create_dir(&root).unwrap(); + let first = onestore::create_section("First.one", "First page", "Author").unwrap(); + std::fs::File::create_new(root.join("First.one")) + .unwrap() + .write_all(&first) + .unwrap(); + let first_id = onestore::Store::parse(&first).unwrap().header.file_id; + std::fs::write( + root.join("Open Notebook.onetoc2"), + onestore::create_table_of_contents("Open Notebook.onetoc2", &[("First.one", first_id)]) + .unwrap(), + ) + .unwrap(); + let mut watcher = Notebook::open(&root, temporary.path().join("watcher")).unwrap(); + let mut other = Notebook::open(&root, temporary.path().join("other")).unwrap(); + assert_eq!(watcher.refresh().unwrap(), []); + + other.create_section("", "Second", "Author").unwrap(); + other.create_group("", "Group").unwrap(); + other.create_section("Group", "Inner", "Author").unwrap(); + assert_eq!( + watcher.refresh().unwrap(), + [ + Change::Added("Second.one".into()), + Change::Added("Group".into()), + Change::Added("Group/Inner.one".into()), + ] + ); + + other.rename("Second.one", "Renamed").unwrap(); + assert_eq!( + watcher.refresh().unwrap(), + [Change::Moved { + from: "Second.one".into(), + to: "Renamed.one".into() + }] + ); + + other.reorder("", &["Renamed.one", "First.one"]).unwrap(); + assert_eq!(watcher.refresh().unwrap(), [Change::Reordered("".into())]); + + other.delete("First.one").unwrap(); + assert_eq!( + watcher.refresh().unwrap(), + [ + Change::Moved { + from: "First.one".into(), + to: "OneNote_RecycleBin/First.one".into() + }, + Change::Added("OneNote_RecycleBin".into()), + ] + ); + std::fs::remove_dir_all(root.join("OneNote_RecycleBin")).unwrap(); + assert_eq!( + watcher.refresh().unwrap(), + [ + Change::Removed("OneNote_RecycleBin".into()), + Change::Removed("OneNote_RecycleBin/First.one".into()), + ] + ); + + let away = temporary.path().join("away"); + std::fs::rename(&root, &away).unwrap(); + assert!(watcher.refresh().is_err()); + assert_eq!(watcher.catalog().sections.len(), 1); + std::fs::rename(&away, &root).unwrap(); + assert_eq!(watcher.refresh().unwrap(), []); + assert_eq!(watcher.refresh().unwrap(), []); +} diff --git a/crates/notebook/tests/links.rs b/crates/notebook/tests/links.rs new file mode 100644 index 0000000000000000000000000000000000000000..39307dc6bc5a259f238786ca8bec97d9dab4946c --- /dev/null +++ b/crates/notebook/tests/links.rs @@ -0,0 +1,78 @@ +use notebook::session::Notebook; +use onestore::page::link::{LinkTarget, internal_link}; +use std::io::Write; + +fn write_section(root: &std::path::Path, name: &str, text: &str) -> [u8; 16] { + let bytes = onestore::create_section(name, text, "Author").unwrap(); + std::fs::File::create_new(root.join(name)) + .unwrap() + .write_all(&bytes) + .unwrap(); + onestore::Store::parse(&bytes).unwrap().header.file_id +} + +/// Links resolve by identity: a page keeps its link when its section is renamed or moved, +/// and a link whose section identity is stale still finds the page in another section. +#[test] +fn internal_links_find_their_page_by_identity() { + let temporary = tempfile::tempdir().unwrap(); + let root = temporary.path().join("notebook"); + std::fs::create_dir(&root).unwrap(); + let first = write_section(&root, "First.one", "Target page"); + let second = write_section(&root, "Second.one", "Other page"); + std::fs::write( + root.join("Open Notebook.onetoc2"), + onestore::create_table_of_contents( + "Open Notebook.onetoc2", + &[("First.one", first), ("Second.one", second)], + ) + .unwrap(), + ) + .unwrap(); + let mut notebook = Notebook::open(&root, temporary.path().join("cache")).unwrap(); + let section = notebook.section("First.one", || {}).unwrap(); + let (space, _) = section.pages().unwrap()[0]; + let page = section.page(space).unwrap(); + section.close().unwrap(); + let target = LinkTarget::Page { + identity: page.identity.unwrap(), + title: &page.title, + }; + let link = internal_link(first, "C:\\notebook\\First.one", target); + assert_eq!( + notebook.find_page(&link).unwrap(), + Some(("First.one".to_owned(), Some(space))) + ); + assert_eq!( + notebook + .find_page(&internal_link(first, "", LinkTarget::Section)) + .unwrap(), + Some(("First.one".to_owned(), None)) + ); + assert_eq!( + notebook.find_page("https://example.invalid/").unwrap(), + None + ); + + notebook.create_group("", "Group").unwrap(); + notebook.rename("First.one", "Renamed").unwrap(); + assert_eq!( + notebook.find_page(&link).unwrap(), + Some(("Renamed.one".to_owned(), Some(space))) + ); + + let stale = internal_link(second, "", target); + assert_eq!( + notebook.find_page(&stale).unwrap(), + Some(("Renamed.one".to_owned(), Some(space))) + ); + let unknown = internal_link( + second, + "", + LinkTarget::Page { + identity: [7; 16], + title: "Gone", + }, + ); + assert_eq!(notebook.find_page(&unknown).unwrap(), None); +} diff --git a/crates/notebook/tests/protected.rs b/crates/notebook/tests/protected.rs new file mode 100644 index 0000000000000000000000000000000000000000..9fe75e76a21ce67bce41efb8e85eb0d7eeaf9c9d --- /dev/null +++ b/crates/notebook/tests/protected.rs @@ -0,0 +1,29 @@ +#![cfg(feature = "protected")] + +use notebook::{discover::SectionState, session::Notebook}; +use std::path::Path; + +/// A locked section is discovered as such and reads with its password, page by page. +#[test] +fn a_locked_section_unlocks_for_reading() { + let root = Path::new(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../corpus/native-protected-boundaries" + )); + let manifest: serde_json::Value = + serde_json::from_slice(&std::fs::read(root.join("manifest.json")).unwrap()).unwrap(); + let password = manifest["password"].as_str().unwrap(); + let temporary = tempfile::tempdir().unwrap(); + let notebook = Notebook::open(root.join("notebook"), temporary.path()).unwrap(); + let section = ¬ebook.catalog().sections[0]; + assert!(matches!(section.state, SectionState::Locked)); + let pages = notebook.unlock(§ion.path, password).unwrap(); + assert_eq!(pages.len(), 11); + assert!(pages.iter().all(|(_, page)| !page.title.is_empty())); + assert!(matches!( + notebook.unlock(§ion.path, "wrong"), + Err(notebook::Error::Protected( + onestore::protected::Error::PasswordMismatch + )) + )); +} diff --git a/crates/notebook/tests/session.rs b/crates/notebook/tests/session.rs index 2d221c31d5c24564a079b187656bb8de04028e4a..260862ab46f589dc05b28140c92d11a8a8047cad 100644 --- a/crates/notebook/tests/session.rs +++ b/crates/notebook/tests/session.rs @@ -680,3 +680,100 @@ fn a_conflicting_save_is_reviewed_against_the_remote_page_and_archived_for_recov assert!(section.queue().unwrap().is_empty()); section.close().unwrap(); } + +#[path = "support/server.rs"] +mod server; + +/// An uncertain attempt survives a restart as `AwaitingConfirmation`; after exporting the +/// archive the user either continues from a reviewed page or abandons the branch. Neither +/// path records a receipt for the uncertain attempt. +#[test] +fn an_uncertain_attempt_is_released_after_restart_by_review() { + for continued in [true, false] { + let directory = tempfile::tempdir().unwrap(); + let file = directory.path().join("notes.one"); + let cache = directory.path().join("cache.sqlite"); + let source = onestore::create_section("notes.one", "Original", "Author").unwrap(); + std::fs::write(&file, &source).unwrap(); + let replica = notebook::Replica::create(&cache, &source).unwrap(); + let space = space_of(&source); + let page = model_ops::page_of(&source, space); + let text = first_text(&page); + let local = edited(&page, "Uncertain "); + let id = replica + .save(&source, space, &local, "Author") + .unwrap() + .unwrap(); + let mut faulty = server::Server::new(&source); + faulty.fault = server::Fault::UnknownBefore; + assert!(replica.sync_once(&mut faulty).is_err()); + assert!(matches!( + replica.status(id).unwrap(), + Some(EditStatus::AwaitingConfirmation { .. }) + )); + drop(replica); + + let notified = Arc::new(AtomicUsize::new(0)); + let counter = Arc::clone(¬ified); + let section = Section::resume(&file, notebook::Replica::open(&cache).unwrap(), move || { + counter.fetch_add(1, Ordering::SeqCst); + }) + .unwrap(); + wait( + §ion, + |event| matches!(event, Event::Attempt { id: n, status: EditStatus::AwaitingConfirmation { .. } } if *n == id), + ); + assert_eq!( + section + .queue() + .unwrap() + .iter() + .map(|edit| edit.id) + .collect::>(), + [id] + ); + assert_same(section.page(space).unwrap(), &local); + let archive = directory.path().join("review.sqlite"); + if continued { + let mut reviewed = section.remote_page(space).unwrap(); + model_ops::replace_text(&mut reviewed, text, 0..0, "Reviewed "); + section.release(id, &archive, Some(&reviewed)).unwrap(); + assert_eq!(section.status(id).unwrap(), Some(EditStatus::Pending)); + published(§ion, id); + assert_same(stored_page(&file, space), &reviewed); + } else { + section.release(id, &archive, None).unwrap(); + assert_eq!( + section.status(id).unwrap(), + Some(EditStatus::Archived { + archive: archive.to_string_lossy().into_owned() + }) + ); + assert_same(section.page(space).unwrap(), &page); + assert_eq!(stored_page(&file, space), page); + } + let recovery = Recovery::open(&archive).unwrap(); + assert_eq!(recovery.pending().unwrap().len(), 1); + assert!(matches!( + recovery.status(id).unwrap(), + Some(EditStatus::AwaitingConfirmation { .. }) + )); + assert!(section.queue().unwrap().is_empty()); + assert!( + section + .release(id, directory.path().join("again.sqlite"), None) + .is_err() + ); + section.close().unwrap(); + } +} + +fn space_of(source: &[u8]) -> ExGuid { + let store = onestore::Store::parse(source).unwrap(); + let index = onestore::RevisionIndex::parse(&store).unwrap(); + onestore::document::Document::parse(&index) + .unwrap() + .pages() + .unwrap()[0] + .0 +} diff --git a/crates/notebook/tests/structure.rs b/crates/notebook/tests/structure.rs new file mode 100644 index 0000000000000000000000000000000000000000..6274848d901a6ee49db5374bd193c0b9b0f6ae3d --- /dev/null +++ b/crates/notebook/tests/structure.rs @@ -0,0 +1,173 @@ +use notebook::session::Notebook; +use std::io::Write; + +/// Each file's header names its parent TOC and carries the name CRC OneNote wrote for the +/// same name in `corpus/notebook-edit/{structured,deleted}/cold`. +fn placement(root: &std::path::Path, expected: &[(&str, u32)]) { + let header = |path: std::path::PathBuf| { + onestore::Store::parse(&std::fs::read(path).unwrap()) + .unwrap() + .header + }; + for (path, crc) in expected { + let (folder, name) = path.rsplit_once('/').unwrap_or(("", path)); + let parent = header(root.join(folder).join("Open Notebook.onetoc2")).file_id; + let file = if name.ends_with(".one") { + root.join(path) + } else { + root.join(path).join("Open Notebook.onetoc2") + }; + let header = header(file); + assert_eq!((header.ancestor, header.name_crc), (parent, *crc), "{path}"); + } +} + +fn names(folder: ¬ebook::discover::Folder) -> (Vec, Vec) { + ( + folder.sections.iter().map(|s| s.path.clone()).collect(), + folder.groups.iter().map(|g| g.path.clone()).collect(), + ) +} + +/// `NOTEBOOK_STRUCTURE_EXPORT` names a new directory receiving the final notebook for a +/// cold reopen. +#[test] +fn sections_and_groups_are_created_renamed_coloured_ordered_and_deleted() { + let temporary = tempfile::tempdir().unwrap(); + let root = temporary.path().join("notebook"); + std::fs::create_dir(&root).unwrap(); + let first = onestore::create_section("First.one", "First page", "Author").unwrap(); + std::fs::File::create_new(root.join("First.one")) + .unwrap() + .write_all(&first) + .unwrap(); + let first_id = onestore::Store::parse(&first).unwrap().header.file_id; + std::fs::write( + root.join("Open Notebook.onetoc2"), + onestore::create_table_of_contents("Open Notebook.onetoc2", &[("First.one", first_id)]) + .unwrap(), + ) + .unwrap(); + let toc = onestore::read_file(root.join("Open Notebook.onetoc2")).unwrap(); + let toc_id = onestore::Store::parse(&toc).unwrap().header.file_id; + onestore::place_file(root.join("First.one"), toc_id, "First.one").unwrap(); + let mut notebook = Notebook::open(&root, temporary.path().join("cache")).unwrap(); + assert_eq!( + notebook.create_section("", "Second", "Author").unwrap(), + "Second.one" + ); + assert_eq!(notebook.create_group("", "Archive").unwrap(), "Archive"); + assert_eq!( + notebook + .create_section("Archive", "Inner", "Author") + .unwrap(), + "Archive/Inner.one" + ); + assert_eq!( + names(notebook.catalog()), + ( + vec!["First.one".to_owned(), "Second.one".to_owned()], + vec!["Archive".to_owned()] + ) + ); + assert_eq!( + names(¬ebook.catalog().groups[0]), + (vec!["Archive/Inner.one".to_owned()], vec![]) + ); + assert_eq!( + notebook.rename("Second.one", "Renamed").unwrap(), + "Renamed.one" + ); + assert_eq!(notebook.rename("Archive", "Kept").unwrap(), "Kept"); + notebook + .set_section_color("Renamed.one", Some(0x5ed7ff)) + .unwrap(); + notebook.reorder("", &["Kept", "Renamed.one"]).unwrap(); + assert_eq!( + names(notebook.catalog()), + ( + vec!["Renamed.one".to_owned(), "First.one".to_owned()], + vec!["Kept".to_owned()] + ) + ); + let toc = notebook.catalog().toc.as_ref().unwrap(); + let orders: Vec<(Option, u32)> = toc + .unresolved + .iter() + .map(|entry| (entry.filename.clone(), entry.order)) + .collect(); + assert!(orders.is_empty(), "{orders:?}"); + let colour = onestore::read_file(root.join("Renamed.one")).unwrap(); + let store = onestore::Store::parse(&colour).unwrap(); + let index = onestore::RevisionIndex::parse(&store).unwrap(); + let document = onestore::document::Document::parse(&index).unwrap(); + let revision = document.active(document.root).unwrap(); + let onestore::document::Kind::SectionMetadata { color, .. } = + &revision.nodes[&revision.roots[&2]].kind + else { + panic!() + }; + assert_eq!(*color, Some(0x5ed7ff)); + placement( + &root, + &[ + ("First.one", 0x2e1b24f3), + ("Renamed.one", 0x6108912a), + ("Kept", 0xf2a400f4), + ("Kept/Inner.one", 0x83e261da), + ], + ); + if let Some(directory) = std::env::var_os("NOTEBOOK_STRUCTURE_EXPORT") { + copy_dir(&root, std::path::Path::new(&directory)); + } + notebook.delete("First.one").unwrap(); + assert_eq!( + names(notebook.catalog()), + ( + vec!["Renamed.one".to_owned()], + vec!["Kept".to_owned(), "OneNote_RecycleBin".to_owned()] + ) + ); + assert_eq!( + names(¬ebook.catalog().groups[1]), + (vec!["OneNote_RecycleBin/First.one".to_owned()], vec![]) + ); + assert!(root.join("OneNote_RecycleBin/First.one").exists()); + let bin = onestore::read_file(root.join("OneNote_RecycleBin/Open Notebook.onetoc2")).unwrap(); + assert!( + String::from_utf16_lossy( + &bin.chunks_exact(2) + .map(|c| u16::from_le_bytes([c[0], c[1]])) + .collect::>() + ) + .contains("First.one") + ); + assert!(notebook.rename("Kept/Inner.one", "First").is_ok()); + assert!(notebook.delete("Missing.one").is_err()); + placement( + &root, + &[ + ("Renamed.one", 0x6108912a), + ("Kept", 0xf2a400f4), + ("Kept/First.one", 0x2e1b24f3), + ("OneNote_RecycleBin", 0x986b646a), + ("OneNote_RecycleBin/First.one", 0x2e1b24f3), + ], + ); + if let Some(directory) = std::env::var_os("NOTEBOOK_STRUCTURE_EXPORT_DELETED") { + copy_dir(&root, std::path::Path::new(&directory)); + } +} + +fn copy_dir(from: &std::path::Path, to: &std::path::Path) { + std::fs::create_dir_all(to).unwrap(); + for entry in std::fs::read_dir(from).unwrap() { + let entry = entry.unwrap(); + let target = to.join(entry.file_name()); + if entry.file_type().unwrap().is_dir() { + copy_dir(&entry.path(), &target); + } else { + std::fs::copy(entry.path(), target).unwrap(); + } + } +} diff --git a/crates/notebook/tests/sync.rs b/crates/notebook/tests/sync.rs index 047b59de8873a97452f134398402d525a4724027..f2702fdfd81e39fc01c58932a45e2d5250ca3c1b 100644 --- a/crates/notebook/tests/sync.rs +++ b/crates/notebook/tests/sync.rs @@ -81,7 +81,7 @@ fn recovery_archive_preserves_the_queue_uncertainty_and_receipts_without_becomin assert_eq!(archive.snapshot().unwrap(), working); assert_eq!(archive.remote_snapshot().unwrap(), remote); assert_eq!(archive.pending().unwrap(), pending); - assert_eq!(archive.status(published).unwrap(), Some(receipt)); + assert_eq!(archive.status(published).unwrap(), Some(receipt.clone())); assert_eq!(archive.status(attempted).unwrap(), uncertain); for edit in &pending { assert_eq!( @@ -89,7 +89,7 @@ fn recovery_archive_preserves_the_queue_uncertainty_and_receipts_without_becomin cache.status(edit.id).unwrap() ); } - let EditStatus::Published { revision } = receipt else { + let EditStatus::Published { revision } = receipt.clone() else { panic!() }; assert_eq!(archive.receipts().unwrap(), [(published, revision)].into()); @@ -158,7 +158,7 @@ fn recovery_archive_retains_conflict_images_and_rejects_foreign_or_future_archiv let archive = Recovery::open(&path).unwrap(); assert_eq!(text(&archive.snapshot().unwrap()).2, "Local"); assert_eq!(archive.remote_snapshot().unwrap(), changed); - assert_eq!(archive.status(id).unwrap(), Some(outcome.1)); + assert_eq!(archive.status(id).unwrap(), Some(outcome.1.clone())); assert_eq!(archive.summary().unwrap().conflicts, 1); assert_eq!(archive.summary().unwrap().uncertain_edits, 0); drop(archive); @@ -173,7 +173,7 @@ fn recovery_archive_retains_conflict_images_and_rejects_foreign_or_future_archiv assert!(Recovery::open(&path).is_err()); assert_eq!(std::fs::read(&path).unwrap(), before); } - assert_eq!(cache.status(id).unwrap(), Some(outcome.1)); + assert_eq!(cache.status(id).unwrap(), Some(outcome.1.clone())); assert_eq!(text(&cache.snapshot().unwrap()).2, "Local"); } @@ -195,7 +195,7 @@ fn disjoint_remote_changes_merge_and_persist_the_remote_receipt() { assert!(cache.pending().unwrap().is_empty()); drop(cache); let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.status(id).unwrap(), Some(outcome.1)); + assert_eq!(cache.status(id).unwrap(), Some(outcome.1.clone())); assert_eq!(cache.sync_once(&mut server).unwrap(), None); assert_eq!(server.publications, 1); assert_eq!(cache.status(id + 1).unwrap(), None); @@ -849,17 +849,17 @@ fn remote_restore_retains_historical_receipts_without_replaying_them() { assert_eq!(cache.sync_once(&mut server).unwrap(), None); assert_eq!(cache.snapshot().unwrap(), source); assert_eq!(cache.remote_snapshot().unwrap(), source); - assert_eq!(cache.status(published).unwrap(), Some(receipt)); + assert_eq!(cache.status(published).unwrap(), Some(receipt.clone())); assert_eq!(server.publications, 1); assert!(cache.pending().unwrap().is_empty()); let archive = notebook::Recovery::open(directory.path().join("published.sqlite")).unwrap(); assert_eq!(archive.snapshot().unwrap(), published_image); - assert_eq!(archive.status(published).unwrap(), Some(receipt)); + assert_eq!(archive.status(published).unwrap(), Some(receipt.clone())); assert_eq!(text(&archive.snapshot().unwrap()).2, "Published Original"); drop(cache); let cache = Replica::open(path).unwrap(); assert_eq!(text(&cache.snapshot().unwrap()).2, "Original"); - assert_eq!(cache.status(published).unwrap(), Some(receipt)); + assert_eq!(cache.status(published).unwrap(), Some(receipt.clone())); } #[test] @@ -892,10 +892,10 @@ fn remote_restore_rebases_unsent_work_but_never_replays_an_uncertain_attempt() { let attempts = server.publications; let cache = Replica::open(&path).unwrap(); let (_, status) = cache.sync_once(&mut server).unwrap().unwrap(); - assert_eq!(cache.status(published).unwrap(), Some(receipt)); + assert_eq!(cache.status(published).unwrap(), Some(receipt.clone())); if uncertain { assert!(matches!(status, EditStatus::AwaitingConfirmation { .. })); - assert_eq!(Some(status), prior_status); + assert_eq!(Some(status.clone()), prior_status); assert_eq!(server.publications, attempts); assert_eq!(server.visible, source); assert_eq!(cache.snapshot().unwrap(), local); @@ -913,8 +913,8 @@ fn remote_restore_rebases_unsent_work_but_never_replays_an_uncertain_attempt() { assert_eq!(archive.status(queued).unwrap(), prior_status); drop(cache); let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.status(queued).unwrap(), Some(status)); - assert_eq!(cache.status(published).unwrap(), Some(receipt)); + assert_eq!(cache.status(queued).unwrap(), Some(status.clone())); + assert_eq!(cache.status(published).unwrap(), Some(receipt.clone())); } } @@ -934,7 +934,7 @@ fn restore_conflicts_keep_dependent_work_and_reject_stale_review() { let conflict = EditStatus::Conflict(ConflictKind::ContentChanged); assert_eq!( cache.sync_once(&mut server).unwrap(), - Some((head, conflict)) + Some((head, conflict.clone())) ); let end = u32::try_from(text(&cache.snapshot().unwrap()).2.encode_utf16().count()).unwrap(); let dependent = save(&cache, object, end..end, " Later").unwrap(); @@ -947,7 +947,7 @@ fn restore_conflicts_keep_dependent_work_and_reject_stale_review() { .unwrap(); drop(cache); let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.status(head).unwrap(), Some(conflict)); + assert_eq!(cache.status(head).unwrap(), Some(conflict.clone())); assert_eq!(cache.pending().unwrap(), pending); assert_eq!(cache.snapshot().unwrap(), local); let reviewed = edited(&source, object, 0..0, "Revised "); @@ -956,7 +956,7 @@ fn restore_conflicts_keep_dependent_work_and_reject_stale_review() { server.durable.clone_from(&changed); assert_eq!( cache.sync_once(&mut server).unwrap(), - Some((head, conflict)) + Some((head, conflict.clone())) ); assert!( matches!(cache.review_page(head, &local, &source, &reviewed), @@ -975,7 +975,7 @@ fn restore_conflicts_keep_dependent_work_and_reject_stale_review() { Some((actual, EditStatus::Published { .. })) if actual == head)); assert_eq!( cache.sync_once(&mut server).unwrap(), - Some((dependent, conflict)) + Some((dependent, conflict.clone())) ); assert_eq!(server.publications, 2); let dependent_local = cache.snapshot().unwrap(); @@ -989,11 +989,11 @@ fn restore_conflicts_keep_dependent_work_and_reject_stale_review() { assert!(matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == dependent)); assert_eq!(text(&server.durable).2, "Revised Original Later Remote"); - assert_eq!(cache.status(published).unwrap(), Some(receipt)); + assert_eq!(cache.status(published).unwrap(), Some(receipt.clone())); let archive = notebook::Recovery::open(directory.path().join("conflicted.sqlite")).unwrap(); assert_eq!(archive.snapshot().unwrap(), local); assert_eq!(archive.pending().unwrap(), pending); - assert_eq!(archive.status(head).unwrap(), Some(conflict)); + assert_eq!(archive.status(head).unwrap(), Some(conflict.clone())); } #[test] diff --git a/crates/notebook/tests/sync_formatting.rs b/crates/notebook/tests/sync_formatting.rs index a45e04dceafbdd0ca61bca9600abee508d190558..4396f27daef2375976ad5cb673f4f125fc0c4271 100644 --- a/crates/notebook/tests/sync_formatting.rs +++ b/crates/notebook/tests/sync_formatting.rs @@ -391,10 +391,13 @@ fn uncertain_formatting_attempts_keep_the_original_attempt_and_never_replay() { assert!(matches!(state, EditStatus::AwaitingConfirmation { .. })); drop(cache); let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.status(id).unwrap(), Some(state)); + assert_eq!(cache.status(id).unwrap(), Some(state.clone())); if matches!(fault, Fault::UnknownBefore) { for _ in 0..3 { - assert_eq!(cache.sync_once(&mut server).unwrap(), Some((id, state))); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, state.clone())) + ); } assert_eq!(server.publications, 1); assert_eq!(server.durable, OUTLINES); diff --git a/crates/notebook/tests/sync_outline.rs b/crates/notebook/tests/sync_outline.rs index 210b6f1f3519787f1c5ea440cba70863cd3f9281..f83c7b95204621a9e4716416b987c6ce416a4354 100644 --- a/crates/notebook/tests/sync_outline.rs +++ b/crates/notebook/tests/sync_outline.rs @@ -410,11 +410,17 @@ fn an_uncertain_layout_attempt_confirms_by_revision_or_by_an_equal_remote_page() assert_eq!((server.publications, server.confirmations), (1, 1)); continue; } - assert_eq!(cache.sync_once(&mut server).unwrap(), Some((id, status))); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, status.clone())) + ); server.visible = remote_with(&source, space, |page| { resized(216.0, true)(outline_mut(page, outline)); }); - assert_eq!(cache.sync_once(&mut server).unwrap(), Some((id, status))); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, status.clone())) + ); assert_eq!(cache.snapshot().unwrap(), local); server.visible = remote_with(&source, space, |page| { resized(144.0, true)(outline_mut(page, outline)); diff --git a/crates/notebook/tests/sync_page.rs b/crates/notebook/tests/sync_page.rs index 5f157c4955f5b2d417b885dfe86c992ec713e390..93101f65f5f916dab60a1ef358379d73db35cdac 100644 --- a/crates/notebook/tests/sync_page.rs +++ b/crates/notebook/tests/sync_page.rs @@ -140,7 +140,7 @@ fn uncertain_page_publication_retains_both_revisions_and_never_replays() { cache.export_recovery(&archive).unwrap(); assert_eq!( Recovery::open(&archive).unwrap().status(id).unwrap(), - Some(attempted) + Some(attempted.clone()) ); drop(cache); let db = rusqlite::Connection::open(&path).unwrap(); @@ -163,7 +163,7 @@ fn uncertain_page_publication_retains_both_revisions_and_never_replays() { assert!(observed[..212] == server.durable[..212]); assert!(observed[252..] == server.durable[252..]); } else { - assert_eq!(result, (id, attempted)); + assert_eq!(result, (id, attempted.clone())); assert_eq!(cache.snapshot().unwrap(), local); } } @@ -219,7 +219,10 @@ fn surviving_page_revision_alone_does_not_confirm_section_publication() { .contains_key(&revisions[&page.space()]) ); let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.sync_once(&mut server).unwrap(), Some((id, attempted))); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, attempted.clone())) + ); assert_eq!(server.confirmations, 0); assert_eq!(server.publications, 1); server.visible = complete; diff --git a/crates/notebook/tests/sync_pages.rs b/crates/notebook/tests/sync_pages.rs index 9110551c573e86f28ce274f524e89c45958c2de2..baffaa9274d7f4ea701e5d7ad69f131fdd12456d 100644 --- a/crates/notebook/tests/sync_pages.rs +++ b/crates/notebook/tests/sync_pages.rs @@ -302,7 +302,7 @@ fn uncertain_page_batches_survive_recovery_and_do_not_replay() { cache.export_recovery(&archive).unwrap(); let recovery = Recovery::open(&archive).unwrap(); assert_eq!(recovery.pending().unwrap(), queue); - assert_eq!(recovery.status(id).unwrap(), Some(attempted)); + assert_eq!(recovery.status(id).unwrap(), Some(attempted.clone())); assert_eq!(recovery.snapshot().unwrap(), local); assert!( cache @@ -315,7 +315,7 @@ fn uncertain_page_batches_survive_recovery_and_do_not_replay() { assert!(matches!(result.1, EditStatus::Published { .. })); assert_eq!(server.confirmations, 1); } else { - assert_eq!(result, (id, attempted)); + assert_eq!(result, (id, attempted.clone())); assert_eq!(server.confirmations, 0); } } @@ -366,7 +366,10 @@ fn an_unchanged_section_revision_cannot_confirm_a_changed_page() { let cache = Replica::open(&path).unwrap(); let complete = server.visible.clone(); server.visible = source; - assert_eq!(cache.sync_once(&mut server).unwrap(), Some((id, attempted))); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, attempted.clone())) + ); assert_eq!(server.confirmations, 0); server.visible = complete; assert!(matches!( diff --git a/crates/notebook/tests/sync_paragraph.rs b/crates/notebook/tests/sync_paragraph.rs index dc3e19bd3042cdd8128fd03f169c6a66f08e15cc..a8967ed404d913f808a78988aed7f3dc92ba877b 100644 --- a/crates/notebook/tests/sync_paragraph.rs +++ b/crates/notebook/tests/sync_paragraph.rs @@ -732,7 +732,7 @@ fn uncertain_splits_and_joins_keep_the_original_attempt_across_reopen() { let state = cache.status(id).unwrap().unwrap(); drop(cache); let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.status(id).unwrap(), Some(state)); + assert_eq!(cache.status(id).unwrap(), Some(state.clone())); match fault { Fault::Before => { assert_eq!(state, EditStatus::Pending); @@ -748,7 +748,10 @@ fn uncertain_splits_and_joins_keep_the_original_attempt_across_reopen() { assert_eq!(cache.pending().unwrap(), pending); assert_eq!(cache.snapshot().unwrap(), local); for _ in 0..3 { - assert_eq!(cache.sync_once(&mut server).unwrap(), Some((id, state))); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, state.clone())) + ); } assert_eq!(server.publications, 1); assert_eq!(server.durable, BEFORE); diff --git a/crates/notebook/tests/sync_worker.rs b/crates/notebook/tests/sync_worker.rs index aa85f40166a8e544cb10f40910826ddb66a62b60..1dced50807904d233f60917b3e42c38a3d13d7bc 100644 --- a/crates/notebook/tests/sync_worker.rs +++ b/crates/notebook/tests/sync_worker.rs @@ -108,7 +108,7 @@ fn reconnects_after_connect_read_and_uncertain_publish_without_replaying() { }, move |result| { observed_tx - .send(result.as_ref().copied().map_err(|error| error.to_string())) + .send(result.as_ref().cloned().map_err(|error| error.to_string())) .unwrap(); }, ) @@ -163,7 +163,7 @@ fn local_saves_wake_an_idle_worker_and_publish_every_writer_marker() { Duration::from_secs(3600), move || Ok(shared.clone()), move |result| { - observed_tx.send(*result.as_ref().unwrap()).unwrap(); + observed_tx.send(result.as_ref().unwrap().clone()).unwrap(); if first { first = false; resume_rx.recv_timeout(Duration::from_secs(5)).unwrap(); @@ -305,7 +305,7 @@ fn dropping_during_publication_is_nonblocking_and_retains_ownership_until_recove Duration::from_secs(3600), move || Ok(shared.clone()), move |result| { - tx.send(*result.as_ref().unwrap()).unwrap(); + tx.send(result.as_ref().unwrap().clone()).unwrap(); }, ) .unwrap(); @@ -377,7 +377,7 @@ fn polling_preserves_conflicts_and_absent_uncertain_revisions_without_replay() { Duration::from_millis(10), move || Ok(shared.clone()), move |result| { - tx.send(result.as_ref().copied().map_err(|_| ())).unwrap(); + tx.send(result.as_ref().cloned().map_err(|_| ())).unwrap(); }, ) .unwrap(); @@ -401,7 +401,7 @@ fn polling_preserves_conflicts_and_absent_uncertain_revisions_without_replay() { if let Some(previous) = previous { assert_eq!(previous, status); } - previous = Some(status); + previous = Some(status.clone()); } assert!( started.elapsed() >= Duration::from_millis(30), @@ -508,7 +508,7 @@ fn invalid_intervals_and_callback_panics_leave_worker_ownership_recoverable() { Duration::from_secs(3600), move || Ok(shared.clone()), move |result| { - tx.send(*result.as_ref().unwrap()).unwrap(); + tx.send(result.as_ref().unwrap().clone()).unwrap(); }, ) .unwrap(); @@ -535,7 +535,7 @@ fn reviewed_conflict_wakes_the_worker_and_publishes_the_original_intent_once() { Duration::from_secs(3600), move || Ok(shared.clone()), move |result| { - tx.send(*result.as_ref().unwrap()).unwrap(); + tx.send(result.as_ref().unwrap().clone()).unwrap(); if first { first = false; resume_rx.recv_timeout(Duration::from_secs(5)).unwrap(); @@ -615,7 +615,7 @@ fn ordinary_read_and_unpublished_write_contention_reuse_the_connection() { Duration::from_millis(1), move || Ok(remote.take().expect("Contention caused a reconnect")), move |result| { - tx.send(result.as_ref().copied().map_err(|error| error.to_string())) + tx.send(result.as_ref().cloned().map_err(|error| error.to_string())) .unwrap(); }, ) @@ -678,7 +678,7 @@ fn publication_backoff_drains_local_wakes_without_waiting_for_the_idle_poll() { .unwrap(); tx.send((second, None)).unwrap(); } - Ok(Some((id, status))) => tx.send((*id, Some(*status))).unwrap(), + Ok(Some((id, status))) => tx.send((*id, Some(status.clone()))).unwrap(), Ok(None) => {} other => panic!("Unexpected worker result: {other:?}"), }, diff --git a/crates/onestore/README.md b/crates/onestore/README.md index bc267d205c498f34e9ea2a67ccec8b8f6cc6da13..59b3843b71b8bc68bb4870da4cef7bec8d013357 100644 --- a/crates/onestore/README.md +++ b/crates/onestore/README.md @@ -43,6 +43,7 @@ harness also accepts `--client-profile release`. | `Object::file_reference`, `Store::file_data` | Identify internal/external payloads and read internal payload bytes | | `document::Document`, `Revision::text_runs` | Interpret document objects and inherited text formatting while retaining unknown properties and revision identities | | `Document::active`, `Document::pages_in`, `Revision::parents`, `RevisionIndex::active` | Resolve the active revision, the pages of a space and parent links without repeating the lookups | +| `page::link::internal_link`, `page::link::parse_internal_link` | Build and read the `onenote:#…` URLs OneNote stores for links to sections, pages and paragraphs, by identity | | `page::Page`, `page::Paragraph`, `page::Ink`, `page::Math` | Build an editable page model (title, outlines, paragraphs with coalesced text spans, tables, images, attachments, ink drawings and handwriting decoded to stroke polylines in page points) with stored identities; equations parse from their linear text and run data into a tree that renders the MathML OneNote exports; content outside the model is retained as `Unsupported` | | `protected::UnlockedSection` (optional feature) | Own decoded buffers for explicit known-password inspection; clear those buffers on drop; derived document strings/exports remain caller-owned | | `create_section` | Create one page containing one plain-text paragraph and an author, including Unicode | @@ -50,6 +51,8 @@ harness also accepts `--client-profile release`. | `PageEdit`, `PreparedEdit::pages` | Publish explicitly selected page moves and indentation changes together, preserving page content and historical revisions | | `PreparedEdit::delete_pages_permanently` | Remove explicit pages and their section references atomically while retaining stored revisions | | `create_table_of_contents` | Create ordered section entries from filenames and file identities | +| `TocEdit`, `PreparedEdit::table_of_contents` | Add, rename, colour, order and remove a table of contents' section and group entries as one revision | +| `place_file` | Name a file for its notebook as OneNote does on adoption (parent TOC identity and name CRC in the header), so OneNote keeps its identity | | `replace_property_bytes` | Append one scalar-property revision; preserve prior revisions and unrelated property values and references | | `replace_text`, `commit_text`, `commit_file_text` | Replace a UTF-16 range across ordinary text runs; publish text, run boundaries and modification time together | | `Insertion`, `PreparedEdit::insert` | Insert paragraphs into editable containers or positioned outlines into a page, retaining intent identities across rebases | @@ -57,7 +60,7 @@ harness also accepts `--client-profile release`. | `ParagraphJoin`, `PreparedEdit::join` | Join adjacent ordinary text while preserving inherited character styles and native text-identity rules | | `TextAttribute`, `PreparedEdit::format` | Change character formatting over a UTF-16 range while sharing immutable styles; preserve unselected runs | | `OutlineEdit`, `PreparedEdit::outline` | Change ordinary outline position/width or a paragraph's saved expansion default, preserving identities and content | -| `PreparedEdit::page` | Publish an edited `page::Page` as one revision per changed space: text, character and paragraph formatting, hyperlinks (external and, via `page::link`, to pages, paragraphs and sections), bullets and numbering, note tags, table rows, columns, cell shading and indents, nested tables, inserted pictures (in paragraphs or on the page) and attachments, picture position, size and description, ink drawings and handwriting (strokes added and erased), equations (built from `page::Math` trees), paragraph insertion/split/join/move/deletion, outline insertion/deletion/position/width and saved collapse state, lowered onto the typed writers with the model's identities | +| `PreparedEdit::page` | Publish an edited `page::Page` as one revision per changed space: text, character and paragraph formatting, hyperlinks (external and, via `page::link`, to pages, paragraphs and sections), bullets and numbering, note tags, table rows, columns, cell shading and indents, nested tables, inserted pictures (in paragraphs or on the page) and attachments, picture position, size and description, a stored attachment's shown name, source path and icon size, ink drawings and handwriting (strokes added and erased), equations (built from `page::Math` trees), paragraph insertion/split/join/move/deletion, outline insertion/deletion/position/width and saved collapse state, lowered onto the typed writers with the model's identities | | `TreeEdit`, `PreparedEdit::tree` | Move or delete a subtree on one page, normalize surviving containers, and replace an emptied table cell's paragraph atomically | | `PreparedEdit::commit`, `PreparedEdit::commit_file` | Publish the exact prepared image under caller-held exclusion or the conservative filesystem adapter | | `read_file` | Read a snapshot under whole-file exclusion | diff --git a/crates/onestore/src/commit.rs b/crates/onestore/src/commit.rs index ad2abf55a7f16bff33ae5a85f753a617095c0f12..48e65c2cb419e6036caebd5331b64b23bc69f583 100644 --- a/crates/onestore/src/commit.rs +++ b/crates/onestore/src/commit.rs @@ -71,6 +71,31 @@ impl Drop for FileIo { } } +/// Places a file in its notebook the way OneNote does on adoption: the header's +/// `guidAncestor` becomes the parent table of contents' file identity and `crcName` the CRC +/// of `name` (a section's file name, a group's folder name). OneNote re-identifies a file +/// whose header disagrees with its location, which orphans its TOC entry. +#[cfg(any(unix, windows))] +pub fn place_file(path: impl AsRef, ancestor: [u8; 16], name: &str) -> io::Result<()> { + let mut io = FileIo::open(path, true)?; + let mut header = [0; 1024]; + let result = (|| { + if io.read_at(0, &mut header)? != header.len() { + return Err(io::Error::from(ErrorKind::UnexpectedEof)); + } + crate::Header::parse(&header) + .map_err(|error| io::Error::new(ErrorKind::InvalidData, error.message))?; + let placement = crate::create::placement(ancestor, name); + if io.write_at(128, &placement)? != placement.len() { + return Err(io::Error::from(ErrorKind::WriteZero)); + } + io.flush() + })(); + let released = io.release(); + result?; + released +} + /// Reads a snapshot under the same whole-file exclusion used for commits. /// Native writers can expose incomplete graphs to unlocked filesystem reads. #[cfg(any(unix, windows))] @@ -192,6 +217,18 @@ impl<'a> PreparedEdit<'a> { }) } + /// Applies table-of-contents edits (sections and section groups: add, rename, colour, + /// order, remove) as one revision of a `.onetoc2` file. + pub fn table_of_contents( + source: &'a [u8], + edits: &[crate::TocEdit], + ) -> Result { + Ok(Self { + source, + written: crate::toc::edit_table_of_contents(source, edits)?, + }) + } + /// Permanently removes explicitly selected pages from the section in one transaction. /// Subpages must be selected explicitly; a surviving first subpage becomes top-level. /// Creates no recycle-bin copies. Prior revisions remain stored; this is not secure erasure. diff --git a/crates/onestore/src/create.rs b/crates/onestore/src/create.rs index df06699aafcf8ed0767db5095557602542406a0c..062d894eadd3e0b641e36b8ef78318b2f4ac3927 100644 --- a/crates/onestore/src/create.rs +++ b/crates/onestore/src/create.rs @@ -101,6 +101,15 @@ struct NewSpace { objects: Vec, } +/// Header bytes 128..148: `guidAncestor` and `crcName`, which OneNote checks against a +/// file's location and name; a mismatch makes it re-identify the file. +pub(crate) fn placement(ancestor: [u8; 16], name: &str) -> [u8; 20] { + let mut bytes = [0; 20]; + bytes[..16].copy_from_slice(&ancestor); + bytes[16..].copy_from_slice(&(!crc(u32::MAX, &string(name), FileType::Section)).to_le_bytes()); + bytes +} + pub(crate) fn current_timestamps() -> Result<(u32, u64)> { let now = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -537,8 +546,7 @@ fn create(file_name: &str, file_type: FileType, spaces: Vec) -> Result output[at..at + 4].copy_from_slice(&u32::MAX.to_le_bytes()); } output[96..100].copy_from_slice(&1_u32.to_le_bytes()); - output[144..148] - .copy_from_slice(&(!crc(u32::MAX, &string(file_name), FileType::Section)).to_le_bytes()); + output[128..148].copy_from_slice(&placement([0; 16], file_name)); for (at, chunk) in hashed_chunk .map(|chunk| (148, chunk)) .into_iter() diff --git a/crates/onestore/src/lib.rs b/crates/onestore/src/lib.rs index 06a63f2e28bb773430ac9fd77976a81aaf077298..b40bd2dd500c157b342581fc669defb8ac4a6502 100644 --- a/crates/onestore/src/lib.rs +++ b/crates/onestore/src/lib.rs @@ -21,6 +21,7 @@ pub mod protected; mod revisions; mod snapshot; mod store; +mod toc; mod tree; mod write; @@ -30,7 +31,8 @@ pub use commit::{ }; #[cfg(any(unix, windows))] pub use commit::{ - commit_file_property, commit_file_text, confirm_file_snapshot, read_file, read_file_limited, + commit_file_property, commit_file_text, confirm_file_snapshot, place_file, read_file, + read_file_limited, }; pub use create::{create_section, create_table_of_contents}; pub use edit::replace_text; @@ -45,5 +47,6 @@ pub use properties::{IdStream, Property, PropertySets, Value}; pub use revisions::{ExGuid, ObjectSpace, Revision, RevisionIndex}; pub use snapshot::{read_snapshot, read_storage_snapshot}; pub use store::{Chunk, Error, FileType, Header, Node, NodeList, Reference, Store}; +pub use toc::TocEdit; pub use tree::TreeEdit; pub use write::replace_property_bytes; diff --git a/crates/onestore/src/page/link.rs b/crates/onestore/src/page/link.rs index c2a84e773fdbdd031e1701d869c3cc17e78eeb3f..e33e4b7b6057f0d3f71ca12d45048003d47530a4 100644 --- a/crates/onestore/src/page/link.rs +++ b/crates/onestore/src/page/link.rs @@ -62,6 +62,49 @@ pub fn internal_link(section: [u8; 16], base_path: &str, target: LinkTarget<'_>) url } +/// The identities a stored internal link names. OneNote resolves links by identity, so +/// the title and base path in the URL are hints only. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct InternalLink { + pub section: [u8; 16], + pub page: Option<[u8; 16]>, + pub object: Option, +} + +/// The identities in an `onenote:#…` link, or `None` for any other URL. +pub fn parse_internal_link(url: &str) -> Option { + let guid = |value: &str| { + format!("{value},0") + .parse::() + .ok() + .map(|id| id.guid) + }; + let parts: Vec<&str> = url.strip_prefix("onenote:#")?.split('&').collect(); + let value = |name: &str| { + parts.iter().find_map(|part| { + part.strip_prefix(name) + .and_then(|rest| rest.strip_prefix('=')) + }) + }; + let section = guid(value("section-id")?)?; + let page = match value("page-id") { + Some(id) => Some(guid(id)?), + None => None, + }; + let object = match parts.iter().position(|part| part.starts_with("object-id=")) { + Some(at) => Some(ExGuid { + guid: guid(&parts[at]["object-id=".len()..])?, + n: parts.get(at + 1)?.parse().ok()?, + }), + None => None, + }; + Some(InternalLink { + section, + page, + object, + }) +} + /// Percent-encodes a page title the way OneNote does in a link fragment. fn encoded(title: &str) -> String { let mut out = String::new(); diff --git a/crates/onestore/src/page/mod.rs b/crates/onestore/src/page/mod.rs index 642e28bea9cca331e8adafe3f23e96e5fb2b9e4e..ea7d612c5f944047551fad0f00ec8e6c38d6284d 100644 --- a/crates/onestore/src/page/mod.rs +++ b/crates/onestore/src/page/mod.rs @@ -331,6 +331,22 @@ impl Page { Self::from_revision(revision, id) } + /// The notebook-management identity of the page in an active page revision, which + /// internal links name. + pub fn identity_of(revision: &Revision<'_>) -> Option<[u8; 16]> { + revision + .roots + .get(&2) + .and_then(|id| revision.nodes.get(id)) + .filter(|node| matches!(node.kind, Kind::Metadata { .. })) + .and_then(|node| node.extra.first()) + .and_then(|fields| fields.iter().find(|field| field.id == 0x1c001c30)) + .and_then(|field| match field.value { + FieldValue::Bytes(bytes) => bytes.try_into().ok(), + _ => None, + }) + } + /// The single active page declared in one page object space. pub fn from_space(document: &Document<'_>, space: ExGuid) -> Result { let pages = document.pages_in(space)?; @@ -365,12 +381,7 @@ impl Page { .and_then(|node| node.extra.first()); let mut page = Self { title: page_title(revision, id).unwrap_or_default().to_owned(), - identity: metadata - .and_then(|fields| fields.iter().find(|field| field.id == 0x1c001c30)) - .and_then(|field| match field.value { - FieldValue::Bytes(bytes) => bytes.try_into().ok(), - _ => None, - }), + identity: Self::identity_of(revision), created: metadata .and_then(|fields| fields.iter().find(|field| field.id == 0x18001c65)) .map(|field| { diff --git a/crates/onestore/src/page/write.rs b/crates/onestore/src/page/write.rs index 8cb3217bf672b5da8e43c65ec37a6a45bb12d9b5..b6b037a8e48b82f8bc4d61040df7ea73fb636cef 100644 --- a/crates/onestore/src/page/write.rs +++ b/crates/onestore/src/page/write.rs @@ -1,7 +1,9 @@ //! Publishes an edited page model by lowering the difference from the stored page onto //! the typed writers, then squashing their transactions into one revision per space. -use super::{Image, Ink, Math, Outline, Page, PageObject, PageParagraph, ParagraphContent, Table}; +use super::{ + Attachment, Image, Ink, Math, Outline, Page, PageObject, PageParagraph, ParagraphContent, Table, +}; use crate::{ Error, ExGuid, Insertion, ObjectData, OutlineEdit, ParagraphJoin, ParagraphSplit, PropertySets, RevisionIndex, Store, TextAttribute, TreeEdit, Value, @@ -650,8 +652,8 @@ impl Lowering<'_> { } } (ParagraphContent::Attachment(a), ParagraphContent::Attachment(b)) => { - if a != b { - return Err(invalid("A stored attachment cannot be edited")); + if a.id != b.id { + return Err(invalid("Attachment identity cannot change")); } } _ => return Err(invalid("Paragraph content type cannot change")), @@ -904,21 +906,24 @@ impl Lowering<'_> { let ParagraphContent::Attachment(attachment) = ¶graph.content else { continue; }; - if let Some(previous) = old.paragraphs.get(paragraph_id) { - if previous.content != paragraph.content { - return Err(invalid("A stored attachment cannot be edited")); - } - continue; - } - let Some(bytes) = &attachment.bytes else { - return Err(invalid("A new attachment needs its payload")); - }; let name = attachment.filename.as_str(); if name.is_empty() || name.contains(['\0', '/', '\\']) { return Err(invalid( "An attachment needs a file name without path separators", )); } + if let Some(previous) = old.paragraphs.get(paragraph_id) { + let ParagraphContent::Attachment(stored) = &previous.content else { + return Err(invalid("Paragraph content type cannot change")); + }; + if stored != attachment { + self.edit_attachment(stored, attachment)?; + } + continue; + } + let Some(bytes) = &attachment.bytes else { + return Err(invalid("A new attachment needs its payload")); + }; let extension = name .rfind('.') .filter(|dot| *dot > 0) @@ -1314,6 +1319,59 @@ impl Lowering<'_> { /// Writes a moved, resized or described picture the way OneNote stores one: the /// position, the layout width and height with the user flag and the description on /// the picture object, leaving the intrinsic size alone. + /// A stored attachment keeps its payload and preview; its shown name, recorded source + /// path and icon size change in place, as OneNote's rename does. + fn edit_attachment( + &mut self, + stored: &Attachment, + attachment: &Attachment, + ) -> Result<(), Error> { + let mut values: Values = vec![( + 0x14001d7a, + crate::create::current_timestamps()? + .0 + .to_le_bytes() + .to_vec(), + )]; + let mut removed = Vec::new(); + if attachment.filename != stored.filename { + let name = crate::create::string(&attachment.filename); + values.push((0x1c001c22, name.clone())); + values.push((0x1c001d9c, name)); + } + if attachment.source_path != stored.source_path { + match &attachment.source_path { + Some(path) => values.push((0x1c001d9d, crate::create::string(path))), + None => removed.push(0x1c001d9d), + } + } + if attachment.size != stored.size { + match attachment.size { + Some([width, height]) => { + if !(width.is_finite() && height.is_finite() && width > 0.0 && height > 0.0) { + return Err(invalid("Attachment icon size must be positive")); + } + values.push((0x140034cd, (width / 36.0).to_le_bytes().to_vec())); + values.push((0x140034ce, (height / 36.0).to_le_bytes().to_vec())); + } + None => removed.extend([0x140034cd, 0x140034ce]), + } + } + let (space, object) = (self.space, self.id(attachment.id)); + self.apply(|current| { + crate::write::write_revision(current, space, |raw| { + let mut node = PropertyObject::from_object(&raw.objects[&object])?; + node.remove(&removed)?; + let values: Vec<(u32, &[u8])> = values + .iter() + .map(|(id, bytes)| (*id, bytes.as_slice())) + .collect(); + node.set(&values)?; + Ok(BTreeMap::from([(object, node)])) + }) + }) + } + fn edit_image(&mut self, stored: &Image, image: &Image) -> Result<(), Error> { picture_fixed_fields(stored, image)?; let mut values: Values = vec![( diff --git a/crates/onestore/src/store.rs b/crates/onestore/src/store.rs index bcbdefbe89dfaf4fbabeb3a570005ff68764c62f..18c57e2aad6216fe68e30f290daf8353717d60f4 100644 --- a/crates/onestore/src/store.rs +++ b/crates/onestore/src/store.rs @@ -71,6 +71,10 @@ impl Chunk { pub struct Header { pub file_type: FileType, pub file_id: [u8; 16], + /// The parent table of contents' file identity; zero outside a notebook. + pub ancestor: [u8; 16], + /// CRC of the file name (a section's file name, a group's folder name). + pub name_crc: u32, pub transaction_count: u32, pub expected_length: u64, pub version_id: [u8; 16], @@ -166,7 +170,9 @@ impl Header { message: "Missing committed transaction", }); } - c.take(48)?; + c.take(28)?; + let ancestor = c.read()?; + let name_crc = u32::from_le_bytes(c.read()?); let hashed_chunks = c.chunk()?; let transaction_log = c.chunk()?; let root = c.chunk()?; @@ -179,6 +185,8 @@ impl Header { Ok(Self { file_type, file_id, + ancestor, + name_crc, transaction_count, expected_length, version_id, diff --git a/crates/onestore/src/toc.rs b/crates/onestore/src/toc.rs new file mode 100644 index 0000000000000000000000000000000000000000..68db7ee3ff4f7dfdcb6d6e08fc7346a4dbe1bdb7 --- /dev/null +++ b/crates/onestore/src/toc.rs @@ -0,0 +1,226 @@ +//! Edits to a notebook's table of contents: the documented `jcidPersistablePropertyContainerForTOC` +//! root lists section and section-group entries, each carrying a file identity, an ordering +//! number, a filename and (for sections) a colour. + +use crate::{ + Error, ExGuid, PropertySets, Store, Value, + document::{Document, Kind}, + revisions::RevisionIndex, + write::{PropertyObject, fresh_guid, write_revision}, +}; +use std::collections::BTreeMap; + +fn invalid(message: &'static str) -> Error { + Error { offset: 0, message } +} + +/// One change to a table of contents, addressed by the entry's file identity. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TocEdit { + /// A section (`name.one`) or, with `group`, a section-group folder appended last. + Add { + filename: String, + identity: [u8; 16], + group: bool, + }, + Rename { + identity: [u8; 16], + filename: String, + }, + /// Section colour as COLORREF; `None` restores OneNote's "undefined" 0xffffffff. + Color { + identity: [u8; 16], + color: Option, + }, + /// Every entry, in the wanted order; entries left out keep their relative order after these. + Order(Vec<[u8; 16]>), + Remove { + identity: [u8; 16], + }, +} + +fn component(name: &str) -> bool { + !name.is_empty() && !name.contains(['/', '\\', '\0']) && name != "." && name != ".." +} + +/// The table of contents with `edits` applied in order. +type PropertyChange = (u32, Option>); + +pub(crate) fn edit_table_of_contents(source: &[u8], edits: &[TocEdit]) -> Result, Error> { + let store = Store::parse(source)?; + if store.header.file_type != crate::FileType::TableOfContents { + return Err(invalid("Choose a table-of-contents file")); + } + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + let space = document.root; + let revision = document.active(space)?; + let root = *revision + .roots + .get(&1) + .ok_or_else(|| invalid("Missing notebook TOC root"))?; + let Some(Kind::Toc { entries, .. }) = revision.nodes.get(&root).map(|node| &node.kind) else { + return Err(invalid("Missing notebook TOC root")); + }; + // Entries in stored order with their identities and filenames. + let mut listed: Vec<(ExGuid, [u8; 16], String)> = Vec::new(); + for id in entries { + let Some(Kind::Toc { + filename, + identity: Some(identity), + .. + }) = revision.nodes.get(id).map(|node| &node.kind) + else { + return Err(invalid("Incomplete notebook TOC reference")); + }; + listed.push((*id, *identity, filename.clone().unwrap_or_default())); + } + let mut created: BTreeMap = BTreeMap::new(); + // Property values to set (or remove, `None`) on each existing entry. + let mut changes: BTreeMap> = BTreeMap::new(); + for edit in edits { + let position = |identity: &[u8; 16]| { + listed + .iter() + .position(|(_, known, _)| known == identity) + .ok_or_else(|| invalid("The table of contents has no entry with that identity")) + }; + match edit { + TocEdit::Add { + filename, + identity, + group, + } => { + if !component(filename) + || (!group && !filename.to_ascii_lowercase().ends_with(".one")) + || *identity == [0; 16] + { + return Err(invalid( + "A TOC entry needs a section filename or group folder name and an identity", + )); + } + if listed.iter().any(|(_, known, name)| { + known == identity || name.eq_ignore_ascii_case(filename) + }) { + return Err(invalid( + "The table of contents already lists that section or group", + )); + } + let id = ExGuid { + guid: fresh_guid()?, + n: 10, + }; + let mut values: Vec<(u32, Vec)> = vec![ + (0x1c001d94, identity.to_vec()), + (0x14001cb9, 0u32.to_le_bytes().to_vec()), + (0x1c001d6b, crate::create::string(filename)), + ]; + if !group { + values.push((0x14001cbe, vec![0xff; 4])); + } + let mut object = PropertyObject { + jcid: 0x20001, + bytes: crate::create::properties(&values)?, + global_ids: std::sync::Arc::new(BTreeMap::from([(0, id.guid)])), + }; + object.reference(id)?; + created.insert(id, object); + listed.push((id, *identity, filename.clone())); + } + TocEdit::Rename { identity, filename } => { + if !component(filename) { + return Err(invalid( + "A TOC entry needs a filename without path separators", + )); + } + let at = position(identity)?; + if listed + .iter() + .enumerate() + .any(|(i, (_, _, name))| i != at && name.eq_ignore_ascii_case(filename)) + { + return Err(invalid("The table of contents already lists that name")); + } + let id = listed[at].0; + listed[at].2 = filename.clone(); + changes + .entry(id) + .or_default() + .push((0x1c001d6b, Some(crate::create::string(filename)))); + } + TocEdit::Color { identity, color } => { + let id = listed[position(identity)?].0; + changes.entry(id).or_default().push(( + 0x14001cbe, + Some(color.unwrap_or(0xffff_ffff).to_le_bytes().to_vec()), + )); + } + TocEdit::Order(wanted) => { + let mut ordered: Vec<(ExGuid, [u8; 16], String)> = Vec::new(); + for identity in wanted { + let at = position(identity)?; + if ordered.iter().any(|(_, known, _)| known == identity) { + return Err(invalid("An entry is ordered twice")); + } + ordered.push(listed[at].clone()); + } + let rest: Vec<_> = listed + .iter() + .filter(|(_, known, _)| !wanted.contains(known)) + .cloned() + .collect(); + ordered.extend(rest); + listed = ordered; + } + TocEdit::Remove { identity } => { + let at = position(identity)?; + let (id, _, _) = listed.remove(at); + changes.remove(&id); + created.remove(&id); + } + } + } + let listed = listed; + write_revision(source, space, |raw| { + let mut changed = BTreeMap::new(); + let mut root_object = PropertyObject::from_object(&raw.objects[&root])?; + let mut references = Vec::new(); + for (order, (id, _, _)) in listed.iter().enumerate() { + let mut object = match created.remove(id) { + Some(object) => object, + None => PropertyObject::from_object(&raw.objects[id])?, + }; + let stored_order = PropertySets::parse(&object.bytes)?.sets[0] + .iter() + .find(|p| p.id == 0x14001cb9) + .and_then(|p| match p.value { + Value::Bytes(b) => b.try_into().ok().map(u32::from_le_bytes), + _ => None, + }); + let mut updates: Vec<(u32, Vec)> = Vec::new(); + if stored_order != Some(order as u32 + 1) { + updates.push((0x14001cb9, (order as u32 + 1).to_le_bytes().to_vec())); + } + for (property, value) in changes.get(id).into_iter().flatten() { + match value { + Some(bytes) => updates.push((*property, bytes.clone())), + None => object.remove(&[*property])?, + } + } + let updates: Vec<(u32, &[u8])> = updates + .iter() + .map(|(id, bytes)| (*id, bytes.as_slice())) + .collect(); + if !updates.is_empty() { + object.set(&updates)?; + } + references.extend(root_object.reference(*id)?); + if !updates.is_empty() || created.contains_key(id) || !raw.objects.contains_key(id) { + changed.insert(*id, object); + } + } + root_object.set(&[(0x24001cf6, &references)])?; + changed.insert(root, root_object); + Ok(changed) + }) +} diff --git a/crates/onestore/src/write.rs b/crates/onestore/src/write.rs index a63774c575c95c9d6995134c4c40ffea970897cd..e9266a1735575b317ae1e04a743f6a30bc30add8 100644 --- a/crates/onestore/src/write.rs +++ b/crates/onestore/src/write.rs @@ -838,6 +838,7 @@ pub(crate) fn write_revisions_with_payloads( ) } }; + let existing: BTreeSet = revision.objects.keys().copied().collect(); let reachable = if rid.is_some() { revision.reachable()? } else { @@ -859,10 +860,10 @@ pub(crate) fn write_revisions_with_payloads( message: "An existing object's type cannot be changed", }); } - } else if !is_section { + } else if !is_section && replacement.jcid != 0x20001 { return Err(Error { offset: 0, - message: "New objects require a section file", + message: "New objects in a table of contents are its entries", }); } if replacement.global_ids.keys().any(|i| *i > 0xffffff) { @@ -1006,10 +1007,16 @@ pub(crate) fn write_revisions_with_payloads( .iter() .filter(|(id, _)| checkpoint || changed.contains(id)) .collect(); - let toc_table = if checkpoint && !is_section { - if selected.iter().any(|(_, object)| { - object.jcid != 0x20001 || !matches!(object.data, ObjectData::Properties(_)) - }) { + // A table-of-contents manifest has one global id table (sections group objects, + // each group with its own table); OneNote resolves every node against it. + let toc_table = if is_section { + None + } else { + if checkpoint + && selected.iter().any(|(_, object)| { + object.jcid != 0x20001 || !matches!(object.data, ObjectData::Properties(_)) + }) + { return Err(Error { offset: 0, message: "TOC checkpoint requires table-of-contents property objects", @@ -1032,8 +1039,6 @@ pub(crate) fn write_revisions_with_payloads( .map(|(i, guid)| (u32::try_from(i).unwrap(), guid)) .collect::>(), ) - } else { - None }; let mut groups = BTreeMap::<_, Vec<_>>::new(); for (id, object) in selected { @@ -1144,10 +1149,13 @@ pub(crate) fn write_revisions_with_payloads( length: u64::try_from(bytes.len()).unwrap(), } }; + // A table-of-contents object is declared when the revision is a + // checkpoint or the object is new, and revised otherwise. + let declared = checkpoint || !existing.contains(&id); if is_section { declaration.extend_from_slice(&object.jcid.to_le_bytes()); declaration.push(flags); - } else if checkpoint { + } else if declared { let body = 1_u64 | (u64::from(flags & 1) << 16); declaration.extend_from_slice(&body.to_le_bytes()[..6]); } else { @@ -1161,7 +1169,7 @@ pub(crate) fn write_revisions_with_payloads( group.push(node( if is_section { if readonly { 0xc5 } else { 0xa5 } - } else if checkpoint { + } else if declared { 0x2e } else { 0x42 diff --git a/crates/onestore/tests/page_attachments.rs b/crates/onestore/tests/page_attachments.rs index 5feeaa00824a57094e105c775e36467ed1be38dd..89e0d1f3cfdb1bb6394075688ce6e9233df656e4 100644 --- a/crates/onestore/tests/page_attachments.rs +++ b/crates/onestore/tests/page_attachments.rs @@ -169,7 +169,7 @@ fn an_attachment_inserted_on_a_fresh_page_reads_back_and_can_be_removed() { } #[test] -fn attachments_need_a_file_name_and_stored_ones_stay_fixed() { +fn attachments_need_a_file_name_and_stored_ones_are_renamed_in_place() { let source = onestore::create_section("files.one", "Text", "Author").unwrap(); let (space, before) = first_page(&source); let mut after = before.clone(); @@ -192,12 +192,38 @@ fn attachments_need_a_file_name_and_stored_ones_stay_fixed() { if let PageObject::Outline(outline) = object { for paragraph in &mut outline.paragraphs { if let ParagraphContent::Attachment(attachment) = &mut paragraph.content { - attachment.filename = "renamed.txt".into(); + attachment.filename = "renamed \u{1f980}.txt".into(); + attachment.source_path = Some("C:\\inputs\\renamed \u{1f980}.txt".into()); found += 1; } } } } assert_eq!(found, 1); - assert!(PreparedEdit::page(NATIVE, space, &renamed, AUTHOR).is_err()); + let written = PreparedEdit::page(NATIVE, space, &renamed, AUTHOR).unwrap(); + let stored = page_in(written.as_bytes(), space); + let (before, after) = (attachments(&native)[0], attachments(&stored)[0]); + assert_eq!(after.filename, "renamed \u{1f980}.txt"); + assert_eq!( + after.source_path.as_deref(), + Some("C:\\inputs\\renamed \u{1f980}.txt") + ); + assert_eq!((after.id, after.size), (before.id, before.size)); + assert_eq!(after.bytes, before.bytes); + assert_eq!(after.preview, before.preview); + if let Some(directory) = std::env::var_os("ONESTORE_ATTACHMENT_RENAME_EXPORT") { + let directory = std::path::PathBuf::from(directory); + std::fs::create_dir_all(&directory).unwrap(); + std::fs::write(directory.join("synthetic.one"), written.as_bytes()).unwrap(); + let file_id = Store::parse(written.as_bytes()).unwrap().header.file_id; + std::fs::write( + directory.join("Open Notebook.onetoc2"), + onestore::create_table_of_contents( + "Open Notebook.onetoc2", + &[("synthetic.one", file_id)], + ) + .unwrap(), + ) + .unwrap(); + } } diff --git a/crates/onestore/tests/page_links.rs b/crates/onestore/tests/page_links.rs index a0c3be26bfd9c6f0e66005ea13fd66c8d50358af..e315f23c148917354be9a7414cda46504b95938b 100644 --- a/crates/onestore/tests/page_links.rs +++ b/crates/onestore/tests/page_links.rs @@ -3,7 +3,7 @@ use onestore::{ document::Document, page::{ Page, PageObject, PageParagraph, Paragraph, - link::{LinkTarget, internal_link}, + link::{InternalLink, LinkTarget, internal_link, parse_internal_link}, text::Edit, }, }; @@ -258,6 +258,32 @@ fn internal_links_match_what_onenote_stores() { ), "onenote:#A%20b%2Fc§ion-id={00000000-0000-0000-0000-000000000000}&page-id={00000000-0000-0000-0000-000000000000}&end&base-path=p" ); + let identity = target.identity.unwrap(); + assert_eq!( + stored[1..] + .iter() + .map(|url| parse_internal_link(url)) + .collect::>(), + [ + Some(InternalLink { + section, + page: Some(identity), + object: None + }), + Some(InternalLink { + section, + page: Some(identity), + object: Some(paragraph) + }), + Some(InternalLink { + section, + page: None, + object: None + }), + ] + ); + assert_eq!(parse_internal_link(&stored[0]), None); + assert_eq!(parse_internal_link("onenote:#x&page-id={0}"), None); } /// `ONESTORE_INTERNAL_LINK_EXPORT` names a new directory receiving the candidate for a cold diff --git a/crates/onestore/tests/toc.rs b/crates/onestore/tests/toc.rs new file mode 100644 index 0000000000000000000000000000000000000000..7a71d89380e88a3faf41b338025b848435ece205 --- /dev/null +++ b/crates/onestore/tests/toc.rs @@ -0,0 +1,108 @@ +use onestore::{PreparedEdit, RevisionIndex, Store, TocEdit, document::Document}; + +fn entries(bytes: &[u8]) -> Vec<(String, [u8; 16], u32, Option)> { + let store = Store::parse(bytes).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let revision = document.active(document.root).unwrap(); + let root = &revision.nodes[&revision.roots[&1]]; + let onestore::document::Kind::Toc { entries, .. } = &root.kind else { + panic!() + }; + entries + .iter() + .map(|id| match &revision.nodes[id].kind { + onestore::document::Kind::Toc { + filename, + identity, + order, + color, + .. + } => ( + filename.clone().unwrap(), + identity.unwrap(), + order.unwrap(), + *color, + ), + _ => panic!(), + }) + .collect() +} + +#[test] +fn sections_and_groups_are_added_renamed_coloured_ordered_and_removed() { + let a = [1; 16]; + let b = [2; 16]; + let toc = onestore::create_table_of_contents( + "Open Notebook.onetoc2", + &[("Alpha.one", a), ("Beta.one", b)], + ) + .unwrap(); + let c = [3; 16]; + let group = [4; 16]; + let written = PreparedEdit::table_of_contents( + &toc, + &[ + TocEdit::Add { + filename: "Gamma.one".into(), + identity: c, + group: false, + }, + TocEdit::Add { + filename: "Archive".into(), + identity: group, + group: true, + }, + ], + ) + .unwrap(); + assert_eq!( + entries(written.as_bytes()), + [ + ("Alpha.one".to_owned(), a, 1, Some(0xffff_ffff)), + ("Beta.one".to_owned(), b, 2, Some(0xffff_ffff)), + ("Gamma.one".to_owned(), c, 3, Some(0xffff_ffff)), + ("Archive".to_owned(), group, 4, None), + ] + ); + let again = PreparedEdit::table_of_contents( + written.as_bytes(), + &[ + TocEdit::Rename { + identity: b, + filename: "Renamed.one".into(), + }, + TocEdit::Color { + identity: c, + color: Some(0x00d7ff), + }, + TocEdit::Order(vec![group, c]), + TocEdit::Remove { identity: a }, + ], + ) + .unwrap(); + assert_eq!( + entries(again.as_bytes()), + [ + ("Archive".to_owned(), group, 1, None), + ("Gamma.one".to_owned(), c, 2, Some(0x00d7ff)), + ("Renamed.one".to_owned(), b, 3, Some(0xffff_ffff)), + ] + ); + assert!( + PreparedEdit::table_of_contents( + again.as_bytes(), + &[TocEdit::Add { + filename: "renamed.one".into(), + identity: [9; 16], + group: false + }] + ) + .is_err() + ); + assert!( + PreparedEdit::table_of_contents(again.as_bytes(), &[TocEdit::Remove { identity: a }]) + .is_err() + ); +} diff --git a/tools/native/notebook-reorder.ps1 b/tools/native/notebook-reorder.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..b672af93e4abe3eeeaa403c5ff4ed48c775238ac --- /dev/null +++ b/tools/native/notebook-reorder.ps1 @@ -0,0 +1,50 @@ +param([Parameter(Mandatory=$true)][string]$Root, [Parameter(Mandatory=$true)][string]$CloneHost) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +& "$PSScriptRoot\cold-current.ps1" -Root $Root -CloneHost $CloneHost +$namespace = 'http://schemas.microsoft.com/office/onenote/2010/onenote' +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +function Read-Tree { + $deadline = [DateTime]::UtcNow.AddSeconds(60) + do { + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [xml]$tree = $hierarchy + if (@($tree.SelectNodes('//*[local-name()="Section"]')).Count -ge 4) { return $hierarchy } + Start-Sleep -Milliseconds 500 + } while ([DateTime]::UtcNow -lt $deadline) + throw 'The notebook did not finish loading.' +} +try { + $app.OpenHierarchy((Join-Path $Root 'notebook'), '', [ref]$notebookId, 0) + $hierarchy = Read-Tree + [IO.File]::WriteAllText((Join-Path $Root 'reorder-00-opened.xml'), $hierarchy, [Text.Encoding]::UTF8) + [xml]$tree = $hierarchy + $manager = New-Object Xml.XmlNamespaceManager($tree.NameTable) + $manager.AddNamespace('one', $namespace) + $notebookNode = $tree.DocumentElement + $sections = @($notebookNode.SelectNodes('one:Section', $manager)) + if ($sections.Count -lt 3) { throw "Expected three root sections, found $($sections.Count)." } + $last = $sections[$sections.Count - 1] + [void]$notebookNode.RemoveChild($last) + [void]$notebookNode.InsertBefore($last, $sections[0]) + [IO.File]::WriteAllText((Join-Path $Root 'reorder-01-update.xml'), $tree.OuterXml, [Text.Encoding]::UTF8) + $app.UpdateHierarchy($tree.OuterXml, 1) + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $Root 'reorder-02-reordered.xml'), $hierarchy, [Text.Encoding]::UTF8) + $app.SyncHierarchy($notebookId) + $app.CloseNotebook($notebookId, $false) + $notebookId = '' + Copy-Item -Recurse (Join-Path $Root 'notebook') (Join-Path $Root 'notebook-reordered') + $app.OpenHierarchy((Join-Path $Root 'notebook'), '', [ref]$notebookId, 0) + $hierarchy = Read-Tree + [IO.File]::WriteAllText((Join-Path $Root 'reorder-03-reopened.xml'), $hierarchy, [Text.Encoding]::UTF8) + $app.CloseNotebook($notebookId, $false) + $notebookId = '' +} finally { + if ($notebookId) { $app.CloseNotebook($notebookId, $false) } + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + [GC]::Collect() + [GC]::WaitForPendingFinalizers() +} diff --git a/tools/native/notebook-structure.ps1 b/tools/native/notebook-structure.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..caa6737aea0e40f0695880dee51d166868e8ce3e --- /dev/null +++ b/tools/native/notebook-structure.ps1 @@ -0,0 +1,71 @@ +param([Parameter(Mandatory=$true)][string]$Root, [Parameter(Mandatory=$true)][string]$CloneHost) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +& "$PSScriptRoot\cold-current.ps1" -Root $Root -CloneHost $CloneHost +$namespace = 'http://schemas.microsoft.com/office/onenote/2010/onenote' +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$log = @() +try { + $app.OpenHierarchy((Join-Path $Root 'notebook'), '', [ref]$notebookId, 0) + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $Root 'structure-00-opened.xml'), $hierarchy, [Text.Encoding]::UTF8) + $second = '' + $app.OpenHierarchy('Second.one', $notebookId, [ref]$second, 3) + $third = '' + $app.OpenHierarchy('Third.one', $notebookId, [ref]$third, 3) + $group = '' + $app.OpenHierarchy('Group', $notebookId, [ref]$group, 2) + $inner = '' + $app.OpenHierarchy('Inner.one', $group, [ref]$inner, 3) + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $Root 'structure-01-created.xml'), $hierarchy, [Text.Encoding]::UTF8) + $app.SyncHierarchy($notebookId) + $app.CloseNotebook($notebookId, $false) + $notebookId = '' + $app.OpenHierarchy((Join-Path $Root 'notebook'), '', [ref]$notebookId, 0) + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [xml]$tree = $hierarchy + $manager = New-Object Xml.XmlNamespaceManager($tree.NameTable) + $manager.AddNamespace('one', $namespace) + $sections = @($tree.SelectNodes('//one:Section', $manager)) + foreach ($section in $sections) { + switch ($section.GetAttribute('name')) { + 'Second' { $section.SetAttribute('name', 'Renamed'); $section.SetAttribute('color', '#FFD75E') } + 'Third' { $section.SetAttribute('color', '#B7C997') } + } + } + [IO.File]::WriteAllText((Join-Path $Root 'structure-02-update.xml'), $tree.OuterXml, [Text.Encoding]::UTF8) + $app.UpdateHierarchy($tree.OuterXml, 1) + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $Root 'structure-03-renamed.xml'), $hierarchy, [Text.Encoding]::UTF8) + $app.SyncHierarchy($notebookId) + $app.CloseNotebook($notebookId, $false) + $notebookId = '' + Copy-Item -Recurse (Join-Path $Root 'notebook') (Join-Path $Root 'notebook-renamed') + $app.OpenHierarchy((Join-Path $Root 'notebook'), '', [ref]$notebookId, 0) + $deadline = [DateTime]::UtcNow.AddSeconds(60) + do { + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [xml]$tree = $hierarchy + if (@($tree.SelectNodes('//*[local-name()="Section"]')).Count -ge 4) { break } + Start-Sleep -Milliseconds 500 + } while ([DateTime]::UtcNow -lt $deadline) + $manager = New-Object Xml.XmlNamespaceManager($tree.NameTable) + $manager.AddNamespace('one', $namespace) + [IO.File]::WriteAllText((Join-Path $Root 'structure-03b-reopened.xml'), $hierarchy, [Text.Encoding]::UTF8) + $doomed = @($tree.SelectNodes('//*[local-name()="Section"]') | Where-Object { $_.GetAttribute('name') -eq 'Third' }) + if ($doomed.Count -ne 1) { throw "Expected one section named Third, found $($doomed.Count)." } + $app.DeleteHierarchy($doomed[0].GetAttribute('ID'), [DateTime]::MinValue, $false) + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $Root 'structure-04-deleted.xml'), $hierarchy, [Text.Encoding]::UTF8) + $app.SyncHierarchy($notebookId) + $app.CloseNotebook($notebookId, $false) + $notebookId = '' +} finally { + if ($notebookId) { $app.CloseNotebook($notebookId, $false) } + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + [GC]::Collect() + [GC]::WaitForPendingFinalizers() +} diff --git a/tools/test_attachment_edit.py b/tools/test_attachment_edit.py index e257c7611eb8941b6a056d4a47ae6e5576e4cec1..7544f904d9a732ec56178326991abe3b077c1ad8 100644 --- a/tools/test_attachment_edit.py +++ b/tools/test_attachment_edit.py @@ -31,6 +31,15 @@ class AttachmentEditTest(unittest.TestCase): def test_a_rust_inserted_attachment_with_an_icon_preview_renders_natively(self): self.check('icon') + def test_a_stored_attachment_renamed_by_rust_shows_its_new_name_and_source(self): + with TemporaryDirectory() as temporary: + read = Path(temporary) / 'read' + shutil.copytree(FIXTURE / 'renamed/cold/read', read) + compare(FIXTURE / 'renamed/candidate', read) + page, = (ET.parse(path).getroot() for path in sorted((FIXTURE / 'renamed/cold/read').glob('page-*.xml'))) + inserted, = page.iter('{%s}InsertedFile' % ns['one']) + self.assertEqual((inserted.get('preferredName'), inserted.get('pathSource')), ('renamed 🦀.txt', 'C:\\inputs\\renamed 🦀.txt')) + if __name__ == '__main__': unittest.main() diff --git a/tools/test_notebook_edit.py b/tools/test_notebook_edit.py new file mode 100644 index 0000000000000000000000000000000000000000..911f96ce81aacf3c6098115519d981392fae640b --- /dev/null +++ b/tools/test_notebook_edit.py @@ -0,0 +1,65 @@ +from pathlib import Path +import runpy +import shutil +from tempfile import TemporaryDirectory +import unittest +import uuid +import xml.etree.ElementTree as ET + +from native_xml import ns + +ROOT = Path(__file__).resolve().parent.parent +FIXTURE = ROOT / 'corpus/notebook-edit' +compare = runpy.run_path(str(ROOT / 'tools/verify-document.py'))['compare'] + + +def file_identity(path): + return uuid.UUID(bytes_le=path.read_bytes()[16:32]) + + +def entries(node): + return [(child.tag.rsplit('}', 1)[-1], child.get('name'), child.get('color'), entries(child)) + for child in node if child.tag.rsplit('}', 1)[-1] in ('Section', 'SectionGroup')] + + +class NotebookEditTest(unittest.TestCase): + def check(self, row, expected): + with TemporaryDirectory() as temporary: + read = Path(temporary) / 'read' + shutil.copytree(FIXTURE / row / 'cold/read', read) + compare(FIXTURE / row / 'candidate', read) + hierarchy = ET.parse(FIXTURE / row / 'cold/read/hierarchy.xml').getroot() + self.assertEqual(entries(hierarchy), expected) + for candidate in sorted((FIXTURE / row / 'candidate').rglob('*.one*')): + relative = candidate.relative_to(FIXTURE / row / 'candidate') + self.assertEqual(file_identity(FIXTURE / row / 'cold/notebook' / relative), file_identity(candidate), str(relative)) + + def test_rust_sections_and_groups_open_natively_in_the_written_order(self): + self.check('structured', [ + ('Section', 'Renamed', '#FFD75E', []), + ('Section', 'First', '#8AA8E4', []), + ('SectionGroup', 'Kept', None, [('Section', 'Inner', '#8AA8E4', [])]), + ]) + + def test_a_rust_deleted_section_sits_in_the_native_recycle_bin(self): + self.check('deleted', [ + ('Section', 'Renamed', '#FFD75E', []), + ('SectionGroup', 'Kept', None, [('Section', 'First', '#8AA8E4', [])]), + ('SectionGroup', 'OneNote_RecycleBin', None, [('Section', 'First', '#8AA8E4', [])]), + ]) + hierarchy = ET.parse(FIXTURE / 'deleted/cold/read/hierarchy.xml').getroot() + bin = hierarchy.find("one:SectionGroup[@name='OneNote_RecycleBin']", ns) + self.assertEqual(bin.get('isRecycleBin'), 'true') + self.assertEqual(bin.find('one:Section', ns).get('isInRecycleBin'), 'true') + + def test_onenote_re_identifies_files_whose_header_disagrees_with_their_place(self): + # The native session lists a Rust section created with a zero ancestor twice: once + # under the identity Rust wrote and once under the identity OneNote assigned. + toc = (FIXTURE / 'native-structure/notebook/Open Notebook.onetoc2').read_bytes() + links = FIXTURE / 'native-structure/notebook/links.one' + self.assertEqual(uuid.UUID(bytes_le=links.read_bytes()[128:144]), file_identity(FIXTURE / 'native-structure/notebook/Open Notebook.onetoc2')) + self.assertIn(file_identity(links).bytes_le, toc) + + +if __name__ == '__main__': + unittest.main()