authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-09-19 23:22:38-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-09-25 20:26:20-07:00
logb26c81f7c036b864e287820c6d3b0498b64159cf
tree9099c8bf67d44eb6888cb8981e5153fe4d07756a
parentbe6b82da9a9e19b3a47c24655949fd2d66db0800
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

fix: review findings in protected saves, the cache and the parser; store only named table entries

Review (three independent passes over the protected-write, shared-GUID, harness and save-cost changes): - Notebook::save_unlocked compared the file with a snapshot it had just read, so a section written since unlock was silently overwritten with the stale model. unlock now returns Unlocked { pages, snapshot } and the save commits against that snapshot. - An acknowledged publication becomes the base that polls trust without validating; it is now validated before it is published. - Global id table imports could grow the pending entry list without bound before the duplicate check at the table end; bounded per entry node. The GUID uniqueness check reuses the list. - A corrupt cache patch length aborted on allocation; bounded by the patch. The recovery summary reads the working length from the patch header and the asset scan holds one image at a time. - Sealed buffers are sized once so growth leaves no plaintext behind; a failing random source is no longer reported as malformed data; file-data strings are decoded checked; protected writing admits exactly one section key and refuses a writer that created a space. The save's working buffers are ordinary memory, and the owner doc now says so. - cache_probe accepts a coalesced insert intent. Growth: a rewritten object group stored the whole global id table of the revision its objects were read from. It now stores the entries they name, and no-op detection compares those entries. OneNote 2010 cold-reads the sparse tables and leaves the file as written (corpus/table-growth). Assisted-by: claude-fable-5.1, claude-opus-5

34 files changed, 606 insertions(+), 93 deletions(-)

corpus/table-growth/README.md created+8
...@@ -0,0 +1,8 @@
1# Table growth
2
3`candidate/notebook` is `native/20260905-05` after `a_text_edit_stores_the_table_entries_it_names`
4(`ONESTORE_TABLE_EXPORT`) appended words to a paragraph of every page. Each rewritten
5object group stores only the global id table entries its objects name, so the stored
6indices have gaps where the source revision's table named GUIDs the edit does not use.
7`cold` is OneNote 2010's cold read of it from a fresh clone; OneNote left the file as
8written.
corpus/table-growth/candidate/notebook/Open Notebook.onetoc2 created
Binary files /dev/null and b/corpus/table-growth/candidate/notebook/Open Notebook.onetoc2 differ
corpus/table-growth/candidate/notebook/synthetic.one created
Binary files /dev/null and b/corpus/table-growth/candidate/notebook/synthetic.one differ
corpus/table-growth/cold/commands.jsonl created+3
...@@ -0,0 +1,3 @@
1{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null}
2{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-6162C217 -ExpectedPages 1", "exit": 0, "stdout": "Read 1 sections and 1 pages.\r\n", "stderr": "", "error": null}
3{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null}
corpus/table-growth/cold/machine.json created+1
...@@ -0,0 +1 @@
1{"name": "m6-6162c217", "hostname": "ONE-M6-6162C217"}
corpus/table-growth/cold/notebook/Open Notebook.onetoc2 created
Binary files /dev/null and b/corpus/table-growth/cold/notebook/Open Notebook.onetoc2 differ
corpus/table-growth/cold/notebook/synthetic.one created
Binary files /dev/null and b/corpus/table-growth/cold/notebook/synthetic.one differ
corpus/table-growth/cold/read/af1af4b8987109ce1e335e538910fb3860b7c57eb3b6110a06aa2114e06d69a7.attachment created+1
...@@ -0,0 +1 @@
1Fictitious attachment for native corpus.
\ No newline at end of file
corpus/table-growth/cold/read/environment.json created+7
...@@ -0,0 +1,7 @@
1{
2 "powershell": "5.1.14409.1005",
3 "schema": "xs2010",
4 "hostname": "ONE-M6-6162C217",
5 "cold": true,
6 "onenote": "14.0.4763.1000"
7}
corpus/table-growth/cold/read/hierarchy.xml created+2
...@@ -0,0 +1,2 @@
1<?xml version="1.0"?>
2<one:Notebook xmlns:one="http://schemas.microsoft.com/office/onenote/2010/onenote" name="notebook" nickname="notebook" ID="{BCD32541-08BB-4A4F-A7D6-91A4DE37385C}{1}{B0}" path="C:\one-tests\runs\capture\notebook" lastModifiedTime="2026-09-20T06:56:45.000Z" color="#9595AA"><one:Section name="synthetic" ID="{7B10731A-3919-06E3-0F51-A7F44821474F}{1}{B0}" path="C:\one-tests\runs\capture\notebook\synthetic.one" lastModifiedTime="2026-09-20T06:56:45.000Z" color="#8AA8E4"><one:Page ID="{5759809C-B0D2-45F2-8836-0F65B4195B50}{1}{B0}" name="Fictitious: café, 東京, مرحبا and a few more words" dateTime="2026-09-05T05:06:54.000Z" lastModifiedTime="2026-09-20T06:56:45.000Z" pageLevel="1"/></one:Section></one:Notebook>
corpus/table-growth/cold/read/page-000.xml created+9
...@@ -0,0 +1,9 @@
1<?xml version="1.0"?>
2<one:Page xmlns:one="http://schemas.microsoft.com/office/onenote/2010/onenote" ID="{5759809C-B0D2-45F2-8836-0F65B4195B50}{1}{B0}" name="Fictitious: café, 東京, مرحبا and a few more words" dateTime="2026-09-05T05:06:54.000Z" lastModifiedTime="2026-09-20T06:56:45.000Z" pageLevel="1" lang="en-US"><one:QuickStyleDef index="0" name="p" fontColor="automatic" highlightColor="automatic" font="Calibri" fontSize="11.0" spaceBefore="0.0" spaceAfter="0.0"/><one:PageSettings RTL="false" color="automatic"><one:PageSize><one:Automatic/></one:PageSize><one:RuleLines visible="false"/></one:PageSettings><one:Outline author="snow" authorInitials="S" lastModifiedBy="snow" lastModifiedByInitials="S" lastModifiedTime="2026-09-20T06:55:06.000Z" objectID="{30952820-FBA7-4644-87C4-83BE4B85BE31}{10}{B0}"><one:Position x="36.0" y="14.40000057220459" z="0"/><one:Size width="127.5136947631836" height="27.72771453857422"/><one:OEChildren><one:OE creationTime="2026-09-05T05:06:55.000Z" lastModifiedTime="2026-09-20T06:55:06.000Z" objectID="{30952820-FBA7-4644-87C4-83BE4B85BE31}{11}{B0}" alignment="left" quickStyleIndex="0" style="font-size:11.0pt;color:#1F4E79"><one:T><![CDATA[<span
3style='font-weight:bold;font-family:Calibri' lang=en-US>Fictitious: café, </span><span
4style='font-weight:bold;font-family:SimSun' lang=en-US>東京</span><span
5style='font-weight:bold;font-family:Calibri' lang=en-US>, </span><span
6style='font-weight:bold;font-family:Arial;direction:rtl;unicode-bidi:embed'
7lang=ar-SA>مرحبا</span><span style='font-weight:bold;font-family:Calibri'
8lang=en-US> and a few more words</span>]]></one:T></one:OE></one:OEChildren></one:Outline><one:Outline author="snow" authorInitials="S" lastModifiedBy="snow" lastModifiedByInitials="S" lastModifiedTime="2026-09-05T05:06:57.000Z" objectID="{6625F435-48F2-4C19-9D72-0136EF895A78}{10}{B0}"><one:Position x="144.0" y="96.0" z="1"/><one:Size width="167.0449523925781" height="13.42771339416504"/><one:OEChildren><one:OE creationTime="2026-09-05T05:06:57.000Z" lastModifiedTime="2026-09-05T05:06:57.000Z" objectID="{6625F435-48F2-4C19-9D72-0136EF895A78}{11}{B0}" alignment="left" quickStyleIndex="0"><one:T><![CDATA[Fictitious positioned outline.]]></one:T></one:OE></one:OEChildren></one:Outline><one:Outline author="snow" authorInitials="S" lastModifiedBy="snow" lastModifiedByInitials="S" lastModifiedTime="2026-09-05T05:06:58.000Z" objectID="{5B0C9CBA-AE68-45B7-AB02-AAC203E412F6}{10}{B0}"><one:Position x="144.0" y="192.0" z="2"/><one:Size width="72.0" height="0.750005722045898"/><one:OEChildren><one:OE creationTime="2026-09-05T05:06:58.000Z" lastModifiedTime="2026-09-05T05:06:58.000Z" objectID="{5B0C9CBA-AE68-45B7-AB02-AAC203E412F6}{11}{B0}" alignment="left"><one:Image format="png" originalPageNumber="0"><one:Data>iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9J1uoA
9AAAASUVORK5CYII=</one:Data></one:Image></one:OE></one:OEChildren></one:Outline><one:Outline author="snow" authorInitials="S" lastModifiedBy="snow" lastModifiedByInitials="S" lastModifiedTime="2026-09-05T05:06:59.000Z" objectID="{DE1684DA-6810-4671-8EAF-CD3C3735DDC5}{10}{B0}"><one:Position x="264.0" y="192.0" z="3"/><one:Size width="72.00001525878906" height="63.0"/><one:OEChildren><one:OE creationTime="2026-09-05T05:06:59.000Z" lastModifiedTime="2026-09-05T05:06:59.000Z" objectID="{DE1684DA-6810-4671-8EAF-CD3C3735DDC5}{11}{B0}" alignment="left"><one:InsertedFile pathCache="C:\Users\clover\AppData\Local\Microsoft\OneNote\14.0\OneNoteOfflineCache_Files\458001d8-0f1e-451f-81cb-621828fc0b39.txt" pathSource="C:\one-tests\runs\20260905-05\assets\fictitious-attachment.txt" preferredName="fictitious-attachment.txt"/></one:OE></one:OEChildren></one:Outline><one:Outline author="snow" authorInitials="S" lastModifiedBy="snow" lastModifiedByInitials="S" lastModifiedTime="2026-09-05T05:07:00.000Z" objectID="{F0C9FD8A-0980-4DF1-91A7-2F691F662683}{10}{B0}"><one:Position x="144.0" y="312.0" z="4"/><one:Size width="92.42181396484374" height="21.94773101806641"/><one:OEChildren><one:OE creationTime="2026-09-05T05:07:00.000Z" lastModifiedTime="2026-09-05T05:07:00.000Z" objectID="{F0C9FD8A-0980-4DF1-91A7-2F691F662683}{11}{B0}" alignment="left"><one:Table bordersVisible="true" lastModifiedTime="2026-09-05T05:07:00.000Z" objectID="{F0C9FD8A-0980-4DF1-91A7-2F691F662683}{12}{B0}"><one:Columns><one:Column index="0" width="39.14614105224609"/><one:Column index="1" width="45.14567184448242"/></one:Columns><one:Row objectID="{F0C9FD8A-0980-4DF1-91A7-2F691F662683}{13}{B0}" lastModifiedTime="2026-09-05T05:07:00.000Z"><one:Cell lastModifiedTime="2026-09-05T05:07:00.000Z" objectID="{F0C9FD8A-0980-4DF1-91A7-2F691F662683}{14}{B0}" lastModifiedByInitials="S"><one:OEChildren><one:OE creationTime="2026-09-05T05:07:00.000Z" lastModifiedTime="2026-09-05T05:07:00.000Z" objectID="{F0C9FD8A-0980-4DF1-91A7-2F691F662683}{17}{B0}" alignment="left" quickStyleIndex="0"><one:T><![CDATA[Left cell]]></one:T></one:OE></one:OEChildren></one:Cell><one:Cell lastModifiedTime="2026-09-05T05:07:00.000Z" objectID="{F0C9FD8A-0980-4DF1-91A7-2F691F662683}{22}{B0}" lastModifiedByInitials="S"><one:OEChildren><one:OE creationTime="2026-09-05T05:07:00.000Z" lastModifiedTime="2026-09-05T05:07:00.000Z" objectID="{F0C9FD8A-0980-4DF1-91A7-2F691F662683}{25}{B0}" alignment="left" quickStyleIndex="0"><one:T><![CDATA[Right cell]]></one:T></one:OE></one:OEChildren></one:Cell></one:Row></one:Table></one:OE></one:OEChildren></one:Outline></one:Page>
corpus/table-growth/cold/read/payloads.json created+10
...@@ -0,0 +1,10 @@
1[
2 {
3 "sha256": "af1af4b8987109ce1e335e538910fb3860b7c57eb3b6110a06aa2114e06d69a7",
4 "name": "fictitious-attachment.txt",
5 "object": "{DE1684DA-6810-4671-8EAF-CD3C3735DDC5}{11}{B0}",
6 "kind": "InsertedFile",
7 "page": "{5759809C-B0D2-45F2-8836-0F65B4195B50}{1}{B0}",
8 "bytes": 43
9 }
10]
\ No newline at end of file
corpus/table-growth/cold/run.json created+18
...@@ -0,0 +1,18 @@
1{
2 "notebook": "/private/tmp/m5/tables/notebook",
3 "expected_pages": 1,
4 "author": null,
5 "author_timeout_seconds": 600,
6 "inspect": false,
7 "collect_notebook": true,
8 "base": {
9 "file": "win7-office-base.qcow2",
10 "format": "qcow2",
11 "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346",
12 "virtual_size": 68719476736
13 },
14 "scripts": {
15 "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331",
16 "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41"
17 }
18}
corpus/table-growth/cold/scripts/cold.ps1 created+27
...@@ -0,0 +1,27 @@
1param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '')
2Set-StrictMode -Version Latest
3$ErrorActionPreference = 'Stop'
4$root = [IO.Path]::GetFullPath($Root).TrimEnd('\')
5if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') {
6 throw 'Choose a run directly below C:\one-tests\runs.'
7}
8if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' }
9$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote'
10if ($CloneHost) {
11 if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) {
12 throw 'The disposable clone hostname does not match this machine.'
13 }
14 New-Item "$key\Options\Paths" -Force | Out-Null
15 New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null
16} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or
17 -not (Test-Path 'C:\one-tests\profile-original-cache')) {
18 throw 'Park the personal OneNote profile before resetting the test cache.'
19}
20$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0'
21$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root))
22if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' }
23New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null
24if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked }
25if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse }
26New-Item "$key\OpenNotebooks" | Out-Null
27New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null
corpus/table-growth/cold/scripts/read.ps1 created+142
...@@ -0,0 +1,142 @@
1param(
2 [Parameter(Mandatory=$true)][string]$Root,
3 [int]$ExpectedPages = -1,
4 [switch]$UseCurrentCache,
5 [switch]$Pdf,
6 [switch]$KeepOpen,
7 [string]$CloneHost = ''
8)
9Set-StrictMode -Version Latest
10$ErrorActionPreference = 'Stop'
11$root = [IO.Path]::GetFullPath($Root).TrimEnd('\')
12if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') {
13 throw 'Choose a run directly below C:\one-tests\runs.'
14}
15$notebook = Join-Path $root 'notebook'
16$output = Join-Path $root 'read'
17if (Test-Path $output) { throw 'Choose a new read destination.' }
18if ($UseCurrentCache) {
19 if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') {
20 throw 'Park the personal OneNote profile before reading test notebooks.'
21 }
22} else {
23 & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost
24}
25New-Item -ItemType Directory -Path $output | Out-Null
26$app = New-Object -ComObject OneNote.Application
27$notebookId = ''
28$failure = $null
29try {
30 $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0)
31 $process = Get-Process ONENOTE
32 @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion;
33 powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } |
34 ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8
35 $sections = @()
36 foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) {
37 $id = ''
38 $app.OpenHierarchy($file.FullName, '', [ref]$id, 0)
39 $sections += $id
40 }
41 $deadline = [DateTime]::UtcNow.AddSeconds(300)
42 $previous = ''
43 $lastChange = ''
44 $stableSince = [DateTime]::UtcNow
45 $settled = $false
46 do {
47 $pages = @{}
48 foreach ($section in $sections) {
49 $hierarchy = ''
50 $app.GetHierarchy($section, 4, [ref]$hierarchy, 1)
51 [xml]$xml = $hierarchy
52 foreach ($node in $xml.SelectNodes('//*[@path]')) {
53 if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) {
54 throw 'OneNote opened a section outside the copied notebook.'
55 }
56 }
57 foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) {
58 $id = $node.GetAttribute('ID')
59 $content = ''
60 $app.GetPageContent($id, [ref]$content, 1, 1)
61 $pages[$id] = $content
62 }
63 }
64 $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] }))
65 if ($signature -ne $previous) {
66 $lastChange = $previous
67 $previous = $signature
68 $stableSince = [DateTime]::UtcNow
69 }
70 if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or
71 ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and
72 ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break }
73 Start-Sleep -Milliseconds 250
74 } while ([DateTime]::UtcNow -lt $deadline)
75 if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) {
76 [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8)
77 $index = 0
78 foreach ($id in @($pages.Keys | Sort-Object)) {
79 [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8)
80 $index++
81 }
82 $hierarchy = ''
83 $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1)
84 [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8)
85 throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled."
86 }
87 $index = 0
88 $payloads = @()
89 foreach ($id in @($pages.Keys | Sort-Object)) {
90 [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8)
91 if ($Pdf) {
92 $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index)
93 $app.NavigateTo($id, '', $false)
94 $app.Publish($id, $pdfPath, 3, '')
95 if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' }
96 }
97 [xml]$page = $pages[$id]
98 foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) {
99 $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache'))
100 $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant()
101 [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes)
102 $payloads += @{ page = $id; object = $file.ParentNode.GetAttribute('objectID');
103 kind = $file.LocalName; name = $file.GetAttribute('preferredName');
104 sha256 = $hash; bytes = $bytes.Length }
105 }
106 $index++
107 }
108 [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8)
109 $all = ''
110 $app.GetHierarchy($notebookId, 4, [ref]$all, 1)
111 [xml]$finalTree = $all
112 $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique)
113 if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) {
114 throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.'
115 }
116 [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8)
117 Write-Output "Read $($sections.Count) sections and $($pages.Count) pages."
118} catch {
119 $failure = $_
120 [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8)
121 throw
122} finally {
123 try {
124 try {
125 if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) }
126 if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) {
127 $app.CloseNotebook($notebookId, $false)
128 }
129 } catch {
130 if ($null -eq $failure) { throw }
131 [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8)
132 }
133 } finally {
134 [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app)
135 }
136 $app = $null
137 [GC]::Collect()
138 [GC]::WaitForPendingFinalizers()
139 if (-not $UseCurrentCache -and -not $CloneHost) {
140 Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10
141 }
142}
corpus/table-growth/cold/source.json created+14
...@@ -0,0 +1,14 @@
1[
2 {
3 "path": "Open Notebook.onetoc2",
4 "bytes": 4816,
5 "sha256": "b2f857961b76258a1582b402c048b340e89962257681426b3bfc58df8e0adbfa",
6 "mtime_ns": 1788585005062916740
7 },
8 {
9 "path": "synthetic.one",
10 "bytes": 25392,
11 "sha256": "9acb9ab3c06daa698c55ca48bc2c100aba4609e9f546839a0aeb99db68296033",
12 "mtime_ns": 1789887306088599720
13 }
14]
corpus/table-growth/cold/teardown.json created+1
...@@ -0,0 +1 @@
1{"absent": true}
crates/notebook/README.md+5-4
...@@ -348,10 +348,11 @@ when the page itself was moved to another section. Other URLs and unknown...@@ -348,10 +348,11 @@ when the page itself was moved to another section. Other URLs and unknown
348targets are `None`.348targets are `None`.
349349
350With the optional `protected` feature, `Notebook::unlock(path, password)`350With the optional `protected` feature, `Notebook::unlock(path, password)`
351reads the pages of a `Locked` section, and `Notebook::save_unlocked(path,351reads a `Locked` section as `Unlocked { pages, .. }`, and
352password, space, page, author)` saves an edited one under the section's key,352`Notebook::save_unlocked(path, password, &mut unlocked, space, page, author)`
353straight to the file: nothing of a protected section is cached or queued, a353saves an edited page under the section's key, straight to the file: nothing of
354section changed since the read fails the save, and a wrong password is354a protected section is cached or queued, a section written since `unlocked`
355was read fails the save, and a wrong password is
355`Error::Protected(PasswordMismatch)`.356`Error::Protected(PasswordMismatch)`.
356357
357`Section::import_page(page, author)` copies a page, usually read from another358`Section::import_page(page, author)` copies a page, usually read from another
crates/notebook/examples/cache_probe.rs+3-2
...@@ -110,8 +110,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {...@@ -110,8 +110,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
110 !intent.before.objects.iter().any(|o| o.id() == object.id())110 !intent.before.objects.iter().any(|o| o.id() == object.id())
111 })111 })
112 .collect();112 .collect();
113 let [PageObject::Outline(outline)] = added.as_slice() else {113 // A save that replaced a pending one carries its outlines too.
114 panic!("Expected one added outline")114 let Some(PageObject::Outline(outline)) = added.last() else {
115 panic!("Expected an added outline")
115 };116 };
116 let text = outline.paragraphs[0].text().unwrap().text.text();117 let text = outline.paragraphs[0].text().unwrap().text.text();
117 let operation: u64 = text.split_once(':').unwrap().0.parse()?;118 let operation: u64 = text.split_once(':').unwrap().0.parse()?;
crates/notebook/src/assets.rs+3-2
...@@ -79,8 +79,9 @@ pub(crate) fn key(filename: &str) -> Result<String> {...@@ -79,8 +79,9 @@ pub(crate) fn key(filename: &str) -> Result<String> {
79}79}
8080
81fn referenced(connection: &Connection, key: &str) -> Result<bool> {81fn referenced(connection: &Connection, key: &str) -> Result<bool> {
82 let (base, working) = crate::images::both(connection)?;82 // One image at a time: the working image usually answers.
83 for image in [working, base] {83 for read in [crate::images::working, crate::images::base] {
84 let image = read(connection)?;
84 let store = Store::parse(&image)?;85 let store = Store::parse(&image)?;
85 let index = RevisionIndex::parse(&store)?;86 let index = RevisionIndex::parse(&store)?;
86 let document = Document::parse(&index)?;87 let document = Document::parse(&index)?;
crates/notebook/src/images.rs+24-4
...@@ -46,10 +46,12 @@ fn restore(mut base: Vec<u8>, patch: &[u8]) -> Result<Vec<u8>> {...@@ -46,10 +46,12 @@ fn restore(mut base: Vec<u8>, patch: &[u8]) -> Result<Vec<u8>> {
46 Err(damaged().into())46 Err(damaged().into())
47 };47 };
48 };48 };
49 base.resize(49 // Every block past the base is a run, so a whole patch bounds the image.
50 usize::try_from(u64::from_le_bytes(*length)).map_err(|_| damaged())?,50 let length = usize::try_from(u64::from_le_bytes(*length))
51 0,51 .ok()
52 );52 .filter(|length| *length <= base.len() + patch.len())
53 .ok_or_else(damaged)?;
54 base.resize(length, 0);
53 while let Some((header, rest)) = runs.split_first_chunk::<16>() {55 while let Some((header, rest)) = runs.split_first_chunk::<16>() {
54 let offset = usize::try_from(u64::from_le_bytes(header[..8].try_into().unwrap()))56 let offset = usize::try_from(u64::from_le_bytes(header[..8].try_into().unwrap()))
55 .map_err(|_| damaged())?;57 .map_err(|_| damaged())?;
...@@ -83,6 +85,21 @@ pub(crate) fn both(connection: &Connection) -> Result<(Vec<u8>, Vec<u8>)> {...@@ -83,6 +85,21 @@ pub(crate) fn both(connection: &Connection) -> Result<(Vec<u8>, Vec<u8>)> {
83 Ok((base, working))85 Ok((base, working))
84}86}
8587
88/// The working image's length, without restoring it.
89pub(crate) fn working_length(connection: &Connection) -> Result<u64> {
90 Ok(connection.query_row(
91 "SELECT length(base), substr(working, 1, 8) FROM replica WHERE id=1",
92 [],
93 |row| {
94 let header: Vec<u8> = row.get(1)?;
95 Ok(match header.first_chunk::<8>() {
96 Some(length) => u64::from_le_bytes(*length),
97 None => row.get::<_, i64>(0)? as u64,
98 })
99 },
100 )?)
101}
102
86pub(crate) fn working(connection: &Connection) -> Result<Vec<u8>> {103pub(crate) fn working(connection: &Connection) -> Result<Vec<u8>> {
87 let (base, patch): (Vec<u8>, Vec<u8>) =104 let (base, patch): (Vec<u8>, Vec<u8>) =
88 connection.query_row("SELECT base, working FROM replica WHERE id=1", [], |row| {105 connection.query_row("SELECT base, working FROM replica WHERE id=1", [], |row| {
...@@ -141,6 +158,9 @@ mod tests {...@@ -141,6 +158,9 @@ mod tests {
141 }158 }
142 assert!(difference(&base, &base).is_empty());159 assert!(difference(&base, &base).is_empty());
143 assert!(difference(&base, &edited).len() < 3 * BLOCK);160 assert!(difference(&base, &edited).len() < 3 * BLOCK);
161 let mut huge = difference(&base, &edited);
162 huge[..8].copy_from_slice(&u64::MAX.to_le_bytes());
163 assert!(restore(base.clone(), &huge).is_err());
144 let patch = difference(&base, &edited);164 let patch = difference(&base, &edited);
145 for cut in 1..patch.len() {165 for cut in 1..patch.len() {
146 if let Ok(image) = restore(base.clone(), &patch[..cut]) {166 if let Ok(image) = restore(base.clone(), &patch[..cut]) {
crates/notebook/src/recovery.rs+1-1
...@@ -144,7 +144,7 @@ fn summary(connection: &Connection) -> Result<RecoverySummary> {...@@ -144,7 +144,7 @@ fn summary(connection: &Connection) -> Result<RecoverySummary> {
144 [],144 [],
145 |row| Ok((unsigned(row, 0)?, unsigned(row, 1)?)),145 |row| Ok((unsigned(row, 0)?, unsigned(row, 1)?)),
146 )?;146 )?;
147 let working_bytes = crate::images::working(connection)?.len() as u64;147 let working_bytes = crate::images::working_length(connection)?;
148 Ok(connection.query_row(148 Ok(connection.query_row(
149 "SELECT (SELECT count(*) FROM edits), (SELECT count(*) FROM conflicts),149 "SELECT (SELECT count(*) FROM edits), (SELECT count(*) FROM conflicts),
150 (SELECT count(*) FROM attempt), (SELECT count(*) FROM receipts),150 (SELECT count(*) FROM attempt), (SELECT count(*) FROM receipts),
crates/notebook/src/session.rs+43-23
...@@ -91,6 +91,14 @@ pub trait Storage: Send + Sync {...@@ -91,6 +91,14 @@ pub trait Storage: Send + Sync {
91 ) -> Result<()>;91 ) -> Result<()>;
92}92}
9393
94/// A password-protected section as `Notebook::unlock` read it.
95#[cfg(feature = "protected")]
96pub struct Unlocked {
97 pub pages: Vec<(ExGuid, Page)>,
98 /// The stored section the pages came from, which a save must still find in place.
99 snapshot: Vec<u8>,
100}
101
94/// A mounted notebook directory.102/// A mounted notebook directory.
95struct Directory(PathBuf);103struct Directory(PathBuf);
96104
...@@ -622,43 +630,55 @@ impl Notebook {...@@ -622,43 +630,55 @@ impl Notebook {
622 Ok(None)630 Ok(None)
623 }631 }
624632
625 /// The pages of a password-protected section, for reading: nothing is cached or633 /// The pages of a password-protected section: nothing is cached, and the decoded
626 /// written, and the decoded buffers go when the pages have been built.634 /// buffers go when the pages have been built.
627 #[cfg(feature = "protected")]635 #[cfg(feature = "protected")]
628 pub fn unlock(&self, path: &str, password: &str) -> Result<Vec<(ExGuid, Page)>> {636 pub fn unlock(&self, path: &str, password: &str) -> Result<Unlocked> {
629 let path = self.section_path(path)?.path.clone();637 let path = self.section_path(path)?.path.clone();
630 let bytes = self.storage.read(&path)?;638 let snapshot = self.storage.read(&path)?;
631 let store = Store::parse(&bytes)?;639 let pages = {
632 let index = RevisionIndex::parse(&store)?;640 let store = Store::parse(&snapshot)?;
633 let unlocked = onestore::protected::UnlockedSection::open(641 let index = RevisionIndex::parse(&store)?;
634 &index,642 let unlocked = onestore::protected::UnlockedSection::open(
635 password,643 &index,
636 onestore::protected::Limits::default(),644 password,
637 )?;645 onestore::protected::Limits::default(),
638 let document = unlocked.document()?;646 )?;
639 document647 let document = unlocked.document()?;
640 .pages()?648 document
641 .into_iter()649 .pages()?
642 .map(|(space, _)| Ok((space, Page::from_space(&document, space)?)))650 .into_iter()
643 .collect()651 .map(|(space, _)| Ok((space, Page::from_space(&document, space)?)))
652 .collect::<Result<_>>()?
653 };
654 Ok(Unlocked { pages, snapshot })
644 }655 }
645656
646 /// Saves a page read through `unlock`, stored under the section's key. The save goes657 /// Saves an edited page of `unlocked` under the section's key, straight to the file:
647 /// straight to the file and fails if the section changed since `unlock` read it;658 /// nothing of a protected section is cached or queued. A section written since
648 /// nothing of a protected section is cached or queued.659 /// `unlocked` was read fails the save; unlock again for its pages.
649 #[cfg(feature = "protected")]660 #[cfg(feature = "protected")]
650 pub fn save_unlocked(661 pub fn save_unlocked(
651 &self,662 &self,
652 path: &str,663 path: &str,
653 password: &str,664 password: &str,
665 unlocked: &mut Unlocked,
654 space: ExGuid,666 space: ExGuid,
655 page: &Page,667 page: &Page,
656 author: &str,668 author: &str,
657 ) -> Result<()> {669 ) -> Result<()> {
658 let path = self.section_path(path)?.path.clone();670 let path = self.section_path(path)?.path.clone();
659 let bytes = self.storage.read(&path)?;671 let edit = onestore::PreparedEdit::page_protected(
660 let edit = onestore::PreparedEdit::page_protected(&bytes, password, space, page, author)?;672 &unlocked.snapshot,
661 self.storage.commit(&path, &edit)673 password,
674 space,
675 page,
676 author,
677 )?;
678 self.storage.commit(&path, &edit)?;
679 let written = edit.as_bytes().to_vec();
680 unlocked.snapshot = written;
681 Ok(())
662 }682 }
663683
664 /// Opens a section of a mounted notebook by its catalog path.684 /// Opens a section of a mounted notebook by its catalog path.
crates/notebook/src/sync.rs+2
...@@ -225,6 +225,8 @@ impl Replica {...@@ -225,6 +225,8 @@ impl Replica {
225 self.acknowledge(intent.id, revision, &snapshot)?;225 self.acknowledge(intent.id, revision, &snapshot)?;
226 return Ok(Some((intent.id, EditStatus::Published { revision })));226 return Ok(Some((intent.id, EditStatus::Published { revision })));
227 }227 }
228 // Once acknowledged this image is the base, which polls trust without validating.
229 validate(prepared.as_bytes())?;
228 let store = Store::parse(prepared.as_bytes())?;230 let store = Store::parse(prepared.as_bytes())?;
229 let index = RevisionIndex::parse(&store)?;231 let index = RevisionIndex::parse(&store)?;
230 let revision = index.active(intent.space)?;232 let revision = index.active(intent.space)?;
crates/notebook/tests/protected.rs+13-5
...@@ -17,7 +17,7 @@ fn a_locked_section_unlocks_for_reading() {...@@ -17,7 +17,7 @@ fn a_locked_section_unlocks_for_reading() {
17 let notebook = Notebook::open(root.join("notebook"), temporary.path()).unwrap();17 let notebook = Notebook::open(root.join("notebook"), temporary.path()).unwrap();
18 let section = &notebook.catalog().sections[0];18 let section = &notebook.catalog().sections[0];
19 assert!(matches!(section.state, SectionState::Locked));19 assert!(matches!(section.state, SectionState::Locked));
20 let pages = notebook.unlock(&section.path, password).unwrap();20 let pages = notebook.unlock(&section.path, password).unwrap().pages;
21 assert_eq!(pages.len(), 11);21 assert_eq!(pages.len(), 11);
22 assert!(pages.iter().all(|(_, page)| !page.title.is_empty()));22 assert!(pages.iter().all(|(_, page)| !page.title.is_empty()));
23 assert!(matches!(23 assert!(matches!(
...@@ -57,7 +57,8 @@ fn an_unlocked_page_is_saved_under_the_section_key() {...@@ -57,7 +57,8 @@ fn an_unlocked_page_is_saved_under_the_section_key() {
57 .unwrap()57 .unwrap()
58 .path58 .path
59 .clone();59 .clone();
60 let (space, mut page) = notebook.unlock(&section, password).unwrap().remove(0);60 let mut unlocked = notebook.unlock(&section, password).unwrap();
61 let (space, mut page) = unlocked.pages[0].clone();
61 let paragraphs = page62 let paragraphs = page
62 .objects63 .objects
63 .iter_mut()64 .iter_mut()
...@@ -95,15 +96,22 @@ fn an_unlocked_page_is_saved_under_the_section_key() {...@@ -95,15 +96,22 @@ fn an_unlocked_page_is_saved_under_the_section_key() {
95 ))96 ))
96 .unwrap();97 .unwrap();
97 assert!(matches!(98 assert!(matches!(
98 notebook.save_unlocked(&section, "wrong", space, &page, "Rust"),99 notebook.save_unlocked(&section, "wrong", &mut unlocked, space, &page, "Rust"),
99 Err(notebook::Error::Protected(100 Err(notebook::Error::Protected(
100 onestore::protected::Error::PasswordMismatch101 onestore::protected::Error::PasswordMismatch
101 ))102 ))
102 ));103 ));
104 let mut stale = notebook.unlock(&section, password).unwrap();
103 notebook105 notebook
104 .save_unlocked(&section, password, space, &page, "Rust")106 .save_unlocked(&section, password, &mut unlocked, space, &page, "Rust")
105 .unwrap();107 .unwrap();
106 let (_, stored) = notebook.unlock(&section, password).unwrap().remove(0);108 // A view read before that save no longer matches the file.
109 assert!(
110 notebook
111 .save_unlocked(&section, password, &mut stale, space, &page, "Rust")
112 .is_err()
113 );
114 let (_, stored) = notebook.unlock(&section, password).unwrap().pages.remove(0);
107 assert!(stored.objects == page.objects);115 assert!(stored.objects == page.objects);
108 if let Some(export) = std::env::var_os("ONESTORE_PROTECTED_EXPORT") {116 if let Some(export) = std::env::var_os("ONESTORE_PROTECTED_EXPORT") {
109 let export = Path::new(&export);117 let export = Path::new(&export);
crates/onestore/src/objects.rs+8-7
...@@ -335,7 +335,6 @@ impl<'a> RevisionIndex<'a> {...@@ -335,7 +335,6 @@ impl<'a> RevisionIndex<'a> {
335 });335 });
336 }336 }
337 table = Arc::new(GlobalIds::new());337 table = Arc::new(GlobalIds::new());
338 defining.clear();
339 defining_table = true;338 defining_table = true;
340 }339 }
341 0x24..=0x26 => {340 0x24..=0x26 => {
...@@ -382,9 +381,12 @@ impl<'a> RevisionIndex<'a> {...@@ -382,9 +381,12 @@ impl<'a> RevisionIndex<'a> {
382 }381 }
383 _ => unreachable!(),382 _ => unreachable!(),
384 }383 }
385 if defining[start..]384 // More entries than indices repeats one; checked here so imports
386 .iter()385 // cannot grow the list without bound.
387 .any(|(index, guid)| *index >= 0xffffff || *guid == [0; 16])386 if defining.len() > 0xffffff
387 || defining[start..]
388 .iter()
389 .any(|(index, guid)| *index >= 0xffffff || *guid == [0; 16])
388 {390 {
389 return Err(Error {391 return Err(Error {
390 offset: node.offset,392 offset: node.offset,
...@@ -407,9 +409,8 @@ impl<'a> RevisionIndex<'a> {...@@ -407,9 +409,8 @@ impl<'a> RevisionIndex<'a> {
407 message: "Invalid or repeated global ID entry",409 message: "Invalid or repeated global ID entry",
408 });410 });
409 }411 }
410 let mut guids: Vec<_> = defining.iter().map(|(_, guid)| *guid).collect();412 defining.sort_unstable_by_key(|(_, guid)| *guid);
411 guids.sort_unstable();413 if defining.windows(2).any(|pair| pair[0].1 == pair[1].1) {
412 if guids.windows(2).any(|pair| pair[0] == pair[1]) {
413 return Err(Error {414 return Err(Error {
414 offset: node.offset,415 offset: node.offset,
415 message: "Global ID table repeats a GUID",416 message: "Global ID table repeats a GUID",
crates/onestore/src/page/write.rs+15-6
...@@ -2987,12 +2987,21 @@ pub(crate) fn squash(...@@ -2987,12 +2987,21 @@ pub(crate) fn squash(
2987 if !live.contains(id) {2987 if !live.contains(id) {
2988 continue;2988 continue;
2989 }2989 }
2990 if before.objects.get(id).is_some_and(|previous| {2990 // Stored tables keep the entries an object names, so those decide.
2991 previous.jcid == object.jcid2991 if let Some(previous) = before.objects.get(id)
2992 && previous.data == object.data2992 && previous.jcid == object.jcid
2993 && previous.global_ids == object.global_ids2993 && previous.data == object.data
2994 }) {2994 {
2995 continue;2995 let entries = match object.data {
2996 ObjectData::Properties(bytes) => crate::write::table_entries(bytes)?,
2997 _ => BTreeSet::new(),
2998 };
2999 if entries
3000 .iter()
3001 .all(|entry| previous.global_ids.get(entry) == object.global_ids.get(entry))
3002 {
3003 continue;
3004 }
2996 }3005 }
2997 let id = rename.get(id).copied().unwrap_or(*id);3006 let id = rename.get(id).copied().unwrap_or(*id);
2998 let mut replacement = match object.data {3007 let mut replacement = match object.data {
crates/onestore/src/protected/crypto.rs+5-2
...@@ -228,7 +228,9 @@ impl Key {...@@ -228,7 +228,9 @@ impl Key {
228 crate::properties::reference_streams(&mut c)?;228 crate::properties::reference_streams(&mut c)?;
229 let prefix = c.offset;229 let prefix = c.offset;
230 let padding = (16 - (2 + clear.len() - prefix) % 16) % 16;230 let padding = (16 - (2 + clear.len() - prefix) % 16) % 16;
231 let mut body = Zeroizing::new((padding as u16).to_le_bytes().to_vec());231 // Sized once: a buffer abandoned by growth would keep plaintext.
232 let mut body = Zeroizing::new(Vec::with_capacity(2 + clear.len() - prefix + padding));
233 body.extend_from_slice(&(padding as u16).to_le_bytes());
232 body.extend_from_slice(&clear[prefix..]);234 body.extend_from_slice(&clear[prefix..]);
233 let length = body.len() + padding;235 let length = body.len() + padding;
234 body.resize(length, 0);236 body.resize(length, 0);
...@@ -248,7 +250,8 @@ impl Key {...@@ -248,7 +250,8 @@ impl Key {
248 if clear.is_empty() {250 if clear.is_empty() {
249 return Vec::new();251 return Vec::new();
250 }252 }
251 let mut output = (clear.len() as u64).to_le_bytes().to_vec();253 let mut output = Vec::with_capacity((8 + clear.len()).next_multiple_of(16));
254 output.extend_from_slice(&(clear.len() as u64).to_le_bytes());
252 output.extend_from_slice(clear);255 output.extend_from_slice(clear);
253 output.resize(output.len().next_multiple_of(16), 0);256 output.resize(output.len().next_multiple_of(16), 0);
254 encrypt(&self.value, &self.file_iv, &mut output);257 encrypt(&self.value, &self.file_iv, &mut output);
crates/onestore/src/protected/mod.rs+37-14
...@@ -83,7 +83,7 @@ struct Decoded<'a> {...@@ -83,7 +83,7 @@ struct Decoded<'a> {
83/// Owns decoded buffers until dropped; returned views cannot outlive this owner.83/// Owns decoded buffers until dropped; returned views cannot outlive this owner.
84///84///
85/// Passwords are not retained. Dropping this owner clears its derived key and85/// Passwords are not retained. Dropping this owner clears its derived key and
86/// decoded buffers. Owned strings or exports made from a `Document` are separate86/// decoded buffers; a protected save works on ordinary buffers, which are not cleared. Owned strings or exports made from a `Document` are separate
87/// caller-owned copies and must be disposed of by the caller when locking.87/// caller-owned copies and must be disposed of by the caller when locking.
88/// Opening never rewrites the source image or changes its protection state.88/// Opening never rewrites the source image or changes its protection state.
89///89///
...@@ -302,15 +302,19 @@ impl UnlockedSection<'_> {...@@ -302,15 +302,19 @@ impl UnlockedSection<'_> {
302 extension,302 extension,
303 } => {303 } => {
304 let text = |bytes: &[u8]| {304 let text = |bytes: &[u8]| {
305 String::from_utf16_lossy(305 String::from_utf16(
306 &bytes306 &bytes
307 .chunks_exact(2)307 .chunks_exact(2)
308 .map(|pair| u16::from_le_bytes([pair[0], pair[1]]))308 .map(|pair| u16::from_le_bytes([pair[0], pair[1]]))
309 .collect::<Vec<_>>(),309 .collect::<Vec<_>>(),
310 )310 )
311 .map_err(|_| crate::Error {
312 offset: 0,
313 message: "Invalid UTF-16 file-data declaration",
314 })
311 };315 };
312 let mut copy =316 let mut copy =
313 PropertyObject::file(*id, &text(reference), &text(extension))?;317 PropertyObject::file(*id, &text(reference)?, &text(extension)?)?;
314 copy.jcid = object.jcid;318 copy.jcid = object.jcid;
315 copy.global_ids = std::sync::Arc::clone(&object.global_ids);319 copy.global_ids = std::sync::Arc::clone(&object.global_ids);
316 copy320 copy
...@@ -336,7 +340,7 @@ impl UnlockedSection<'_> {...@@ -336,7 +340,7 @@ impl UnlockedSection<'_> {
336 };340 };
337 let store = crate::Store::parse(&scaffold)?;341 let store = crate::Store::parse(&scaffold)?;
338 let placeholder = identity(RevisionIndex::parse(&store)?.root);342 let placeholder = identity(RevisionIndex::parse(&store)?.root);
339 for at in 0..scaffold.len() - 20 {343 for at in 0..=scaffold.len() - 20 {
340 if scaffold[at..at + 20] == placeholder {344 if scaffold[at..at + 20] == placeholder {
341 scaffold[at..at + 20].copy_from_slice(&identity(self.index.root));345 scaffold[at..at + 20].copy_from_slice(&identity(self.index.root));
342 }346 }
...@@ -398,14 +402,32 @@ pub(crate) fn write_page(...@@ -398,14 +402,32 @@ pub(crate) fn write_page(
398 let store = crate::Store::parse(source)?;402 let store = crate::Store::parse(source)?;
399 let index = RevisionIndex::parse(&store)?;403 let index = RevisionIndex::parse(&store)?;
400 let unlocked = UnlockedSection::open(&index, password, Limits::default())?;404 let unlocked = UnlockedSection::open(&index, password, Limits::default())?;
405 if unlocked.keys.len() != 1 {
406 return Err(Error::Unsupported);
407 }
401 let twin = unlocked.twin()?;408 let twin = unlocked.twin()?;
402 let applied = Zeroizing::new(crate::page::write::write_page(&twin, space, page, author)?);409 let applied = Zeroizing::new(crate::page::write::write_page(&twin, space, page, author)?);
410 // Squash skips the spaces the source lacks as the twin's scaffold; the writers added none.
411 let spaces = |image: &[u8]| -> Result<Vec<ExGuid>> {
412 let store = crate::Store::parse(image)?;
413 Ok(RevisionIndex::parse(&store)?.spaces.into_keys().collect())
414 };
415 if spaces(&twin)? != spaces(&applied)? {
416 return Err(invalid("Page edits cannot create object spaces"));
417 }
403 let written = crate::page::write::squash(source, &applied, &BTreeMap::new(), Some(&unlocked))?;418 let written = crate::page::write::squash(source, &applied, &BTreeMap::new(), Some(&unlocked))?;
404 let store = crate::Store::parse(&written)?;419 let store = crate::Store::parse(&written)?;
405 UnlockedSection::open(&RevisionIndex::parse(&store)?, password, Limits::default())?;420 UnlockedSection::open(&RevisionIndex::parse(&store)?, password, Limits::default())?;
406 Ok(written)421 Ok(written)
407}422}
408423
424impl UnlockedSection<'_> {
425 /// The section key; `write_page` admits sections with exactly one.
426 fn key(&self) -> &crypto::Key {
427 self.keys.values().next().expect("one section key")
428 }
429}
430
409impl crate::write::Protection for UnlockedSection<'_> {431impl crate::write::Protection for UnlockedSection<'_> {
410 fn resolve(432 fn resolve(
411 &self,433 &self,
...@@ -423,20 +445,21 @@ impl crate::write::Protection for UnlockedSection<'_> {...@@ -423,20 +445,21 @@ impl crate::write::Protection for UnlockedSection<'_> {
423 }445 }
424446
425 fn seal_property(&self, clear: &[u8]) -> std::result::Result<Vec<u8>, crate::Error> {447 fn seal_property(&self, clear: &[u8]) -> std::result::Result<Vec<u8>, crate::Error> {
426 let [key] = self.keys.values().collect::<Vec<_>>()[..] else {448 let random = crate::Error {
427 return Err(crate::Error {449 offset: 0,
428 offset: 0,450 message: "System random source failed",
429 message: "Protected writing needs one section key",
430 });
431 };451 };
432 let failed = |message| crate::Error { offset: 0, message };
433 let mut iv = [0; 16];452 let mut iv = [0; 16];
434 getrandom::fill(&mut iv).map_err(|_| failed("System random source failed"))?;453 getrandom::fill(&mut iv).map_err(|_| random)?;
435 key.seal_property(clear, iv)454 self.key()
436 .map_err(|_| failed("Malformed property object"))455 .seal_property(clear, iv)
456 .map_err(|error| match error {
457 Error::Invalid(error) => error,
458 _ => random,
459 })
437 }460 }
438461
439 fn seal_file(&self, clear: &[u8]) -> Vec<u8> {462 fn seal_file(&self, clear: &[u8]) -> Vec<u8> {
440 self.keys.values().next().unwrap().seal_file(clear)463 self.key().seal_file(clear)
441 }464 }
442}465}
crates/onestore/src/write.rs+57-19
...@@ -104,6 +104,21 @@ fn compact(id: ExGuid, table: &BTreeMap<u32, [u8; 16]>) -> Result<[u8; 4]> {...@@ -104,6 +104,21 @@ fn compact(id: ExGuid, table: &BTreeMap<u32, [u8; 16]>) -> Result<[u8; 4]> {
104 Ok(((index << 8) | id.n).to_le_bytes())104 Ok(((index << 8) | id.n).to_le_bytes())
105}105}
106106
107/// The global id table entries a property object's references name.
108pub(crate) fn table_entries(bytes: &[u8]) -> Result<BTreeSet<u32>> {
109 let mut entries = BTreeSet::new();
110 for property in PropertySets::parse(bytes)?.sets.iter().flatten() {
111 if let Value::References { compact_ids, .. } = property.value {
112 entries.extend(
113 compact_ids
114 .chunks_exact(4)
115 .map(|id| u32::from_le_bytes(id.try_into().unwrap()) >> 8),
116 );
117 }
118 }
119 Ok(entries)
120}
121
107fn field_length(property: &crate::Property<'_>, set_lengths: &[usize]) -> usize {122fn field_length(property: &crate::Property<'_>, set_lengths: &[usize]) -> usize {
108 match &property.value {123 match &property.value {
109 Value::NoData => 0,124 Value::NoData => 0,
...@@ -734,14 +749,21 @@ pub(crate) fn write_revision_with_payloads(...@@ -734,14 +749,21 @@ pub(crate) fn write_revision_with_payloads(
734 payloads: &[([u8; 16], &[u8])],749 payloads: &[([u8; 16], &[u8])],
735 edit: impl FnOnce(&crate::ResolvedRevision<'_>) -> Result<BTreeMap<ExGuid, PropertyObject>>,750 edit: impl FnOnce(&crate::ResolvedRevision<'_>) -> Result<BTreeMap<ExGuid, PropertyObject>>,
736) -> Result<Vec<u8>> {751) -> Result<Vec<u8>> {
737 write_revisions_with_payloads(source, payloads, |index| {752 write_revisions_with_payloads(source, payloads, update(space, edit))
738 let rid = index.active(space)?;753}
739 let revision = index.resolve(space, rid)?;754
755/// One space's active revision edited into its update.
756fn update(
757 space: ExGuid,
758 edit: impl FnOnce(&crate::ResolvedRevision<'_>) -> Result<BTreeMap<ExGuid, PropertyObject>>,
759) -> impl FnOnce(&RevisionIndex<'_>) -> Result<BTreeMap<ExGuid, RevisionEdit>> {
760 move |index| {
761 let revision = index.resolve(space, index.active(space)?)?;
740 Ok(BTreeMap::from([(762 Ok(BTreeMap::from([(
741 space,763 space,
742 RevisionEdit::Update(edit(&revision)?),764 RevisionEdit::Update(edit(&revision)?),
743 )]))765 )]))
744 })766 }
745}767}
746768
747fn append_fragment(769fn append_fragment(
...@@ -853,13 +875,7 @@ pub(crate) fn write_revision_on(...@@ -853,13 +875,7 @@ pub(crate) fn write_revision_on(
853 space: ExGuid,875 space: ExGuid,
854 edit: impl FnOnce(&crate::ResolvedRevision<'_>) -> Result<BTreeMap<ExGuid, PropertyObject>>,876 edit: impl FnOnce(&crate::ResolvedRevision<'_>) -> Result<BTreeMap<ExGuid, PropertyObject>>,
855) -> Result<Vec<u8>> {877) -> Result<Vec<u8>> {
856 let output = build_on(index, &[], None, |index| {878 let output = build_on(index, &[], None, update(space, edit))?;
857 let revision = index.resolve(space, index.active(space)?)?;
858 Ok(BTreeMap::from([(
859 space,
860 RevisionEdit::Update(edit(&revision)?),
861 )]))
862 })?;
863 check(&output, true)?;879 check(&output, true)?;
864 Ok(output)880 Ok(output)
865}881}
...@@ -1079,14 +1095,24 @@ fn build_on(...@@ -1079,14 +1095,24 @@ fn build_on(
1079 object.bytes[offset..offset + 4].copy_from_slice(&reference);1095 object.bytes[offset..offset + 4].copy_from_slice(&reference);
1080 }1096 }
1081 }1097 }
1082 replacements.retain(|id, replacement| {1098 // Detached objects must pass the final reachability check even when unchanged. Equal
1083 // Detached objects must pass the final reachability check even when unchanged.1099 // bytes are the same object when the table entries they name agree: stored tables
1084 !reachable.contains(id)1100 // keep only those.
1085 || !revision.objects.get(id).is_some_and(|object| {1101 let mut unchanged = Vec::new();
1086 object.data == ObjectData::Properties(&replacement.bytes)1102 for (id, replacement) in &replacements {
1087 && object.global_ids == replacement.global_ids1103 if let Some(object) = revision.objects.get(id)
1088 })1104 && reachable.contains(id)
1089 });1105 && object.data == ObjectData::Properties(&replacement.bytes)
1106 && table_entries(&replacement.bytes)?
1107 .iter()
1108 .all(|entry| object.global_ids.get(entry) == replacement.global_ids.get(entry))
1109 {
1110 unchanged.push(*id);
1111 }
1112 }
1113 for id in unchanged {
1114 replacements.remove(&id);
1115 }
1090 if replacements.is_empty() {1116 if replacements.is_empty() {
1091 continue;1117 continue;
1092 }1118 }
...@@ -1244,7 +1270,19 @@ fn build_on(...@@ -1244,7 +1270,19 @@ fn build_on(
1244 } else {1270 } else {
1245 vec![node(0x21, None, &[0])?]1271 vec![node(0x21, None, &[0])?]
1246 };1272 };
1273 // A table read from a long-lived page names every session that edited it; the
1274 // group stores the entries its objects use.
1275 let mut used = BTreeSet::new();
1276 for (id, object) in &objects {
1277 used.insert(u32::from_le_bytes(compact(*id, table)?) >> 8);
1278 if let ObjectData::Properties(bytes) = object.data {
1279 used.extend(table_entries(bytes)?);
1280 }
1281 }
1247 for (id, guid) in table {1282 for (id, guid) in table {
1283 if is_section && !used.contains(id) {
1284 continue;
1285 }
1248 let mut entry = id.to_le_bytes().to_vec();1286 let mut entry = id.to_le_bytes().to_vec();
1249 entry.extend_from_slice(guid);1287 entry.extend_from_slice(guid);
1250 group.push(node(0x24, None, &entry)?);1288 group.push(node(0x24, None, &entry)?);
crates/onestore/tests/edit.rs+7-1
...@@ -264,7 +264,13 @@ fn assert_other_objects_preserved(...@@ -264,7 +264,13 @@ fn assert_other_objects_preserved(
264 }264 }
265 let current = &after.objects[id];265 let current = &after.objects[id];
266 assert_eq!(object.jcid, current.jcid);266 assert_eq!(object.jcid, current.jcid);
267 assert_eq!(object.global_ids, current.global_ids);267 // A rewritten object keeps the table entries it names.
268 assert!(
269 current
270 .global_ids
271 .iter()
272 .all(|(entry, guid)| object.global_ids.get(entry) == Some(guid))
273 );
268 let mut pending = vec![*id];274 let mut pending = vec![*id];
269 let mut visited = std::collections::BTreeSet::new();275 let mut visited = std::collections::BTreeSet::new();
270 while let Some(next) = pending.pop() {276 while let Some(next) = pending.pop() {
crates/onestore/tests/table_growth.rs created+101
...@@ -0,0 +1,101 @@
1//! A rewritten object stores the global id table entries it names, not the table of the
2//! revision it was read from.
3
4use onestore::{
5 PreparedEdit, RevisionIndex, Store,
6 document::Document,
7 page::{Page, PageObject, Paragraph},
8};
9
10const SOURCE: &[u8] = include_bytes!("../../../corpus/native/20260905-05/notebook/synthetic.one");
11
12fn pages(bytes: &[u8]) -> Vec<(onestore::ExGuid, Page)> {
13 let store = Store::parse(bytes).unwrap();
14 let index = RevisionIndex::parse(&store).unwrap();
15 let document = Document::parse(&index).unwrap();
16 document
17 .pages()
18 .unwrap()
19 .into_iter()
20 .map(|(space, _)| (space, Page::from_space(&document, space).unwrap()))
21 .collect()
22}
23
24/// Table entries per object group written past `from`, as `(entries, highest index + 1)`.
25fn tables(bytes: &[u8], from: usize) -> Vec<(usize, u32)> {
26 let store = Store::parse(bytes).unwrap();
27 store
28 .lists
29 .values()
30 .filter(|list| list.nodes.first().is_some_and(|node| node.offset >= from))
31 .map(|list| {
32 let entries: Vec<u32> = list
33 .nodes
34 .iter()
35 .filter(|node| node.id == 0x24)
36 .map(|node| u32::from_le_bytes(node.payload[..4].try_into().unwrap()))
37 .collect();
38 (entries.len(), entries.iter().max().map_or(0, |max| max + 1))
39 })
40 .filter(|(entries, _)| *entries > 0)
41 .collect()
42}
43
44/// `ONESTORE_TABLE_EXPORT` names a directory receiving the edited notebook for a cold reopen.
45#[test]
46fn a_text_edit_stores_the_table_entries_it_names() {
47 let mut bytes = SOURCE.to_vec();
48 let mut sparse = false;
49 let mut edited = Vec::new();
50 for (space, mut page) in pages(SOURCE) {
51 let Some(text) = page
52 .objects
53 .iter_mut()
54 .filter_map(|object| match object {
55 PageObject::Outline(outline) if !outline.title => Some(&mut outline.paragraphs),
56 _ => None,
57 })
58 .flatten()
59 .find_map(|paragraph| paragraph.text_mut())
60 else {
61 continue;
62 };
63 let format = text.text.format_at(0).unwrap().clone();
64 text.text
65 .append(Paragraph::new(" and a few more words".to_owned(), format))
66 .unwrap();
67 let written = PreparedEdit::page(&bytes, space, &page, "Rust")
68 .unwrap()
69 .as_bytes()
70 .to_vec();
71 for (entries, span) in tables(&written, bytes.len()) {
72 sparse |= (entries as u32) < span;
73 }
74 assert_eq!(
75 PreparedEdit::page(&written, space, &page, "Rust")
76 .unwrap()
77 .as_bytes(),
78 written
79 );
80 bytes = written;
81 edited.push((space, page));
82 }
83 // The fixture's pages were written over several sessions, so a text edit names a
84 // subset of its revision's table and the stored indices have gaps.
85 assert!(sparse);
86 let stored = pages(&bytes);
87 for (space, page) in &edited {
88 let (_, stored) = stored.iter().find(|(id, _)| id == space).unwrap();
89 assert!(stored.objects == page.objects);
90 }
91 if let Some(export) = std::env::var_os("ONESTORE_TABLE_EXPORT") {
92 let export = std::path::Path::new(&export);
93 std::fs::create_dir_all(export).unwrap();
94 std::fs::write(export.join("synthetic.one"), &bytes).unwrap();
95 std::fs::copy(
96 "../../corpus/native/20260905-05/notebook/Open Notebook.onetoc2",
97 export.join("Open Notebook.onetoc2"),
98 )
99 .unwrap();
100 }
101}
crates/onestore/tests/writer.rs+15-3
...@@ -55,7 +55,11 @@ fn scalar_edit_appends_a_revision_and_preserves_every_prior_object() {...@@ -55,7 +55,11 @@ fn scalar_edit_appends_a_revision_and_preserves_every_prior_object() {
55 let same = &unchanged.objects[&id];55 let same = &unchanged.objects[&id];
56 assert_eq!(object.jcid, same.jcid);56 assert_eq!(object.jcid, same.jcid);
57 assert_eq!(object.reference_count, same.reference_count);57 assert_eq!(object.reference_count, same.reference_count);
58 assert_eq!(object.global_ids, same.global_ids);58 assert!(
59 same.global_ids
60 .iter()
61 .all(|(entry, guid)| object.global_ids.get(entry) == Some(guid))
62 );
59 assert_eq!(object.data, same.data);63 assert_eq!(object.data, same.data);
60 }64 }
61 }65 }
...@@ -403,7 +407,11 @@ fn checkpoints_bound_dependencies_and_preserve_native_objects_and_history() {...@@ -403,7 +407,11 @@ fn checkpoints_bound_dependencies_and_preserve_native_objects_and_history() {
403 assert_eq!(a.object_spaces, b.object_spaces);407 assert_eq!(a.object_spaces, b.object_spaces);
404 assert_eq!(a.contexts, b.contexts);408 assert_eq!(a.contexts, b.contexts);
405 if section {409 if section {
406 assert_eq!(object.global_ids, same.global_ids);410 assert!(
411 same.global_ids
412 .iter()
413 .all(|(entry, guid)| object.global_ids.get(entry) == Some(guid))
414 );
407 if *id != oid {415 if *id != oid {
408 assert_eq!(object.data, same.data);416 assert_eq!(object.data, same.data);
409 }417 }
...@@ -437,7 +445,11 @@ fn checkpoints_bound_dependencies_and_preserve_native_objects_and_history() {...@@ -437,7 +445,11 @@ fn checkpoints_bound_dependencies_and_preserve_native_objects_and_history() {
437 let same = &preserved.objects[&id];445 let same = &preserved.objects[&id];
438 assert_eq!(object.jcid, same.jcid);446 assert_eq!(object.jcid, same.jcid);
439 assert_eq!(object.reference_count, same.reference_count);447 assert_eq!(object.reference_count, same.reference_count);
440 assert_eq!(object.global_ids, same.global_ids);448 assert!(
449 same.global_ids
450 .iter()
451 .all(|(entry, guid)| object.global_ids.get(entry) == Some(guid))
452 );
441 assert_eq!(object.data, same.data);453 assert_eq!(object.data, same.data);
442 }454 }
443 }455 }
tools/test_table_growth.py created+24
...@@ -0,0 +1,24 @@
1from pathlib import Path
2import runpy
3import shutil
4from tempfile import TemporaryDirectory
5import unittest
6
7ROOT = Path(__file__).resolve().parent.parent
8FIXTURE = ROOT / 'corpus/table-growth'
9compare = runpy.run_path(str(ROOT / 'tools/verify-document.py'))['compare']
10
11
12class TableGrowthTest(unittest.TestCase):
13 def test_onenote_reads_object_groups_whose_id_tables_have_gaps(self):
14 with TemporaryDirectory() as temporary:
15 native = Path(temporary) / 'read'
16 shutil.copytree(FIXTURE / 'cold/read', native)
17 compare(FIXTURE / 'candidate/notebook', native)
18 self.assertIn('and a few more words', (native / 'page-000.xml').read_text(encoding='utf-8-sig'))
19 self.assertEqual((FIXTURE / 'candidate/notebook/synthetic.one').read_bytes(),
20 (FIXTURE / 'cold/notebook/synthetic.one').read_bytes())
21
22
23if __name__ == '__main__':
24 unittest.main()