authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-09-19 23:22:38-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-09-25 20:26:20-07:00
logb26c81f7c036b864e287820c6d3b0498b64159cf
tree9099c8bf67d44eb6888cb8981e5153fe4d07756a
parentbe6b82da9a9e19b3a47c24655949fd2d66db0800
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

fix: review findings in protected saves, the cache and the parser; store only named table entries

Review (three independent passes over the protected-write, shared-GUID, harness and save-cost changes): - Notebook::save_unlocked compared the file with a snapshot it had just read, so a section written since unlock was silently overwritten with the stale model. unlock now returns Unlocked { pages, snapshot } and the save commits against that snapshot. - An acknowledged publication becomes the base that polls trust without validating; it is now validated before it is published. - Global id table imports could grow the pending entry list without bound before the duplicate check at the table end; bounded per entry node. The GUID uniqueness check reuses the list. - A corrupt cache patch length aborted on allocation; bounded by the patch. The recovery summary reads the working length from the patch header and the asset scan holds one image at a time. - Sealed buffers are sized once so growth leaves no plaintext behind; a failing random source is no longer reported as malformed data; file-data strings are decoded checked; protected writing admits exactly one section key and refuses a writer that created a space. The save's working buffers are ordinary memory, and the owner doc now says so. - cache_probe accepts a coalesced insert intent. Growth: a rewritten object group stored the whole global id table of the revision its objects were read from. It now stores the entries they name, and no-op detection compares those entries. OneNote 2010 cold-reads the sparse tables and leaves the file as written (corpus/table-growth). Assisted-by: claude-fable-5.1, claude-opus-5

34 files changed, 606 insertions(+), 93 deletions(-)

corpus/table-growth/README.md created+8
......@@ -0,0 +1,8 @@
1# Table growth
2
3`candidate/notebook` is `native/20260905-05` after `a_text_edit_stores_the_table_entries_it_names`
4(`ONESTORE_TABLE_EXPORT`) appended words to a paragraph of every page. Each rewritten
5object group stores only the global id table entries its objects name, so the stored
6indices have gaps where the source revision's table named GUIDs the edit does not use.
7`cold` is OneNote 2010's cold read of it from a fresh clone; OneNote left the file as
8written.
corpus/table-growth/candidate/notebook/Open Notebook.onetoc2 created
Binary files /dev/null and b/corpus/table-growth/candidate/notebook/Open Notebook.onetoc2 differ
corpus/table-growth/candidate/notebook/synthetic.one created
Binary files /dev/null and b/corpus/table-growth/candidate/notebook/synthetic.one differ
corpus/table-growth/cold/commands.jsonl created+3
......@@ -0,0 +1,3 @@
1{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null}
2{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-6162C217 -ExpectedPages 1", "exit": 0, "stdout": "Read 1 sections and 1 pages.\r\n", "stderr": "", "error": null}
3{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null}
corpus/table-growth/cold/machine.json created+1
......@@ -0,0 +1 @@
1{"name": "m6-6162c217", "hostname": "ONE-M6-6162C217"}
corpus/table-growth/cold/notebook/Open Notebook.onetoc2 created
Binary files /dev/null and b/corpus/table-growth/cold/notebook/Open Notebook.onetoc2 differ
corpus/table-growth/cold/notebook/synthetic.one created
Binary files /dev/null and b/corpus/table-growth/cold/notebook/synthetic.one differ
corpus/table-growth/cold/read/af1af4b8987109ce1e335e538910fb3860b7c57eb3b6110a06aa2114e06d69a7.attachment created+1
......@@ -0,0 +1 @@
1Fictitious attachment for native corpus.
\ No newline at end of file
corpus/table-growth/cold/read/environment.json created+7
......@@ -0,0 +1,7 @@
1{
2 "powershell": "5.1.14409.1005",
3 "schema": "xs2010",
4 "hostname": "ONE-M6-6162C217",
5 "cold": true,
6 "onenote": "14.0.4763.1000"
7}
corpus/table-growth/cold/read/hierarchy.xml created+2
......@@ -0,0 +1,2 @@
1<?xml version="1.0"?>
2<one:Notebook xmlns:one="http://schemas.microsoft.com/office/onenote/2010/onenote" name="notebook" nickname="notebook" ID="{BCD32541-08BB-4A4F-A7D6-91A4DE37385C}{1}{B0}" path="C:\one-tests\runs\capture\notebook" lastModifiedTime="2026-09-20T06:56:45.000Z" color="#9595AA"><one:Section name="synthetic" ID="{7B10731A-3919-06E3-0F51-A7F44821474F}{1}{B0}" path="C:\one-tests\runs\capture\notebook\synthetic.one" lastModifiedTime="2026-09-20T06:56:45.000Z" color="#8AA8E4"><one:Page ID="{5759809C-B0D2-45F2-8836-0F65B4195B50}{1}{B0}" name="Fictitious: café, 東京, مرحبا and a few more words" dateTime="2026-09-05T05:06:54.000Z" lastModifiedTime="2026-09-20T06:56:45.000Z" pageLevel="1"/></one:Section></one:Notebook>
corpus/table-growth/cold/read/page-000.xml created+9
......@@ -0,0 +1,9 @@
1<?xml version="1.0"?>
2<one:Page xmlns:one="http://schemas.microsoft.com/office/onenote/2010/onenote" ID="{5759809C-B0D2-45F2-8836-0F65B4195B50}{1}{B0}" name="Fictitious: café, 東京, مرحبا and a few more words" dateTime="2026-09-05T05:06:54.000Z" lastModifiedTime="2026-09-20T06:56:45.000Z" pageLevel="1" lang="en-US"><one:QuickStyleDef index="0" name="p" fontColor="automatic" highlightColor="automatic" font="Calibri" fontSize="11.0" spaceBefore="0.0" spaceAfter="0.0"/><one:PageSettings RTL="false" color="automatic"><one:PageSize><one:Automatic/></one:PageSize><one:RuleLines visible="false"/></one:PageSettings><one:Outline author="snow" authorInitials="S" lastModifiedBy="snow" lastModifiedByInitials="S" lastModifiedTime="2026-09-20T06:55:06.000Z" objectID="{30952820-FBA7-4644-87C4-83BE4B85BE31}{10}{B0}"><one:Position x="36.0" y="14.40000057220459" z="0"/><one:Size width="127.5136947631836" height="27.72771453857422"/><one:OEChildren><one:OE creationTime="2026-09-05T05:06:55.000Z" lastModifiedTime="2026-09-20T06:55:06.000Z" objectID="{30952820-FBA7-4644-87C4-83BE4B85BE31}{11}{B0}" alignment="left" quickStyleIndex="0" style="font-size:11.0pt;color:#1F4E79"><one:T><![CDATA[<span
3style='font-weight:bold;font-family:Calibri' lang=en-US>Fictitious: café, </span><span
4style='font-weight:bold;font-family:SimSun' lang=en-US>東京</span><span
5style='font-weight:bold;font-family:Calibri' lang=en-US>, </span><span
6style='font-weight:bold;font-family:Arial;direction:rtl;unicode-bidi:embed'
7lang=ar-SA>مرحبا</span><span style='font-weight:bold;font-family:Calibri'
8lang=en-US> and a few more words</span>]]></one:T></one:OE></one:OEChildren></one:Outline><one:Outline author="snow" authorInitials="S" lastModifiedBy="snow" lastModifiedByInitials="S" lastModifiedTime="2026-09-05T05:06:57.000Z" objectID="{6625F435-48F2-4C19-9D72-0136EF895A78}{10}{B0}"><one:Position x="144.0" y="96.0" z="1"/><one:Size width="167.0449523925781" height="13.42771339416504"/><one:OEChildren><one:OE creationTime="2026-09-05T05:06:57.000Z" lastModifiedTime="2026-09-05T05:06:57.000Z" objectID="{6625F435-48F2-4C19-9D72-0136EF895A78}{11}{B0}" alignment="left" quickStyleIndex="0"><one:T><![CDATA[Fictitious positioned outline.]]></one:T></one:OE></one:OEChildren></one:Outline><one:Outline author="snow" authorInitials="S" lastModifiedBy="snow" lastModifiedByInitials="S" lastModifiedTime="2026-09-05T05:06:58.000Z" objectID="{5B0C9CBA-AE68-45B7-AB02-AAC203E412F6}{10}{B0}"><one:Position x="144.0" y="192.0" z="2"/><one:Size width="72.0" height="0.750005722045898"/><one:OEChildren><one:OE creationTime="2026-09-05T05:06:58.000Z" lastModifiedTime="2026-09-05T05:06:58.000Z" objectID="{5B0C9CBA-AE68-45B7-AB02-AAC203E412F6}{11}{B0}" alignment="left"><one:Image format="png" originalPageNumber="0"><one:Data>iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9J1uoA
9AAAASUVORK5CYII=</one:Data></one:Image></one:OE></one:OEChildren></one:Outline><one:Outline author="snow" authorInitials="S" lastModifiedBy="snow" lastModifiedByInitials="S" lastModifiedTime="2026-09-05T05:06:59.000Z" objectID="{DE1684DA-6810-4671-8EAF-CD3C3735DDC5}{10}{B0}"><one:Position x="264.0" y="192.0" z="3"/><one:Size width="72.00001525878906" height="63.0"/><one:OEChildren><one:OE creationTime="2026-09-05T05:06:59.000Z" lastModifiedTime="2026-09-05T05:06:59.000Z" objectID="{DE1684DA-6810-4671-8EAF-CD3C3735DDC5}{11}{B0}" alignment="left"><one:InsertedFile pathCache="C:\Users\clover\AppData\Local\Microsoft\OneNote\14.0\OneNoteOfflineCache_Files\458001d8-0f1e-451f-81cb-621828fc0b39.txt" pathSource="C:\one-tests\runs\20260905-05\assets\fictitious-attachment.txt" preferredName="fictitious-attachment.txt"/></one:OE></one:OEChildren></one:Outline><one:Outline author="snow" authorInitials="S" lastModifiedBy="snow" lastModifiedByInitials="S" lastModifiedTime="2026-09-05T05:07:00.000Z" objectID="{F0C9FD8A-0980-4DF1-91A7-2F691F662683}{10}{B0}"><one:Position x="144.0" y="312.0" z="4"/><one:Size width="92.42181396484374" height="21.94773101806641"/><one:OEChildren><one:OE creationTime="2026-09-05T05:07:00.000Z" lastModifiedTime="2026-09-05T05:07:00.000Z" objectID="{F0C9FD8A-0980-4DF1-91A7-2F691F662683}{11}{B0}" alignment="left"><one:Table bordersVisible="true" lastModifiedTime="2026-09-05T05:07:00.000Z" objectID="{F0C9FD8A-0980-4DF1-91A7-2F691F662683}{12}{B0}"><one:Columns><one:Column index="0" width="39.14614105224609"/><one:Column index="1" width="45.14567184448242"/></one:Columns><one:Row objectID="{F0C9FD8A-0980-4DF1-91A7-2F691F662683}{13}{B0}" lastModifiedTime="2026-09-05T05:07:00.000Z"><one:Cell lastModifiedTime="2026-09-05T05:07:00.000Z" objectID="{F0C9FD8A-0980-4DF1-91A7-2F691F662683}{14}{B0}" lastModifiedByInitials="S"><one:OEChildren><one:OE creationTime="2026-09-05T05:07:00.000Z" lastModifiedTime="2026-09-05T05:07:00.000Z" objectID="{F0C9FD8A-0980-4DF1-91A7-2F691F662683}{17}{B0}" alignment="left" quickStyleIndex="0"><one:T><![CDATA[Left cell]]></one:T></one:OE></one:OEChildren></one:Cell><one:Cell lastModifiedTime="2026-09-05T05:07:00.000Z" objectID="{F0C9FD8A-0980-4DF1-91A7-2F691F662683}{22}{B0}" lastModifiedByInitials="S"><one:OEChildren><one:OE creationTime="2026-09-05T05:07:00.000Z" lastModifiedTime="2026-09-05T05:07:00.000Z" objectID="{F0C9FD8A-0980-4DF1-91A7-2F691F662683}{25}{B0}" alignment="left" quickStyleIndex="0"><one:T><![CDATA[Right cell]]></one:T></one:OE></one:OEChildren></one:Cell></one:Row></one:Table></one:OE></one:OEChildren></one:Outline></one:Page>
corpus/table-growth/cold/read/payloads.json created+10
......@@ -0,0 +1,10 @@
1[
2 {
3 "sha256": "af1af4b8987109ce1e335e538910fb3860b7c57eb3b6110a06aa2114e06d69a7",
4 "name": "fictitious-attachment.txt",
5 "object": "{DE1684DA-6810-4671-8EAF-CD3C3735DDC5}{11}{B0}",
6 "kind": "InsertedFile",
7 "page": "{5759809C-B0D2-45F2-8836-0F65B4195B50}{1}{B0}",
8 "bytes": 43
9 }
10]
\ No newline at end of file
corpus/table-growth/cold/run.json created+18
......@@ -0,0 +1,18 @@
1{
2 "notebook": "/private/tmp/m5/tables/notebook",
3 "expected_pages": 1,
4 "author": null,
5 "author_timeout_seconds": 600,
6 "inspect": false,
7 "collect_notebook": true,
8 "base": {
9 "file": "win7-office-base.qcow2",
10 "format": "qcow2",
11 "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346",
12 "virtual_size": 68719476736
13 },
14 "scripts": {
15 "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331",
16 "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41"
17 }
18}
corpus/table-growth/cold/scripts/cold.ps1 created+27
......@@ -0,0 +1,27 @@
1param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '')
2Set-StrictMode -Version Latest
3$ErrorActionPreference = 'Stop'
4$root = [IO.Path]::GetFullPath($Root).TrimEnd('\')
5if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') {
6 throw 'Choose a run directly below C:\one-tests\runs.'
7}
8if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' }
9$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote'
10if ($CloneHost) {
11 if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) {
12 throw 'The disposable clone hostname does not match this machine.'
13 }
14 New-Item "$key\Options\Paths" -Force | Out-Null
15 New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null
16} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or
17 -not (Test-Path 'C:\one-tests\profile-original-cache')) {
18 throw 'Park the personal OneNote profile before resetting the test cache.'
19}
20$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0'
21$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root))
22if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' }
23New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null
24if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked }
25if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse }
26New-Item "$key\OpenNotebooks" | Out-Null
27New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null
corpus/table-growth/cold/scripts/read.ps1 created+142
......@@ -0,0 +1,142 @@
1param(
2 [Parameter(Mandatory=$true)][string]$Root,
3 [int]$ExpectedPages = -1,
4 [switch]$UseCurrentCache,
5 [switch]$Pdf,
6 [switch]$KeepOpen,
7 [string]$CloneHost = ''
8)
9Set-StrictMode -Version Latest
10$ErrorActionPreference = 'Stop'
11$root = [IO.Path]::GetFullPath($Root).TrimEnd('\')
12if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') {
13 throw 'Choose a run directly below C:\one-tests\runs.'
14}
15$notebook = Join-Path $root 'notebook'
16$output = Join-Path $root 'read'
17if (Test-Path $output) { throw 'Choose a new read destination.' }
18if ($UseCurrentCache) {
19 if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') {
20 throw 'Park the personal OneNote profile before reading test notebooks.'
21 }
22} else {
23 & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost
24}
25New-Item -ItemType Directory -Path $output | Out-Null
26$app = New-Object -ComObject OneNote.Application
27$notebookId = ''
28$failure = $null
29try {
30 $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0)
31 $process = Get-Process ONENOTE
32 @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion;
33 powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } |
34 ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8
35 $sections = @()
36 foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) {
37 $id = ''
38 $app.OpenHierarchy($file.FullName, '', [ref]$id, 0)
39 $sections += $id
40 }
41 $deadline = [DateTime]::UtcNow.AddSeconds(300)
42 $previous = ''
43 $lastChange = ''
44 $stableSince = [DateTime]::UtcNow
45 $settled = $false
46 do {
47 $pages = @{}
48 foreach ($section in $sections) {
49 $hierarchy = ''
50 $app.GetHierarchy($section, 4, [ref]$hierarchy, 1)
51 [xml]$xml = $hierarchy
52 foreach ($node in $xml.SelectNodes('//*[@path]')) {
53 if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) {
54 throw 'OneNote opened a section outside the copied notebook.'
55 }
56 }
57 foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) {
58 $id = $node.GetAttribute('ID')
59 $content = ''
60 $app.GetPageContent($id, [ref]$content, 1, 1)
61 $pages[$id] = $content
62 }
63 }
64 $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] }))
65 if ($signature -ne $previous) {
66 $lastChange = $previous
67 $previous = $signature
68 $stableSince = [DateTime]::UtcNow
69 }
70 if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or
71 ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and
72 ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break }
73 Start-Sleep -Milliseconds 250
74 } while ([DateTime]::UtcNow -lt $deadline)
75 if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) {
76 [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8)
77 $index = 0
78 foreach ($id in @($pages.Keys | Sort-Object)) {
79 [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8)
80 $index++
81 }
82 $hierarchy = ''
83 $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1)
84 [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8)
85 throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled."
86 }
87 $index = 0
88 $payloads = @()
89 foreach ($id in @($pages.Keys | Sort-Object)) {
90 [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8)
91 if ($Pdf) {
92 $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index)
93 $app.NavigateTo($id, '', $false)
94 $app.Publish($id, $pdfPath, 3, '')
95 if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' }
96 }
97 [xml]$page = $pages[$id]
98 foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) {
99 $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache'))
100 $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant()
101 [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes)
102 $payloads += @{ page = $id; object = $file.ParentNode.GetAttribute('objectID');
103 kind = $file.LocalName; name = $file.GetAttribute('preferredName');
104 sha256 = $hash; bytes = $bytes.Length }
105 }
106 $index++
107 }
108 [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8)
109 $all = ''
110 $app.GetHierarchy($notebookId, 4, [ref]$all, 1)
111 [xml]$finalTree = $all
112 $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique)
113 if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) {
114 throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.'
115 }
116 [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8)
117 Write-Output "Read $($sections.Count) sections and $($pages.Count) pages."
118} catch {
119 $failure = $_
120 [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8)
121 throw
122} finally {
123 try {
124 try {
125 if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) }
126 if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) {
127 $app.CloseNotebook($notebookId, $false)
128 }
129 } catch {
130 if ($null -eq $failure) { throw }
131 [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8)
132 }
133 } finally {
134 [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app)
135 }
136 $app = $null
137 [GC]::Collect()
138 [GC]::WaitForPendingFinalizers()
139 if (-not $UseCurrentCache -and -not $CloneHost) {
140 Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10
141 }
142}
corpus/table-growth/cold/source.json created+14
......@@ -0,0 +1,14 @@
1[
2 {
3 "path": "Open Notebook.onetoc2",
4 "bytes": 4816,
5 "sha256": "b2f857961b76258a1582b402c048b340e89962257681426b3bfc58df8e0adbfa",
6 "mtime_ns": 1788585005062916740
7 },
8 {
9 "path": "synthetic.one",
10 "bytes": 25392,
11 "sha256": "9acb9ab3c06daa698c55ca48bc2c100aba4609e9f546839a0aeb99db68296033",
12 "mtime_ns": 1789887306088599720
13 }
14]
corpus/table-growth/cold/teardown.json created+1
......@@ -0,0 +1 @@
1{"absent": true}
crates/notebook/README.md+5-4
......@@ -348,10 +348,11 @@ when the page itself was moved to another section. Other URLs and unknown
348348targets are `None`.
349349
350350With the optional `protected` feature, `Notebook::unlock(path, password)`
351reads the pages of a `Locked` section, and `Notebook::save_unlocked(path,
352password, space, page, author)` saves an edited one under the section's key,
353straight to the file: nothing of a protected section is cached or queued, a
354section changed since the read fails the save, and a wrong password is
351reads a `Locked` section as `Unlocked { pages, .. }`, and
352`Notebook::save_unlocked(path, password, &mut unlocked, space, page, author)`
353saves an edited page under the section's key, straight to the file: nothing of
354a protected section is cached or queued, a section written since `unlocked`
355was read fails the save, and a wrong password is
355356`Error::Protected(PasswordMismatch)`.
356357
357358`Section::import_page(page, author)` copies a page, usually read from another
crates/notebook/examples/cache_probe.rs+3-2
......@@ -110,8 +110,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
110110 !intent.before.objects.iter().any(|o| o.id() == object.id())
111111 })
112112 .collect();
113 let [PageObject::Outline(outline)] = added.as_slice() else {
114 panic!("Expected one added outline")
113 // A save that replaced a pending one carries its outlines too.
114 let Some(PageObject::Outline(outline)) = added.last() else {
115 panic!("Expected an added outline")
115116 };
116117 let text = outline.paragraphs[0].text().unwrap().text.text();
117118 let operation: u64 = text.split_once(':').unwrap().0.parse()?;
crates/notebook/src/assets.rs+3-2
......@@ -79,8 +79,9 @@ pub(crate) fn key(filename: &str) -> Result<String> {
7979}
8080
8181fn referenced(connection: &Connection, key: &str) -> Result<bool> {
82 let (base, working) = crate::images::both(connection)?;
83 for image in [working, base] {
82 // One image at a time: the working image usually answers.
83 for read in [crate::images::working, crate::images::base] {
84 let image = read(connection)?;
8485 let store = Store::parse(&image)?;
8586 let index = RevisionIndex::parse(&store)?;
8687 let document = Document::parse(&index)?;
crates/notebook/src/images.rs+24-4
......@@ -46,10 +46,12 @@ fn restore(mut base: Vec<u8>, patch: &[u8]) -> Result<Vec<u8>> {
4646 Err(damaged().into())
4747 };
4848 };
49 base.resize(
50 usize::try_from(u64::from_le_bytes(*length)).map_err(|_| damaged())?,
51 0,
52 );
49 // Every block past the base is a run, so a whole patch bounds the image.
50 let length = usize::try_from(u64::from_le_bytes(*length))
51 .ok()
52 .filter(|length| *length <= base.len() + patch.len())
53 .ok_or_else(damaged)?;
54 base.resize(length, 0);
5355 while let Some((header, rest)) = runs.split_first_chunk::<16>() {
5456 let offset = usize::try_from(u64::from_le_bytes(header[..8].try_into().unwrap()))
5557 .map_err(|_| damaged())?;
......@@ -83,6 +85,21 @@ pub(crate) fn both(connection: &Connection) -> Result<(Vec<u8>, Vec<u8>)> {
8385 Ok((base, working))
8486}
8587
88/// The working image's length, without restoring it.
89pub(crate) fn working_length(connection: &Connection) -> Result<u64> {
90 Ok(connection.query_row(
91 "SELECT length(base), substr(working, 1, 8) FROM replica WHERE id=1",
92 [],
93 |row| {
94 let header: Vec<u8> = row.get(1)?;
95 Ok(match header.first_chunk::<8>() {
96 Some(length) => u64::from_le_bytes(*length),
97 None => row.get::<_, i64>(0)? as u64,
98 })
99 },
100 )?)
101}
102
86103pub(crate) fn working(connection: &Connection) -> Result<Vec<u8>> {
87104 let (base, patch): (Vec<u8>, Vec<u8>) =
88105 connection.query_row("SELECT base, working FROM replica WHERE id=1", [], |row| {
......@@ -141,6 +158,9 @@ mod tests {
141158 }
142159 assert!(difference(&base, &base).is_empty());
143160 assert!(difference(&base, &edited).len() < 3 * BLOCK);
161 let mut huge = difference(&base, &edited);
162 huge[..8].copy_from_slice(&u64::MAX.to_le_bytes());
163 assert!(restore(base.clone(), &huge).is_err());
144164 let patch = difference(&base, &edited);
145165 for cut in 1..patch.len() {
146166 if let Ok(image) = restore(base.clone(), &patch[..cut]) {
crates/notebook/src/recovery.rs+1-1
......@@ -144,7 +144,7 @@ fn summary(connection: &Connection) -> Result<RecoverySummary> {
144144 [],
145145 |row| Ok((unsigned(row, 0)?, unsigned(row, 1)?)),
146146 )?;
147 let working_bytes = crate::images::working(connection)?.len() as u64;
147 let working_bytes = crate::images::working_length(connection)?;
148148 Ok(connection.query_row(
149149 "SELECT (SELECT count(*) FROM edits), (SELECT count(*) FROM conflicts),
150150 (SELECT count(*) FROM attempt), (SELECT count(*) FROM receipts),
crates/notebook/src/session.rs+43-23
......@@ -91,6 +91,14 @@ pub trait Storage: Send + Sync {
9191 ) -> Result<()>;
9292}
9393
94/// A password-protected section as `Notebook::unlock` read it.
95#[cfg(feature = "protected")]
96pub struct Unlocked {
97 pub pages: Vec<(ExGuid, Page)>,
98 /// The stored section the pages came from, which a save must still find in place.
99 snapshot: Vec<u8>,
100}
101
94102/// A mounted notebook directory.
95103struct Directory(PathBuf);
96104
......@@ -622,43 +630,55 @@ impl Notebook {
622630 Ok(None)
623631 }
624632
625 /// The pages of a password-protected section, for reading: nothing is cached or
626 /// written, and the decoded buffers go when the pages have been built.
633 /// The pages of a password-protected section: nothing is cached, and the decoded
634 /// buffers go when the pages have been built.
627635 #[cfg(feature = "protected")]
628 pub fn unlock(&self, path: &str, password: &str) -> Result<Vec<(ExGuid, Page)>> {
636 pub fn unlock(&self, path: &str, password: &str) -> Result<Unlocked> {
629637 let path = self.section_path(path)?.path.clone();
630 let bytes = self.storage.read(&path)?;
631 let store = Store::parse(&bytes)?;
632 let index = RevisionIndex::parse(&store)?;
633 let unlocked = onestore::protected::UnlockedSection::open(
634 &index,
635 password,
636 onestore::protected::Limits::default(),
637 )?;
638 let document = unlocked.document()?;
639 document
640 .pages()?
641 .into_iter()
642 .map(|(space, _)| Ok((space, Page::from_space(&document, space)?)))
643 .collect()
638 let snapshot = self.storage.read(&path)?;
639 let pages = {
640 let store = Store::parse(&snapshot)?;
641 let index = RevisionIndex::parse(&store)?;
642 let unlocked = onestore::protected::UnlockedSection::open(
643 &index,
644 password,
645 onestore::protected::Limits::default(),
646 )?;
647 let document = unlocked.document()?;
648 document
649 .pages()?
650 .into_iter()
651 .map(|(space, _)| Ok((space, Page::from_space(&document, space)?)))
652 .collect::<Result<_>>()?
653 };
654 Ok(Unlocked { pages, snapshot })
644655 }
645656
646 /// Saves a page read through `unlock`, stored under the section's key. The save goes
647 /// straight to the file and fails if the section changed since `unlock` read it;
648 /// nothing of a protected section is cached or queued.
657 /// Saves an edited page of `unlocked` under the section's key, straight to the file:
658 /// nothing of a protected section is cached or queued. A section written since
659 /// `unlocked` was read fails the save; unlock again for its pages.
649660 #[cfg(feature = "protected")]
650661 pub fn save_unlocked(
651662 &self,
652663 path: &str,
653664 password: &str,
665 unlocked: &mut Unlocked,
654666 space: ExGuid,
655667 page: &Page,
656668 author: &str,
657669 ) -> Result<()> {
658670 let path = self.section_path(path)?.path.clone();
659 let bytes = self.storage.read(&path)?;
660 let edit = onestore::PreparedEdit::page_protected(&bytes, password, space, page, author)?;
661 self.storage.commit(&path, &edit)
671 let edit = onestore::PreparedEdit::page_protected(
672 &unlocked.snapshot,
673 password,
674 space,
675 page,
676 author,
677 )?;
678 self.storage.commit(&path, &edit)?;
679 let written = edit.as_bytes().to_vec();
680 unlocked.snapshot = written;
681 Ok(())
662682 }
663683
664684 /// Opens a section of a mounted notebook by its catalog path.
crates/notebook/src/sync.rs+2
......@@ -225,6 +225,8 @@ impl Replica {
225225 self.acknowledge(intent.id, revision, &snapshot)?;
226226 return Ok(Some((intent.id, EditStatus::Published { revision })));
227227 }
228 // Once acknowledged this image is the base, which polls trust without validating.
229 validate(prepared.as_bytes())?;
228230 let store = Store::parse(prepared.as_bytes())?;
229231 let index = RevisionIndex::parse(&store)?;
230232 let revision = index.active(intent.space)?;
crates/notebook/tests/protected.rs+13-5
......@@ -17,7 +17,7 @@ fn a_locked_section_unlocks_for_reading() {
1717 let notebook = Notebook::open(root.join("notebook"), temporary.path()).unwrap();
1818 let section = &notebook.catalog().sections[0];
1919 assert!(matches!(section.state, SectionState::Locked));
20 let pages = notebook.unlock(&section.path, password).unwrap();
20 let pages = notebook.unlock(&section.path, password).unwrap().pages;
2121 assert_eq!(pages.len(), 11);
2222 assert!(pages.iter().all(|(_, page)| !page.title.is_empty()));
2323 assert!(matches!(
......@@ -57,7 +57,8 @@ fn an_unlocked_page_is_saved_under_the_section_key() {
5757 .unwrap()
5858 .path
5959 .clone();
60 let (space, mut page) = notebook.unlock(&section, password).unwrap().remove(0);
60 let mut unlocked = notebook.unlock(&section, password).unwrap();
61 let (space, mut page) = unlocked.pages[0].clone();
6162 let paragraphs = page
6263 .objects
6364 .iter_mut()
......@@ -95,15 +96,22 @@ fn an_unlocked_page_is_saved_under_the_section_key() {
9596 ))
9697 .unwrap();
9798 assert!(matches!(
98 notebook.save_unlocked(&section, "wrong", space, &page, "Rust"),
99 notebook.save_unlocked(&section, "wrong", &mut unlocked, space, &page, "Rust"),
99100 Err(notebook::Error::Protected(
100101 onestore::protected::Error::PasswordMismatch
101102 ))
102103 ));
104 let mut stale = notebook.unlock(&section, password).unwrap();
103105 notebook
104 .save_unlocked(&section, password, space, &page, "Rust")
106 .save_unlocked(&section, password, &mut unlocked, space, &page, "Rust")
105107 .unwrap();
106 let (_, stored) = notebook.unlock(&section, password).unwrap().remove(0);
108 // A view read before that save no longer matches the file.
109 assert!(
110 notebook
111 .save_unlocked(&section, password, &mut stale, space, &page, "Rust")
112 .is_err()
113 );
114 let (_, stored) = notebook.unlock(&section, password).unwrap().pages.remove(0);
107115 assert!(stored.objects == page.objects);
108116 if let Some(export) = std::env::var_os("ONESTORE_PROTECTED_EXPORT") {
109117 let export = Path::new(&export);
crates/onestore/src/objects.rs+8-7
......@@ -335,7 +335,6 @@ impl<'a> RevisionIndex<'a> {
335335 });
336336 }
337337 table = Arc::new(GlobalIds::new());
338 defining.clear();
339338 defining_table = true;
340339 }
341340 0x24..=0x26 => {
......@@ -382,9 +381,12 @@ impl<'a> RevisionIndex<'a> {
382381 }
383382 _ => unreachable!(),
384383 }
385 if defining[start..]
386 .iter()
387 .any(|(index, guid)| *index >= 0xffffff || *guid == [0; 16])
384 // More entries than indices repeats one; checked here so imports
385 // cannot grow the list without bound.
386 if defining.len() > 0xffffff
387 || defining[start..]
388 .iter()
389 .any(|(index, guid)| *index >= 0xffffff || *guid == [0; 16])
388390 {
389391 return Err(Error {
390392 offset: node.offset,
......@@ -407,9 +409,8 @@ impl<'a> RevisionIndex<'a> {
407409 message: "Invalid or repeated global ID entry",
408410 });
409411 }
410 let mut guids: Vec<_> = defining.iter().map(|(_, guid)| *guid).collect();
411 guids.sort_unstable();
412 if guids.windows(2).any(|pair| pair[0] == pair[1]) {
412 defining.sort_unstable_by_key(|(_, guid)| *guid);
413 if defining.windows(2).any(|pair| pair[0].1 == pair[1].1) {
413414 return Err(Error {
414415 offset: node.offset,
415416 message: "Global ID table repeats a GUID",
crates/onestore/src/page/write.rs+15-6
......@@ -2987,12 +2987,21 @@ pub(crate) fn squash(
29872987 if !live.contains(id) {
29882988 continue;
29892989 }
2990 if before.objects.get(id).is_some_and(|previous| {
2991 previous.jcid == object.jcid
2992 && previous.data == object.data
2993 && previous.global_ids == object.global_ids
2994 }) {
2995 continue;
2990 // Stored tables keep the entries an object names, so those decide.
2991 if let Some(previous) = before.objects.get(id)
2992 && previous.jcid == object.jcid
2993 && previous.data == object.data
2994 {
2995 let entries = match object.data {
2996 ObjectData::Properties(bytes) => crate::write::table_entries(bytes)?,
2997 _ => BTreeSet::new(),
2998 };
2999 if entries
3000 .iter()
3001 .all(|entry| previous.global_ids.get(entry) == object.global_ids.get(entry))
3002 {
3003 continue;
3004 }
29963005 }
29973006 let id = rename.get(id).copied().unwrap_or(*id);
29983007 let mut replacement = match object.data {
crates/onestore/src/protected/crypto.rs+5-2
......@@ -228,7 +228,9 @@ impl Key {
228228 crate::properties::reference_streams(&mut c)?;
229229 let prefix = c.offset;
230230 let padding = (16 - (2 + clear.len() - prefix) % 16) % 16;
231 let mut body = Zeroizing::new((padding as u16).to_le_bytes().to_vec());
231 // Sized once: a buffer abandoned by growth would keep plaintext.
232 let mut body = Zeroizing::new(Vec::with_capacity(2 + clear.len() - prefix + padding));
233 body.extend_from_slice(&(padding as u16).to_le_bytes());
232234 body.extend_from_slice(&clear[prefix..]);
233235 let length = body.len() + padding;
234236 body.resize(length, 0);
......@@ -248,7 +250,8 @@ impl Key {
248250 if clear.is_empty() {
249251 return Vec::new();
250252 }
251 let mut output = (clear.len() as u64).to_le_bytes().to_vec();
253 let mut output = Vec::with_capacity((8 + clear.len()).next_multiple_of(16));
254 output.extend_from_slice(&(clear.len() as u64).to_le_bytes());
252255 output.extend_from_slice(clear);
253256 output.resize(output.len().next_multiple_of(16), 0);
254257 encrypt(&self.value, &self.file_iv, &mut output);
crates/onestore/src/protected/mod.rs+37-14
......@@ -83,7 +83,7 @@ struct Decoded<'a> {
8383/// Owns decoded buffers until dropped; returned views cannot outlive this owner.
8484///
8585/// Passwords are not retained. Dropping this owner clears its derived key and
86/// decoded buffers. Owned strings or exports made from a `Document` are separate
86/// decoded buffers; a protected save works on ordinary buffers, which are not cleared. Owned strings or exports made from a `Document` are separate
8787/// caller-owned copies and must be disposed of by the caller when locking.
8888/// Opening never rewrites the source image or changes its protection state.
8989///
......@@ -302,15 +302,19 @@ impl UnlockedSection<'_> {
302302 extension,
303303 } => {
304304 let text = |bytes: &[u8]| {
305 String::from_utf16_lossy(
305 String::from_utf16(
306306 &bytes
307307 .chunks_exact(2)
308308 .map(|pair| u16::from_le_bytes([pair[0], pair[1]]))
309309 .collect::<Vec<_>>(),
310310 )
311 .map_err(|_| crate::Error {
312 offset: 0,
313 message: "Invalid UTF-16 file-data declaration",
314 })
311315 };
312316 let mut copy =
313 PropertyObject::file(*id, &text(reference), &text(extension))?;
317 PropertyObject::file(*id, &text(reference)?, &text(extension)?)?;
314318 copy.jcid = object.jcid;
315319 copy.global_ids = std::sync::Arc::clone(&object.global_ids);
316320 copy
......@@ -336,7 +340,7 @@ impl UnlockedSection<'_> {
336340 };
337341 let store = crate::Store::parse(&scaffold)?;
338342 let placeholder = identity(RevisionIndex::parse(&store)?.root);
339 for at in 0..scaffold.len() - 20 {
343 for at in 0..=scaffold.len() - 20 {
340344 if scaffold[at..at + 20] == placeholder {
341345 scaffold[at..at + 20].copy_from_slice(&identity(self.index.root));
342346 }
......@@ -398,14 +402,32 @@ pub(crate) fn write_page(
398402 let store = crate::Store::parse(source)?;
399403 let index = RevisionIndex::parse(&store)?;
400404 let unlocked = UnlockedSection::open(&index, password, Limits::default())?;
405 if unlocked.keys.len() != 1 {
406 return Err(Error::Unsupported);
407 }
401408 let twin = unlocked.twin()?;
402409 let applied = Zeroizing::new(crate::page::write::write_page(&twin, space, page, author)?);
410 // Squash skips the spaces the source lacks as the twin's scaffold; the writers added none.
411 let spaces = |image: &[u8]| -> Result<Vec<ExGuid>> {
412 let store = crate::Store::parse(image)?;
413 Ok(RevisionIndex::parse(&store)?.spaces.into_keys().collect())
414 };
415 if spaces(&twin)? != spaces(&applied)? {
416 return Err(invalid("Page edits cannot create object spaces"));
417 }
403418 let written = crate::page::write::squash(source, &applied, &BTreeMap::new(), Some(&unlocked))?;
404419 let store = crate::Store::parse(&written)?;
405420 UnlockedSection::open(&RevisionIndex::parse(&store)?, password, Limits::default())?;
406421 Ok(written)
407422}
408423
424impl UnlockedSection<'_> {
425 /// The section key; `write_page` admits sections with exactly one.
426 fn key(&self) -> &crypto::Key {
427 self.keys.values().next().expect("one section key")
428 }
429}
430
409431impl crate::write::Protection for UnlockedSection<'_> {
410432 fn resolve(
411433 &self,
......@@ -423,20 +445,21 @@ impl crate::write::Protection for UnlockedSection<'_> {
423445 }
424446
425447 fn seal_property(&self, clear: &[u8]) -> std::result::Result<Vec<u8>, crate::Error> {
426 let [key] = self.keys.values().collect::<Vec<_>>()[..] else {
427 return Err(crate::Error {
428 offset: 0,
429 message: "Protected writing needs one section key",
430 });
448 let random = crate::Error {
449 offset: 0,
450 message: "System random source failed",
431451 };
432 let failed = |message| crate::Error { offset: 0, message };
433452 let mut iv = [0; 16];
434 getrandom::fill(&mut iv).map_err(|_| failed("System random source failed"))?;
435 key.seal_property(clear, iv)
436 .map_err(|_| failed("Malformed property object"))
453 getrandom::fill(&mut iv).map_err(|_| random)?;
454 self.key()
455 .seal_property(clear, iv)
456 .map_err(|error| match error {
457 Error::Invalid(error) => error,
458 _ => random,
459 })
437460 }
438461
439462 fn seal_file(&self, clear: &[u8]) -> Vec<u8> {
440 self.keys.values().next().unwrap().seal_file(clear)
463 self.key().seal_file(clear)
441464 }
442465}
crates/onestore/src/write.rs+57-19
......@@ -104,6 +104,21 @@ fn compact(id: ExGuid, table: &BTreeMap<u32, [u8; 16]>) -> Result<[u8; 4]> {
104104 Ok(((index << 8) | id.n).to_le_bytes())
105105}
106106
107/// The global id table entries a property object's references name.
108pub(crate) fn table_entries(bytes: &[u8]) -> Result<BTreeSet<u32>> {
109 let mut entries = BTreeSet::new();
110 for property in PropertySets::parse(bytes)?.sets.iter().flatten() {
111 if let Value::References { compact_ids, .. } = property.value {
112 entries.extend(
113 compact_ids
114 .chunks_exact(4)
115 .map(|id| u32::from_le_bytes(id.try_into().unwrap()) >> 8),
116 );
117 }
118 }
119 Ok(entries)
120}
121
107122fn field_length(property: &crate::Property<'_>, set_lengths: &[usize]) -> usize {
108123 match &property.value {
109124 Value::NoData => 0,
......@@ -734,14 +749,21 @@ pub(crate) fn write_revision_with_payloads(
734749 payloads: &[([u8; 16], &[u8])],
735750 edit: impl FnOnce(&crate::ResolvedRevision<'_>) -> Result<BTreeMap<ExGuid, PropertyObject>>,
736751) -> Result<Vec<u8>> {
737 write_revisions_with_payloads(source, payloads, |index| {
738 let rid = index.active(space)?;
739 let revision = index.resolve(space, rid)?;
752 write_revisions_with_payloads(source, payloads, update(space, edit))
753}
754
755/// One space's active revision edited into its update.
756fn update(
757 space: ExGuid,
758 edit: impl FnOnce(&crate::ResolvedRevision<'_>) -> Result<BTreeMap<ExGuid, PropertyObject>>,
759) -> impl FnOnce(&RevisionIndex<'_>) -> Result<BTreeMap<ExGuid, RevisionEdit>> {
760 move |index| {
761 let revision = index.resolve(space, index.active(space)?)?;
740762 Ok(BTreeMap::from([(
741763 space,
742764 RevisionEdit::Update(edit(&revision)?),
743765 )]))
744 })
766 }
745767}
746768
747769fn append_fragment(
......@@ -853,13 +875,7 @@ pub(crate) fn write_revision_on(
853875 space: ExGuid,
854876 edit: impl FnOnce(&crate::ResolvedRevision<'_>) -> Result<BTreeMap<ExGuid, PropertyObject>>,
855877) -> Result<Vec<u8>> {
856 let output = build_on(index, &[], None, |index| {
857 let revision = index.resolve(space, index.active(space)?)?;
858 Ok(BTreeMap::from([(
859 space,
860 RevisionEdit::Update(edit(&revision)?),
861 )]))
862 })?;
878 let output = build_on(index, &[], None, update(space, edit))?;
863879 check(&output, true)?;
864880 Ok(output)
865881}
......@@ -1079,14 +1095,24 @@ fn build_on(
10791095 object.bytes[offset..offset + 4].copy_from_slice(&reference);
10801096 }
10811097 }
1082 replacements.retain(|id, replacement| {
1083 // Detached objects must pass the final reachability check even when unchanged.
1084 !reachable.contains(id)
1085 || !revision.objects.get(id).is_some_and(|object| {
1086 object.data == ObjectData::Properties(&replacement.bytes)
1087 && object.global_ids == replacement.global_ids
1088 })
1089 });
1098 // Detached objects must pass the final reachability check even when unchanged. Equal
1099 // bytes are the same object when the table entries they name agree: stored tables
1100 // keep only those.
1101 let mut unchanged = Vec::new();
1102 for (id, replacement) in &replacements {
1103 if let Some(object) = revision.objects.get(id)
1104 && reachable.contains(id)
1105 && object.data == ObjectData::Properties(&replacement.bytes)
1106 && table_entries(&replacement.bytes)?
1107 .iter()
1108 .all(|entry| object.global_ids.get(entry) == replacement.global_ids.get(entry))
1109 {
1110 unchanged.push(*id);
1111 }
1112 }
1113 for id in unchanged {
1114 replacements.remove(&id);
1115 }
10901116 if replacements.is_empty() {
10911117 continue;
10921118 }
......@@ -1244,7 +1270,19 @@ fn build_on(
12441270 } else {
12451271 vec![node(0x21, None, &[0])?]
12461272 };
1273 // A table read from a long-lived page names every session that edited it; the
1274 // group stores the entries its objects use.
1275 let mut used = BTreeSet::new();
1276 for (id, object) in &objects {
1277 used.insert(u32::from_le_bytes(compact(*id, table)?) >> 8);
1278 if let ObjectData::Properties(bytes) = object.data {
1279 used.extend(table_entries(bytes)?);
1280 }
1281 }
12471282 for (id, guid) in table {
1283 if is_section && !used.contains(id) {
1284 continue;
1285 }
12481286 let mut entry = id.to_le_bytes().to_vec();
12491287 entry.extend_from_slice(guid);
12501288 group.push(node(0x24, None, &entry)?);
crates/onestore/tests/edit.rs+7-1
......@@ -264,7 +264,13 @@ fn assert_other_objects_preserved(
264264 }
265265 let current = &after.objects[id];
266266 assert_eq!(object.jcid, current.jcid);
267 assert_eq!(object.global_ids, current.global_ids);
267 // A rewritten object keeps the table entries it names.
268 assert!(
269 current
270 .global_ids
271 .iter()
272 .all(|(entry, guid)| object.global_ids.get(entry) == Some(guid))
273 );
268274 let mut pending = vec![*id];
269275 let mut visited = std::collections::BTreeSet::new();
270276 while let Some(next) = pending.pop() {
crates/onestore/tests/table_growth.rs created+101
......@@ -0,0 +1,101 @@
1//! A rewritten object stores the global id table entries it names, not the table of the
2//! revision it was read from.
3
4use onestore::{
5 PreparedEdit, RevisionIndex, Store,
6 document::Document,
7 page::{Page, PageObject, Paragraph},
8};
9
10const SOURCE: &[u8] = include_bytes!("../../../corpus/native/20260905-05/notebook/synthetic.one");
11
12fn pages(bytes: &[u8]) -> Vec<(onestore::ExGuid, Page)> {
13 let store = Store::parse(bytes).unwrap();
14 let index = RevisionIndex::parse(&store).unwrap();
15 let document = Document::parse(&index).unwrap();
16 document
17 .pages()
18 .unwrap()
19 .into_iter()
20 .map(|(space, _)| (space, Page::from_space(&document, space).unwrap()))
21 .collect()
22}
23
24/// Table entries per object group written past `from`, as `(entries, highest index + 1)`.
25fn tables(bytes: &[u8], from: usize) -> Vec<(usize, u32)> {
26 let store = Store::parse(bytes).unwrap();
27 store
28 .lists
29 .values()
30 .filter(|list| list.nodes.first().is_some_and(|node| node.offset >= from))
31 .map(|list| {
32 let entries: Vec<u32> = list
33 .nodes
34 .iter()
35 .filter(|node| node.id == 0x24)
36 .map(|node| u32::from_le_bytes(node.payload[..4].try_into().unwrap()))
37 .collect();
38 (entries.len(), entries.iter().max().map_or(0, |max| max + 1))
39 })
40 .filter(|(entries, _)| *entries > 0)
41 .collect()
42}
43
44/// `ONESTORE_TABLE_EXPORT` names a directory receiving the edited notebook for a cold reopen.
45#[test]
46fn a_text_edit_stores_the_table_entries_it_names() {
47 let mut bytes = SOURCE.to_vec();
48 let mut sparse = false;
49 let mut edited = Vec::new();
50 for (space, mut page) in pages(SOURCE) {
51 let Some(text) = page
52 .objects
53 .iter_mut()
54 .filter_map(|object| match object {
55 PageObject::Outline(outline) if !outline.title => Some(&mut outline.paragraphs),
56 _ => None,
57 })
58 .flatten()
59 .find_map(|paragraph| paragraph.text_mut())
60 else {
61 continue;
62 };
63 let format = text.text.format_at(0).unwrap().clone();
64 text.text
65 .append(Paragraph::new(" and a few more words".to_owned(), format))
66 .unwrap();
67 let written = PreparedEdit::page(&bytes, space, &page, "Rust")
68 .unwrap()
69 .as_bytes()
70 .to_vec();
71 for (entries, span) in tables(&written, bytes.len()) {
72 sparse |= (entries as u32) < span;
73 }
74 assert_eq!(
75 PreparedEdit::page(&written, space, &page, "Rust")
76 .unwrap()
77 .as_bytes(),
78 written
79 );
80 bytes = written;
81 edited.push((space, page));
82 }
83 // The fixture's pages were written over several sessions, so a text edit names a
84 // subset of its revision's table and the stored indices have gaps.
85 assert!(sparse);
86 let stored = pages(&bytes);
87 for (space, page) in &edited {
88 let (_, stored) = stored.iter().find(|(id, _)| id == space).unwrap();
89 assert!(stored.objects == page.objects);
90 }
91 if let Some(export) = std::env::var_os("ONESTORE_TABLE_EXPORT") {
92 let export = std::path::Path::new(&export);
93 std::fs::create_dir_all(export).unwrap();
94 std::fs::write(export.join("synthetic.one"), &bytes).unwrap();
95 std::fs::copy(
96 "../../corpus/native/20260905-05/notebook/Open Notebook.onetoc2",
97 export.join("Open Notebook.onetoc2"),
98 )
99 .unwrap();
100 }
101}
crates/onestore/tests/writer.rs+15-3
......@@ -55,7 +55,11 @@ fn scalar_edit_appends_a_revision_and_preserves_every_prior_object() {
5555 let same = &unchanged.objects[&id];
5656 assert_eq!(object.jcid, same.jcid);
5757 assert_eq!(object.reference_count, same.reference_count);
58 assert_eq!(object.global_ids, same.global_ids);
58 assert!(
59 same.global_ids
60 .iter()
61 .all(|(entry, guid)| object.global_ids.get(entry) == Some(guid))
62 );
5963 assert_eq!(object.data, same.data);
6064 }
6165 }
......@@ -403,7 +407,11 @@ fn checkpoints_bound_dependencies_and_preserve_native_objects_and_history() {
403407 assert_eq!(a.object_spaces, b.object_spaces);
404408 assert_eq!(a.contexts, b.contexts);
405409 if section {
406 assert_eq!(object.global_ids, same.global_ids);
410 assert!(
411 same.global_ids
412 .iter()
413 .all(|(entry, guid)| object.global_ids.get(entry) == Some(guid))
414 );
407415 if *id != oid {
408416 assert_eq!(object.data, same.data);
409417 }
......@@ -437,7 +445,11 @@ fn checkpoints_bound_dependencies_and_preserve_native_objects_and_history() {
437445 let same = &preserved.objects[&id];
438446 assert_eq!(object.jcid, same.jcid);
439447 assert_eq!(object.reference_count, same.reference_count);
440 assert_eq!(object.global_ids, same.global_ids);
448 assert!(
449 same.global_ids
450 .iter()
451 .all(|(entry, guid)| object.global_ids.get(entry) == Some(guid))
452 );
441453 assert_eq!(object.data, same.data);
442454 }
443455 }
tools/test_table_growth.py created+24
......@@ -0,0 +1,24 @@
1from pathlib import Path
2import runpy
3import shutil
4from tempfile import TemporaryDirectory
5import unittest
6
7ROOT = Path(__file__).resolve().parent.parent
8FIXTURE = ROOT / 'corpus/table-growth'
9compare = runpy.run_path(str(ROOT / 'tools/verify-document.py'))['compare']
10
11
12class TableGrowthTest(unittest.TestCase):
13 def test_onenote_reads_object_groups_whose_id_tables_have_gaps(self):
14 with TemporaryDirectory() as temporary:
15 native = Path(temporary) / 'read'
16 shutil.copytree(FIXTURE / 'cold/read', native)
17 compare(FIXTURE / 'candidate/notebook', native)
18 self.assertIn('and a few more words', (native / 'page-000.xml').read_text(encoding='utf-8-sig'))
19 self.assertEqual((FIXTURE / 'candidate/notebook/synthetic.one').read_bytes(),
20 (FIXTURE / 'cold/notebook/synthetic.one').read_bytes())
21
22
23if __name__ == '__main__':
24 unittest.main()