diff --git a/Cargo.lock b/Cargo.lock index 1c8b8268362b464f9eaa30a2ca0c8c66c5c8f1c4..6e5c6d204ad123c550aef66d2de29e52bfbdb520 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1560,6 +1560,7 @@ dependencies = [ "onestore", "parley", "pollster", + "serde_json", "wgpu", ] @@ -1657,6 +1658,7 @@ dependencies = [ name = "notebook" version = "0.1.0" dependencies = [ + "libc", "onestore", "rusqlite", "serde", diff --git a/apps/ios/Snowbound/CanvasView.swift b/apps/ios/Snowbound/CanvasView.swift index ef59eaa3fedca7b425b9d89f0b582df180894a77..75eb32851ea184dd48d19775463e56ca2c51d890 100644 --- a/apps/ios/Snowbound/CanvasView.swift +++ b/apps/ios/Snowbound/CanvasView.swift @@ -241,6 +241,7 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI inputDelegate?.textWillChange(self) } let changed = change() + dropEdits() if external { inputDelegate?.textDidChange(self) inputDelegate?.selectionDidChange(self) @@ -251,6 +252,13 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI } } + /// Takes the edit the page recorded, so its ops do not pile up; the host drops it + /// until it holds a notebook session that stores edits. + private func dropEdits() { + guard let handle, let edit = sb_view_edit(handle) else { return } + sb_string_free(edit) + } + private func revealCaret() { guard let handle, isFirstResponder else { return } var range: [UInt32] = [0, 0] @@ -354,6 +362,7 @@ final class CanvasView: UIScrollView, UIScrollViewDelegate, UITextInput, UITextI func unmarkText() { guard let handle else { return } sb_unmark(handle) + dropEdits() dirty = true } diff --git a/corpus/page-date/README.md b/corpus/page-date/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f571772840d7d92fbbd4f562ef4b17301f37ccb1 --- /dev/null +++ b/corpus/page-date/README.md @@ -0,0 +1,19 @@ +# Page date + +`native` is OneNote 2010 changing a page's date (`tools/native/page-date.ps1` on +`picture-edit/native-page-level/notebook`): it creates two pages, "Kept date" and +"Changed date", each with OneNote's own title, date and time, then gives "Changed date" +the `dateTime` 2024-03-05T14:30:00Z through the COM API (`before.xml`, `update.xml`, +`after.xml`; `kept.xml` is the untouched page). In `notebook/`, the revision that change +stored sets the page metadata's `TopologyCreationTimeStamp` (`0x18001c65`), rewrites the +date and time fields' text (`TextExtendedAscii`, `0x1c003498`, in the clone's Pacific +time: "Tuesday, March 05, 2024", "7:30 AM"), and gives the two fields' elements the +change time as `CreationTimeStamp` (`0x14001d09`) and `LastModifiedTime`, with the +date outline's `LastModifiedTime`. The title, the page and the section's page metadata +keep theirs. + +`candidate` is `a_page_date_stores_what_onenote_stores` in `crates/onestore/src/op/tests.rs` +(`ONESTORE_PAGE_DATE_EXPORT`): the `Date` op dates "Kept date" 2025-07-04T16:45:00Z, +showing "Friday, July 04, 2025" and "9:45 AM", and changes the same objects OneNote did. +`cold` is its fresh OneNote 2010 read: the page reads back with that `dateTime`, and +`read/page-001.png` shows the new date and time under the title. diff --git a/corpus/page-date/candidate/Open Notebook.onetoc2 b/corpus/page-date/candidate/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..a7bcb6b4cf63c11f7984c722d32485a2ba522578 Binary files /dev/null and b/corpus/page-date/candidate/Open Notebook.onetoc2 differ diff --git a/corpus/page-date/candidate/pictures.one b/corpus/page-date/candidate/pictures.one new file mode 100644 index 0000000000000000000000000000000000000000..ffd10d36c6059003feeb168663d492f3cf08bcf4 Binary files /dev/null and b/corpus/page-date/candidate/pictures.one differ diff --git a/corpus/page-date/cold/commands.jsonl b/corpus/page-date/cold/commands.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8edc2a9242976b1dc7c1f46f6d488d2f820133d5 --- /dev/null +++ b/corpus/page-date/cold/commands.jsonl @@ -0,0 +1,3 @@ +{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-814B0AC0 -ExpectedPages 3 -KeepOpen", "exit": 0, "stdout": "Read 1 sections and 3 pages.\r\n", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\read -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null} diff --git a/corpus/page-date/cold/machine.json b/corpus/page-date/cold/machine.json new file mode 100644 index 0000000000000000000000000000000000000000..b31dd420258d510a0f12aedd1ddfa0236e4140b8 --- /dev/null +++ b/corpus/page-date/cold/machine.json @@ -0,0 +1 @@ +{"name": "m6-814b0ac0", "hostname": "ONE-M6-814B0AC0"} diff --git a/corpus/page-date/cold/notebook/Open Notebook.onetoc2 b/corpus/page-date/cold/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..a7bcb6b4cf63c11f7984c722d32485a2ba522578 Binary files /dev/null and b/corpus/page-date/cold/notebook/Open Notebook.onetoc2 differ diff --git a/corpus/page-date/cold/notebook/pictures.one b/corpus/page-date/cold/notebook/pictures.one new file mode 100644 index 0000000000000000000000000000000000000000..ffd10d36c6059003feeb168663d492f3cf08bcf4 Binary files /dev/null and b/corpus/page-date/cold/notebook/pictures.one differ diff --git a/corpus/page-date/cold/read/environment.json b/corpus/page-date/cold/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..443e6dd40b2bd8c1bee5891e6a51ca8e38570fe0 --- /dev/null +++ b/corpus/page-date/cold/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-814B0AC0", + "cold": true, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/page-date/cold/read/hierarchy.xml b/corpus/page-date/cold/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..2e3a8cfaf29dce9fcb86fef34985dfc88b8724de --- /dev/null +++ b/corpus/page-date/cold/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/page-date/cold/read/page-000.navigation.json b/corpus/page-date/cold/read/page-000.navigation.json new file mode 100644 index 0000000000000000000000000000000000000000..b9d4d9124849ecffacac7e0b640250c408a9afcd --- /dev/null +++ b/corpus/page-date/cold/read/page-000.navigation.json @@ -0,0 +1,13 @@ +{ + "exit": 0, + "stdout": "", + "stderr": "", + "w": 800, + "h": 600, + "error": null, + "win": { + "title": "Changed date - Microsoft OneNote", + "class": "Framework::CFrame", + "dialog": false + } +} \ No newline at end of file diff --git a/corpus/page-date/cold/read/page-000.png b/corpus/page-date/cold/read/page-000.png new file mode 100644 index 0000000000000000000000000000000000000000..3ebdf650758fc364e2b5a5f916019149b77afec1 Binary files /dev/null and b/corpus/page-date/cold/read/page-000.png differ diff --git a/corpus/page-date/cold/read/page-000.xml b/corpus/page-date/cold/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..79bff96685cbc3d2a0e7e9904bd6124fc106fae4 --- /dev/null +++ b/corpus/page-date/cold/read/page-000.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/page-date/cold/read/page-001.navigation.json b/corpus/page-date/cold/read/page-001.navigation.json new file mode 100644 index 0000000000000000000000000000000000000000..7337ad212809f6ef8ffa0a1c57c120a2574507da --- /dev/null +++ b/corpus/page-date/cold/read/page-001.navigation.json @@ -0,0 +1,13 @@ +{ + "exit": 0, + "stdout": "", + "stderr": "", + "w": 800, + "h": 600, + "error": null, + "win": { + "title": "Kept date - Microsoft OneNote", + "class": "Framework::CFrame", + "dialog": false + } +} \ No newline at end of file diff --git a/corpus/page-date/cold/read/page-001.png b/corpus/page-date/cold/read/page-001.png new file mode 100644 index 0000000000000000000000000000000000000000..8f1a1560e286fc7970be279a4d06c4f7a7ebe267 Binary files /dev/null and b/corpus/page-date/cold/read/page-001.png differ diff --git a/corpus/page-date/cold/read/page-001.xml b/corpus/page-date/cold/read/page-001.xml new file mode 100644 index 0000000000000000000000000000000000000000..ee7e861468376d19b58f7c4aea5909db67c50ca9 --- /dev/null +++ b/corpus/page-date/cold/read/page-001.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/page-date/cold/read/page-002.navigation.json b/corpus/page-date/cold/read/page-002.navigation.json new file mode 100644 index 0000000000000000000000000000000000000000..b65a0c422e83efcb89dd26be239dd1906202f810 --- /dev/null +++ b/corpus/page-date/cold/read/page-002.navigation.json @@ -0,0 +1,13 @@ +{ + "exit": 0, + "stdout": "", + "stderr": "", + "w": 800, + "h": 600, + "error": null, + "win": { + "title": "Before the picture - Microsoft OneNote", + "class": "Framework::CFrame", + "dialog": false + } +} \ No newline at end of file diff --git a/corpus/page-date/cold/read/page-002.png b/corpus/page-date/cold/read/page-002.png new file mode 100644 index 0000000000000000000000000000000000000000..148935b7ed8d1c1ee988154c7dfb5f3c730f7d6c Binary files /dev/null and b/corpus/page-date/cold/read/page-002.png differ diff --git a/corpus/page-date/cold/read/page-002.xml b/corpus/page-date/cold/read/page-002.xml new file mode 100644 index 0000000000000000000000000000000000000000..23d00b0f6993e7bdb6f1de14576f8b9e7cb9f134 --- /dev/null +++ b/corpus/page-date/cold/read/page-002.xml @@ -0,0 +1,6 @@ + +iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4z8AAAAMBAQAY3Y2w +AAAAAElFTkSuQmCC +iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4z8AAAAMBAQAY3Y2w +AAAAAElFTkSuQmCC + diff --git a/corpus/page-date/cold/read/payloads.json b/corpus/page-date/cold/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..1ce91a413d6fff4d58765187b76756f0e97ae843 --- /dev/null +++ b/corpus/page-date/cold/read/payloads.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/corpus/page-date/cold/run.json b/corpus/page-date/cold/run.json new file mode 100644 index 0000000000000000000000000000000000000000..9e148fa431dd55ae9c8edd5133b0e12554ec74a1 --- /dev/null +++ b/corpus/page-date/cold/run.json @@ -0,0 +1,18 @@ +{ + "notebook": "/private/tmp/pagedate/candidate/notebook", + "expected_pages": 3, + "author": null, + "author_timeout_seconds": 600, + "inspect": false, + "collect_notebook": false, + "base": { + "file": "win7-office-base.qcow2", + "format": "qcow2", + "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346", + "virtual_size": 68719476736 + }, + "scripts": { + "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331", + "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41" + } +} diff --git a/corpus/page-date/cold/scripts/cold.ps1 b/corpus/page-date/cold/scripts/cold.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..a0b99e7aeb86f4dc10032537800501a52ad65b4f --- /dev/null +++ b/corpus/page-date/cold/scripts/cold.ps1 @@ -0,0 +1,27 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '') +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' } +$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote' +if ($CloneHost) { + if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) { + throw 'The disposable clone hostname does not match this machine.' + } + New-Item "$key\Options\Paths" -Force | Out-Null + New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null +} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or + -not (Test-Path 'C:\one-tests\profile-original-cache')) { + throw 'Park the personal OneNote profile before resetting the test cache.' +} +$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0' +$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root)) +if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' } +New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null +if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked } +if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse } +New-Item "$key\OpenNotebooks" | Out-Null +New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null diff --git a/corpus/page-date/cold/scripts/read.ps1 b/corpus/page-date/cold/scripts/read.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..476094ab1e0c47077988a6461db2080cf208642b --- /dev/null +++ b/corpus/page-date/cold/scripts/read.ps1 @@ -0,0 +1,142 @@ +param( + [Parameter(Mandatory=$true)][string]$Root, + [int]$ExpectedPages = -1, + [switch]$UseCurrentCache, + [switch]$Pdf, + [switch]$KeepOpen, + [string]$CloneHost = '' +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +$notebook = Join-Path $root 'notebook' +$output = Join-Path $root 'read' +if (Test-Path $output) { throw 'Choose a new read destination.' } +if ($UseCurrentCache) { + if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') { + throw 'Park the personal OneNote profile before reading test notebooks.' + } +} else { + & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost +} +New-Item -ItemType Directory -Path $output | Out-Null +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$failure = $null +try { + $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0) + $process = Get-Process ONENOTE + @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion; + powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } | + ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8 + $sections = @() + foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) { + $id = '' + $app.OpenHierarchy($file.FullName, '', [ref]$id, 0) + $sections += $id + } + $deadline = [DateTime]::UtcNow.AddSeconds(300) + $previous = '' + $lastChange = '' + $stableSince = [DateTime]::UtcNow + $settled = $false + do { + $pages = @{} + foreach ($section in $sections) { + $hierarchy = '' + $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) + [xml]$xml = $hierarchy + foreach ($node in $xml.SelectNodes('//*[@path]')) { + if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) { + throw 'OneNote opened a section outside the copied notebook.' + } + } + foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) { + $id = $node.GetAttribute('ID') + $content = '' + $app.GetPageContent($id, [ref]$content, 1, 1) + $pages[$id] = $content + } + } + $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] })) + if ($signature -ne $previous) { + $lastChange = $previous + $previous = $signature + $stableSince = [DateTime]::UtcNow + } + if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or + ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and + ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break } + Start-Sleep -Milliseconds 250 + } while ([DateTime]::UtcNow -lt $deadline) + if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) { + [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8) + $index = 0 + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + $index++ + } + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8) + throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled." + } + $index = 0 + $payloads = @() + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + if ($Pdf) { + $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index) + $app.NavigateTo($id, '', $false) + $app.Publish($id, $pdfPath, 3, '') + if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' } + } + [xml]$page = $pages[$id] + foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) { + $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache')) + $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant() + [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes) + $payloads += @{ page = $id; object = $file.ParentNode.GetAttribute('objectID'); + kind = $file.LocalName; name = $file.GetAttribute('preferredName'); + sha256 = $hash; bytes = $bytes.Length } + } + $index++ + } + [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8) + $all = '' + $app.GetHierarchy($notebookId, 4, [ref]$all, 1) + [xml]$finalTree = $all + $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique) + if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) { + throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.' + } + [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8) + Write-Output "Read $($sections.Count) sections and $($pages.Count) pages." +} catch { + $failure = $_ + [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + throw +} finally { + try { + try { + if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) } + if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) { + $app.CloseNotebook($notebookId, $false) + } + } catch { + if ($null -eq $failure) { throw } + [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + } + } finally { + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + } + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + if (-not $UseCurrentCache -and -not $CloneHost) { + Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10 + } +} diff --git a/corpus/page-date/cold/source.json b/corpus/page-date/cold/source.json new file mode 100644 index 0000000000000000000000000000000000000000..362e409c5d0b3ad88042c418479bd136cfefaec6 --- /dev/null +++ b/corpus/page-date/cold/source.json @@ -0,0 +1,14 @@ +[ + { + "path": "Open Notebook.onetoc2", + "bytes": 3764, + "sha256": "8bf472d0a9c5829b049c69b817f40b766025431a61c71689bca583a60d28365d", + "mtime_ns": 1790452811018151759 + }, + { + "path": "pictures.one", + "bytes": 24184, + "sha256": "0fea376be55a62c76dd61e311513b41a4212d671a9316eb8d18897a5baa5a125", + "mtime_ns": 1790453075875161561 + } +] diff --git a/corpus/page-date/cold/teardown.json b/corpus/page-date/cold/teardown.json new file mode 100644 index 0000000000000000000000000000000000000000..05a47793de40f322e745c4c4183727e3590ecf70 --- /dev/null +++ b/corpus/page-date/cold/teardown.json @@ -0,0 +1 @@ +{"absent": true} diff --git a/corpus/page-date/native/after.xml b/corpus/page-date/native/after.xml new file mode 100644 index 0000000000000000000000000000000000000000..8268312536edeccf9d167357fb577761f5be4bd0 --- /dev/null +++ b/corpus/page-date/native/after.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/page-date/native/before.xml b/corpus/page-date/native/before.xml new file mode 100644 index 0000000000000000000000000000000000000000..3bd9a83fe09ff593da13c5f2b154817263b5e593 --- /dev/null +++ b/corpus/page-date/native/before.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/page-date/native/commands.jsonl b/corpus/page-date/native/commands.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..9be10279c7fff972b889d4664b0e049a1bed9863 --- /dev/null +++ b/corpus/page-date/native/commands.jsonl @@ -0,0 +1,4 @@ +{"command": "powershell -NoProfile -Command \"Expand-Archive -LiteralPath C:\\one-tests\\transfer.zip -DestinationPath C:\\one-tests\\runs\\capture\\notebook\"", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\author.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-CE0C154F", "exit": 0, "stdout": "", "stderr": "", "error": null} +{"command": "powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-M6-CE0C154F -ExpectedPages 3 -UseCurrentCache", "exit": 0, "stdout": "Read 1 sections and 3 pages.\r\n", "stderr": "", "error": null} +{"command": "powershell -NoProfile -Command \"Compress-Archive -Force -Path C:\\one-tests\\runs\\capture\\* -DestinationPath C:\\one-tests\\captured.zip\"", "exit": 0, "stdout": "", "stderr": "", "error": null} diff --git a/corpus/page-date/native/kept.xml b/corpus/page-date/native/kept.xml new file mode 100644 index 0000000000000000000000000000000000000000..d8f81f6e11887924b4a0506d51269c4ff8539a0e --- /dev/null +++ b/corpus/page-date/native/kept.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/page-date/native/machine.json b/corpus/page-date/native/machine.json new file mode 100644 index 0000000000000000000000000000000000000000..9a4e440a3afaae97d40590a5d8b7e6690d5e8953 --- /dev/null +++ b/corpus/page-date/native/machine.json @@ -0,0 +1 @@ +{"name": "m6-ce0c154f", "hostname": "ONE-M6-CE0C154F"} diff --git a/corpus/page-date/native/notebook/Open Notebook.onetoc2 b/corpus/page-date/native/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..a7bcb6b4cf63c11f7984c722d32485a2ba522578 Binary files /dev/null and b/corpus/page-date/native/notebook/Open Notebook.onetoc2 differ diff --git a/corpus/page-date/native/notebook/pictures.one b/corpus/page-date/native/notebook/pictures.one new file mode 100644 index 0000000000000000000000000000000000000000..afa9af53962f787548e1b29e3e3f12075df9f5c2 Binary files /dev/null and b/corpus/page-date/native/notebook/pictures.one differ diff --git a/corpus/page-date/native/read/environment.json b/corpus/page-date/native/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..3decb628b4205e11247b739b79595a80660a1980 --- /dev/null +++ b/corpus/page-date/native/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-CE0C154F", + "cold": false, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/page-date/native/read/hierarchy.xml b/corpus/page-date/native/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..f5aba1f698660d2972e1b09d90c385483ce7c475 --- /dev/null +++ b/corpus/page-date/native/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/page-date/native/read/page-000.xml b/corpus/page-date/native/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..5556b84c2dfb85dae83861ce2d88dc220a38401c --- /dev/null +++ b/corpus/page-date/native/read/page-000.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/page-date/native/read/page-001.xml b/corpus/page-date/native/read/page-001.xml new file mode 100644 index 0000000000000000000000000000000000000000..f58c14ea32d3a52a94abdea710da3495d416c3bf --- /dev/null +++ b/corpus/page-date/native/read/page-001.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/page-date/native/read/page-002.xml b/corpus/page-date/native/read/page-002.xml new file mode 100644 index 0000000000000000000000000000000000000000..22b9a3272304419443e0ab40794ce4a70f74c776 --- /dev/null +++ b/corpus/page-date/native/read/page-002.xml @@ -0,0 +1,6 @@ + +iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4z8AAAAMBAQAY3Y2w +AAAAAElFTkSuQmCC +iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4z8AAAAMBAQAY3Y2w +AAAAAElFTkSuQmCC + diff --git a/corpus/page-date/native/read/payloads.json b/corpus/page-date/native/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..1ce91a413d6fff4d58765187b76756f0e97ae843 --- /dev/null +++ b/corpus/page-date/native/read/payloads.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/corpus/page-date/native/run.json b/corpus/page-date/native/run.json new file mode 100644 index 0000000000000000000000000000000000000000..9d8d8c78273d3bff01aabb508a50b161f1f1587b --- /dev/null +++ b/corpus/page-date/native/run.json @@ -0,0 +1,19 @@ +{ + "notebook": "/Users/clo/dev/one/corpus/picture-edit/native-page-level/notebook", + "expected_pages": 3, + "author": "tools/native/page-date.ps1", + "author_timeout_seconds": 600, + "inspect": false, + "collect_notebook": true, + "base": { + "file": "win7-office-base.qcow2", + "format": "qcow2", + "sha256": "a1a4f8fab782ee14885ff801ca2f6347c208fdcfc3513637096f314315c89346", + "virtual_size": 68719476736 + }, + "scripts": { + "author.ps1": "f7a55c35305dc63b41efdff5bba9a5c0eef621948727f7630c2ebda3248c3490", + "cold.ps1": "c177fc72ae6c2634d186f5671a880de20b09f9716532c37c69cb13aa15c7e331", + "read.ps1": "04013bfcccee40a17e2a225f9b9e96f40a3a8daad9e350609654f8eda3ccbb41" + } +} diff --git a/corpus/page-date/native/scripts/author.ps1 b/corpus/page-date/native/scripts/author.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..4003849226b50011740d4a45c123ff734d64f7ce --- /dev/null +++ b/corpus/page-date/native/scripts/author.ps1 @@ -0,0 +1,46 @@ +param([Parameter(Mandatory=$true)][string]$Root, [Parameter(Mandatory=$true)][string]$CloneHost) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +& "$PSScriptRoot\cold-current.ps1" -Root $Root -CloneHost $CloneHost +$namespace = 'http://schemas.microsoft.com/office/onenote/2010/onenote' +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +function Save-Page([string]$pageId, [string]$name) { + $content = '' + $app.GetPageContent($pageId, [ref]$content, 0, 1) + [IO.File]::WriteAllText((Join-Path $Root $name), $content, [Text.Encoding]::UTF8) +} +try { + $app.OpenHierarchy((Join-Path $Root 'notebook'), '', [ref]$notebookId, 0) + $sectionId = '' + $app.OpenHierarchy('pictures.one', $notebookId, [ref]$sectionId, 0) + # Two new pages with OneNote's own title, date and time: one keeps its date, the + # other has its date and time changed after it was saved once. + $ids = @() + foreach ($title in @('Kept date', 'Changed date')) { + $pageId = '' + $app.CreateNewPage($sectionId, [ref]$pageId, 0) + $escaped = [Security.SecurityElement]::Escape($pageId) + $xml = "$titleBody of $title" + $app.UpdatePageContent($xml, [DateTime]::MinValue, 1, $false) + $ids += $pageId + } + $app.SyncHierarchy($notebookId) + Start-Sleep -Seconds 3 + Save-Page $ids[1] 'before.xml' + $escaped = [Security.SecurityElement]::Escape($ids[1]) + $update = "" + [IO.File]::WriteAllText((Join-Path $Root 'update.xml'), $update, [Text.Encoding]::UTF8) + $app.UpdatePageContent($update, [DateTime]::MinValue, 1, $false) + $app.SyncHierarchy($notebookId) + Start-Sleep -Seconds 3 + Save-Page $ids[1] 'after.xml' + Save-Page $ids[0] 'kept.xml' + $app.CloseNotebook($notebookId, $false) + $notebookId = '' +} finally { + if ($notebookId) { $app.CloseNotebook($notebookId, $false) } + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + [GC]::Collect() + [GC]::WaitForPendingFinalizers() +} diff --git a/corpus/page-date/native/scripts/cold.ps1 b/corpus/page-date/native/scripts/cold.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..a0b99e7aeb86f4dc10032537800501a52ad65b4f --- /dev/null +++ b/corpus/page-date/native/scripts/cold.ps1 @@ -0,0 +1,27 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '') +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +if (Get-Process ONENOTE -ErrorAction SilentlyContinue) { throw 'Close OneNote before resetting its test cache.' } +$key = 'HKCU:\Software\Microsoft\Office\14.0\OneNote' +if ($CloneHost) { + if ($CloneHost -notmatch '^ONE-[A-Z0-9-]+$' -or [Environment]::MachineName -ne $CloneHost) { + throw 'The disposable clone hostname does not match this machine.' + } + New-Item "$key\Options\Paths" -Force | Out-Null + New-ItemProperty "$key\Options\Paths" -Name UnfiledNotesSection -PropertyType ExpandString -Value 'C:\one-tests\Loose.one' -Force | Out-Null +} elseif ((Get-ItemProperty "$key\Options\Paths").UnfiledNotesSection -ne 'C:\one-tests\Loose.one' -or + -not (Test-Path 'C:\one-tests\profile-original-cache')) { + throw 'Park the personal OneNote profile before resetting the test cache.' +} +$cache = Join-Path $env:LOCALAPPDATA 'Microsoft\OneNote\14.0' +$parked = Join-Path 'C:\one-tests\caches' ([IO.Path]::GetFileName($root)) +if (Test-Path $parked) { throw 'Choose a new run; its parked cache already exists.' } +New-Item -ItemType Directory -Path 'C:\one-tests\caches' -Force | Out-Null +if (Test-Path $cache) { Move-Item -LiteralPath $cache -Destination $parked } +if (Test-Path "$key\OpenNotebooks") { Remove-Item "$key\OpenNotebooks" -Recurse } +New-Item "$key\OpenNotebooks" | Out-Null +New-ItemProperty "$key\OpenNotebooks" -Name '1' -PropertyType String -Value "$root\notebook" | Out-Null diff --git a/corpus/page-date/native/scripts/read.ps1 b/corpus/page-date/native/scripts/read.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..476094ab1e0c47077988a6461db2080cf208642b --- /dev/null +++ b/corpus/page-date/native/scripts/read.ps1 @@ -0,0 +1,142 @@ +param( + [Parameter(Mandatory=$true)][string]$Root, + [int]$ExpectedPages = -1, + [switch]$UseCurrentCache, + [switch]$Pdf, + [switch]$KeepOpen, + [string]$CloneHost = '' +) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$root = [IO.Path]::GetFullPath($Root).TrimEnd('\') +if ([IO.Path]::GetDirectoryName($root) -ne 'C:\one-tests\runs') { + throw 'Choose a run directly below C:\one-tests\runs.' +} +$notebook = Join-Path $root 'notebook' +$output = Join-Path $root 'read' +if (Test-Path $output) { throw 'Choose a new read destination.' } +if ($UseCurrentCache) { + if ((Get-ItemProperty 'HKCU:\Software\Microsoft\Office\14.0\OneNote\Options\Paths').UnfiledNotesSection -ne 'C:\one-tests\Loose.one') { + throw 'Park the personal OneNote profile before reading test notebooks.' + } +} else { + & "$PSScriptRoot\cold-current.ps1" -Root $root -CloneHost $CloneHost +} +New-Item -ItemType Directory -Path $output | Out-Null +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +$failure = $null +try { + $app.OpenHierarchy($notebook, '', [ref]$notebookId, 0) + $process = Get-Process ONENOTE + @{ hostname = [Environment]::MachineName; onenote = $process.MainModule.FileVersionInfo.FileVersion; + powershell = $PSVersionTable.PSVersion.ToString(); schema = 'xs2010'; cold = (-not $UseCurrentCache.IsPresent) } | + ConvertTo-Json | Set-Content (Join-Path $output 'environment.json') -Encoding UTF8 + $sections = @() + foreach ($file in @(Get-ChildItem $notebook -Recurse | Where-Object { $_.Extension -eq '.one' })) { + $id = '' + $app.OpenHierarchy($file.FullName, '', [ref]$id, 0) + $sections += $id + } + $deadline = [DateTime]::UtcNow.AddSeconds(300) + $previous = '' + $lastChange = '' + $stableSince = [DateTime]::UtcNow + $settled = $false + do { + $pages = @{} + foreach ($section in $sections) { + $hierarchy = '' + $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) + [xml]$xml = $hierarchy + foreach ($node in $xml.SelectNodes('//*[@path]')) { + if (-not $node.GetAttribute('path').StartsWith("$notebook\", [StringComparison]::OrdinalIgnoreCase)) { + throw 'OneNote opened a section outside the copied notebook.' + } + } + foreach ($node in $xml.SelectNodes('//*[local-name()="Page"]')) { + $id = $node.GetAttribute('ID') + $content = '' + $app.GetPageContent($id, [ref]$content, 1, 1) + $pages[$id] = $content + } + } + $signature = [String]::Join('|', @($pages.Keys | Sort-Object | ForEach-Object { $_ + $pages[$_] })) + if ($signature -ne $previous) { + $lastChange = $previous + $previous = $signature + $stableSince = [DateTime]::UtcNow + } + if ((($ExpectedPages -ge 0 -and $pages.Count -eq $ExpectedPages) -or + ($ExpectedPages -lt 0 -and $pages.Count -gt 0)) -and + ([DateTime]::UtcNow - $stableSince).TotalSeconds -ge 2) { $settled = $true; break } + Start-Sleep -Milliseconds 250 + } while ([DateTime]::UtcNow -lt $deadline) + if (-not $settled -or ($ExpectedPages -ge 0 -and $pages.Count -ne $ExpectedPages) -or ($ExpectedPages -lt 0 -and $pages.Count -eq 0)) { + [IO.File]::WriteAllText((Join-Path $output 'previous-signature.txt'), $lastChange, [Text.Encoding]::UTF8) + $index = 0 + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('unsettled-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + $index++ + } + $hierarchy = '' + $app.GetHierarchy($notebookId, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText((Join-Path $output 'unsettled-hierarchy.xml'), $hierarchy, [Text.Encoding]::UTF8) + throw "Expected $ExpectedPages stable pages; OneNote returned $($pages.Count), settled=$settled." + } + $index = 0 + $payloads = @() + foreach ($id in @($pages.Keys | Sort-Object)) { + [IO.File]::WriteAllText((Join-Path $output ('page-{0:d3}.xml' -f $index)), $pages[$id], [Text.Encoding]::UTF8) + if ($Pdf) { + $pdfPath = Join-Path $output ('page-{0:d3}.pdf' -f $index) + $app.NavigateTo($id, '', $false) + $app.Publish($id, $pdfPath, 3, '') + if (-not (Test-Path $pdfPath) -or (Get-Item $pdfPath).Length -eq 0) { throw 'OneNote did not publish the page PDF.' } + } + [xml]$page = $pages[$id] + foreach ($file in $page.SelectNodes('//*[local-name()="InsertedFile" or local-name()="MediaFile"]')) { + $bytes = [IO.File]::ReadAllBytes($file.GetAttribute('pathCache')) + $hash = [BitConverter]::ToString([Security.Cryptography.SHA256]::Create().ComputeHash($bytes)).Replace('-', '').ToLowerInvariant() + [IO.File]::WriteAllBytes((Join-Path $output ($hash + '.attachment')), $bytes) + $payloads += @{ page = $id; object = $file.ParentNode.GetAttribute('objectID'); + kind = $file.LocalName; name = $file.GetAttribute('preferredName'); + sha256 = $hash; bytes = $bytes.Length } + } + $index++ + } + [IO.File]::WriteAllText((Join-Path $output 'payloads.json'), (ConvertTo-Json -InputObject $payloads -Depth 4), [Text.Encoding]::UTF8) + $all = '' + $app.GetHierarchy($notebookId, 4, [ref]$all, 1) + [xml]$finalTree = $all + $finalIds = @($finalTree.SelectNodes('//*[local-name()="Page"]') | ForEach-Object { $_.GetAttribute('ID') } | Sort-Object -Unique) + if ($finalIds.Count -ne $pages.Count -or @($finalIds | Where-Object { -not $pages.ContainsKey($_) }).Count -ne 0) { + throw 'The notebook hierarchy changed while collecting page evidence; repeat the cold read.' + } + [IO.File]::WriteAllText((Join-Path $output 'hierarchy.xml'), $all, [Text.Encoding]::UTF8) + Write-Output "Read $($sections.Count) sections and $($pages.Count) pages." +} catch { + $failure = $_ + [IO.File]::WriteAllText((Join-Path $output 'failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + throw +} finally { + try { + try { + if ($notebookId -and $CloneHost) { $app.SyncHierarchy($notebookId) } + if ($notebookId -and -not $KeepOpen -and (-not $UseCurrentCache -or $CloneHost)) { + $app.CloseNotebook($notebookId, $false) + } + } catch { + if ($null -eq $failure) { throw } + [IO.File]::WriteAllText((Join-Path $output 'cleanup-failure.txt'), ($_ | Out-String), [Text.Encoding]::UTF8) + } + } finally { + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + } + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() + if (-not $UseCurrentCache -and -not $CloneHost) { + Get-Process ONENOTE -ErrorAction SilentlyContinue | Wait-Process -Timeout 10 + } +} diff --git a/corpus/page-date/native/source.json b/corpus/page-date/native/source.json new file mode 100644 index 0000000000000000000000000000000000000000..eb9b3087f7a5fd4919a32627e71c4a8b8ee191cd --- /dev/null +++ b/corpus/page-date/native/source.json @@ -0,0 +1,14 @@ +[ + { + "path": "Open Notebook.onetoc2", + "bytes": 3764, + "sha256": "8bf472d0a9c5829b049c69b817f40b766025431a61c71689bca583a60d28365d", + "mtime_ns": 1789249099590321649 + }, + { + "path": "pictures.one", + "bytes": 17504, + "sha256": "ea585992f7c450e426e0b98d8d0ce972ecdaf7602df962c938166d3d1b54c820", + "mtime_ns": 1789249099590147690 + } +] diff --git a/corpus/page-date/native/teardown.json b/corpus/page-date/native/teardown.json new file mode 100644 index 0000000000000000000000000000000000000000..05a47793de40f322e745c4c4183727e3590ecf70 --- /dev/null +++ b/corpus/page-date/native/teardown.json @@ -0,0 +1 @@ +{"absent": true} diff --git a/corpus/page-date/native/update.xml b/corpus/page-date/native/update.xml new file mode 100644 index 0000000000000000000000000000000000000000..6d64b05e7b1c6bf275d56190109235fb439a291c --- /dev/null +++ b/corpus/page-date/native/update.xml @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/crates/canvas/src/editor.rs b/crates/canvas/src/editor.rs index dc9e1c2458938d2b05f6bc1b109ab5b7f0d23eee..46fbe3fbfbfca8c9a725832dffe20881fa980c71 100644 --- a/crates/canvas/src/editor.rs +++ b/crates/canvas/src/editor.rs @@ -7,6 +7,7 @@ use crate::{ use draw::edit::{self, Movement, SelectionUnit}; use onestore::ExGuid; use onestore::document::{Format, Kind}; +use onestore::op::PageOp; use onestore::page::text::{EditError, Paragraph}; use onestore::page::{ Definition, Outline, Page, PageObject, PageParagraph, ParagraphContent, Title, @@ -26,6 +27,7 @@ pub const DEFAULT_OUTLINE_WIDTH: f32 = 468.0; #[cfg(test)] mod evidence; mod format; +mod ops; pub(crate) mod page; mod table; pub use format::{Alignment, FormatState, Formatting, NoteTag, Toggle}; @@ -92,6 +94,12 @@ pub struct CanvasEditor { preferred_x: Option, /// Formatting chosen at a caret for the text typed there next, until an edit. pending: Option<(ExGuid, TextPosition, onestore::document::Format)>, + /// What `take_ops` hands over next. + ops: Result, onestore::Error>, + /// The page as stored when the editor last read it and the ops `take_ops` handed out + /// since, which `refresh` compares a changed stored page with; none once the editor + /// holds an edit that could not be stored. + stored: Option<(Page, Vec)>, } /// Imported page state the editable content does not carry. @@ -660,10 +668,13 @@ impl CanvasEditor { composition: None, preferred_x: None, pending: None, + ops: Ok(Vec::new()), + stored: None, }) } pub fn from_page(mut page: Page, engine: &mut TextEngine) -> Result { + let stored = page.clone(); let page::Import { objects, mut outlines, @@ -706,9 +717,175 @@ impl CanvasEditor { if !editor.object_layouts().all(|(id, _)| ids.insert(id)) { return Err(EditError::InvalidStructure.into()); } + editor.stored = Some((stored, Vec::new())); Ok(editor) } + /// Shows `page`, the stored page after a change made elsewhere. What the change did not + /// reach stays as it is, history included: the page as stored before it (as last read, + /// with the ops handed out since) is compared with `page`; outlines the change reached + /// show anew with the caret and selection kept by paragraph identity, and history entries + /// editing them are dropped. False when the change reached nothing shown. Marked text + /// must be committed or cancelled first. + pub fn refresh(&mut self, page: Page, engine: &mut TextEngine) -> Result { + let known = self.stored.take().and_then(|(mut stored, sent)| { + sent.iter() + .try_for_each(|op| onestore::op::predict(&mut stored, op)) + .ok() + .map(|()| stored) + }); + // Without that, the editor's page stands in, which storage may normalize apart. + let known = match known { + Some(known) => known, + None => self.page()?, + }; + self.stored = Some((page.clone(), Vec::new())); + if known == page { + return Ok(false); + } + fn outlines(page: &Page) -> BTreeMap { + page.objects + .iter() + .flat_map(|object| match object { + PageObject::Outline(outline) => std::slice::from_ref(outline), + PageObject::Title(title) => title.outlines.as_slice(), + _ => &[], + }) + .map(|outline| (outline.id, outline)) + .collect() + } + let (before, after) = (outlines(&known), outlines(&page)); + let changed: BTreeSet = before + .keys() + .chain(after.keys()) + .filter(|id| before.get(id) != after.get(id)) + .copied() + .collect(); + // Everything but outline content: the page's objects, geometry and date. + let frame = |page: &Page| { + let hollow = |outline: &Outline| Outline { + id: outline.id, + title: false, + min_width: None, + layout: Default::default(), + indents: Vec::new(), + paragraphs: Vec::new(), + unsupported: Vec::new(), + }; + let objects: Vec = page + .objects + .iter() + .map(|object| match object { + PageObject::Outline(outline) => PageObject::Outline(hollow(outline)), + PageObject::Title(title) => PageObject::Title(Title { + id: title.id, + date: title.date, + layout: title.layout.clone(), + outlines: title.outlines.iter().map(hollow).collect(), + }), + object => object.clone(), + }) + .collect(); + (page.created, page.margin_origin, objects) + }; + let objects_changed = frame(&known) != frame(&page); + let date_changed = known.created != page.created + || self + .date + .as_ref() + .is_some_and(|date| changed.contains(&date.source().id)); + let mut fresh = Self::from_page(page, engine)?; + let shown = self.active_outline(); + let (id, selection) = (shown.id, shown.selection); + let reached = changed.contains(&id); + let document = reached.then(|| shown.document.clone()); + let mut own: BTreeMap = std::mem::take(&mut self.outlines) + .into_iter() + .map(|outline| (outline.id, outline)) + .collect(); + for outline in &mut fresh.outlines { + if !changed.contains(&outline.id) + && let Some(kept) = own.remove(&outline.id) + { + *outline = kept; + } + } + let position = fresh.outlines.iter().position(|outline| outline.id == id); + match ( + std::mem::replace(&mut self.active, Focus::Outline(0)), + position, + ) { + (Focus::Caret { outline, index }, _) => { + fresh.active = Focus::Caret { + outline, + index: index.min(fresh.outlines.len()), + }; + } + (Focus::Draft { outline, .. }, Some(index)) if !reached => { + fresh.active = Focus::Draft { index, outline }; + } + (_, Some(index)) if !reached => fresh.active = Focus::Outline(index), + (_, Some(index)) => { + fresh.active = Focus::Outline(index); + let document = document.expect("a reached outline's document was kept"); + let mapped = Selection { + positions: selection.positions.map(|position| { + follow(&document, &fresh.outlines[index].document, position) + }), + affinities: selection.affinities, + }; + let _ = fresh.select(mapped); + self.pending = self.pending.take().filter(|(_, at, _)| { + follow(&document, &fresh.active_outline().document, *at) == *at + }); + } + (_, None) => self.pending = None, + } + if !date_changed { + fresh.date = self.date.take(); + } + if !objects_changed { + fresh.objects = std::mem::take(&mut self.objects); + } + for (id, definition) in std::mem::take(&mut self.definitions) { + fresh.definitions.entry(id).or_insert(definition); + } + // Undoing an outline's creation focuses the outline focused before, if it is there. + let gone = |focus: &RestoreFocus| matches!(focus, RestoreFocus::Outline(id) if !fresh.outlines.iter().any(|o| o.id == *id)); + let reaches = |history: &History| match history { + History::Date(_) => date_changed, + History::Image { .. } | History::Picture { .. } => objects_changed, + History::Draft { outlines, .. } => changed.contains(&outlines[0].id), + History::Text { outline, change } => { + changed.contains(outline) + || change + .positions + .iter() + .any(|placement| objects_changed || changed.contains(&placement.id)) + } + History::Position { outline, .. } | History::Layout { outline, .. } => { + changed.contains(outline) + } + History::Remove { outline, focus } => changed.contains(outline) || gone(focus), + History::Insert { outline, focus, .. } => changed.contains(&outline.id) || gone(focus), + History::Restore { source, focus, .. } => changed.contains(&source.id) || gone(focus), + }; + fresh.undo = std::mem::take(&mut self.undo) + .into_iter() + .filter(|history| !reaches(history)) + .collect(); + fresh.redo = std::mem::take(&mut self.redo) + .into_iter() + .filter(|history| !reaches(history)) + .collect(); + fresh.pending = self.pending.take(); + fresh.preferred_x = self.preferred_x; + fresh.ops = std::mem::replace(&mut self.ops, Ok(Vec::new())); + fresh.stored = self.stored.take(); + *self = fresh; + Ok(true) + } + /// Rebuilds the stored page, restoring the title areas and read-only objects import split up. pub fn page(&self) -> Result { let mut objects: Vec = Vec::new(); @@ -781,18 +958,7 @@ impl CanvasEditor { .iter() .flat_map(|outline| descendants(&outline.paragraphs, None)) { - let content_tags = match &node.content { - ParagraphContent::Text(text) => text.tags.as_slice(), - ParagraphContent::Table(table) => table.tags.as_slice(), - _ => &[], - }; - referenced.extend(node.lists.iter().copied().chain(node.style)); - referenced.extend( - node.tags - .iter() - .chain(content_tags) - .filter_map(|tag| tag.definition), - ); + referenced.extend(ops::references(node)); } } let mut definitions = self.definitions.clone(); @@ -1027,6 +1193,8 @@ impl CanvasEditor { composition: None, preferred_x: None, pending: None, + ops: Ok(Vec::new()), + stored: None, }) } @@ -1055,6 +1223,7 @@ impl CanvasEditor { self.undo .push(History::Date(Box::new(self.date.replace(updated).unwrap()))); self.redo.clear(); + self.record(Ok(self.date_ops())); Ok(true) } @@ -1261,6 +1430,7 @@ impl CanvasEditor { self.redo.clear(); self.active = Focus::Outline(self.outlines.len()); self.outlines.push(outline); + self.record(self.outline_ops(id, None)); self.preferred_x = None; Ok(id) } @@ -1301,13 +1471,18 @@ impl CanvasEditor { .position(|outline| outline.id == id) .unwrap(); let layout = &mut self.outlines[index].layout; + let previous = [layout.x, layout.y]; self.undo.push(History::Position { outline: id, - position: [layout.x, layout.y], + position: previous, }); self.redo.clear(); layout.x = Some(position[0]); layout.y = Some(position[1]); + self.record(self.placement_ops(&Placement { + id, + position: previous, + })); self.active = Focus::Outline(index); self.preferred_x = None; Ok(()) @@ -1429,6 +1604,8 @@ impl CanvasEditor { layout: previous, }); self.redo.clear(); + let image = self.image(id).unwrap(); + self.record(Ok(ops::picture_layout(image))); Ok(()) } @@ -1531,6 +1708,7 @@ impl CanvasEditor { image: Some(Box::new(image)), }); self.redo.clear(); + self.record(Ok(vec![PageOp::Delete { object: id }])); Ok(()) } @@ -1559,6 +1737,11 @@ impl CanvasEditor { let id = resized.id; *self.active_outline_mut() = resized; if self.caret_outline().is_none() { + self.record(ops::layout_ops( + id, + &previous, + &self.active_outline().layout, + )); self.undo.push(History::Layout { outline: id, layout: previous, @@ -2669,7 +2852,11 @@ impl CanvasEditor { return Err((history, EditError::InvalidRange.into())); } let inverse = match history { - History::Date(date) => History::Date(Box::new(self.date.replace(*date).unwrap())), + History::Date(date) => { + let shown = self.date.replace(*date).unwrap(); + self.record(Ok(self.date_ops())); + History::Date(Box::new(shown)) + } History::Draft { outlines, index, @@ -2700,13 +2887,12 @@ impl CanvasEditor { None }; let outline = outlines[usize::from(!restore_caret)].clone(); - if self + let id = outline.id; + let stored = self .outlines .get(index) - .is_some_and(|item| item.id == outline.id) - { - self.outlines.remove(index); - } + .is_some_and(|item| item.id == id) + .then(|| self.outlines.remove(index)); self.active = if outline.is_empty() { Focus::Caret { outline: Box::new(outline), @@ -2722,6 +2908,7 @@ impl CanvasEditor { None => Focus::Outline(index), } }; + self.record(self.outline_ops(id, stored.as_ref())); History::Draft { outlines, index, @@ -2750,6 +2937,7 @@ impl CanvasEditor { return Err((History::Text { outline, change }, error)); } }; + self.record(self.change_ops(&self.outlines[index], &inverse)); History::Text { outline, change: Box::new(inverse), @@ -2762,13 +2950,17 @@ impl CanvasEditor { .position(|item| item.id == outline) .unwrap(); let layout = &mut self.outlines[index].layout; - let inverse = History::Position { - outline, - position: [layout.x, layout.y], - }; + let previous = [layout.x, layout.y]; [layout.x, layout.y] = position; self.active = Focus::Outline(index); - inverse + self.record(self.placement_ops(&Placement { + id: outline, + position: previous, + })); + History::Position { + outline, + position: previous, + } } History::Layout { outline, layout } => { let index = self @@ -2793,25 +2985,46 @@ impl CanvasEditor { }; resized.selection = self.outlines[index].selection; self.outlines[index] = resized; + self.record(ops::layout_ops( + outline, + &previous, + &self.outlines[index].layout, + )); self.active = Focus::Outline(index); History::Layout { outline, layout: previous, } } - History::Image { image, layout } => History::Image { - image, - layout: std::mem::replace(&mut self.image_mut(image).unwrap().layout, layout), - }, + History::Image { image, layout } => { + let previous = + std::mem::replace(&mut self.image_mut(image).unwrap().layout, layout); + self.record(Ok(ops::picture_layout(self.image(image).unwrap()))); + History::Image { + image, + layout: previous, + } + } History::Picture { index, image } => History::Picture { index, image: match image { Some(image) => { self.objects.insert(index, page::Content::Image(*image)); + let page::Content::Image(image) = &self.objects[index] else { + unreachable!() + }; + let ops = vec![PageOp::Add { + object: PageObject::Image(image.clone()), + before: self.successor(image.id), + }]; + self.record(Ok(ops)); None } None => match self.objects.remove(index) { - page::Content::Image(image) => Some(Box::new(image)), + page::Content::Image(image) => { + self.record(Ok(vec![PageOp::Delete { object: image.id }])); + Some(Box::new(image)) + } _ => unreachable!(), }, }, @@ -2847,6 +3060,7 @@ impl CanvasEditor { } }; let outline = Box::new(self.outlines.remove(index)); + self.record(Ok(vec![PageOp::Delete { object: outline.id }])); self.active = next_focus; History::Insert { index, @@ -2892,6 +3106,7 @@ impl CanvasEditor { } => { let id = outline.id; self.outlines.insert(index, *outline); + self.record(self.outline_ops(id, None)); self.active = Focus::Outline(index); History::Remove { outline: id, focus } } @@ -3152,6 +3367,7 @@ impl CanvasEditor { } else { self.outlines[index] = *outline; } + self.record(self.outline_ops(versions[0].id, Some(&versions[0]))); self.undo.push(History::Draft { outlines: versions, index, @@ -3173,8 +3389,9 @@ impl CanvasEditor { }; let mut source = outline.snapshot(); source.paragraphs = change.edit.replacement; + let id = outline.id; self.undo.push(History::Remove { - outline: outline.id, + outline: id, focus: RestoreFocus::Caret { source: Box::new(source), selection: change.selection, @@ -3182,11 +3399,13 @@ impl CanvasEditor { }, }); self.outlines.insert(index, *outline); + self.record(self.outline_ops(id, None)); } else if self.active_outline().is_empty() && !self.active_outline().title { let Focus::Outline(index) = self.active else { unreachable!() }; let outline = self.outlines.remove(index); + self.record(Ok(vec![PageOp::Delete { object: outline.id }])); debug_assert_eq!(change.edit.range, 0..1); let mut source = outline.snapshot(); source.paragraphs = change.edit.replacement; @@ -3205,6 +3424,7 @@ impl CanvasEditor { index, }; } else { + self.record(self.change_ops(self.active_outline(), &change)); self.undo.push(History::Text { outline: self.active_outline().id, change: Box::new(change), @@ -3311,6 +3531,59 @@ impl CanvasEditor { } } +/// Where `position` in `old` lies in `new`, the same outline changed elsewhere: in the same +/// paragraph, past what changed in its text when it lies after it; at the start of the +/// paragraph now at its place when that one is gone. +fn follow(old: &TextDocument, new: &TextDocument, position: TextPosition) -> TextPosition { + let found = old.leaf(position.paragraph).and_then(|(_, _, node)| { + let paragraph = new + .text_nodes() + .position(|candidate| candidate.id == node.id)?; + Some((paragraph, node, new.leaf(paragraph)?.2)) + }); + let Some((paragraph, before, after)) = found else { + let count = new.text_nodes().count(); + return TextPosition { + paragraph: position.paragraph.min(count.saturating_sub(1)), + offset: 0, + }; + }; + let units = |text: &str| text.encode_utf16().count() as u32; + let (a, b) = ( + before.text().unwrap().text.text(), + after.text().unwrap().text.text(), + ); + let prefix = units( + &a[..a + .char_indices() + .zip(b.chars()) + .find(|((_, x), y)| x != y) + .map_or(a.len().min(b.len()), |((at, _), _)| at)], + ); + let suffix = a + .chars() + .rev() + .zip(b.chars().rev()) + .take_while(|(x, y)| x == y) + .map(|(x, _)| x.len_utf16() as u32) + .scan(0, |sum, units| { + *sum += units; + Some(*sum) + }) + .take_while(|sum| prefix + sum <= units(a).min(units(b))) + .last() + .unwrap_or(0); + let (old_length, new_length) = (units(a), units(b)); + let offset = if position.offset <= prefix { + position.offset + } else if position.offset >= old_length - suffix { + new_length - (old_length - position.offset) + } else { + new_length - suffix + }; + TextPosition { paragraph, offset } +} + fn inserted_position( mut position: TextPosition, text: &str, @@ -8360,4 +8633,65 @@ mod tests { assert_eq!(body_text(&reread, body).as_deref(), Some(edited.as_str())); assert_ne!(body_text(&source, body).as_deref(), Some(edited.as_str())); } + + /// A page changed elsewhere shows in place: the caret keeps its paragraph and moves + /// past text inserted before it; history goes, unless nothing changed. + #[test] + fn a_refresh_keeps_the_caret_by_identity() { + let mut engine = TextEngine::default(); + let lines = + ["Hello world", "Second"].map(|line| Paragraph::new(line.into(), Format::default())); + let document = TextDocument::new(lines.to_vec()).unwrap(); + let mut editor = CanvasEditor::new(&mut engine, document, 400.0).unwrap(); + editor + .select( + [TextPosition { + paragraph: 1, + offset: 3, + }; 2] + .into(), + ) + .unwrap(); + editor.insert(&mut engine, "x").unwrap(); + editor + .select( + [TextPosition { + paragraph: 0, + offset: 8, + }; 2] + .into(), + ) + .unwrap(); + let _ = editor.take_ops(); + let page = editor.page().unwrap(); + assert!(!editor.refresh(page.clone(), &mut engine).unwrap()); + assert!(!editor.undo.is_empty(), "an unchanged page keeps history"); + + let mut remote = page; + let PageObject::Outline(outline) = &mut remote.objects[0] else { + unreachable!() + }; + for (paragraph, text) in outline + .paragraphs + .iter_mut() + .zip(["Hey, Hello world", "Other"]) + { + paragraph.text_mut().unwrap().text = Paragraph::new(text.into(), Format::default()); + } + let (shown, affinities) = (editor.active_outline().id, editor.selection().affinities); + assert!(editor.refresh(remote, &mut engine).unwrap()); + assert_eq!(editor.active_outline().id, shown); + assert_eq!( + editor.selection(), + Selection { + positions: [TextPosition { + paragraph: 0, + offset: 13 + }; 2], + affinities, + } + ); + assert!(editor.undo.is_empty()); + assert_eq!(editor.take_ops().unwrap(), []); + } } diff --git a/crates/canvas/src/editor/ops.rs b/crates/canvas/src/editor/ops.rs new file mode 100644 index 0000000000000000000000000000000000000000..f28952bc4503632331c0ba5666d9cc542095b4fd --- /dev/null +++ b/crates/canvas/src/editor/ops.rs @@ -0,0 +1,743 @@ +//! The ops each stored change lowers to, recorded as the change applies so a page is never +//! rebuilt to be saved. + +use super::*; +use onestore::op; +use onestore::{OutlineEdit, document::Layout}; + +type Lowered = Result, onestore::Error>; + +fn refused(message: &'static str) -> onestore::Error { + onestore::Error { offset: 0, message } +} + +fn stale(_: EditError) -> onestore::Error { + refused("The editor lost track of the stored page") +} + +/// Definitions a paragraph names: its lists, style and note tags. +pub(super) fn references(node: &PageParagraph) -> impl Iterator + '_ { + let content_tags = match &node.content { + ParagraphContent::Text(text) => text.tags.as_slice(), + ParagraphContent::Table(table) => table.tags.as_slice(), + _ => &[], + }; + node.lists.iter().copied().chain(node.style).chain( + node.tags + .iter() + .chain(content_tags) + .filter_map(|tag| tag.definition), + ) +} + +/// The definitions `paragraphs` and their cells name. +fn named( + definitions: &BTreeMap, + paragraphs: &[&[PageParagraph]], +) -> BTreeMap { + paragraphs + .iter() + .flat_map(|list| descendants(list, None)) + .flat_map(|(_, _, node)| references(node)) + .filter_map(|id| Some((id, definitions.get(&id)?.clone()))) + .collect() +} + +/// `op::lower` of paragraphs `before` of `container` replaced by `nodes[range]` and the +/// definitions they name, so it stays O(edit). When the structure changes, it also sees the +/// unchanged paragraphs an op may split, join, move or anchor to: the ones either side and +/// their ancestors, the range's descendants past it, and after them the next child of each +/// ancestor. +fn lower( + container: ExGuid, + nodes: &[PageParagraph], + range: Range, + before: &[PageParagraph], + definitions: &BTreeMap, +) -> Lowered { + let after = &nodes[range.clone()]; + if before.is_empty() && after.is_empty() { + return Ok(Vec::new()); + } + let mut context = BTreeSet::new(); + let shape = |node: &PageParagraph| (node.id, node.parent, node.level); + if before.iter().map(shape).ne(after.iter().map(shape)) { + context.extend(range.start.checked_sub(1)); + context.extend((range.end < nodes.len()).then_some(range.end)); + let held: BTreeSet = before.iter().chain(after).map(|node| node.id).collect(); + let mut wanted: Vec = before + .iter() + .chain(after) + .chain(context.iter().map(|&at| &nodes[at])) + .filter_map(|node| node.parent) + .collect(); + let mut open = BTreeMap::from([(None, 0)]); + while let Some(id) = wanted.pop() { + if held.contains(&id) || context.iter().any(|&at| nodes[at].id == id) { + continue; + } + if let Some(at) = nodes[..range.start].iter().rposition(|node| node.id == id) { + context.insert(at); + open.insert(Some(id), nodes[at].level); + wanted.extend(nodes[at].parent); + } + } + // Descendants of the range move with the paragraphs it splits, joins and moves. + let mut members = held; + let mut end = range.end; + while let Some(node) = nodes.get(end) + && node.parent.is_some_and(|parent| members.contains(&parent)) + { + members.insert(node.id); + context.insert(end); + end += 1; + } + for (at, node) in nodes.iter().enumerate().skip(end) { + if open.remove(&node.parent).is_some() { + context.insert(at); + } + // A container's children end where its subtree does. + open.retain(|_, level| *level < node.level); + if open.is_empty() { + break; + } + } + } + let around = |middle: &[PageParagraph]| { + let (head, tail): (Vec, Vec) = + context.iter().partition(|&&at| at < range.start); + head.iter() + .map(|&at| &nodes[at]) + .chain(middle) + .chain(tail.iter().map(|&at| &nodes[at])) + .cloned() + .collect::>() + }; + let (before, after) = (around(before), around(after)); + let named = named(definitions, &[&before, &after]); + // What `op::lower` cannot move or change in place is removed and inserted again under + // the identities it had: stored paragraphs a new table cell holds, and, where lowering + // fails, a paragraph whose text object a join replaced outside what `Join` expresses + // (between empty paragraphs, or back on undo) with the paragraph whose text it takes. + let text = |node: &PageParagraph| node.text().map(|text| text.id); + let stored: BTreeMap> = descendants(&before, None) + .map(|(_, _, node)| (node.id, text(node))) + .collect(); + let cells: BTreeSet = descendants(&before, None) + .filter_map(|(cell, _, _)| cell) + .collect(); + let mut removed: BTreeSet = descendants(&after, None) + .filter(|(cell, _, node)| { + stored.contains_key(&node.id) && cell.is_some_and(|cell| !cells.contains(&cell)) + }) + .map(|(_, _, node)| node.id) + .collect(); + let lowered = |removed: &BTreeSet| { + let mut ops = Vec::new(); + let mut deleted = Vec::new(); + let mut kept = without(&before, removed, &mut ops, &mut deleted).map_err(stale)?; + if kept.is_empty() { + let anchor = anchor().map_err(stale)?; + ops.push(PageOp::Insert { + container, + before: None, + paragraphs: vec![anchor.clone()], + }); + kept.push(anchor); + } + ops.extend(deleted.into_iter().map(|object| PageOp::Delete { object })); + ops.extend(op::lower(container, &kept, &after, None, &named)?); + Ok(ops) + }; + let error = match lowered(&removed) { + Ok(ops) => return Ok(ops), + Err(error) => error, + }; + let taken: BTreeSet = descendants(&after, None) + .filter(|(_, _, node)| stored.get(&node.id).is_some_and(|old| *old != text(node))) + .filter_map(|(_, _, node)| { + removed.insert(node.id); + text(node) + }) + .collect(); + if taken.is_empty() { + return Err(error); + } + removed.extend( + descendants(&before, None) + .filter(|(_, _, node)| text(node).is_some_and(|id| taken.contains(&id))) + .map(|(_, _, node)| node.id), + ); + lowered(&removed) +} + +/// An empty paragraph keeping a container from emptying while its paragraphs are removed +/// and inserted again; the lowering removes it after. +fn anchor() -> Result { + crate::document::node( + Paragraph::new(String::new(), Format::default()), + Format::default(), + ) +} + +/// `nodes` without the subtrees of `removed`, whose roots `deleted` receives; a table cell +/// they would empty first receives an anchor, which `ops` inserts. +fn without( + nodes: &[PageParagraph], + removed: &BTreeSet, + ops: &mut Vec, + deleted: &mut Vec, +) -> Result, EditError> { + let mut gone = BTreeSet::new(); + let mut kept = Vec::new(); + for node in nodes { + if node.parent.is_some_and(|parent| gone.contains(&parent)) { + gone.insert(node.id); + } else if removed.contains(&node.id) { + gone.insert(node.id); + deleted.push(node.id); + } else { + let mut node = node.clone(); + if let ParagraphContent::Table(table) = &mut node.content { + for cell in table.rows.iter_mut().flat_map(|row| &mut row.cells) { + cell.paragraphs = without(&cell.paragraphs, removed, ops, deleted)?; + if cell.paragraphs.is_empty() { + let anchor = anchor()?; + ops.push(PageOp::Insert { + container: cell.id, + before: None, + paragraphs: vec![anchor.clone()], + }); + cell.paragraphs.push(anchor); + } + } + } + kept.push(node); + } + } + Ok(kept) +} + +/// A page-level outline's position and width, as `op::lower_page` compares them. +pub(super) fn layout_ops(id: ExGuid, old: &Layout, new: &Layout) -> Lowered { + let mut ops = Vec::new(); + if (new.x, new.y) != (old.x, old.y) { + let (Some(x), Some(y)) = (new.x, new.y) else { + return Err(refused("An outline position needs both coordinates")); + }; + ops.push(PageOp::Outline { + object: id, + edit: OutlineEdit::Position { x, y }, + }); + } + if (new.max_width, new.width_set_by_user) != (old.max_width, old.width_set_by_user) { + ops.push(PageOp::Outline { + object: id, + edit: OutlineEdit::Width { + points: new + .max_width + .ok_or_else(|| refused("An outline width cannot be removed"))?, + user_set: new.width_set_by_user == Some(true), + }, + }); + } + Ok(ops) +} + +/// A page picture's stored position, size and description. +pub(super) fn picture_layout(image: &onestore::page::Image) -> Vec { + vec![PageOp::Picture { + picture: image.id, + layout: image.layout.clone(), + alt: image.alt.clone(), + }] +} + +impl CanvasEditor { + /// The ops the stored page took since the last call, in order; an error means an edit + /// since then cannot be stored, and the page should be reopened from storage. + pub fn take_ops(&mut self) -> Result, onestore::Error> { + let taken = std::mem::replace(&mut self.ops, Ok(Vec::new())); + match (&taken, &mut self.stored) { + (Ok(ops), Some((_, sent))) => sent.extend(ops.iter().cloned()), + (Err(_), stored) => *stored = None, + (Ok(_), None) => {} + } + taken + } + + pub(super) fn record(&mut self, lowered: Lowered) { + if let Ok(ops) = &mut self.ops { + match lowered { + Ok(mut lowered) => ops.append(&mut lowered), + Err(error) => self.ops = Err(error), + } + } + } + + /// Ops for `change`, the inverse of an edit just applied to stored outline `outline`. + pub(super) fn change_ops(&self, outline: &TextOutline, change: &TextChange) -> Lowered { + let edit = &change.edit; + let nodes = outline.document.container(edit.container).map_err(stale)?; + let mut ops = lower( + edit.container.unwrap_or(outline.id), + nodes, + edit.range.clone(), + &edit.replacement, + &self.definitions, + )?; + for (_, _, node) in descendants(outline.document.nodes(), None) { + if let ParagraphContent::Table(table) = &node.content + && edit.columns.contains_key(&table.id) + { + ops.push(PageOp::Table { + table: table.id, + edit: op::TableEdit::Columns(table.columns.clone()), + }); + } + } + for placement in &change.positions { + ops.extend(self.placement_ops(placement)?); + } + Ok(ops) + } + + /// A page object's move from `placement`, as title flow makes it. + pub(super) fn placement_ops(&self, placement: &Placement) -> Lowered { + let old = Layout { + x: placement.position[0], + y: placement.position[1], + ..Default::default() + }; + let moved = |layout: &Layout| Layout { + x: layout.x, + y: layout.y, + ..Default::default() + }; + let picture = |image: &onestore::page::Image| { + Ok(if [image.layout.x, image.layout.y] == placement.position { + Vec::new() + } else { + picture_layout(image) + }) + }; + if let Some(outline) = self.outlines.iter().find(|item| item.id == placement.id) { + return layout_ops(outline.id, &old, &moved(&outline.layout)); + } + for object in &self.objects { + match object { + page::Content::Image(image) if image.id == placement.id => return picture(image), + page::Content::Outline { source, .. } if source.id == placement.id => { + return layout_ops(source.id, &old, &moved(&source.layout)); + } + page::Content::ReadOnly(object) if object.source.id() == placement.id => { + return match &object.source { + PageObject::Outline(source) => { + layout_ops(source.id, &old, &moved(&source.layout)) + } + PageObject::Image(image) => picture(image), + _ => Err(refused("Unsupported objects cannot be moved")), + }; + } + _ => {} + } + } + Err(refused("The editor lost track of the stored page")) + } + + /// Ops turning stored outline `old` into what `outlines` holds under `id` now: added, + /// removed, or its changed paragraphs and geometry. + pub(super) fn outline_ops(&self, id: ExGuid, old: Option<&TextOutline>) -> Lowered { + let new = self.outlines.iter().find(|outline| outline.id == id); + match (old, new) { + (None, None) => Ok(Vec::new()), + (Some(_), None) => Ok(vec![PageOp::Delete { object: id }]), + (None, Some(new)) => { + let outline = new.snapshot(); + let page = |objects| Page { + title: String::new(), + identity: None, + created: None, + margin_origin: [0.0; 2], + objects, + definitions: named(&self.definitions, &[&outline.paragraphs]), + }; + let mut ops = op::lower_page( + &page(Vec::new()), + &page(vec![PageObject::Outline(outline.clone())]), + )?; + let successor = self.successor(id); + for op in &mut ops { + if let PageOp::Add { before, .. } = op { + *before = successor; + } + } + Ok(ops) + } + (Some(old), Some(new)) => { + let (a, b) = (old.document.nodes(), new.document.nodes()); + let prefix = a.iter().zip(b).take_while(|(x, y)| x == y).count(); + let suffix = a[prefix..] + .iter() + .rev() + .zip(b[prefix..].iter().rev()) + .take_while(|(x, y)| x == y) + .count(); + let mut ops = lower( + id, + b, + prefix..b.len() - suffix, + &a[prefix..a.len() - suffix], + &self.definitions, + )?; + ops.extend(layout_ops(id, &old.layout, &new.layout)?); + Ok(ops) + } + } + } + + /// The page's date as the editor shows it. + pub(super) fn date_ops(&self) -> Vec { + let Some(date) = &self.date else { + return Vec::new(); + }; + vec![PageOp::Date { + created: date.timestamp(), + fields: date + .fields() + .filter_map(|(_, paragraph)| { + let text = paragraph.text()?; + Some((text.id, text.text.text().to_owned())) + }) + .collect(), + }] + } + + /// The page child `page()` places after `id`, titles aside. + pub(super) fn successor(&self, id: ExGuid) -> Option { + let mut children: Vec = Vec::new(); + for content in &self.objects { + let child = match content { + page::Content::Editable(outline) + if self.outlines.iter().any(|o| o.id == *outline) => + { + *outline + } + page::Content::Editable(_) => continue, + page::Content::Outline { source, .. } => source.id, + page::Content::Date { .. } => continue, + page::Content::Image(image) => image.id, + page::Content::Ink(ink) => ink.id, + page::Content::ReadOnly(object) + if matches!(object.source, PageObject::Title(_)) => + { + continue; + } + page::Content::ReadOnly(object) => object.source.id(), + }; + // Titles, the date's among them, are listed apart from what an op places. + if !self + .header + .areas + .iter() + .any(|area| area.origins.contains_key(&child)) + { + children.push(child); + } + } + children.extend( + self.outlines + .iter() + .filter(|outline| !self.has_page_outline(outline.id)) + .map(|outline| outline.id), + ); + let at = children.iter().position(|child| *child == id)?; + children.get(at + 1).copied() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::layout::TextEngine; + use std::time::{Duration, Instant}; + + fn median(mut samples: Vec) -> Duration { + samples.sort(); + samples[samples.len() / 2] + } + + /// The rows of the data-layer plan's table: each edit and its undo as ops, the undo + /// of a join splitting the paragraph back under its own identities. + #[test] + fn edits_and_their_undo_record_the_plan_s_ops() { + let mut engine = TextEngine::default(); + let lines = ["Hello", "World"].map(|line| Paragraph::new(line.into(), Format::default())); + let document = crate::document::TextDocument::new(lines.to_vec()).unwrap(); + let mut editor = CanvasEditor::new(&mut engine, document, 400.0).unwrap(); + let nodes = editor.outlines()[0].document().nodes().to_vec(); + let text = |index: usize| nodes[index].text().unwrap().id; + let at = |paragraph, offset| [TextPosition { paragraph, offset }; 2].into(); + let undone = |editor: &mut CanvasEditor, engine: &mut TextEngine| { + editor.undo(engine).unwrap(); + editor.take_ops().unwrap() + }; + + editor.select(at(0, 3)).unwrap(); + editor.insert(&mut engine, "ab").unwrap(); + let typed = PageOp::Text { + text: text(0), + range: 3..3, + with: "ab".into(), + }; + assert_eq!(editor.take_ops().unwrap(), [typed]); + let erased = PageOp::Text { + text: text(0), + range: 3..5, + with: String::new(), + }; + assert_eq!(undone(&mut editor, &mut engine), [erased]); + + editor.select(at(1, 0)).unwrap(); + editor.delete(&mut engine, true).unwrap(); + let joined = PageOp::Join { + left: text(0), + right: text(1), + }; + assert_eq!(editor.take_ops().unwrap(), [joined]); + let split = PageOp::Split { + text: text(0), + at: 5, + paragraph: nodes[1].id, + right: text(1), + lists: Vec::new(), + }; + assert_eq!(undone(&mut editor, &mut engine), [split]); + + editor.select(at(1, 0)).unwrap(); + editor.tab(&mut engine, false).unwrap(); + let [PageOp::Move { object, parent, .. }] = &editor.take_ops().unwrap()[..] else { + panic!("Tab moves the paragraph under its sibling") + }; + assert_eq!((*object, *parent), (nodes[1].id, Some(nodes[0].id))); + // A moved paragraph keeps its level where it lies deeper than its new parent. + let ops = undone(&mut editor, &mut engine); + let [ + PageOp::Move { object, parent, .. }, + PageOp::Level { + paragraph, + level: 1, + }, + ] = &ops[..] + else { + panic!("undoing Tab moves it back and outdents it: {ops:?}") + }; + assert_eq!( + (*object, *parent, *paragraph), + (nodes[1].id, Some(editor.outlines()[0].id), nodes[1].id) + ); + + editor + .select( + [ + TextPosition { + paragraph: 0, + offset: 1, + }, + TextPosition { + paragraph: 0, + offset: 4, + }, + ] + .into(), + ) + .unwrap(); + editor + .format(&mut engine, Formatting::Toggle(format::Toggle::Bold)) + .unwrap(); + let bold = PageOp::Format { + text: text(0), + range: 1..4, + set: vec![onestore::TextAttribute::Bold(true)], + clear: Vec::new(), + }; + assert_eq!(editor.take_ops().unwrap(), [bold]); + let [ + PageOp::Format { + text: target, + range, + .. + }, + ] = &undone(&mut editor, &mut engine)[..] + else { + panic!("undoing bold formats the range back") + }; + assert_eq!((*target, range.clone()), (text(0), 1..4)); + } + + /// A change from elsewhere reaches only what it changed: a refresh whose stored page is + /// what the editor's ops left (though storage normalized it) changes nothing, history + /// included; one that changes an outline drops that outline's history and keeps the + /// rest, which still undoes into the section. + #[test] + fn a_refresh_keeps_what_the_change_did_not_reach() { + use onestore::op::{Edit, Op}; + let image = + include_bytes!("../../../../corpus/paragraph-edit/before/notebook/synthetic.one"); + let arena = onestore::Arena::default(); + let mut section = onestore::Section::open(&arena, image.to_vec()).unwrap(); + let space = section.pages().unwrap()[0].0; + let mut engine = TextEngine::default(); + let mut editor = + CanvasEditor::from_page(section.page(space).unwrap(), &mut engine).unwrap(); + let mut at = 133_000_000_000_000_000; + let mut store = |section: &mut onestore::Section<'_>, ops: Vec| { + at += 10_000_000; + let ops = ops.into_iter().map(|op| Op::Page { space, op }).collect(); + section.apply("Test", &Edit { at, ops }).unwrap(); + }; + let local = editor.outlines().iter().find(|o| !o.title).unwrap().id; + editor.focus_outline(local).unwrap(); + editor + .select( + [TextPosition { + paragraph: 0, + offset: 0, + }; 2] + .into(), + ) + .unwrap(); + editor.insert(&mut engine, "Local ").unwrap(); + let created = editor + .create_outline(&mut engine, [400.0, 600.0], 300.0) + .unwrap(); + editor.insert(&mut engine, "Mine").unwrap(); + store(&mut section, editor.take_ops().unwrap()); + + let (shown, history) = (editor.page().unwrap(), editor.undo.len()); + let stored = section.page(space).unwrap(); + assert_ne!( + Page { + title: shown.title.clone(), + ..stored.clone() + }, + shown, + "storage normalizes" + ); + assert!(!editor.refresh(stored, &mut engine).unwrap()); + assert_eq!( + ( + editor.page().unwrap(), + editor.undo.len(), + editor.active_outline().id + ), + (shown, history, created) + ); + + let text = section + .page(space) + .unwrap() + .objects + .iter() + .find_map(|object| match object { + onestore::page::PageObject::Outline(outline) if outline.id == local => outline + .paragraphs + .last()? + .text() + .map(|text| (text.id, text.text.text().encode_utf16().count() as u32)), + _ => None, + }); + let (text, length) = text.unwrap(); + store( + &mut section, + vec![PageOp::Text { + text, + range: length..length, + with: " remote".into(), + }], + ); + assert!( + editor + .refresh(section.page(space).unwrap(), &mut engine) + .unwrap() + ); + assert_eq!( + editor.undo.len(), + history - 1, + "the typing in the changed outline goes" + ); + assert_eq!(editor.active_outline().id, created); + assert!(editor.undo(&mut engine).unwrap()); + assert!(editor.undo(&mut engine).unwrap()); + store(&mut section, editor.take_ops().unwrap()); + assert!(!editor.undo(&mut engine).unwrap()); + let page = section.page(space).unwrap(); + assert!(!page.objects.iter().any(|object| object.id() == created)); + let texts: Vec = editor + .outlines() + .iter() + .flat_map(|o| o.document().paragraphs().map(|p| p.text().to_owned())) + .collect(); + assert!(texts.iter().any(|text| text.starts_with("Local "))); + assert!(texts.iter().any(|text| text.ends_with(" remote"))); + } + + /// One keystroke on the 3000-paragraph probe page, on the frame thread: + /// `SECTION_PROBE=path cargo test --release -p canvas --lib keystroke_recording -- --ignored --nocapture`. + #[test] + #[ignore] + fn keystroke_recording() { + let path = std::env::var("SECTION_PROBE").unwrap_or("/tmp/probe3000.one".into()); + let image = std::fs::read(&path).unwrap(); + let arena = onestore::Arena::default(); + let mut section = onestore::Section::open(&arena, image).unwrap(); + let page = section + .pages() + .unwrap() + .into_iter() + .map(|(space, ..)| section.page(space).unwrap()) + .max_by_key(|page| format!("{page:?}").len()) + .unwrap(); + let mut engine = TextEngine::default(); + let mut editor = CanvasEditor::from_page(page, &mut engine).unwrap(); + let outline = editor + .outlines() + .iter() + .max_by_key(|outline| outline.document().nodes().len()) + .unwrap(); + let (id, count) = (outline.id, outline.document().nodes().len()); + editor.focus_outline(id).unwrap(); + let caret = TextPosition { + paragraph: count / 2, + offset: 0, + }; + editor.select([caret; 2].into()).unwrap(); + let (mut typing, mut recording, mut rebuilding) = (Vec::new(), Vec::new(), Vec::new()); + let mut previous = editor.page().unwrap(); + for _ in 0..200 { + let start = Instant::now(); + editor.insert(&mut engine, "x").unwrap(); + typing.push(start.elapsed()); + let Some(History::Text { change, .. }) = editor.undo.last() else { + unreachable!() + }; + let start = Instant::now(); + let ops = editor.change_ops(editor.active_outline(), change).unwrap(); + recording.push(start.elapsed()); + assert_eq!(ops.len(), 1, "{ops:?}"); + assert_eq!(editor.take_ops().unwrap().len(), 1); + let start = Instant::now(); + let page = editor.page().unwrap(); + assert!(page != previous); + rebuilding.push(start.elapsed()); + previous = page; + } + println!( + "{path}: {count} paragraphs; keystroke {:?}, of which recording its op {:?}; \ + the page rebuilt and compared as persist did {:?}", + median(typing), + median(recording), + median(rebuilding) + ); + } +} diff --git a/crates/canvas/src/gpu/page.rs b/crates/canvas/src/gpu/page.rs index b49ccd5091da072c02cc7cab8a98818336be046c..49af7234797d07cb248cb404c35f6b31f6085a4e 100644 --- a/crates/canvas/src/gpu/page.rs +++ b/crates/canvas/src/gpu/page.rs @@ -213,6 +213,33 @@ impl PageScene { Ok((scene, editor)) } + /// Follows `editor` after it refreshed from storage, keeping the rasters of pictures + /// whose bytes did not change so they do not blink. + pub fn refresh( + &mut self, + editor: &mut CanvasEditor, + engine: &mut TextEngine, + ) -> Result<(), SceneError> { + let mut scene = Self::pictures(&editor.objects, Some(editor))?; + for (id, picture) in &mut scene.pictures { + if let Some(shown) = self.pictures.remove(id) + && shown.same(picture) + { + *picture = shown; + } + } + for (id, art) in &mut scene.backgrounds { + if let Some(shown) = self.backgrounds.remove(id) + && shown.same(art) + { + *art = shown; + } + } + scene.mark_unavailable(&mut editor.objects, engine)?; + *self = scene; + Ok(()) + } + /// Turns page pictures the renderer cannot decode into placeholders; their stored data /// is kept. fn mark_unavailable( diff --git a/crates/canvas/src/gpu/page/background.rs b/crates/canvas/src/gpu/page/background.rs index 8b34f76d6df5d07ed956381f9bc8f06f7b879b97..1cd2c1445aaafc81e1da53fd8359ae3aa57698ec 100644 --- a/crates/canvas/src/gpu/page/background.rs +++ b/crates/canvas/src/gpu/page/background.rs @@ -330,6 +330,11 @@ struct Variant { } impl Background { + /// Whether both recreate the same template picture at the same size. + pub fn same(&self, other: &Self) -> bool { + self.svg == other.svg && self.size == other.size + } + pub fn recognise(picture: &onestore::page::Image) -> Option { let digest: String = Sha256::digest(picture.bytes.as_deref()?) .iter() diff --git a/crates/canvas/src/gpu/page/picture.rs b/crates/canvas/src/gpu/page/picture.rs index 18516bef894d37e7b9684a6910d0faa872bab01e..bd44b77d4d43ec0c89ff94ed6ecce472864b688b 100644 --- a/crates/canvas/src/gpu/page/picture.rs +++ b/crates/canvas/src/gpu/page/picture.rs @@ -31,6 +31,11 @@ pub(super) struct Picture { impl Picture { /// Reads the size from the picture's header; None when the renderer cannot decode it. + /// Whether both show the same stored bytes. + pub fn same(&self, other: &Self) -> bool { + self.bytes == other.bytes + } + pub fn new(bytes: &Arc<[u8]>) -> Option { Some(Self { native: RasterImage::measure(bytes).ok()?, diff --git a/crates/canvas/src/interaction/mod.rs b/crates/canvas/src/interaction/mod.rs index d4ca7e656af4367f77cb57af9af4386847b2c0da..8d5f66f9874d43c8c6ed406f8c210f7088a139dd 100644 --- a/crates/canvas/src/interaction/mod.rs +++ b/crates/canvas/src/interaction/mod.rs @@ -257,6 +257,21 @@ impl PageView { self.object_focus = None; } + /// Shows the stored page after a change made elsewhere in place of the one shown, + /// keeping the scroll, the caret and selection, and the pictures already drawn. Marked + /// text must be committed or cancelled first. + pub fn refresh(&mut self, page: onestore::page::Page) -> Result { + if !self.editor.refresh(page, &mut self.engine)? { + return Ok(Response::default()); + } + if let Some((scene, _)) = &mut self.scene { + scene.refresh(&mut self.editor, &mut self.engine)?; + } + self.drag = None; + self.object_focus = None; + self.moved() + } + /// Shows another page as OneNote opens one, keeping the zoom. pub fn open(&mut self, editor: CanvasEditor, scene: Option<(PageScene, [f32; 2])>) { self.replace(editor, scene); diff --git a/crates/canvas/tests/ops_differential.rs b/crates/canvas/tests/ops_differential.rs new file mode 100644 index 0000000000000000000000000000000000000000..5b2654946e68b12f9e7ea5205d9f2a0d16b43f4a --- /dev/null +++ b/crates/canvas/tests/ops_differential.rs @@ -0,0 +1,670 @@ +//! The ops the editor records for each edit, applied to the section, store what the edited +//! page lowered whole (`Section::apply_page(editor.page())`) stores: every editor operation +//! alone and in random sequences with undo and redo, on every page of the corpus sections +//! below and of any named in `OPS_SWEEP_SECTIONS` (`:`-separated paths; the structural +//! probe section, which `structural_roundtrip` sweeps, takes minutes). + +use canvas::{ + document::TextPosition, + editor::{Alignment, CanvasEditor, Formatting, NoteTag, Toggle}, + layout::TextEngine, +}; +use draw::edit::Movement; +use onestore::{ + Arena, ExGuid, Section, + op::{Edit, Op}, + page::Page, +}; +use std::{collections::BTreeMap, path::Path}; + +const SECTIONS: [&str; 15] = [ + "corpus/outline-edit/before/notebook/synthetic.one", + "corpus/paragraph-edit/before/notebook/synthetic.one", + "corpus/outline-edit/tree/before/notebook/synthetic.one", + "corpus/m6/native-features-01/notebook/Features.one", + "corpus/table-edit/nested/cold/notebook/synthetic.one", + "corpus/m6/native-table-controls-01/notebook/synthetic.one", + "corpus/list-edit/cold/notebook/lists.one", + "corpus/tag-edit/cold/notebook/tags.one", + "corpus/picture-edit/native-page-level/notebook/pictures.one", + "corpus/attachment-edit/plain/cold/notebook/files.one", + "corpus/ink-edit/drawing/cold/notebook/ink.one", + "corpus/math-edit/native-editor/notebook/links.one", + "corpus/link-edit/native-links/notebook/links.one", + "corpus/paragraph-format/cold/notebook/synthetic.one", + "corpus/canvas/baseline-anchors.one", +]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +enum Kind { + Type, + TypeEnd, + Compose, + ComposeCancel, + DeleteBackward, + DeleteForward, + DeleteRange, + DeleteWord, + EnterStart, + EnterMiddle, + EnterEnd, + SoftEnter, + Backspace, + Delete, + Tab, + ShiftTab, + TabMiddle, + Paste, + PasteLine, + Bold, + Italic, + Font, + FontSize, + Color, + Highlight, + Clear, + Center, + Bullets, + Numbering, + Tag, + Check, + ClickCheck, + MoveOutline, + ResizeOutline, + CreateOutline, + EmptyOutline, + ArrowPastEnd, + PlaceImage, + RemoveImage, + Date, + Undo, + Redo, +} + +const KINDS: [Kind; 42] = [ + Kind::Type, + Kind::TypeEnd, + Kind::Compose, + Kind::ComposeCancel, + Kind::DeleteBackward, + Kind::DeleteForward, + Kind::DeleteRange, + Kind::DeleteWord, + Kind::EnterStart, + Kind::EnterMiddle, + Kind::EnterEnd, + Kind::SoftEnter, + Kind::Backspace, + Kind::Delete, + Kind::Tab, + Kind::ShiftTab, + Kind::TabMiddle, + Kind::Paste, + Kind::PasteLine, + Kind::Bold, + Kind::Italic, + Kind::Font, + Kind::FontSize, + Kind::Color, + Kind::Highlight, + Kind::Clear, + Kind::Center, + Kind::Bullets, + Kind::Numbering, + Kind::Tag, + Kind::Check, + Kind::ClickCheck, + Kind::MoveOutline, + Kind::ResizeOutline, + Kind::CreateOutline, + Kind::EmptyOutline, + Kind::ArrowPastEnd, + Kind::PlaceImage, + Kind::RemoveImage, + Kind::Date, + Kind::Undo, + Kind::Redo, +]; + +struct Random(u64); + +impl Random { + fn below(&mut self, bound: usize) -> usize { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + (self.0 % bound.max(1) as u64) as usize + } +} + +/// A random caret in the active outline: the paragraph and its UTF-16 length. +fn caret(editor: &CanvasEditor, random: &mut Random) -> (usize, u32) { + let count = editor.active_outline().document().paragraphs().count(); + let paragraph = random.below(count); + let text = editor + .active_outline() + .document() + .paragraphs() + .nth(paragraph) + .unwrap(); + (paragraph, text.text().encode_utf16().count() as u32) +} + +/// A scalar boundary of the paragraph near its middle, or a random one. +fn inside(editor: &CanvasEditor, paragraph: usize, random: &mut Random) -> u32 { + let text = editor + .active_outline() + .document() + .paragraphs() + .nth(paragraph) + .unwrap(); + let chars: Vec = text.text().char_indices().map(|(at, _)| at).collect(); + let at = chars + .get(random.below(chars.len() + 1)) + .copied() + .unwrap_or(text.text().len()); + text.utf16_offset(at).unwrap() +} + +/// Performs `kind` somewhere on the page; false when the editor declines it. +fn perform( + editor: &mut CanvasEditor, + engine: &mut TextEngine, + kind: Kind, + random: &mut Random, +) -> bool { + let outlines: Vec = editor.outlines().iter().map(|outline| outline.id).collect(); + if !matches!( + kind, + Kind::Undo + | Kind::Redo + | Kind::PlaceImage + | Kind::RemoveImage + | Kind::CreateOutline + | Kind::Date + ) { + if outlines.is_empty() { + return false; + } + let outline = outlines[random.below(outlines.len())]; + if editor.focus_outline(outline).is_err() { + return false; + } + } + let at = |paragraph, offset| [TextPosition { paragraph, offset }; 2].into(); + let (paragraph, length) = if outlines.is_empty() { + (0, 0) + } else { + caret(editor, random) + }; + let middle = if outlines.is_empty() { + 0 + } else { + inside(editor, paragraph, random) + }; + let place = |editor: &mut CanvasEditor, offset| editor.select(at(paragraph, offset)).is_ok(); + let range = |editor: &mut CanvasEditor, random: &mut Random| { + let (last, _) = caret(editor, random); + let (first, last) = (paragraph.min(last), paragraph.max(last)); + let start = inside(editor, first, random); + let end = inside(editor, last, random); + let (start, end) = if first == last { + (start.min(end), start.max(end)) + } else { + (start, end) + }; + editor + .select( + [ + TextPosition { + paragraph: first, + offset: start, + }, + TextPosition { + paragraph: last, + offset: end, + }, + ] + .into(), + ) + .is_ok() + }; + let format = + |editor: &mut CanvasEditor, engine: &mut TextEngine, random: &mut Random, command| { + range(editor, random) && editor.format(engine, command).is_ok() + }; + match kind { + Kind::Type => place(editor, middle) && editor.insert(engine, "ab").is_ok(), + Kind::TypeEnd => place(editor, length) && editor.commit_text(engine, "z".into()).is_ok(), + Kind::Compose => { + place(editor, middle) + && editor.compose(engine, "k".into(), 1..1).is_ok() + && editor.compose(engine, "ka".into(), 2..2).is_ok() + && editor.commit_text(engine, "か".into()).is_ok() + } + Kind::ComposeCancel => { + place(editor, middle) + && editor.compose(engine, "k".into(), 1..1).is_ok() + && editor.cancel_composition(engine).is_ok() + } + Kind::DeleteBackward => place(editor, middle) && editor.delete(engine, true).is_ok(), + Kind::DeleteForward => place(editor, middle) && editor.delete(engine, false).is_ok(), + Kind::DeleteRange => range(editor, random) && editor.delete(engine, true).is_ok(), + Kind::DeleteWord => { + place(editor, middle) && editor.delete_to(engine, Movement::WordLeft).is_ok() + } + Kind::EnterStart => place(editor, 0) && editor.enter(engine, false).is_ok(), + Kind::EnterMiddle => place(editor, middle) && editor.enter(engine, false).is_ok(), + Kind::EnterEnd => place(editor, length) && editor.enter(engine, false).is_ok(), + Kind::SoftEnter => place(editor, middle) && editor.enter(engine, true).is_ok(), + Kind::Backspace => place(editor, 0) && editor.delete(engine, true).is_ok(), + Kind::Delete => place(editor, length) && editor.delete(engine, false).is_ok(), + Kind::Tab => place(editor, 0) && editor.tab(engine, false).is_ok(), + Kind::ShiftTab => place(editor, 0) && editor.tab(engine, true).is_ok(), + Kind::TabMiddle => place(editor, middle) && editor.tab(engine, false).is_ok(), + Kind::Paste => { + range(editor, random) && editor.paste(engine, "Line one\r\nLine two", 1033).is_ok() + } + Kind::PasteLine => place(editor, middle) && editor.paste(engine, "pasted", 1033).is_ok(), + Kind::Bold => format(editor, engine, random, Formatting::Toggle(Toggle::Bold)), + Kind::Italic => format(editor, engine, random, Formatting::Toggle(Toggle::Italic)), + Kind::Font => format(editor, engine, random, Formatting::Font("Georgia".into())), + Kind::FontSize => format(editor, engine, random, Formatting::FontSize(14.0)), + Kind::Color => format(editor, engine, random, Formatting::Color(Some(0x0000_00ff))), + Kind::Highlight => format( + editor, + engine, + random, + Formatting::Highlight(Some(0x0000_ffff)), + ), + Kind::Clear => format(editor, engine, random, Formatting::Clear), + Kind::Center => format(editor, engine, random, Formatting::Align(Alignment::Center)), + Kind::Bullets => format(editor, engine, random, Formatting::Bullets), + Kind::Numbering => format(editor, engine, random, Formatting::Numbering), + Kind::Tag => { + let tag = [ + NoteTag::ToDo, + NoteTag::Important, + NoteTag::Question, + NoteTag::RememberForLater, + NoteTag::Definition, + NoteTag::Highlight, + NoteTag::Contact, + NoteTag::Address, + NoteTag::PhoneNumber, + ][random.below(9)]; + format(editor, engine, random, Formatting::Tag(tag)) + } + Kind::Check => format(editor, engine, random, Formatting::Check), + Kind::ClickCheck => { + let outline = editor.active_outline(); + let id = outline.document().text_nodes().nth(paragraph).unwrap().id; + let outline = outline.id; + editor.click_check(engine, outline, id).is_ok() + } + Kind::MoveOutline => { + let outline = editor.active_outline(); + let [x, y] = outline.origin(); + let id = outline.id; + editor.move_outline(id, [x + 36.0, y + 18.0]).is_ok() + } + Kind::ResizeOutline => { + let width = editor.active_outline().wrap_width(); + editor.resize(engine, width + 72.0).is_ok() + } + Kind::CreateOutline => { + editor + .create_outline( + engine, + [400.0, 600.0 + 20.0 * random.below(10) as f32], + 300.0, + ) + .is_ok() + && editor.insert(engine, "Fresh").is_ok() + } + Kind::EmptyOutline => editor.select_all().is_ok() && editor.delete(engine, true).is_ok(), + Kind::ArrowPastEnd => { + editor + .move_selection(engine, Movement::DocumentEnd, false) + .is_ok() + && editor.move_selection(engine, Movement::Down, false).is_ok() + && editor.move_selection(engine, Movement::Down, false).is_ok() + && editor.insert(engine, "Below").is_ok() + } + Kind::PlaceImage | Kind::RemoveImage => { + let images: Vec = editor + .object_layouts() + .map(|(id, _)| id) + .filter(|id| editor.image_placement(*id).is_some()) + .collect(); + if images.is_empty() { + return false; + } + let id = images[random.below(images.len())]; + if kind == Kind::RemoveImage { + return editor.remove_image(engine, id).is_ok(); + } + let (origin, size) = editor.image_placement(id).unwrap(); + editor + .place_image( + engine, + id, + [origin[0] + 18.0, origin[1] + 9.0], + [size[0] * 1.5, size[1] * 1.5], + ) + .is_ok() + } + Kind::Date => { + let Some(date) = editor.date() else { + return false; + }; + let timestamp = date.timestamp() + 864_000_000_000 * (1 + random.below(30) as u64); + editor + .change_date( + engine, + timestamp, + ["Friday, July 04, 2025".into(), "9:45 AM".into()], + ) + .unwrap_or(false) + } + Kind::Undo => editor.undo(engine).unwrap_or(false), + Kind::Redo => editor.redo(engine).unwrap_or(false), + } +} + +/// How a step's ops compared with its page lowered whole. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +enum Outcome { + /// The ops store the editor's page, or what that page lowered whole stores. + Same, + /// Both refuse the edit. + Refused, + /// The ops store; the page lowered whole cannot reach the editor's page from what is + /// stored. + OpsOnly, + /// The paths part where the section already stored the page differently from the + /// editor's model, as the writers normalize some values. + Normalized, + Differs(String), +} + +/// The section the editor's ops are applied to, sealed after each step. +struct Stored<'a> { + section: Section<'a>, + space: ExGuid, + at: u64, +} + +impl Stored<'_> { + /// Stores the editor's step as its ops and, unless they store the editor's page, + /// compares with that page lowered whole onto the section as it was before the step. + fn step(&mut self, editor: &mut CanvasEditor, model: &Page) -> Outcome { + self.at += 10_000_000; + let image = self.section.image(); + let reads_as = |stored: Page, model: &Page| { + Page { + title: model.title.clone(), + ..stored + } == *model + }; + let normalized = !reads_as(self.section.page(self.space).unwrap(), model); + let ops = editor.take_ops().map_err(|error| error.to_string()); + let edit = ops.clone().map(|ops| Edit { + at: self.at, + ops: ops + .into_iter() + .map(|op| Op::Page { + space: self.space, + op, + }) + .collect(), + }); + let applied = edit.and_then(|edit| { + if edit.ops.is_empty() { + return Ok(()); + } + self.section + .apply("Sweep", &edit) + .map_err(|error| error.to_string()) + }); + let edited = editor.page().unwrap(); + if applied.is_ok() && reads_as(self.section.page(self.space).unwrap(), &edited) { + self.section.seal().unwrap(); + return Outcome::Same; + } + let arena = Arena::default(); + let mut whole = Section::open(&arena, image).unwrap(); + let lowered = onestore::op::lower_page(&whole.page(self.space).unwrap(), &edited) + .map_err(|error| error.to_string()) + .and_then(|ops| { + let edit = Edit { + at: self.at, + ops: ops + .iter() + .map(|op| Op::Page { + space: self.space, + op: op.clone(), + }) + .collect(), + }; + whole + .apply("Sweep", &edit) + .map_err(|error| error.to_string())?; + Ok(ops) + }); + let outcome = match (applied, &lowered) { + (Err(error), Err(whole)) => { + if std::env::var_os("OPS_SWEEP_DEBUG").is_some() { + eprintln!("BOTH {error} | {whole}"); + } + Outcome::Refused + } + (Ok(()), Ok(_)) => { + let (a, b) = ( + self.section.page(self.space).unwrap(), + whole.page(self.space).unwrap(), + ); + if a == b { + Outcome::Same + } else if normalized { + Outcome::Normalized + } else { + Outcome::Differs(difference(&a, &b)) + } + } + (Err(_), Ok(_)) if normalized => Outcome::Normalized, + (Err(error), Ok(_)) => Outcome::Differs(format!("ops refused: {error}")), + (Ok(()), Err(_)) => { + if std::env::var_os("OPS_SWEEP_DEBUG").is_some() { + let stored = self.section.page(self.space).unwrap(); + eprintln!("OPSONLY-MODEL {}", difference(&stored, &edited)); + } + Outcome::OpsOnly + } + }; + if !matches!(outcome, Outcome::Same | Outcome::Refused) + && std::env::var_os("OPS_SWEEP_DEBUG").is_some() + { + let show = |ops: &[onestore::op::PageOp]| { + for op in ops { + let op = format!("{op:?}"); + eprintln!(" {}", &op[..op.len().min(600)]); + } + }; + eprintln!("DIFFERS {outcome:?}\n ops:"); + if let Ok(ops) = &ops { + show(ops); + } + eprintln!(" whole:"); + if let Ok(ops) = &lowered { + show(ops); + } + } + self.section.seal().unwrap(); + outcome + } +} + +/// The first place two pages part: an object missing from one, then objects by identity, +/// then their order. +fn difference(a: &Page, b: &Page) -> String { + let objects = |page: &Page| -> BTreeMap { + page.objects + .iter() + .map(|object| (object.id(), format!("{object:?}"))) + .collect() + }; + let (x, y) = (objects(a), objects(b)); + if let Some(id) = x + .keys() + .chain(y.keys()) + .find(|id| x.contains_key(id) != y.contains_key(id)) + { + return format!("object {id} on one side only"); + } + for (id, x) in &x { + let y = &y[id]; + if x != y { + let at = x.bytes().zip(y.bytes()).take_while(|(p, q)| p == q).count(); + let from = at.saturating_sub(160); + return format!( + "object {id} differs\n one: …{}\n other: …{}", + &x[from..(at + 160).min(x.len())], + &y[from..(at + 160).min(y.len())] + ); + } + } + if a.objects + .iter() + .map(|o| o.id()) + .ne(b.objects.iter().map(|o| o.id())) + { + return "object order".into(); + } + if a.definitions != b.definitions { + return "definitions".into(); + } + "page fields".into() +} + +#[derive(Default)] +struct Tally { + outcomes: BTreeMap<(Kind, Outcome), usize>, + examples: BTreeMap, +} + +fn sweep( + path: &Path, + random: &mut Random, + engine: &mut TextEngine, + tally: &mut Tally, + sequences: usize, +) { + let image = std::fs::read(path).unwrap(); + let arena = Arena::default(); + let mut section = Section::open(&arena, image.clone()).unwrap(); + for (space, title, _) in section.pages().unwrap() { + let Ok(page) = section.page(space) else { + continue; + }; + if CanvasEditor::from_page(page.clone(), engine).is_err() { + continue; + } + let plans = KINDS + .iter() + .map(|kind| vec![*kind]) + .chain(KINDS.iter().map(|kind| vec![*kind, Kind::Undo, Kind::Redo])) + .chain((0..sequences).map(|_| { + (0..3 + random.below(8)) + .map(|_| KINDS[random.below(KINDS.len())]) + .collect() + })) + .collect::>>(); + for plan in plans { + let arena = Arena::default(); + let mut stored = Stored { + section: Section::open(&arena, image.clone()).unwrap(), + space, + at: 133_000_000_000_000_000, + }; + let mut editor = CanvasEditor::from_page(page.clone(), engine).unwrap(); + for kind in &plan { + let model = editor.page().unwrap(); + if !perform(&mut editor, engine, *kind, random) { + continue; + } + if std::env::var_os("OPS_SWEEP_DEBUG").is_some() { + eprintln!("STEP {kind:?} of {plan:?} on {title:?}"); + } + let outcome = stored.step(&mut editor, &model); + *tally.outcomes.entry((*kind, outcome.clone())).or_default() += 1; + if let Outcome::Differs(reason) = &outcome { + tally + .examples + .entry(reason.clone()) + .or_insert_with(|| format!("{plan:?} on {title:?} in {}", path.display())); + } + if outcome != Outcome::Same { + break; + } + } + } + } +} + +#[test] +fn editor_ops_store_what_the_page_lowered_whole_stores() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let mut paths: Vec<_> = SECTIONS.iter().map(|path| root.join(path)).collect(); + if let Some(extra) = std::env::var_os("OPS_SWEEP_SECTIONS") { + paths.extend(std::env::split_paths(&extra)); + } + let seed = std::env::var("OPS_SWEEP_SEED").map_or(1, |seed| seed.parse().unwrap()); + let sequences = std::env::var("OPS_SWEEP_SEQUENCES").map_or(4, |n| n.parse().unwrap()); + let mut random = Random(seed); + let mut engine = TextEngine::default(); + let mut tally = Tally::default(); + for path in &paths { + let start = std::time::Instant::now(); + sweep(path, &mut random, &mut engine, &mut tally, sequences); + if std::env::var_os("OPS_SWEEP_DEBUG").is_some() { + eprintln!("TIME {:?} {}", start.elapsed(), path.display()); + } + } + let mut report = String::new(); + let mut per_kind: BTreeMap = BTreeMap::new(); + for ((kind, outcome), count) in &tally.outcomes { + let slot = match outcome { + Outcome::Same => 0, + Outcome::Refused => 1, + Outcome::OpsOnly => 2, + Outcome::Normalized => 3, + Outcome::Differs(_) => 4, + }; + per_kind.entry(*kind).or_default()[slot] += count; + } + let mut totals = [0; 5]; + for (kind, counts) in &per_kind { + let [same, refused, only, normalized, differs] = counts; + report += &format!( + "{kind:?}: {same} same, {refused} both refused, {only} only as ops, {normalized} after normalization, {differs} differ\n" + ); + for (total, count) in totals.iter_mut().zip(counts) { + *total += count; + } + } + report += &format!( + "all: {totals:?} (same, both refused, only as ops, after normalization, differ)\n" + ); + for ((kind, outcome), count) in &tally.outcomes { + if let Outcome::Differs(reason) = outcome { + report += &format!( + "{count} x {kind:?} {reason}\n e.g. {}\n", + tally.examples.get(reason).map_or("", String::as_str) + ); + } + } + println!("{report}"); + let differs = totals[4]; + assert_eq!(differs, 0, "{report}"); +} diff --git a/crates/canvas/tests/structural_roundtrip.rs b/crates/canvas/tests/structural_roundtrip.rs index 0eefdb43a1876d99934f5cfe19a057f66af64373..78c330f6aa39ac99ba2c9aa088c32204d206be2f 100644 --- a/crates/canvas/tests/structural_roundtrip.rs +++ b/crates/canvas/tests/structural_roundtrip.rs @@ -1,7 +1,7 @@ -//! Every structural edit the editor makes saves through the page writer and rereads as the -//! model it saved: random Enter, Backspace, Delete, Tab, Shift+Tab, list and paste edits, alone -//! and several to a save, on every outline of the probe section and of any sections named in -//! `CANVAS_SWEEP_SECTIONS` (`:`-separated paths). +//! Every structural edit the editor makes saves, through the page writer and as the ops the +//! editor recorded, and rereads as the model it saved: random Enter, Backspace, Delete, Tab, +//! Shift+Tab, list and paste edits, alone and several to a save, on every outline of the probe +//! section and of any sections named in `CANVAS_SWEEP_SECTIONS` (`:`-separated paths). use canvas::{ document::TextPosition, @@ -9,7 +9,7 @@ use canvas::{ layout::TextEngine, }; use onestore::{ - ExGuid, PreparedEdit, RevisionIndex, Store, + Arena, ExGuid, PreparedEdit, RevisionIndex, Section, Store, document::Document, page::{Outline, Page, PageObject, PageParagraph, ParagraphContent}, }; @@ -334,26 +334,43 @@ fn sweep(path: &Path, random: &mut Random, engine: &mut TextEngine, tally: &mut ) .unwrap(); } + // An untitled page's title follows its first line as the writer stores it. + let rereads = |reread: &Page| { + Page { + title: after.title.clone(), + ..reread.clone() + } == after + }; + let arena = Arena::default(); + let mut stored = Section::open(&arena, section.clone()).unwrap(); + let edit = onestore::op::Edit { + at: 133_000_000_000_000_000, + ops: editor + .take_ops() + .unwrap() + .into_iter() + .map(|op| onestore::op::Op::Page { space, op }) + .collect(), + }; + let ops = match stored.apply("Sweep", &edit) { + Err(error) => Some(format!("ops: {error}")), + Ok(()) => { + let reread = stored.page(space).unwrap(); + (!rereads(&reread)).then(|| format!("ops: {}", difference(&after, &reread))) + } + }; let failure = match PreparedEdit::page(§ion, space, &after, "Sweep") { Err(error) => Some(error.message.to_owned()), Ok(saved) => match pages(saved.as_bytes()) .into_iter() .find(|(id, _)| *id == space) { - // An untitled page's title follows its first line as the writer stores it. - Some((_, reread)) - if Page { - title: after.title.clone(), - ..reread.clone() - } == after => - { - None - } + Some((_, reread)) if rereads(&reread) => None, Some((_, reread)) => Some(difference(&after, &reread)), None => Some("the page does not reread".into()), }, }; - if let Some(failure) = failure { + for failure in failure.into_iter().chain(ops) { tally.failures.entry(failure).or_default().push(format!( "{plan:?} on {:?} in {}", page.title, diff --git a/crates/mobile/Cargo.toml b/crates/mobile/Cargo.toml index a0d7d497d249ed0e144ff821c4438acd73d25715..08a0a19a63b232cd1d1f282c8472cc8ab8bdefb7 100644 --- a/crates/mobile/Cargo.toml +++ b/crates/mobile/Cargo.toml @@ -15,3 +15,4 @@ onestore = { path = "../onestore" } wgpu.workspace = true parley.workspace = true pollster = "0.4" +serde_json = "1" diff --git a/crates/mobile/include/snowbound.h b/crates/mobile/include/snowbound.h index 83fc816a3b1376404b412a395fd90d21bf6650f8..726e45fbef99d7ab41948a9a78dfc511bea9acaa 100644 --- a/crates/mobile/include/snowbound.h +++ b/crates/mobile/include/snowbound.h @@ -22,6 +22,7 @@ void sb_view_content(View *view, float bounds[4]); void sb_view_set_transform(View *view, float zoom, float x, float y); bool sb_view_tap(View *view, float x, float y); bool sb_view_in_active_text(const View *view, float x, float y); +char *sb_view_edit(View *view); uint32_t sb_text_length(const View *view); char *sb_text(const View *view, uint32_t start, uint32_t end); diff --git a/crates/mobile/src/lib.rs b/crates/mobile/src/lib.rs index ff9b79c1e1462bd4b14fff642abc2cb85aac528e..da46f88aed76a848482c779d784621f072b549c8 100644 --- a/crates/mobile/src/lib.rs +++ b/crates/mobile/src/lib.rs @@ -14,7 +14,12 @@ use canvas::{ layout::TextEngine, }; use draw::edit::{Key, NamedKey, SelectionUnit}; -use onestore::{RevisionIndex, Store, document::Document, page::Page}; +use onestore::{ + ExGuid, RevisionIndex, Store, + document::Document, + op::{Edit, Op}, + page::Page, +}; use parley::{Affinity, BoundingBox}; use std::{ error::Error, @@ -40,7 +45,8 @@ const COLORS: TextColors = TextColors { }; pub struct Section { - pages: Vec, + /// Each page with the object space holding it. + pages: Vec<(ExGuid, Page)>, titles: Vec, } @@ -52,11 +58,11 @@ impl Section { let pages = document .pages()? .into_iter() - .map(|(space, id)| Page::from_revision(document.active(space)?, id)) - .collect::, _>>()?; + .map(|(space, id)| Ok((space, Page::from_revision(document.active(space)?, id)?))) + .collect::, onestore::Error>>()?; let titles = pages .iter() - .map(|page| CString::new(page.title.replace('\0', ""))) + .map(|(_, page)| CString::new(page.title.replace('\0', ""))) .collect::>()?; Ok(Self { pages, titles }) } @@ -74,6 +80,8 @@ impl Wake for Frame { pub struct View { page: PageView, + /// The object space of the page shown, which its edits name. + space: ExGuid, surface: wgpu::Surface<'static>, config: wgpu::SurfaceConfiguration, renderer: draw::Renderer, @@ -89,7 +97,12 @@ fn moved(response: Response) -> bool { } impl View { - fn new(layer: *mut c_void, page: Page, size: [f32; 2], scale: f32) -> Result { + fn new( + layer: *mut c_void, + (space, page): (ExGuid, Page), + size: [f32; 2], + scale: f32, + ) -> Result { let instance = wgpu::Instance::new(wgpu::InstanceDescriptor::new_without_display_handle()); // SAFETY: the host passes a live CAMetalLayer that outlives the view. let surface = unsafe { @@ -122,6 +135,7 @@ impl View { scale, Duration::from_millis(350), ), + space, surface, config, renderer, @@ -129,6 +143,27 @@ impl View { }) } + /// What the page took since the last call as one edit, or none when it took nothing. + fn edit(&mut self) -> Result> { + let ops = self.page.editor.take_ops()?; + if ops.is_empty() { + return Ok(None); + } + let unix = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)?; + Ok(Some(Edit { + // FILETIME: 100 ns ticks since 1601. + at: (unix.as_secs() + 11_644_473_600) * 10_000_000 + + u64::from(unix.subsec_nanos() / 100), + ops: ops + .into_iter() + .map(|op| Op::Page { + space: self.space, + op, + }) + .collect(), + })) + } + /// Device pixels per host point. fn display_scale(&self) -> f32 { self.page.viewport.scale / (self.page.zoom() * POINT) @@ -515,8 +550,19 @@ pub extern "C" fn sb_text(view: &View, start: u32, end: u32) -> *mut c_char { .map_or(std::ptr::null_mut(), CString::into_raw) } +/// The edit the page took since the last call, as `onestore::op::Edit` JSON for the +/// section to apply, freed with `sb_string_free`; null when it took none or cannot be +/// stored, in which case the page should be opened again. +#[unsafe(no_mangle)] +pub extern "C" fn sb_view_edit(view: &mut View) -> *mut c_char { + report(view.edit()) + .flatten() + .and_then(|edit| CString::new(serde_json::to_string(&edit).ok()?).ok()) + .map_or(std::ptr::null_mut(), CString::into_raw) +} + /// # Safety -/// `text` came from `sb_text` and is not used again. +/// `text` came from `sb_text` or `sb_view_edit` and is not used again. #[unsafe(no_mangle)] pub unsafe extern "C" fn sb_string_free(text: *mut c_char) { drop(unsafe { CString::from_raw(text) }); diff --git a/crates/notebook/README.md b/crates/notebook/README.md index 02f47cf807b27b685101f351a8ee6d8c1992fc9c..aca6350886a28468c7c89088df71beb923831914 100644 --- a/crates/notebook/README.md +++ b/crates/notebook/README.md @@ -38,6 +38,10 @@ assert_eq!(reopened.pending()?.last().map(|edit| edit.id), Some(id)); # } ``` +Page reads never wait for the sync thread: a rebase or reread of the queue runs on a +new thread while the current one keeps answering `page` and `pages` from the section as +it was, then hands its requests over; edits sent meanwhile apply to the rebuilt section. + Share one `Replica` between threads. Keep the cache on a local filesystem: the connection holds exclusive ownership between transactions, and a second open fails busy. No network wait occurs in a local edit. After a database error the section diff --git a/crates/notebook/examples/open_probe.rs b/crates/notebook/examples/open_probe.rs new file mode 100644 index 0000000000000000000000000000000000000000..d093914bf64d997dbad657b4246d415fcf8f7639 --- /dev/null +++ b/crates/notebook/examples/open_probe.rs @@ -0,0 +1,236 @@ +//! `open_probe SECTION | --paragraphs N`: times opening a section through a fresh and a +//! reopened cache up to its page list and first page, then how long page reads wait while +//! the sync thread rebases the queue onto a changed remote. + +use notebook::{Remote, Replica, session::Section}; +use onestore::{ + CommitError, ExGuid, Stamp, Transaction, + document::{Format, Layout}, + op::{Edit, Op, PageOp, SectionOp}, + page::{ + Outline, Page, PageObject, PageParagraph, Paragraph, ParagraphContent, TextObject, + text::new_id, + }, +}; +use std::{ + io, + time::{Duration, Instant}, +}; + +/// A section holding one page of `count` paragraphs. +fn generated(count: usize) -> Result, Box> { + let paragraph = |i: usize| PageParagraph { + id: new_id().unwrap(), + parent: None, + level: 1, + style: None, + format: Format::default(), + content: ParagraphContent::Text(TextObject { + id: new_id().unwrap(), + date_field: None, + text: Paragraph::new(format!("Paragraph {i} of the probe"), Format::default()), + tags: Vec::new(), + }), + lists: Vec::new(), + tags: Vec::new(), + media: Default::default(), + collapsed: false, + }; + let arena = onestore::Arena::default(); + let mut section = + onestore::Section::open(&arena, onestore::create_section("probe.one", "", "Probe")?)?; + let page = Page { + title: "Probe".into(), + identity: None, + created: None, + margin_origin: [0.0; 2], + objects: vec![PageObject::Outline(Outline { + id: new_id()?, + title: false, + min_width: None, + layout: Layout { + x: Some(36.0), + y: Some(86.4), + ..Layout::default() + }, + indents: Vec::new(), + paragraphs: (0..count).map(paragraph).collect(), + unsupported: Vec::new(), + })], + definitions: Default::default(), + }; + let creation = onestore::PageCreation::new(None, Some("Probe"), "Probe")?; + section.apply( + "Probe", + &Edit { + at: 133_000_000_000_000_000, + ops: vec![Op::Section(SectionOp::Import { creation, page })], + }, + )?; + section.seal()?; + Ok(section.image()) +} + +/// The first text of the page with the most outline paragraphs, and that page's space. +fn text(page: &Page) -> Option { + page.objects.iter().find_map(|object| match object { + PageObject::Outline(outline) => outline + .paragraphs + .iter() + .find_map(|p| p.text().map(|t| t.id)), + _ => None, + }) +} + +struct Memory(Vec); + +impl onestore::CommitIo for Memory { + fn read_at(&mut self, offset: u64, output: &mut [u8]) -> io::Result { + let offset = offset as usize; + let size = output.len().min(self.0.len().saturating_sub(offset)); + output[..size].copy_from_slice(&self.0[offset..offset + size]); + Ok(size) + } + fn write_at(&mut self, offset: u64, bytes: &[u8]) -> io::Result { + let offset = offset as usize; + self.0.resize(self.0.len().max(offset + bytes.len()), 0); + self.0[offset..offset + bytes.len()].copy_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl Remote for Memory { + fn read(&mut self) -> io::Result> { + Ok(self.0.clone()) + } + fn stamp(&mut self) -> io::Result> { + Ok(Stamp::of(&self.0).ok()) + } + fn publish(&mut self, transaction: &Transaction) -> Result<(), CommitError> { + transaction.commit(self) + } + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + onestore::confirm_snapshot(self, snapshot) + } +} + +fn main() -> Result<(), Box> { + let mut args = std::env::args().skip(1); + let source = match (args.next().ok_or("SECTION | --paragraphs N")?, args.next()) { + (flag, Some(count)) if flag == "--paragraphs" => generated(count.parse()?)?, + (path, _) => std::fs::read(path)?, + }; + let directory = tempfile::tempdir()?; + let file = directory.path().join("section.one"); + std::fs::write(&file, &source)?; + let cache = directory.path().join("cache"); + println!("section {} bytes", source.len()); + for label in ["fresh cache", "reopened cache"] { + let start = Instant::now(); + let section = Section::open(&file, &cache, || {})?; + let opened = start.elapsed(); + let pages = section.pages()?; + let listed = start.elapsed(); + let page = section.page(pages[0].0)?; + let read = start.elapsed(); + println!( + "{label}: open {opened:.2?}, page list ({} pages) at {listed:.2?}, first page ({} objects) at {read:.2?}", + pages.len(), + page.objects.len() + ); + section.close()?; + } + + // A local edit queued against a remote another writer changed elsewhere: the next + // step rebases, while this thread keeps reading the page list and a page. + let replica = std::sync::Arc::new(Replica::create( + directory.path().join("rebase.sqlite"), + &source, + )?); + let (space, ..) = replica + .pages()? + .into_iter() + .max_by_key(|(space, ..)| { + replica + .page(*space) + .map(|page| page.objects.len()) + .unwrap_or(0) + }) + .ok_or("a page")?; + let target = text(&replica.page(space)?).ok_or("a text")?; + let typed = |with: &str| Edit { + at: 133_000_000_000_000_000, + ops: vec![Op::Page { + space, + op: PageOp::Text { + text: target, + range: 0..0, + with: with.into(), + }, + }], + }; + replica.apply("Probe", typed("Local "))?; + let remote = { + let arena = onestore::Arena::default(); + let mut section = onestore::Section::open(&arena, source.clone())?; + let root = section.root(); + let other = section + .pages()? + .into_iter() + .map(|(space, ..)| space) + .find(|other| *other != space); + match other.and_then(|other| Some((other, text(§ion.page(other).ok()?)?))) { + Some((other, text)) => section.apply( + "Native", + &Edit { + at: 133_000_000_000_000_000, + ops: vec![Op::Page { + space: other, + op: PageOp::Text { + text, + range: 0..0, + with: "Remote ".into(), + }, + }], + }, + )?, + None => section.apply( + "Native", + &Edit { + at: 133_000_000_000_000_000, + ops: vec![Op::Section(SectionOp::Create(onestore::PageCreation::new( + None, + Some("Remote"), + "Native", + )?))], + }, + )?, + } + let _ = root; + section.seal()?; + section.image() + }; + let syncing = std::sync::Arc::clone(&replica); + let sync = std::thread::spawn(move || { + let start = Instant::now(); + let result = syncing.sync_once(&mut Memory(remote)); + (start.elapsed(), result.map(|synced| synced.edit.is_some())) + }); + let (mut slowest, mut reads) = (Duration::ZERO, 0); + while !sync.is_finished() { + let start = Instant::now(); + replica.pages()?; + replica.page(space)?; + slowest = slowest.max(start.elapsed()); + reads += 1; + } + let (rebase, published) = sync.join().map_err(|_| "the sync step panicked")?; + println!( + "rebase and publish {rebase:.2?} (published: {:?}); {reads} page-list + page reads meanwhile, slowest {slowest:.2?}", + published? + ); + Ok(()) +} diff --git a/crates/notebook/src/lib.rs b/crates/notebook/src/lib.rs index 15ed5979a9d09bc29ffcd15f6b9164fc52f14c76..75c9457dbe646828d4687291cc1c9576df7e6a4c 100644 --- a/crates/notebook/src/lib.rs +++ b/crates/notebook/src/lib.rs @@ -16,7 +16,6 @@ use std::{ io, path::Path, sync::{Arc, Mutex, MutexGuard, mpsc}, - thread::JoinHandle, time::Duration, }; @@ -97,8 +96,7 @@ pub enum Resolution { /// exclusive connection retains ownership between local transactions. pub struct Replica { connection: Arc>, - working: Option>, - thread: Option>, + section: Arc, synchronization: Mutex<()>, worker: Arc>>, /// The section's root object space, which names the document. @@ -182,11 +180,10 @@ impl Replica { fn start(connection: Connection) -> Result { let connection = Arc::new(Mutex::new(connection)); let worker = Arc::new(Mutex::new(std::sync::Weak::new())); - let (sender, thread, root) = working::spawn(Arc::clone(&connection), Arc::clone(&worker))?; + let (section, root) = working::spawn(Arc::clone(&connection), Arc::clone(&worker))?; Ok(Self { connection, - working: Some(sender), - thread: Some(thread), + section, synchronization: Mutex::new(()), worker, root, @@ -199,10 +196,7 @@ impl Replica { /// Hands `request` to the section thread. fn send(&self, request: working::Request) -> Result<()> { - self.working - .as_ref() - .and_then(|working| working.send(request).ok()) - .ok_or_else(|| io::Error::other("The section thread stopped").into()) + self.section.send(request) } /// Asks the section thread and waits for its answer. @@ -257,7 +251,8 @@ impl Replica { /// The section image the queued edits leave, the unsealed ones sealed as one more /// revision whose identities differ per call: O(section), for tests and diagnostics. pub fn snapshot(&self) -> Result> { - self.ask(|reply| working::Request::Image { reply }) + self.ask(|reply| working::Request::Flush { reply })?; + working::image(&*self.lock()?) } /// The section file's identity, which internal links name as `section-id`. @@ -279,10 +274,7 @@ impl Replica { impl Drop for Replica { fn drop(&mut self) { - drop(self.working.take()); - if let Some(thread) = self.thread.take() { - let _ = thread.join(); - } + self.section.stop(); } } diff --git a/crates/notebook/src/merge.rs b/crates/notebook/src/merge.rs index 1d22b0fd051443e7c5906bb349faad6e909ce4d3..c29a016bd5a9f1af16ffb12c22e80e5860801cd5 100644 --- a/crates/notebook/src/merge.rs +++ b/crates/notebook/src/merge.rs @@ -532,6 +532,11 @@ impl Diff { PageOp::Link { text, .. } | PageOp::Equation { text, .. } => { require(self.region(*text)?.is_none())?; } + PageOp::Date { fields, .. } => { + for (text, _) in fields.iter() { + require(self.region(*text)?.is_none())?; + } + } PageOp::Split { text, at, right, .. } => { diff --git a/crates/notebook/src/working.rs b/crates/notebook/src/working.rs index 6a2411809aec020e3182584f7a1697f89079641e..ec937105e69c6ba4a4d9fba33a3483b46e8ed16d 100644 --- a/crates/notebook/src/working.rs +++ b/crates/notebook/src/working.rs @@ -44,8 +44,9 @@ pub(crate) enum Request { Pages { reply: Reply>, }, - Image { - reply: Reply>, + /// Answers once the edits before it are written. + Flush { + reply: Reply<()>, }, /// Seals the open batch, when no sealed batch waits for publication, recording the /// attempt to publish it. @@ -62,8 +63,15 @@ pub(crate) enum Request { Reopen { reply: Reply<()>, }, + /// From the thread that rebuilt the section: hand it the requests. + Handover(mpsc::SyncSender), + /// From the thread that rebuilt nothing: carry on with the section as it is. + Resume, } +/// The request channel and the requests held while the section was rebuilt. +type Takeover = (mpsc::Receiver, VecDeque); + /// A sealed batch: the transaction publishing it, or none when its edits changed nothing. pub(crate) struct Sealed { pub batch: i64, @@ -71,7 +79,7 @@ pub(crate) struct Sealed { } /// A conflict's resolution: the batch it holds and every later edit drop their ops on -/// `space`; `Mine` first rewrites the rebased page to the local one (or to `page`). +/// `space`; `Mine` first rewrites the rebased page to `page`, the local one unless given. pub(crate) struct Resolve { pub first: u64, pub space: ExGuid, @@ -98,34 +106,88 @@ impl Request { Self::Save { reply, .. } => reply(Err(error())), Self::Page { reply, .. } => reply(Err(error())), Self::Pages { reply } => reply(Err(error())), - Self::Image { reply } => reply(Err(error())), + Self::Flush { reply } => reply(Err(error())), Self::Seal { reply } => reply(Err(error())), Self::Rebase { reply, .. } => reply(Err(error())), Self::Reopen { reply } => reply(Err(error())), + Self::Handover(_) | Self::Resume => {} } } } type Worker = Arc>>; +/// The section thread as the replica sees it. Rereading the section after the queue was +/// replaced (a rebase, a released attempt) happens on a new thread while the current one +/// keeps answering page reads from the section as it was; the new thread then takes the +/// requests over, so reads never wait for a rebuild. +pub(crate) struct Thread { + connection: Arc>, + worker: Worker, + /// Taken when the replica drops, which ends every section thread. + sender: Mutex>>, + threads: Mutex>>, +} + +impl Thread { + pub(crate) fn send(&self, request: Request) -> Result<()> { + self.sender + .lock() + .ok() + .and_then(|sender| sender.as_ref()?.send(request).ok()) + .ok_or_else(|| io::Error::other("The section thread stopped").into()) + } + + /// Ends the section threads and waits for them, so the cache is released. + pub(crate) fn stop(&self) { + if let Ok(mut sender) = self.sender.lock() { + sender.take(); + } + while let Some(thread) = self + .threads + .lock() + .ok() + .and_then(|mut threads| threads.pop()) + { + let _ = thread.join(); + } + } + + fn start(self: &Arc, run: impl FnOnce(Arc) + Send + 'static) -> Result<()> { + let shared = Arc::clone(self); + let thread = thread::Builder::new() + .name("onestore-section".into()) + .spawn(move || run(shared))?; + self.threads + .lock() + .map_err(|_| io::Error::other("The section thread panicked"))? + .push(thread); + Ok(()) + } +} + /// Starts the section thread once the queue opens. pub(crate) fn spawn( connection: Arc>, worker: Worker, -) -> Result<(mpsc::Sender, thread::JoinHandle<()>, ExGuid)> { - let (sender, receiver) = mpsc::channel(); +) -> Result<(Arc, ExGuid)> { + let (sender, requests) = mpsc::channel(); + let shared = Arc::new(Thread { + connection, + worker, + sender: Mutex::new(Some(sender)), + threads: Mutex::new(Vec::new()), + }); let (ready, opened) = mpsc::sync_channel(1); - let thread = thread::Builder::new() - .name("onestore-section".into()) - .spawn(move || run(&connection, &worker, &receiver, ready))?; + shared.start(move |shared| run(shared, requests, VecDeque::new(), Some(ready)))?; match opened.recv() { - Ok(Ok(root)) => Ok((sender, thread, root)), + Ok(Ok(root)) => Ok((shared, root)), Ok(Err(error)) => { - let _ = thread.join(); + shared.stop(); Err(error) } Err(_) => { - let _ = thread.join(); + shared.stop(); Err(io::Error::other("The section thread panicked").into()) } } @@ -134,28 +196,24 @@ pub(crate) fn spawn( enum Next { Stop, Reopen, + Handover(mpsc::SyncSender, VecDeque), } fn run( - connection: &Mutex, - worker: &Worker, - requests: &mpsc::Receiver, - ready: mpsc::SyncSender>, + shared: Arc, + requests: mpsc::Receiver, + mut backlog: VecDeque, + mut ready: Option>>, ) { - let mut ready = Some(ready); - let mut backlog = VecDeque::new(); loop { let arena = Arena::default(); - let working = match Working::open(&arena, connection) { + let working = match Working::open(&arena, &shared.connection) { Ok(working) => working, Err(error) => { - let message = error.to_string(); if let Some(ready) = ready.take() { let _ = ready.send(Err(error)); - return; - } - for request in backlog.drain(..).chain(requests.iter()) { - request.fail(&message); + } else { + fail(error, backlog, requests); } return; } @@ -163,9 +221,75 @@ fn run( if let Some(ready) = ready.take() { let _ = ready.send(Ok(working.section.root())); } - match working.serve(connection, worker, requests, &mut backlog) { + match working.serve(&shared, &requests, &mut backlog) { Next::Stop => return, Next::Reopen => {} + Next::Handover(to, held) => { + let _ = to.send((requests, held)); + return; + } + } + } +} + +/// Answers every request with `error` until the replica drops. +fn fail(error: crate::Error, backlog: VecDeque, requests: mpsc::Receiver) { + let message = error.to_string(); + for request in backlog.into_iter().chain(requests.iter()) { + request.fail(&message); + } +} + +/// What a rebuilding thread does before it rereads the section. +enum Job { + Rebase { + image: Option>, + resolve: Option, + reply: Reply, + }, + Reopen { + reply: Reply<()>, + }, +} + +/// Runs `job`, rereads the section and takes the requests over from the thread that +/// started it; a rebase that found a conflict changed nothing that thread serves. +fn build(shared: Arc, job: Job, signal: mpsc::Sender) { + let answer: Box = match job { + Job::Reopen { reply } => Box::new(move || reply(Ok(()))), + Job::Rebase { + image, + resolve, + reply, + } => match rebase(&shared.connection, image, resolve) { + Ok(applied @ Rebased::Applied { .. }) => Box::new(move || reply(Ok(applied))), + other => { + reply(other); + let _ = signal.send(Request::Resume); + return; + } + }, + }; + let arena = Arena::default(); + let opened = Working::open(&arena, &shared.connection); + let (to, from) = mpsc::sync_channel(1); + if signal.send(Request::Handover(to)).is_err() { + return; + } + drop(signal); + answer(); + let Ok((requests, mut backlog)) = from.recv() else { + return; + }; + let working = match opened { + Ok(working) => working, + Err(error) => return fail(error, backlog, requests), + }; + match working.serve(&shared, &requests, &mut backlog) { + Next::Stop => {} + Next::Reopen => run(shared, requests, backlog, None), + Next::Handover(to, held) => { + let _ = to.send((requests, held)); } } } @@ -214,11 +338,14 @@ impl<'a> Working<'a> { fn serve( mut self, - connection: &Mutex, - worker: &Worker, + shared: &Arc, requests: &mpsc::Receiver, backlog: &mut VecDeque, ) -> Next { + let (connection, worker) = (&*shared.connection, &shared.worker); + // While another thread rebuilds the section, reads answer from this one and every + // other request waits for the rebuilt section. + let mut held: Option> = None; loop { let first = match backlog.pop_front() { Some(request) => request, @@ -231,7 +358,28 @@ impl<'a> Working<'a> { burst.extend(backlog.drain(..)); burst.extend(requests.try_iter()); let mut accepted = Vec::new(); - while let Some(request) = burst.pop_front() { + // Requests that write the queue beyond the burst's edits run after its reads. + let mut later = VecDeque::new(); + while let Some(request) = burst.pop_front().or_else(|| later.pop_front()) { + if let Some(waiting) = &mut held { + match request { + Request::Page { space, reply } => { + reply(self.section.page(space).map_err(Into::into)) + } + Request::Pages { reply } => reply(self.section.pages().map_err(Into::into)), + Request::Handover(to) => { + let mut waiting = held.take().unwrap_or_default(); + waiting.extend(burst.drain(..).chain(later.drain(..))); + return Next::Handover(to, waiting); + } + Request::Resume => { + let waiting = held.take().unwrap_or_default(); + burst = waiting.into_iter().chain(burst.drain(..)).collect(); + } + other => waiting.push_back(other), + } + continue; + } let reopen = match request { Request::Apply { author, @@ -326,16 +474,20 @@ impl<'a> Working<'a> { reply(self.section.pages().map_err(Into::into)); false } - Request::Image { reply } => { - if self.flush(connection, worker, &mut accepted) { - reply(lock(connection).and_then(|connection| image(&connection))); - false + request @ (Request::Flush { .. } | Request::Seal { .. }) + if !burst.is_empty() => + { + later.push_back(request); + false + } + Request::Flush { reply } => { + let written = self.flush(connection, worker, &mut accepted); + reply(if written { + Ok(()) } else { - reply(Err( - io::Error::other("The queue could not be written").into() - )); - true - } + Err(io::Error::other("The queue could not be written").into()) + }); + !written } Request::Seal { reply } => { if !self.flush(connection, worker, &mut accepted) { @@ -352,29 +504,61 @@ impl<'a> Working<'a> { } Request::Rebase { image, - resolve, + mut resolve, reply, } => { - if !self.flush(connection, worker, &mut accepted) { - reply(Err( - io::Error::other("The queue could not be written").into() - )); - true - } else { - let rebased = self.rebase(connection, image, resolve); - let reopen = !matches!(rebased, Ok(Rebased::Conflict { .. })); - reply(rebased); - reopen + let written = self.flush(connection, worker, &mut accepted); + // The page a `Mine` resolution keeps is read here, before the + // rebuild; the root space lists pages and holds none. + let root = self.section.root(); + let kept = match &mut resolve { + Some(Resolve { + keep: Resolution::Mine, + page: page @ None, + space, + .. + }) if *space != root => { + self.section.page(*space).map(|local| *page = Some(local)) + } + _ => Ok(()), + }; + match (written, kept) { + (true, Ok(())) => { + held = self + .rebuild( + shared, + Job::Rebase { + image, + resolve, + reply, + }, + ) + .then(VecDeque::new); + false + } + (_, Err(error)) => { + reply(Err(error.into())); + !written + } + (false, _) => { + reply(Err( + io::Error::other("The queue could not be written").into() + )); + true + } } } Request::Reopen { reply } => { self.flush(connection, worker, &mut accepted); - reply(Ok(())); - true + held = self + .rebuild(shared, Job::Reopen { reply }) + .then(VecDeque::new); + false } + Request::Handover(_) | Request::Resume => false, }; if reopen { - backlog.extend(burst); + backlog.extend(burst.drain(..).chain(later.drain(..))); return Next::Reopen; } } @@ -384,6 +568,30 @@ impl<'a> Working<'a> { } } + /// Starts a thread that runs `job` and rereads the section; false when the replica is + /// stopping, which answers the job's request. + fn rebuild(&self, shared: &Arc, job: Job) -> bool { + let signal = shared + .sender + .lock() + .ok() + .and_then(|sender| sender.as_ref().cloned()); + let Some(signal) = signal else { + match job { + Job::Rebase { reply, .. } => { + reply(Err(io::Error::other("The section thread stopped").into())) + } + Job::Reopen { reply } => { + reply(Err(io::Error::other("The section thread stopped").into())) + } + } + return false; + }; + shared + .start(move |shared| build(shared, job, signal)) + .is_ok() + } + /// Applies an edit, keeping it to be written with the burst. A refused edit is answered /// here; `Err(true)` when it failed part way and the section must be reread. fn accept( @@ -536,150 +744,145 @@ impl<'a> Working<'a> { self.touched.clear(); Ok(Some(Sealed { batch, transaction })) } +} - /// Replays every queued edit on `image`, keeping it as the base when all apply. - fn rebase( - &mut self, - connection: &Mutex, - image: Option>, - resolve: Option, - ) -> Result { - let (base_image, image, edits) = { - let connection = lock(connection)?; - if connection.query_row( - "SELECT EXISTS(SELECT 1 FROM batches WHERE attempted=1)", - [], - |row| row.get::<_, bool>(0), - )? { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "An uncertain attempt is never rebased", - ) - .into()); +/// Replays every queued edit on `image`, keeping it as the base when all apply. +fn rebase( + connection: &Mutex, + image: Option>, + resolve: Option, +) -> Result { + let (base_image, image, edits) = { + let connection = lock(connection)?; + if connection.query_row( + "SELECT EXISTS(SELECT 1 FROM batches WHERE attempted=1)", + [], + |row| row.get::<_, bool>(0), + )? { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "An uncertain attempt is never rebased", + ) + .into()); + } + let base_image = base::read(&connection, base::Image::Base)?.ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "The cache has no base image") + })?; + let image = match image { + Some(image) => image, + None => { + base::read(&connection, base::Image::Remote)?.unwrap_or_else(|| base_image.clone()) } - let base_image = base::read(&connection, base::Image::Base)?.ok_or_else(|| { - io::Error::new(io::ErrorKind::InvalidData, "The cache has no base image") - })?; - let image = match image { - Some(image) => image, - None => base::read(&connection, base::Image::Remote)? - .unwrap_or_else(|| base_image.clone()), - }; - (base_image, image, queue::load(&connection, None)?) }; - // The page the local edits leave, which `Mine` rewrites the remote page to; the - // root space lists pages and holds none. - let target = match &resolve { - Some(Resolve { - keep: Resolution::Mine, - page: None, - space, - .. - }) if *space != self.section.root() => Some(self.section.page(*space)?), - Some(Resolve { page, .. }) => page.clone(), - None => None, - }; - let old_arena = Arena::default(); - let mut old = Section::open(&old_arena, base_image.clone())?; - let before: BTreeMap = old.revisions().collect(); - let mut after: BTreeMap; - // Pages the remote already holds as the local edits leave them: their ops are done. - let mut converged = BTreeSet::new(); - let outcome = loop { - let arena = Arena::default(); - let mut new = Section::open(&arena, image.clone())?; - if old.root() != new.root() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Remote snapshot belongs to another document", - ) - .into()); - } - after = new.revisions().collect(); - let outcome = merge::rebase( - &mut old, - &mut new, - &edits, - &converged, - resolve.as_ref().map(|resolve| merge::Resolving { - first: resolve.first, - space: resolve.space, - mine: resolve.keep == Resolution::Mine, - target: target.as_ref(), - }), + (base_image, image, queue::load(&connection, None)?) + }; + let old_arena = Arena::default(); + let mut old = Section::open(&old_arena, base_image.clone())?; + let before: BTreeMap = old.revisions().collect(); + let mut after: BTreeMap; + // Pages the remote already holds as the local edits leave them: their ops are done. + let mut converged = BTreeSet::new(); + let outcome = loop { + let arena = Arena::default(); + let mut new = Section::open(&arena, image.clone())?; + if old.root() != new.root() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Remote snapshot belongs to another document", + ) + .into()); + } + after = new.revisions().collect(); + let outcome = merge::rebase( + &mut old, + &mut new, + &edits, + &converged, + resolve.as_ref().map(|resolve| merge::Resolving { + first: resolve.first, + space: resolve.space, + mine: resolve.keep == Resolution::Mine, + target: resolve.page.as_ref(), + }), + )?; + if let merge::Outcome::Conflict { space, .. } = &outcome + && !converged.contains(space) + && let Some(local) = local_page(connection, *space)? + && Section::open(&Arena::default(), image.clone())? + .page(*space) + .is_ok_and(|remote| remote == local) + { + converged.insert(*space); + continue; + } + break outcome; + }; + let mut connection = lock(connection)?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let rebased = match outcome { + merge::Outcome::Conflict { id, space, kind } => { + transaction.execute("UPDATE batches SET conflict=NULL, space=NULL", [])?; + transaction.execute( + "UPDATE batches SET conflict=?1, space=?2 WHERE id=(SELECT batch FROM edits WHERE id=?3)", + params![kind as i64, space.to_string(), signed(id)?], )?; - if let merge::Outcome::Conflict { space, .. } = &outcome - && !converged.contains(space) - && let Ok(local) = self.section.page(*space) - && Section::open(&Arena::default(), image.clone())? - .page(*space) - .is_ok_and(|remote| remote == local) - { - converged.insert(*space); - continue; - } - break outcome; - }; - let mut connection = lock(connection)?; - let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; - let rebased = match outcome { - merge::Outcome::Conflict { id, space, kind } => { - transaction.execute("UPDATE batches SET conflict=NULL, space=NULL", [])?; - transaction.execute( - "UPDATE batches SET conflict=?1, space=?2 WHERE id=(SELECT batch FROM edits WHERE id=?3)", - params![kind as i64, space.to_string(), signed(id)?], - )?; - if image == base_image { - base::clear(&transaction, base::Image::Remote)?; - } else { - base::write(&transaction, base::Image::Remote, &image)?; - } - // A conflict is reported under its batch's newest edit. - let id = crate::unsigned(transaction.query_row( - "SELECT max(id) FROM edits WHERE batch=(SELECT batch FROM edits WHERE id=?1)", - [signed(id)?], - |row| row.get(0), - )?)?; - Rebased::Conflict { id, space, kind } - } - merge::Outcome::Applied(rewritten) => { - base::write(&transaction, base::Image::Base, &image)?; + if image == base_image { base::clear(&transaction, base::Image::Remote)?; - transaction.execute("INSERT INTO batches DEFAULT VALUES", [])?; - let batch = transaction.last_insert_rowid(); - transaction.execute("UPDATE edits SET batch=?1", [batch])?; - transaction.execute("DELETE FROM batches WHERE id<>?1", [batch])?; - for (id, edit) in rewritten { - queue::rewrite(&transaction, id, &edit)?; - } - if transaction - .query_row("SELECT count(*) FROM edits", [], |row| row.get::<_, i64>(0))? - == 0 - { - transaction.execute("DELETE FROM batches", [])?; - } - queue::collect(&transaction)?; - let mut changed: BTreeSet = before - .iter() - .filter(|(space, rid)| after.get(space) != Some(rid)) - .map(|(space, _)| *space) - .chain( - after - .keys() - .filter(|space| !before.contains_key(space)) - .copied(), - ) - .collect(); - changed.extend(resolve.as_ref().map(|resolve| resolve.space)); - changed.extend(converged); - Rebased::Applied { - changed: changed.into_iter().collect(), - } + } else { + base::write(&transaction, base::Image::Remote, &image)?; } - }; - transaction.commit()?; - Ok(rebased) - } + // A conflict is reported under its batch's newest edit. + let id = crate::unsigned(transaction.query_row( + "SELECT max(id) FROM edits WHERE batch=(SELECT batch FROM edits WHERE id=?1)", + [signed(id)?], + |row| row.get(0), + )?)?; + Rebased::Conflict { id, space, kind } + } + merge::Outcome::Applied(rewritten) => { + base::write(&transaction, base::Image::Base, &image)?; + base::clear(&transaction, base::Image::Remote)?; + transaction.execute("INSERT INTO batches DEFAULT VALUES", [])?; + let batch = transaction.last_insert_rowid(); + transaction.execute("UPDATE edits SET batch=?1", [batch])?; + transaction.execute("DELETE FROM batches WHERE id<>?1", [batch])?; + for (id, edit) in rewritten { + queue::rewrite(&transaction, id, &edit)?; + } + if transaction + .query_row("SELECT count(*) FROM edits", [], |row| row.get::<_, i64>(0))? + == 0 + { + transaction.execute("DELETE FROM batches", [])?; + } + queue::collect(&transaction)?; + let mut changed: BTreeSet = before + .iter() + .filter(|(space, rid)| after.get(space) != Some(rid)) + .map(|(space, _)| *space) + .chain( + after + .keys() + .filter(|space| !before.contains_key(space)) + .copied(), + ) + .collect(); + changed.extend(resolve.as_ref().map(|resolve| resolve.space)); + changed.extend(converged); + Rebased::Applied { + changed: changed.into_iter().collect(), + } + } + }; + transaction.commit()?; + Ok(rebased) +} + +/// A page as the queued edits leave it, read from the cache: O(section). +fn local_page(connection: &Mutex, space: ExGuid) -> Result> { + let arena = Arena::default(); + let (section, ..) = replay(&arena, &*lock(connection)?)?; + Ok(section.page(space).ok()) } enum Stale { diff --git a/crates/notebook/tests/session.rs b/crates/notebook/tests/session.rs index 0592efe480ceee679ec5bdf9b4918d37e6cc7e73..1f3800abc52273f37826eb2687a9ee06b76c0fdb 100644 --- a/crates/notebook/tests/session.rs +++ b/crates/notebook/tests/session.rs @@ -115,7 +115,12 @@ fn a_section_opens_through_its_replica_and_a_save_reaches_the_file_and_survives_ }; assert_same(section.page(space).unwrap(), &after); published(§ion, id); - assert!(notified.load(Ordering::SeqCst) > 0); + // The worker reports the step after writing its receipt. + let deadline = Instant::now() + Duration::from_secs(20); + while notified.load(Ordering::SeqCst) == 0 { + assert!(Instant::now() < deadline, "the host was not woken"); + std::thread::sleep(Duration::from_millis(10)); + } assert_same(stored_page(&file, space), &after); assert!(matches!( section.status(id).unwrap(), diff --git a/crates/onestore/src/edit.rs b/crates/onestore/src/edit.rs index 61be7cc4df0f8faf574b328f7131f39bed057223..4e97286956a4152a579493ae92dca906b56380a4 100644 --- a/crates/onestore/src/edit.rs +++ b/crates/onestore/src/edit.rs @@ -53,6 +53,58 @@ pub(crate) fn text_changes( object: ExGuid, range: Range, replacement: &str, +) -> Result { + rewrite(active, object, range, replacement, false) +} + +/// The title's date or time field showing `text`, as OneNote 2010 rewrites both when a +/// page's date changes: the field's element takes the change time as its creation time. +pub(crate) fn field_changes( + active: &ActivePage<'_>, + object: ExGuid, + text: &str, +) -> Result { + let length = crate::page::text_of(&active.view, &active.parents, object)? + .text() + .encode_utf16() + .count(); + let mut changes = rewrite(active, object, 0..length as u32, text, true)?; + let [element] = active + .parents + .get(&object) + .map(Vec::as_slice) + .unwrap_or_default() + else { + return Err(Error { + offset: 0, + message: "Select a date field belonging to one element", + }); + }; + // OneNote 2010 dates the field's element and outline and leaves the title and page + // above them as they were. + let outline = active + .parents + .get(element) + .and_then(|parents| parents.first()) + .copied(); + changes.retain(|id, _| *id == object || id == element || Some(*id) == outline); + let modified = crate::create::current_timestamps()?.0.to_le_bytes(); + let element = match changes.entry(*element) { + std::collections::btree_map::Entry::Occupied(entry) => entry.into_mut(), + std::collections::btree_map::Entry::Vacant(entry) => entry.insert( + crate::write::PropertyObject::from_object(&active.live.revision.objects[element])?, + ), + }; + element.set(&[(0x14001d09, &modified)])?; + Ok(changes) +} + +fn rewrite( + active: &ActivePage<'_>, + object: ExGuid, + range: Range, + replacement: &str, + field: bool, ) -> Result { let invalid = |message| Error { offset: 0, message }; let raw = &active.live.revision; @@ -60,7 +112,12 @@ pub(crate) fn text_changes( return Err(invalid("Object is not reachable in the active revision")); } let revision = &active.view; - let parents = active.editable_parents(object)?; + // Generated fields are read-only to ordinary edits; only a date change rewrites them. + let parents = if field { + &active.parents + } else { + active.editable_parents(object)? + }; let node = revision .nodes .get(&object) @@ -74,10 +131,12 @@ pub(crate) fn text_changes( else { return Err(invalid("Select a rich-text object")); }; - if *boilerplate { - return Err(invalid( - "Generated title fields cannot be edited as ordinary text", - )); + if *boilerplate != field { + return Err(invalid(if field { + "Select the title's date or time" + } else { + "Generated title fields cannot be edited as ordinary text" + })); } let selected = runs .iter() @@ -103,7 +162,7 @@ pub(crate) fn text_changes( let properties = PropertySets::parse(blob)?; if properties.sets[0] .iter() - .any(|p| p.id == 0x88001cde || p.id == 0x24003458) + .any(|p| (p.id == 0x88001cde && !field) || p.id == 0x24003458) { return Err(invalid( "This text object is read-only or contains associated run data", diff --git a/crates/onestore/src/op/apply.rs b/crates/onestore/src/op/apply.rs index dbbe31ef9e3761bc88b6fa968da0291194759b60..d82c371bd17b36a2e86298e7d4b2981d446390cc 100644 --- a/crates/onestore/src/op/apply.rs +++ b/crates/onestore/src/op/apply.rs @@ -326,6 +326,28 @@ impl<'a> Writer<'_, 'a> { self.target(*text)?; self.write(|page| content::equation_changes(page, *text, math)) } + PageOp::Date { created, fields } => { + for (text, shown) in fields { + self.target(*text)?; + if shown.contains(['\0', '\n', '\r', '\u{fffc}']) { + return Err(OpError::Unsupported("Use a date as one line of text").into()); + } + self.write(|page| crate::edit::field_changes(page, *text, shown))?; + } + self.write(|page| { + let metadata = page + .view + .roots + .get(&2) + .filter(|id| matches!(page.view.nodes[id].kind, Kind::Metadata { .. })) + .ok_or_else(|| invalid("The page has no metadata to date"))?; + let mut object = crate::write::PropertyObject::from_object( + &page.live.revision.objects[metadata], + )?; + object.set(&[(0x18001c65, &created.to_le_bytes())])?; + Ok(BTreeMap::from([(*metadata, object)])) + }) + } PageOp::Insert { container, before, diff --git a/crates/onestore/src/op/lower.rs b/crates/onestore/src/op/lower.rs index ccdb388c0eef145e51c55b07c561ea0334be69e9..249ac6809d2244831cbf5feb195cf5cc73494ed8 100644 --- a/crates/onestore/src/op/lower.rs +++ b/crates/onestore/src/op/lower.rs @@ -236,8 +236,8 @@ pub(crate) fn validate( old: &View<'_>, new: &View<'_>, ) -> Result<(), Error> { - if after.created != before.created || after.margin_origin != before.margin_origin { - return Err(invalid("Page creation time and margins cannot be edited")); + if after.margin_origin != before.margin_origin { + return Err(invalid("Page margins cannot be edited")); } for (id, definition) in &after.definitions { if before @@ -806,6 +806,7 @@ impl Lowering { self.delete(old, new, &consumed)?; self.levels(new)?; self.equations(new)?; + self.date(before, after)?; self.text(new)?; self.styles(after, new)?; self.lists(after, new)?; @@ -832,7 +833,10 @@ impl Lowering { for paragraph in document { if old.paragraphs.contains_key(¶graph.id) { left = paragraph.text().is_some().then_some(paragraph.id); - } else if let Some(left) = left { + } else if let Some(left) = left + // A new cell arrives with its paragraphs. + && old.children.contains_key(&root(new, paragraph.id)) + { let container = new.container[¶graph.id]; let at = new.children[&container] .iter() @@ -1542,6 +1546,24 @@ impl Lowering { Ok(()) } + /// The page's date, with every date and time field of the title showing it. + fn date(&mut self, before: &Page, after: &Page) -> Result<(), Error> { + let fields: Vec<(ExGuid, String)> = model::date_fields(after) + .into_iter() + .map(|text| (text.id, text.text.text().to_owned())) + .collect(); + let shown = fields.iter().all(|(id, shown)| { + model::paragraph_text(&self.current, *id).is_some_and(|text| text.text.text() == shown) + }); + if before.created == after.created && shown { + return Ok(()); + } + let created = after + .created + .ok_or_else(|| invalid("A page date cannot be removed"))?; + self.emit(PageOp::Date { created, fields }) + } + fn text(&mut self, new: &View<'_>) -> Result<(), Error> { for (id, paragraph) in &new.paragraphs { let Some(text) = paragraph.text() else { diff --git a/crates/onestore/src/op/mod.rs b/crates/onestore/src/op/mod.rs index ce0ac5b4bfda3a172ad7e216be2c8c1c3b79dca2..32eefd577750ba90de58345808041dca1fb91851 100644 --- a/crates/onestore/src/op/mod.rs +++ b/crates/onestore/src/op/mod.rs @@ -22,6 +22,9 @@ pub(crate) mod table; mod tests; pub use lower::{lower, lower_page}; +/// `op` applied to `page`, the model of what the section holds: the page `Section::apply` +/// leaves, as reading it back shows it. +pub use model::apply as predict; /// Property identifiers with their encoded values. pub(crate) type Values = Vec<(u32, Vec)>; @@ -71,6 +74,12 @@ pub enum PageOp { }, /// Rewrites an equation's linear text and spans whole. Equation { text: ExGuid, math: Paragraph }, + /// The page's date: its creation time, FILETIME, and the text each of the title's date + /// and time fields shows for it, as OneNote 2010 writes both when the date changes. + Date { + created: u64, + fields: Vec<(ExGuid, String)>, + }, /// Inserts paragraphs before a direct child of `container`, or last. Children follow /// their parents in `paragraphs`; levels are absolute. Text keeps its spans' formats; /// lists, tags, styles and collapse state are set by their own ops. diff --git a/crates/onestore/src/op/model.rs b/crates/onestore/src/op/model.rs index c6e0a7b86edaffe4e27a7a1e5d6da04a485f03ec..474982b04a5f52b4a6e518f2b65fa24b3e8be954 100644 --- a/crates/onestore/src/op/model.rs +++ b/crates/onestore/src/op/model.rs @@ -203,6 +203,29 @@ fn text_holder(page: &Page, text: ExGuid) -> Option<(Path, usize)> { }) } +/// The text of the title's date and time fields, in order. +pub(crate) fn date_fields(page: &Page) -> Vec<&TextObject> { + page.objects + .iter() + .filter_map(|object| match object { + PageObject::Title(title) => { + let date = title.date?; + title.outlines.iter().find(|outline| outline.id == date) + } + _ => None, + }) + .flat_map(|outline| outline.paragraphs.iter().filter_map(PageParagraph::text)) + .collect() +} + +/// Text object `text` wherever the page holds it. +pub(crate) fn paragraph_text(page: &Page, text: ExGuid) -> Option<&TextObject> { + lists(page).into_iter().find_map(|(_, _, list)| { + list.iter() + .find_map(|paragraph| paragraph.text().filter(|object| object.id == text)) + }) +} + fn text_mut(page: &mut Page, text: ExGuid) -> Result<&mut TextObject, Error> { let (path, index) = text_holder(page, text).ok_or_else(|| invalid("Text is not on the page"))?; Ok(list_at(page, &path)[index].text_mut().unwrap()) @@ -547,7 +570,7 @@ fn automatic(page: &mut Page) { } /// Applies `op` to `page` as reading the page back after `Section::apply` would show it. -pub(crate) fn apply(page: &mut Page, op: &PageOp) -> Result<(), Error> { +pub fn apply(page: &mut Page, op: &PageOp) -> Result<(), Error> { interpret(page, op)?; // A page lists the definitions its paragraphs reference. let mut referenced = BTreeSet::new(); @@ -569,6 +592,17 @@ pub(crate) fn apply(page: &mut Page, op: &PageOp) -> Result<(), Error> { fn interpret(page: &mut Page, op: &PageOp) -> Result<(), Error> { match op { + PageOp::Date { created, fields } => { + for (text, shown) in fields { + if !date_fields(page).iter().any(|field| field.id == *text) { + return Err(invalid("Select the title's date or time")); + } + let object = text_mut(page, *text)?; + let format = object.text.spans()[0].format.clone(); + object.text = Paragraph::new(shown.clone(), format); + } + page.created = Some(*created); + } PageOp::Text { text, range, with } => { let object = text_mut(page, *text)?; object.text = replace_text(&object.text, range.clone(), with)?; diff --git a/crates/onestore/src/op/tests.rs b/crates/onestore/src/op/tests.rs index 0928cf607cc8a7963b7a502fb866267280495ecf..4501ae30eb9dadd83ce5bbb2c3e23f7027025eab 100644 --- a/crates/onestore/src/op/tests.rs +++ b/crates/onestore/src/op/tests.rs @@ -2148,3 +2148,101 @@ fn op_gate_candidate() { std::fs::write(directory.join("expected-count.txt"), count.to_string()).unwrap(); println!("{} imported pages, {edits} lowered edits, {count} pages", imported.len()); } + +/// OneNote 2010's own date change (`corpus/page-date/native`, COM `UpdatePageContent` +/// with a new `dateTime`): the page's second page, "Changed date", took it; "Kept date" +/// did not. +const DATED: &[u8] = include_bytes!("../../../../corpus/page-date/native/notebook/pictures.one"); + +/// The objects of `space`'s active revision that differ from its previous one, by jcid. +fn changed_jcids(image: &[u8], space: ExGuid) -> Vec { + let store = Store::parse(image).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let active = index.active(space).unwrap(); + let previous = index.spaces[&space].revisions[&active].dependency.unwrap(); + let (old, new) = ( + index.resolve(space, previous).unwrap(), + index.resolve(space, active).unwrap(), + ); + let mut jcids: Vec = new + .objects + .iter() + .filter(|(id, object)| { + old.objects.get(id).is_none_or(|old| { + format!("{:?}", old.data) != format!("{:?}", object.data) + }) + }) + .map(|(_, object)| object.jcid) + .collect(); + jcids.sort(); + jcids +} + +/// The title's date and time fields of `page`, by text object. +fn date_fields(page: &Page) -> Vec<(ExGuid, String)> { + model::date_fields(page) + .into_iter() + .map(|text| (text.id, text.text.text().to_owned())) + .collect() +} + +/// FILETIME of 2025-07-04 16:45 UTC, which a Pacific clock shows as 9:45 AM. +const DATED_AT: u64 = 133_961_211_000_000_000; + +/// The `Date` op writes what OneNote writes for a new page date: the metadata's creation +/// time, the date and time field text, and the modification times above them. +#[test] +fn a_page_date_stores_what_onenote_stores() { + let spaces = pages(DATED); + let title = |space: &ExGuid| read(DATED, *space).title; + let kept = *spaces.iter().find(|space| title(space) == "Kept date").unwrap(); + let changed = *spaces.iter().find(|space| title(space) == "Changed date").unwrap(); + let native = changed_jcids(DATED, changed); + assert_eq!(native, [0x20030, 0x6000c, 0x6000d, 0x6000d, 0x6000e, 0x6000e]); + + let arena = Arena::default(); + let mut section = Section::open(&arena, DATED.to_vec()).unwrap(); + let before = section.page(kept).unwrap(); + // The date field precedes the time field. + let fields: Vec<(ExGuid, String)> = date_fields(&before) + .into_iter() + .zip(["Friday, July 04, 2025", "9:45 AM"]) + .map(|((id, _), shown)| (id, shown.to_owned())) + .collect(); + assert_eq!(fields.len(), 2); + let op = PageOp::Date { + created: DATED_AT, + fields: fields.clone(), + }; + let mut predicted = before.clone(); + model::apply(&mut predicted, &op).unwrap(); + section + .apply("Author", &Edit { at: AT, ops: vec![Op::Page { space: kept, op }] }) + .unwrap(); + section.seal().unwrap(); + let image = section.image(); + let stored = read(&image, kept); + assert_eq!(stored.created, Some(DATED_AT)); + assert_eq!(date_fields(&stored), fields); + assert_eq!(stored, predicted); + assert_eq!(changed_jcids(&image, kept), native); + let store = Store::parse(&image).unwrap(); + RevisionIndex::parse(&store).unwrap().validate_current().unwrap(); + + // The whole-page lowering reaches the same page through the same op. + let mut after = before.clone(); + after.created = Some(DATED_AT); + model::apply(&mut after, &PageOp::Date { created: DATED_AT, fields }).unwrap(); + assert!(matches!(&lower_page(&before, &after).unwrap()[..], [PageOp::Date { .. }])); + + if let Some(directory) = std::env::var_os("ONESTORE_PAGE_DATE_EXPORT") { + let directory = std::path::Path::new(&directory); + std::fs::create_dir_all(directory).unwrap(); + std::fs::write(directory.join("pictures.one"), &image).unwrap(); + std::fs::copy( + concat!(env!("CARGO_MANIFEST_DIR"), "/../../corpus/page-date/native/notebook/Open Notebook.onetoc2"), + directory.join("Open Notebook.onetoc2"), + ) + .unwrap(); + } +} diff --git a/crates/onestore/src/page/write.rs b/crates/onestore/src/page/write.rs index 7a1190d904d25ea88ade3b8888b213a679f24e1e..edc1011d82cc4fe8892b6ae0a304ad74c389f8fc 100644 --- a/crates/onestore/src/page/write.rs +++ b/crates/onestore/src/page/write.rs @@ -142,6 +142,9 @@ impl<'a> Lowering<'a> { let old = View::new(before, false)?; let new = View::new(after, false)?; validate(before, after, &old, &new)?; + if after.created != before.created { + return Err(invalid("Page creation time cannot be edited")); + } for id in new.outlines.keys().chain(new.paragraphs.keys()) { if !old.outlines.contains_key(id) && !old.paragraphs.contains_key(id) diff --git a/crates/snowbound/src/macos.rs b/crates/snowbound/src/macos.rs index 929ef7ede236b3c0d095f721b68670465b1d7169..f35307d3c5254383e11aae8099e1b51856e0b0b8 100644 --- a/crates/snowbound/src/macos.rs +++ b/crates/snowbound/src/macos.rs @@ -458,6 +458,20 @@ pub fn discard_changes() -> bool { } } +/// Tells the user an edit could not be stored and where their text went. +pub fn change_not_saved() { + let mtm = MainThreadMarker::new().expect("Window events run on the main thread"); + // The alert and its strings stay on AppKit's main thread for the modal call. + unsafe { + let alert = NSAlert::new(mtm); + alert.setMessageText(&NSString::from_str("Change not saved")); + alert.setInformativeText(&NSString::from_str( + "The page now shows what was last saved. Your text is on the clipboard to paste back.", + )); + alert.runModal(); + } +} + /// End AppKit preedit after a canvas action commits or cancels the composition. pub fn clear_marked_text(window: &Window) { MainThreadMarker::new().expect("Text input belongs to the main thread"); diff --git a/crates/snowbound/src/main.rs b/crates/snowbound/src/main.rs index 3a1aecb10e056809f7a4104083624c40539da393..161ee6900abc621c0cca0d6cf641c6f9dc2c0239 100644 --- a/crates/snowbound/src/main.rs +++ b/crates/snowbound/src/main.rs @@ -198,14 +198,13 @@ impl Library { } } -/// The open section and the stored model the editor's page was loaded from. +/// The open section and the page the editor shows. struct Session { section: notebook::session::Section, tab: usize, /// Spaces, titles and outline levels in section order. pages: Vec<(ExGuid, String, u32)>, space: ExGuid, - before: Page, status: &'static str, /// The editor's page has an unreviewed conflict with another machine's change. conflict: bool, @@ -222,21 +221,20 @@ impl Session { fn refresh_conflict(&mut self) -> Result<(), Box> { self.conflict = self .section - .conflicts()? - .iter() - .any(|(edit, _)| edit.space == self.space); + .conflict()? + .is_some_and(|conflict| conflict.space == self.space); Ok(()) } } -/// A section or page read on a loader thread. +/// What a loader thread opened: a section, or another page of the open one. enum Loaded { - Section(Session), - Page { space: ExGuid, page: Page }, + Section(Box), + Page(ExGuid), } -/// A loader thread's request number and what it read. -type Read = (u64, Result); +/// A loader thread's request number, what it opened and the page it read. +type Read = (u64, Result<(Loaded, Page), String>); /// The newest page read, laid out and waiting for the pictures it shows first, so it /// never appears without them. @@ -289,6 +287,9 @@ struct State { changed: bool, /// Only the view scrolled or zoomed during this frame, to be announced after it. moved: bool, + /// The stored page changed elsewhere while marked text was pending; it is shown once + /// the composition ends. + stale: bool, /// Whether the page last heard it had keyboard focus. page_focused: bool, /// The pointer in logical pixels, for window drags from the strip. @@ -425,8 +426,8 @@ impl State { .ok_or_else(|| format!("No page titled {title:?} in {}", file.display()))? .0; library = Some(Arc::new(opened)); - let opened = read_session(section, 0, Some(space))?; - let (scene, editor) = PageScene::from_page(opened.before.clone(), &mut engine)?; + let (opened, page) = read_session(section, 0, Some(space))?; + let (scene, editor) = PageScene::from_page(page, &mut engine)?; session = Some(opened); (editor, Some((scene, [0.0; 2]))) } @@ -444,8 +445,8 @@ impl State { }; let section = opened.open(0, proxy.clone())?; library = Some(Arc::new(opened)); - let opened = read_session(section, 0, None)?; - let (scene, editor) = PageScene::from_page(opened.before.clone(), &mut engine)?; + let (opened, page) = read_session(section, 0, None)?; + let (scene, editor) = PageScene::from_page(page, &mut engine)?; session = Some(opened); (editor, Some((scene, [0.0; 2]))) } @@ -511,6 +512,7 @@ impl State { commands: Vec::new(), changed: false, moved: false, + stale: false, page_focused: true, pointer: [0.0; 2], strip_press: None, @@ -587,6 +589,9 @@ impl State { self.after_move()?; } self.moved = false; + if self.stale { + self.refresh()?; + } // A covered window shows nothing, so animations wait for it to be uncovered. if follow && !self.occluded { self.window.request_redraw(); @@ -1483,18 +1488,14 @@ impl State { let proxy = self.proxy.clone(); self.load(move || { let section = library.open(tab, proxy)?; - Ok(Loaded::Section(read_session(section, tab, None)?)) + let (session, page) = read_session(section, tab, None)?; + Ok((Loaded::Section(Box::new(session)), page)) }); } Command::OpenPage(space) => { let session = self.session.as_ref().ok_or("No section is open")?; let replica = Arc::clone(session.section.replica()); - self.load(move || { - Ok(Loaded::Page { - space, - page: replica.page(space)?, - }) - }); + self.load(move || Ok((Loaded::Page(space), replica.page(space)?))); } Command::Resolve { keep_mine } => self.resolve_conflict(keep_mine)?, Command::Page(Request::EditDate(field)) => self.edit_date(field)?, @@ -1512,7 +1513,10 @@ impl State { /// Runs `read` on a thread of its own; `open_loaded` shows what it read unless a newer /// read was asked for meanwhile. - fn load(&mut self, read: impl FnOnce() -> Result> + Send + 'static) { + fn load( + &mut self, + read: impl FnOnce() -> Result<(Loaded, Page), Box> + Send + 'static, + ) { self.loading += 1; let (id, sender, redraw) = (self.loading, self.loads.0.clone(), self.redraw.clone()); std::thread::spawn(move || { @@ -1528,12 +1532,8 @@ impl State { if id != self.loading { continue; } - let loaded = loaded?; - let page = match &loaded { - Loaded::Section(session) => &session.before, - Loaded::Page { page, .. } => page, - }; - let (scene, editor) = PageScene::from_page(page.clone(), &mut self.view.engine)?; + let (loaded, page) = loaded?; + let (scene, editor) = PageScene::from_page(page, &mut self.view.engine)?; self.opening = Some(Opening { loaded, scene: (scene, [0.0; 2]), @@ -1556,15 +1556,16 @@ impl State { self.opening = Some(opening); return Ok(()); } + // The page shown so far keeps what was typed while the next one loaded. + self.persist()?; match opening.loaded { Loaded::Section(session) => { - self.session = Some(session); + self.session = Some(*session); self.filter.clear(); } - Loaded::Page { space, page } => { + Loaded::Page(space) => { let session = self.session.as_mut().ok_or("No section is open")?; session.space = space; - session.before = page; session.refresh_conflict()?; } } @@ -1663,104 +1664,131 @@ impl State { Ok(()) } - /// Saves the edited page to the section's replica; a page changed underneath the - /// editor is reloaded in place of the edit. + /// Hands the section the ops the editor recorded since the last call, as one edit; the + /// section thread stores it. fn persist(&mut self) -> Result<(), Box> { + let ops = match self.view.editor.take_ops() { + Ok(ops) => ops, + Err(error) => return self.refused(&error.to_string()), + }; let Some(session) = &mut self.session else { return Ok(()); }; - let after = self.view.editor.page()?; - if after == session.before { + if ops.is_empty() { return Ok(()); } - if let Some(entry) = session - .pages - .iter_mut() - .find(|(space, ..)| *space == session.space) - { - entry.1.clone_from(&after.title); - } - // Queued saves chain: each one's `after` is the next one's `before`. - let before = std::mem::replace(&mut session.before, after.clone()); - session - .section - .queue_save(session.space, before, after, "snowbound")?; + let space = session.space; + session.section.apply( + "snowbound", + onestore::op::Edit { + at: filetime(), + ops: ops + .into_iter() + .map(|op| onestore::op::Op::Page { space, op }) + .collect(), + }, + )?; + session.status = "Saving"; Ok(()) } - /// Replaces the editor with the page currently stored in the section. - fn reload(&mut self) -> Result<(), Box> { - let Some(session) = &mut self.session else { + /// Shows the page as stored after a change made elsewhere, in place: the caret, + /// selection, scroll and drawn pictures stay. Waits for marked text to end. + fn refresh(&mut self) -> Result<(), Box> { + if self.view.editor.marked_range().is_some() { + self.stale = true; + return Ok(()); + } + self.stale = false; + self.persist()?; + let Some(session) = &self.session else { return Ok(()); }; let page = session.section.page(session.space)?; - let (scene, editor) = PageScene::from_page(page.clone(), &mut self.view.engine)?; - session.before = page; - self.view.replace(editor, Some((scene, [0.0; 2]))); + let response = self.view.refresh(page)?; + self.respond(response); self.title(); - self.update_accessibility()?; self.window.request_redraw(); Ok(()) } - /// Reviews the oldest conflict on this page: `keep_mine` publishes the editor's page - /// over the remote change, otherwise the remote page replaces the editor's. + /// An edit the section could not store: the page returns to what is stored, and the + /// text the editor showed goes on the clipboard so nothing typed is lost unannounced. + fn refused(&mut self, error: &str) -> Result<(), Box> { + eprintln!("Saving failed: {error}"); + self.clipboard.set_text(page_text(&self.view.editor))?; + // What the editor recorded since builds on the refused edit. + let _ = self.view.editor.take_ops(); + self.view.editor.cancel_composition(&mut self.view.engine)?; + macos::clear_marked_text(&self.window); + if let Some(session) = &mut self.session { + session.status = "Not saving"; + } + self.refresh()?; + macos::change_not_saved(); + Ok(()) + } + + /// Ends the conflict on this page: `keep_mine` publishes the local page over the remote + /// change, otherwise the remote page replaces it. fn resolve_conflict(&mut self, keep_mine: bool) -> Result<(), Box> { + self.persist()?; let Some(session) = &mut self.session else { return Ok(()); }; - let Some((edit, _)) = session + let Some(conflict) = session .section - .conflicts()? - .into_iter() - .find(|(edit, _)| edit.space == session.space) + .conflict()? + .filter(|conflict| conflict.space == session.space) else { return Ok(()); }; - let reviewed = if keep_mine { - self.view.editor.page()? + let resolution = if keep_mine { + notebook::Resolution::Mine } else { - session.section.remote_page(session.space)? + notebook::Resolution::Theirs }; - session.section.review(edit.id, &reviewed)?; + session.section.resolve(conflict.id, resolution)?; session.status = "Saving"; session.refresh_conflict()?; if keep_mine { - session.before = session.section.page(session.space)?; self.window.request_redraw(); Ok(()) } else { - self.reload() + self.refresh() } } - /// Applies what the synchronization thread reported since the last poll. + /// Applies what the section reported since the last poll. fn synced(&mut self) -> Result<(), Box> { let Some(session) = &mut self.session else { return Ok(()); }; let shown = (session.status, session.conflict); - let mut refreshed = false; - let mut stale = false; + let mut listed = false; + let mut changed = false; + let mut rejected = None; let mut conflicted = session.conflict; - for (space, outcome) in session.section.saved() { - match outcome { - Ok(notebook::session::Save::Queued(_)) => session.status = "Saving", - Ok(notebook::session::Save::Unchanged) => {} - Ok(notebook::session::Save::Stale) => stale |= space == session.space, - Err(error) => { - eprintln!("Saving failed: {error}"); - session.status = "Not saving"; - } - } - } for event in session.section.events() { use notebook::session::Event; session.status = match event { Event::Refreshed => { - refreshed = true; + listed = true; continue; } + Event::Changed(spaces) => { + listed = true; + changed |= spaces.contains(&session.space); + continue; + } + Event::Rejected { spaces, error } => { + if spaces.contains(&session.space) { + rejected = Some(error); + } else { + eprintln!("Saving failed: {error}"); + } + "Not saving" + } Event::Attempt { status: notebook::EditStatus::Published { .. }, .. @@ -1780,15 +1808,15 @@ impl State { } }; } - // Reading the queue waits on the save thread, so only a conflict shown or reported - // sends the frame thread there. + // Reading the queue waits on the section thread, so only a conflict shown or + // reported sends the frame thread there. if conflicted { session.refresh_conflict()?; } - if refreshed || shown != (session.status, session.conflict) { + if listed || changed || shown != (session.status, session.conflict) { self.window.request_redraw(); } - if refreshed { + if listed { session.pages = session.section.pages()?; if !session .pages @@ -1797,13 +1825,13 @@ impl State { { let first = session.pages.first().ok_or("The section has no pages")?.0; self.commands.push(Command::OpenPage(first)); - } else if !session.section.saving() - && session.section.page(session.space)? != session.before - { - self.reload()?; + return Ok(()); } - } else if stale { - self.reload()?; + } + if let Some(error) = rejected { + self.refused(&error)?; + } else if changed { + self.refresh()?; } Ok(()) } @@ -2206,29 +2234,65 @@ fn tabs(notebook: ¬ebook::session::Notebook) -> Vec { .collect() } -/// The session for `section` showing `space`, or its first page. +/// The session for `section` showing `space`, or its first page, and that page. fn read_session( section: notebook::session::Section, tab: usize, space: Option, -) -> Result> { +) -> Result<(Session, Page), Box> { let pages = section.pages()?; let space = match space { Some(space) => space, None => pages.first().ok_or("The section has no pages")?.0, }; - let before = section.page(space)?; + let page = section.page(space)?; let mut session = Session { section, tab, pages, space, - before, status: "", conflict: false, }; session.refresh_conflict()?; - Ok(session) + Ok((session, page)) +} + +/// The text the editor shows, outline by outline, without hidden field codes. +fn page_text(editor: &CanvasEditor) -> String { + editor + .visible_outlines() + .chain(editor.caret_outline()) + .map(|outline| { + outline + .document() + .paragraphs() + .map(|paragraph| { + let mut start = 0; + paragraph + .spans() + .iter() + .filter_map(|span| { + let run = ¶graph.text()[start..span.end]; + start = span.end; + (span.format.hidden != Some(true)).then_some(run) + }) + .collect::() + }) + .collect::>() + .join("\n") + }) + .filter(|text| !text.trim().is_empty()) + .collect::>() + .join("\n\n") +} + +/// FILETIME now: when an edit happened, which its modification times record. +fn filetime() -> u64 { + let unix = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default(); + (unix.as_secs() + 11_644_473_600) * 10_000_000 + u64::from(unix.subsec_nanos() / 100) } impl App { @@ -2716,3 +2780,199 @@ fn main() -> Result<(), Box> { event_loop.run_app(&mut app)?; app.startup_error.map_or(Ok(()), Err) } + +#[cfg(test)] +mod tests { + use super::*; + use canvas::document::TextPosition; + + /// The ops an editing session records reach the section through `apply` and read back + /// as the editor's page. + #[test] + fn recorded_ops_reach_the_section() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let directory = std::env::temp_dir().join(format!("snowbound-ops-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&directory); + std::fs::create_dir_all(directory.join("cache")).unwrap(); + let file = directory.join("section.one"); + std::fs::copy( + root.join("corpus/paragraph-edit/before/notebook/synthetic.one"), + &file, + ) + .unwrap(); + let section = + notebook::session::Section::open(&file, directory.join("cache"), || {}).unwrap(); + let mut engine = TextEngine::default(); + for (space, ..) in section.pages().unwrap().into_iter().take(3) { + let mut editor = + CanvasEditor::from_page(section.page(space).unwrap(), &mut engine).unwrap(); + let outline = editor.outlines().iter().find(|o| !o.title).unwrap().id; + editor.focus_outline(outline).unwrap(); + let at = |paragraph, offset| [TextPosition { paragraph, offset }; 2].into(); + editor.select(at(0, 0)).unwrap(); + editor.insert(&mut engine, "Typed ").unwrap(); + editor.enter(&mut engine, false).unwrap(); + editor.delete(&mut engine, true).unwrap(); + editor.undo(&mut engine).unwrap(); + let ops = editor.take_ops().unwrap(); + section + .apply( + "Test", + onestore::op::Edit { + at: filetime(), + ops: ops + .into_iter() + .map(|op| onestore::op::Op::Page { space, op }) + .collect(), + }, + ) + .unwrap(); + let (stored, model) = (section.page(space).unwrap(), editor.page().unwrap()); + assert_eq!( + Page { + title: model.title.clone(), + ..stored + }, + model + ); + } + assert!( + !section + .events() + .iter() + .any(|event| matches!(event, notebook::session::Event::Rejected { .. })) + ); + section.close().unwrap(); + std::fs::remove_dir_all(&directory).unwrap(); + } + + use notebook::session::{Event, Section}; + + /// Edits `editor` made, as one edit of `space`. + fn edit(editor: &mut CanvasEditor, space: ExGuid) -> onestore::op::Edit { + onestore::op::Edit { + at: filetime(), + ops: editor + .take_ops() + .unwrap() + .into_iter() + .map(|op| onestore::op::Op::Page { space, op }) + .collect(), + } + } + + /// Polls `section`, keeping what it reports in `seen`, until `done` holds of an event. + fn wait(section: &Section, seen: &mut Vec, done: impl Fn(&Event) -> bool) { + let deadline = Instant::now() + std::time::Duration::from_secs(120); + loop { + let events = section.events(); + let finished = events.iter().any(&done); + seen.extend(events); + if finished { + return; + } + assert!(Instant::now() < deadline, "no such event: {seen:?}"); + section.wake(); + std::thread::sleep(std::time::Duration::from_millis(50)); + } + } + + fn published(event: &Event) -> bool { + matches!( + event, + Event::Attempt { + status: notebook::EditStatus::Published { .. }, + .. + } + ) + } + + /// Publishing this machine's edits never reports the page it edits as changed, which + /// would reload the editor under the caret; another writer's edit does, and the editor + /// takes it in place with the caret where it was. + #[test] + fn only_another_writer_changes_the_open_page() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let directory = + std::env::temp_dir().join(format!("snowbound-changed-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&directory); + std::fs::create_dir_all(&directory).unwrap(); + let file = directory.join("section.one"); + std::fs::copy( + root.join("corpus/paragraph-edit/before/notebook/synthetic.one"), + &file, + ) + .unwrap(); + let ours = Section::open(&file, directory.join("ours"), || {}).unwrap(); + let space = ours.pages().unwrap()[0].0; + let mut engine = TextEngine::default(); + let mut editor = CanvasEditor::from_page(ours.page(space).unwrap(), &mut engine).unwrap(); + let outline = editor.outlines().iter().find(|o| !o.title).unwrap().id; + editor.focus_outline(outline).unwrap(); + let at = |paragraph, offset| [TextPosition { paragraph, offset }; 2].into(); + editor.select(at(0, 0)).unwrap(); + let mut seen = Vec::new(); + for text in ["One ", "two ", "three "] { + editor.insert(&mut engine, text).unwrap(); + ours.apply("Ours", edit(&mut editor, space)).unwrap(); + wait(&ours, &mut seen, published); + } + editor.enter(&mut engine, false).unwrap(); + editor.delete(&mut engine, true).unwrap(); + ours.apply("Ours", edit(&mut editor, space)).unwrap(); + wait(&ours, &mut seen, published); + for _ in 0..3 { + ours.wake(); + std::thread::sleep(std::time::Duration::from_millis(500)); + seen.extend(ours.events()); + } + assert!( + !seen + .iter() + .any(|event| matches!(event, Event::Changed(spaces) if spaces.contains(&space))), + "{seen:?}" + ); + + let theirs = Section::open(&file, directory.join("theirs"), || {}).unwrap(); + let mut other = CanvasEditor::from_page(theirs.page(space).unwrap(), &mut engine).unwrap(); + other.focus_outline(outline).unwrap(); + let last = other.active_outline().document().paragraphs().count() - 1; + let end = other + .active_outline() + .document() + .paragraphs() + .last() + .unwrap() + .text() + .encode_utf16() + .count() as u32; + other.select(at(last, end)).unwrap(); + other.insert(&mut engine, " and theirs").unwrap(); + theirs.apply("Theirs", edit(&mut other, space)).unwrap(); + wait(&theirs, &mut Vec::new(), published); + wait( + &ours, + &mut seen, + |event| matches!(event, Event::Changed(spaces) if spaces.contains(&space)), + ); + + let (shown, selection) = (editor.active_outline().id, editor.selection()); + editor + .refresh(ours.page(space).unwrap(), &mut engine) + .unwrap(); + assert_eq!( + (editor.active_outline().id, editor.selection()), + (shown, selection) + ); + assert!( + editor + .active_outline() + .document() + .paragraphs() + .any(|paragraph| paragraph.text().ends_with(" and theirs")) + ); + ours.close().unwrap(); + theirs.close().unwrap(); + std::fs::remove_dir_all(&directory).unwrap(); + } +} diff --git a/tools/native/page-date.ps1 b/tools/native/page-date.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..4003849226b50011740d4a45c123ff734d64f7ce --- /dev/null +++ b/tools/native/page-date.ps1 @@ -0,0 +1,46 @@ +param([Parameter(Mandatory=$true)][string]$Root, [Parameter(Mandatory=$true)][string]$CloneHost) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +& "$PSScriptRoot\cold-current.ps1" -Root $Root -CloneHost $CloneHost +$namespace = 'http://schemas.microsoft.com/office/onenote/2010/onenote' +$app = New-Object -ComObject OneNote.Application +$notebookId = '' +function Save-Page([string]$pageId, [string]$name) { + $content = '' + $app.GetPageContent($pageId, [ref]$content, 0, 1) + [IO.File]::WriteAllText((Join-Path $Root $name), $content, [Text.Encoding]::UTF8) +} +try { + $app.OpenHierarchy((Join-Path $Root 'notebook'), '', [ref]$notebookId, 0) + $sectionId = '' + $app.OpenHierarchy('pictures.one', $notebookId, [ref]$sectionId, 0) + # Two new pages with OneNote's own title, date and time: one keeps its date, the + # other has its date and time changed after it was saved once. + $ids = @() + foreach ($title in @('Kept date', 'Changed date')) { + $pageId = '' + $app.CreateNewPage($sectionId, [ref]$pageId, 0) + $escaped = [Security.SecurityElement]::Escape($pageId) + $xml = "$titleBody of $title" + $app.UpdatePageContent($xml, [DateTime]::MinValue, 1, $false) + $ids += $pageId + } + $app.SyncHierarchy($notebookId) + Start-Sleep -Seconds 3 + Save-Page $ids[1] 'before.xml' + $escaped = [Security.SecurityElement]::Escape($ids[1]) + $update = "" + [IO.File]::WriteAllText((Join-Path $Root 'update.xml'), $update, [Text.Encoding]::UTF8) + $app.UpdatePageContent($update, [DateTime]::MinValue, 1, $false) + $app.SyncHierarchy($notebookId) + Start-Sleep -Seconds 3 + Save-Page $ids[1] 'after.xml' + Save-Page $ids[0] 'kept.xml' + $app.CloseNotebook($notebookId, $false) + $notebookId = '' +} finally { + if ($notebookId) { $app.CloseNotebook($notebookId, $false) } + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + [GC]::Collect() + [GC]::WaitForPendingFinalizers() +} diff --git a/tools/test_page_date.py b/tools/test_page_date.py new file mode 100644 index 0000000000000000000000000000000000000000..b12fe329fe7fabf3b2f9d8b97bc32183f29432d9 --- /dev/null +++ b/tools/test_page_date.py @@ -0,0 +1,39 @@ +from pathlib import Path +import runpy +import shutil +from tempfile import TemporaryDirectory +import unittest +import xml.etree.ElementTree as ET + +from native_xml import ns + +ROOT = Path(__file__).resolve().parent.parent +FIXTURE = ROOT / 'corpus/page-date' +compare = runpy.run_path(str(ROOT / 'tools/verify-document.py'))['compare'] + + +def native_pages(notebook, capture): + with TemporaryDirectory() as temporary: + read = Path(temporary) / 'read' + shutil.copytree(capture / 'read', read) + compare(notebook, read) + pages = (ET.parse(path).getroot() for path in sorted((capture / 'read').glob('page-*.xml'))) + return {page.get('name'): page for page in pages} + + +class PageDateTest(unittest.TestCase): + def test_onenote_dates_a_page_through_its_creation_time(self): + pages = native_pages(FIXTURE / 'native/notebook', FIXTURE / 'native') + self.assertEqual(pages['Changed date'].get('dateTime'), '2024-03-05T14:30:00.000Z') + self.assertNotEqual(pages['Kept date'].get('dateTime'), '2024-03-05T14:30:00.000Z') + + def test_a_rust_dated_page_reads_back_with_its_new_date(self): + pages = native_pages(FIXTURE / 'candidate', FIXTURE / 'cold') + self.assertEqual(pages['Kept date'].get('dateTime'), '2025-07-04T16:45:00.000Z') + self.assertEqual(pages['Changed date'].get('dateTime'), '2024-03-05T14:30:00.000Z') + title = pages['Kept date'].find('one:Title/one:OE/one:T', ns) + self.assertEqual(title.text, 'Kept date') + + +if __name__ == '__main__': + unittest.main()