From b253974a925fe86b3f2ee31e969477fe2ae43fd3 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Sun, 6 Sep 2026 15:46:15 -0700 Subject: [PATCH] feat: offline support and real smb client --- API-AUDIT.md | 69 - Cargo.lock | 844 +++++- FEATURES.md | 29 - M6-ACCEPTANCE.md | 102 - MILESTONE6.md | 183 -- MILESTONE7.md | 429 --- MILESTONE8.md | 84 - PROGRESS.md | 227 -- crates/onestore-diagnostic/Cargo.toml | 10 + crates/onestore-diagnostic/src/main.rs | 107 + crates/onestore-offline/Cargo.toml | 23 + crates/onestore-offline/README.md | 191 ++ .../onestore-offline/examples/cache_probe.rs | 230 ++ .../examples/recovery_probe.rs | 160 + .../examples/smb_offline_client.rs | 593 ++++ .../onestore-offline/examples/support/view.rs | 57 + crates/onestore-offline/src/formatting.rs | 204 ++ crates/onestore-offline/src/lib.rs | 362 +++ crates/onestore-offline/src/rebase.rs | 297 ++ crates/onestore-offline/src/schema.rs | 119 + crates/onestore-offline/src/smb.rs | 36 + crates/onestore-offline/src/sync.rs | 458 +++ crates/onestore-offline/src/worker.rs | 167 ++ crates/onestore-offline/tests/cache.rs | 685 +++++ crates/onestore-offline/tests/sync.rs | 2618 +++++++++++++++++ crates/onestore-smb/Cargo.toml | 13 + crates/onestore-smb/README.md | 43 + .../examples/smb_concurrent_client.rs | 22 + .../examples/smb_reconnect_client.rs | 267 ++ crates/onestore-smb/src/lib.rs | 466 +++ crates/onestore-smb/src/tests.rs | 340 +++ crates/onestore-smb/src/tests/faults.rs | 444 +++ README.md => crates/onestore/README.md | 74 +- crates/onestore/examples/concurrent_client.rs | 203 +- crates/onestore/examples/document.rs | 26 +- crates/onestore/examples/insert.rs | 52 + .../onestore/examples/maintenance_fixture.rs | 63 + crates/onestore/examples/power_loss.rs | 81 +- .../onestore/examples/support/concurrent.rs | 258 ++ crates/onestore/src/commit.rs | 186 +- crates/onestore/src/create.rs | 18 +- crates/onestore/src/document.rs | 2 +- crates/onestore/src/edit.rs | 238 +- crates/onestore/src/formatting.rs | 288 ++ crates/onestore/src/insertion.rs | 343 +++ crates/onestore/src/lib.rs | 13 +- crates/onestore/src/objects.rs | 23 +- crates/onestore/src/revisions.rs | 44 + crates/onestore/src/snapshot.rs | 76 + crates/onestore/src/store.rs | 2 +- crates/onestore/src/write.rs | 455 ++- crates/onestore/src/write/tests.rs | 680 +++++ crates/onestore/tests/edit.rs | 205 +- crates/onestore/tests/formatting.rs | 407 +++ crates/onestore/tests/insertion.rs | 377 +++ crates/onestore/tests/revisions.rs | 62 + crates/onestore/tests/shared_snapshot.rs | 420 +++ crates/onestore/tests/support/current.rs | 26 + crates/onestore/tests/support/trace.rs | 36 + readme.md | 1 + tools/codex_usage_report.example.json | 18 + tools/codex_usage_report.mjs | 129 + tools/concurrent_rust.py | 20 +- tools/crash_recovery.py | 15 + tools/diagnostic/editor.css | 24 + tools/diagnostic/editor.js | 246 ++ tools/native/network.ps1 | 6 +- tools/native/probe.ps1 | 42 +- tools/native/stress.ps1 | 8 + tools/native_collaboration.py | 148 +- tools/native_disconnect.py | 137 + tools/native_maintenance.py | 210 ++ tools/native_probe.py | 9 +- tools/native_stress.py | 168 +- tools/notebook_editor.py | 280 ++ tools/notebook_report.py | 49 +- tools/offline_cache_crash.py | 130 + tools/offline_document_history.py | 163 + tools/offline_history.py | 73 + tools/offline_outage.py | 313 ++ tools/offline_publication_crash.py | 237 ++ tools/smb-proxy.py | 95 +- tools/smb_faults.py | 82 + tools/test_concurrent_rust.py | 37 +- tools/test_crash_recovery.py | 29 +- tools/test_native_disconnect.py | 143 + tools/test_native_maintenance.py | 53 + tools/test_native_stress.py | 48 + tools/test_notebook_editor.py | 237 ++ tools/test_notebook_report.py | 33 + tools/test_offline_confirmation.py | 88 + tools/test_offline_document_history.py | 151 + tools/test_offline_history.py | 156 + tools/test_offline_outage.py | 216 ++ tools/test_offline_publication_crash.py | 81 + tools/test_smb_overlap.py | 144 + tools/test_smb_proxy.py | 196 ++ tools/test_verify_smb_faults.py | 97 + tools/verify-collaboration.py | 2 +- tools/verify-reader.py | 2 +- tools/verify-writer.py | 4 +- tools/verify_offline.py | 71 + tools/verify_offline_confirmation.py | 95 + tools/verify_offline_recovery.py | 56 + tools/verify_smb_faults.py | 81 + tools/verify_smb_overlap.py | 123 + 106 files changed, 17504 insertions(+), 1779 deletions(-) delete mode 100644 API-AUDIT.md delete mode 100644 FEATURES.md delete mode 100644 M6-ACCEPTANCE.md delete mode 100644 MILESTONE6.md delete mode 100644 MILESTONE7.md delete mode 100644 MILESTONE8.md delete mode 100644 PROGRESS.md create mode 100644 crates/onestore-diagnostic/Cargo.toml create mode 100644 crates/onestore-diagnostic/src/main.rs create mode 100644 crates/onestore-offline/Cargo.toml create mode 100644 crates/onestore-offline/README.md create mode 100644 crates/onestore-offline/examples/cache_probe.rs create mode 100644 crates/onestore-offline/examples/recovery_probe.rs create mode 100644 crates/onestore-offline/examples/smb_offline_client.rs create mode 100644 crates/onestore-offline/examples/support/view.rs create mode 100644 crates/onestore-offline/src/formatting.rs create mode 100644 crates/onestore-offline/src/lib.rs create mode 100644 crates/onestore-offline/src/rebase.rs create mode 100644 crates/onestore-offline/src/schema.rs create mode 100644 crates/onestore-offline/src/smb.rs create mode 100644 crates/onestore-offline/src/sync.rs create mode 100644 crates/onestore-offline/src/worker.rs create mode 100644 crates/onestore-offline/tests/cache.rs create mode 100644 crates/onestore-offline/tests/sync.rs create mode 100644 crates/onestore-smb/Cargo.toml create mode 100644 crates/onestore-smb/README.md create mode 100644 crates/onestore-smb/examples/smb_concurrent_client.rs create mode 100644 crates/onestore-smb/examples/smb_reconnect_client.rs create mode 100644 crates/onestore-smb/src/lib.rs create mode 100644 crates/onestore-smb/src/tests.rs create mode 100644 crates/onestore-smb/src/tests/faults.rs rename README.md => crates/onestore/README.md (69%) create mode 100644 crates/onestore/examples/insert.rs create mode 100644 crates/onestore/examples/maintenance_fixture.rs create mode 100644 crates/onestore/examples/support/concurrent.rs create mode 100644 crates/onestore/src/formatting.rs create mode 100644 crates/onestore/src/insertion.rs create mode 100644 crates/onestore/src/snapshot.rs create mode 100644 crates/onestore/src/write/tests.rs create mode 100644 crates/onestore/tests/formatting.rs create mode 100644 crates/onestore/tests/insertion.rs create mode 100644 crates/onestore/tests/shared_snapshot.rs create mode 100644 crates/onestore/tests/support/current.rs create mode 100644 crates/onestore/tests/support/trace.rs create mode 100644 readme.md create mode 100644 tools/codex_usage_report.example.json create mode 100644 tools/codex_usage_report.mjs create mode 100644 tools/diagnostic/editor.css create mode 100644 tools/diagnostic/editor.js create mode 100644 tools/native_disconnect.py create mode 100644 tools/native_maintenance.py create mode 100644 tools/notebook_editor.py create mode 100644 tools/offline_cache_crash.py create mode 100644 tools/offline_document_history.py create mode 100644 tools/offline_history.py create mode 100644 tools/offline_outage.py create mode 100644 tools/offline_publication_crash.py create mode 100644 tools/smb_faults.py create mode 100644 tools/test_native_disconnect.py create mode 100644 tools/test_native_maintenance.py create mode 100644 tools/test_notebook_editor.py create mode 100644 tools/test_offline_confirmation.py create mode 100644 tools/test_offline_document_history.py create mode 100644 tools/test_offline_history.py create mode 100644 tools/test_offline_outage.py create mode 100644 tools/test_offline_publication_crash.py create mode 100644 tools/test_smb_overlap.py create mode 100644 tools/test_smb_proxy.py create mode 100644 tools/test_verify_smb_faults.py create mode 100644 tools/verify_offline.py create mode 100644 tools/verify_offline_confirmation.py create mode 100644 tools/verify_offline_recovery.py create mode 100644 tools/verify_smb_faults.py create mode 100644 tools/verify_smb_overlap.py diff --git a/API-AUDIT.md b/API-AUDIT.md deleted file mode 100644 index 7e696f24ab7f6948b45feec8a7589e6d75793dcd..0000000000000000000000000000000000000000 --- a/API-AUDIT.md +++ /dev/null @@ -1,69 +0,0 @@ -# Public API audit - -The current API is reasonable for a Rust reader/canvas prototype and the supported -text-edit diagnostic tool. It is an experimental interoperability API, not yet a -stable application SDK. The workspace move preserves every public name and its -behavior; this audit adds contract documentation without introducing wrappers or -changing serialization. - -## Consumer boundary - -```text -owned snapshot bytes - └─ Store::parse committed storage; checksum diagnostics - └─ RevisionIndex::parse revision identities and dependencies - └─ Document::parse semantic views; checksum damage rejected - ├─ pages() active section pages in stored order - └─ text_runs() text with inherited formatting and links - -snapshot + typed identities + UTF-16 range + replacement - └─ commit_file_text lock → compare snapshot → append → flush - └─ Result<(), CommitError> success or explicit publication state -``` - -A document represents one `.one` or `.onetoc2` file, not a notebook directory. -The directory, section ordering, native references and report assets are currently -assembled by the tooling. A second crate can use the semantic model directly and -keep its layout/cache state separate. Mutating the public model changes only the -inspection view; writers reparse the supplied snapshot and enforce their own -invariants. - -## Findings and decisions - -| Area | Assessment | Recommendation | -| --- | --- | --- | -| Read/write separation | Good: parsed views cannot accidentally save themselves. Snapshot comparison rejects stale writes before publication. | Keep explicit commit operations; do not add a mutable document plus generic `save()`. | -| Commit outcomes | Good: `NotCommitted`, `Unknown` and `Committed` distinguish retry behavior. An error can still mean the edit persisted. | Keep this distinction at every UI/FFI boundary. Variant documentation now states the caller action. | -| Borrowing and lifetimes | The model owns decoded strings but borrows payloads. Consumers must retain its source snapshot/store. An application object holding both bytes and borrowed views would require a different ownership design. | Let the canvas prototype determine whether its actual access pattern needs an owned model. Do not add a self-referential owner or duplicate document DTO in advance. | -| Semantic error classification | `Error` exposes diagnostic text and an offset, with no typed reason. I/O failures already have `ErrorKind`; semantic distinctions such as unsupported edits and temporarily missing contexts require text matching today. | Before adding automatic semantic recovery, choose a small typed classification based on the recovery actions it needs. A detailed variant for every parser message would enlarge the compatibility burden. | -| Editable text discovery | Readable text is broader than editable text: hidden fields, hyperlinks, equations, generated content, conflicts and protected content can be readable but rejected for writing. Currently callers can try `replace_text` and inspect its result; candidate discovery duplicates some checks in the random-edit example. | For the diagnostic tool, obtain eligibility from the writer's actual validation. If a public eligibility API is added, share that validation rather than maintain a second list of rules. | -| Identity transport | `ExGuid` is typed, ordered, hashable and serializes to its display string. It has no `FromStr` or `Deserialize`. Existing tools retain or look up typed IDs instead of parsing them. | A Rust canvas can keep typed IDs. A JSON editing endpoint should either map strings back to IDs from its snapshot or justify a canonical parser with round-trip tests. | -| Raw storage exports | `Header`, nodes, references, property arenas and both revision layers are public. They are useful for diagnostics, but expose implementation details to consumers. | Keep them available during interoperability work; have the UI depend on `onestore::document` plus edit functions. Moving them into a separate module now would create churn without an established consumer requirement. | -| Public fields and enums | Inspection fields are mutable and enums are exhaustive. Downstream code can depend on the exact shape. | Treat the current Rust and JSON shapes as experimental. Whether to restrict construction or allow future enum variants is an API-evolution tradeoff to decide with the first consumer, not an assumed stability promise. | -| Document boundaries | `pages()` excludes conflicts/history, returns no visible pages for encryption, and rejects TOC files. All referenced contexts remain in the model. | These boundaries are now documented on the method. Readers must inspect the root kind when distinguishing a locked section from an empty section. | -| Units and defaults | Coordinates are points, edit/run offsets are UTF-16 units, Time32 values use the 1980 epoch, and `Format` preserves absence separately from explicit false. | Keep the native distinctions; renderers should use resolved runs and convert offsets explicitly when crossing into UTF-8 or browser selection APIs. | -| In-memory replacements | `replace_text` and `replace_property_bytes` return complete byte images without locking or persistence. Overwriting a live file with those bytes would bypass the commit protocol. | Their documentation now directs existing-file updates to the corresponding commit functions. Raw scalar editing still requires the caller to maintain document semantics. | -| Scope and dependencies | The library stays native and synchronous, with no network runtime, renderer, process management or platform UI dependency. Internal writer helpers remain crate-private. | Keep the core boundary. Put diagnostic serving and rendering in consumers; expose a C ABI only against concrete embedding needs. | - -The two decisions worth reviewing before the diagnostic tool are semantic error -classification and edit eligibility. Neither requires a broad API redesign. -The lifetime/ownership question can be informed by the separate canvas prototype. - -## Verification - -- Root workspace tests: `evidence/m8/workspace-tests.log`. -- Clippy and formatting: `workspace-clippy.log`, `workspace-format.log`. -- Rustdoc and compiled documentation example: `workspace-docs.log`, - `workspace-doctests.log`. -- All eight cargo-fuzz targets build against the relocated library: - `workspace-fuzz-build.log`. -- Python verification tools: 36 tests in `workspace-python.log`. -- An independent crate under `evidence/m8/api-consumer` uses only public APIs to - create a section, traverse pages/runs, serialize identities, commit a Unicode - edit, reject a stale snapshot through typed commit/I/O state, and reopen the - exact resulting text. Output: `evidence/m8/api-consumer.log`. - -The audit examined exported declarations and their implementations in storage, -revision resolution, document interpretation, creation, editing and persistence. -It does not freeze the API or turn the earlier native compatibility evidence into -a new platform guarantee. diff --git a/Cargo.lock b/Cargo.lock index dcab381a17e62e126615d9f449b7f3f18915a5d8..83b5d064c58b1b4cc4ba9e45b3c9d91fa9334ba8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,101 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aead" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "aes" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32" +dependencies = [ + "cipher", + "cpubits", + "cpufeatures", +] + +[[package]] +name = "aes-gcm" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ctutils", + "ghash", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + [[package]] name = "bitflags" version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "ccm" +version = "0.6.0-rc.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4edea5ea70a1285565ac264767613d6c88351a9a0557e7af793a0942590baaed" +dependencies = [ + "aead", + "cipher", + "ctr", + "subtle", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -20,6 +109,175 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "cipher" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" +dependencies = [ + "block-buffer", + "crypto-common", + "inout", +] + +[[package]] +name = "cmac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac78aa94ce13e432b332a4d1bf2eff167d3a2520188ee05b337180a42fd2e62e" +dependencies = [ + "cipher", + "dbl", + "digest", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "getrandom", + "hybrid-array", + "rand_core", +] + +[[package]] +name = "ctr" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" +dependencies = [ + "cipher", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + +[[package]] +name = "dbl" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d7a944e61df464668c5f51f56cc667396a8821434273112948ea0b66e405d7" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "ctutils", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -29,6 +287,80 @@ dependencies = [ "cfg-if", "libc", "r-efi", + "rand_core", +] + +[[package]] +name = "ghash" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" +dependencies = [ + "polyval", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32069d97bb81e38fa67eab65e3393bf804bb85969f2bc06bf13f64aef5aba248" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest", +] + +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "inout" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" +dependencies = [ + "hybrid-array", ] [[package]] @@ -37,12 +369,73 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + [[package]] name = "libc" version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libsqlite3-sys" +version = "0.38.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "md4" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd76fb0fd6b2e4be62a73f8e0858ca97f81babcb1af322dcaca196f735f17f80" +dependencies = [ + "digest", +] + [[package]] name = "md5" version = "0.8.1" @@ -55,6 +448,17 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "mio" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + [[package]] name = "nix" version = "0.31.3" @@ -67,6 +471,34 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + [[package]] name = "onestore" version = "0.1.0" @@ -78,6 +510,80 @@ dependencies = [ "serde_json", ] +[[package]] +name = "onestore-diagnostic" +version = "0.1.0" +dependencies = [ + "onestore", + "serde", + "serde_json", +] + +[[package]] +name = "onestore-offline" +version = "0.1.0" +dependencies = [ + "onestore", + "onestore-smb", + "rusqlite", + "serde", + "serde_json", + "tempfile", + "thiserror", +] + +[[package]] +name = "onestore-smb" +version = "0.1.0" +dependencies = [ + "onestore", + "serde_json", + "smb2", + "tokio", +] + +[[package]] +name = "pbkdf2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "polyval" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" +dependencies = [ + "cpubits", + "cpufeatures", + "universal-hash", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -102,6 +608,56 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror", +] + +[[package]] +name = "rusqlite" +version = "0.40.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + [[package]] name = "serde" version = "1.0.229" @@ -129,7 +685,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] @@ -145,6 +701,112 @@ dependencies = [ "zmij", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" + +[[package]] +name = "smb2" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ef4f20cff3d39a131335d17df6146eada7851d7705252f2765b8717a0f52db8" +dependencies = [ + "aes", + "aes-gcm", + "async-trait", + "ccm", + "cmac", + "digest", + "futures-util", + "getrandom", + "hmac", + "log", + "lz4_flex", + "md-5", + "md4", + "num_enum", + "pbkdf2", + "sha1", + "sha2", + "thiserror", + "tokio", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "3.0.5" @@ -156,12 +818,192 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.13+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow", +] + +[[package]] +name = "twox-hash" +version = "2.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "universal-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" +dependencies = [ + "crypto-common", + "ctutils", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.5", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + [[package]] name = "zmij" version = "1.0.23" diff --git a/FEATURES.md b/FEATURES.md deleted file mode 100644 index 7a156aa4db2204639b7d0394bab79c61cfb58293..0000000000000000000000000000000000000000 --- a/FEATURES.md +++ /dev/null @@ -1,29 +0,0 @@ -# Milestone 6 feature matrix - -The matrix follows MS-ONE's document families, including features absent from the -personal corpus. Each row requires an independently authored native fixture, -Rust interpretation checks, and a review-report inspection before acceptance. -A retained opaque payload is evidence of preservation, not interpreted coverage. - -| Family | Required cases | Specification Evidence | -| --- | --- | --- --- | -| Notebook structure | Multiple notebooks, section groups, section ordering/colors, duplicate names, empty sections | 2.2.14–18, 2.2.91–96 [Native feature controls](corpus/m6/native-features-01/ORACLE.md); [Rust scale](evidence/m6/rust-scale-final-01.log) | -| Pages | Titles/alternate titles, order, subpages, duplicate text/titles, authors/timestamps, RTL | 2.2.19, 2.2.29–31, 2.2.58, 2.3.74 [Direction/origin controls](corpus/m6/native-page-direction-03); [private review](evidence/m6/private-current-report-05/qa/review.json) | -| Outline hierarchy | Positioned outlines, nesting, outline groups, indentation, collapsed/hidden content | 2.2.20–23, 2.3.8, 2.3.18–19 [Structure](corpus/m6/native-structure-01/ORACLE.md); [saved collapse](corpus/m6/rust-collapse-control-01/ORACLE.md) | -| Text | ASCII/Unicode preference, surrogate pairs, combining marks, mixed scripts, empty/long paragraphs | 2.1.4, 2.2.5, 2.2.23, 2.2.89 [100 native histories](evidence/m6/native-histories-independent-03.log); [break controls](corpus/m6/native-break-controls-02) | -| Character formatting | Mixed runs, font/size/color/highlight, bold/italic/underline/strike, super/subscript, language | 2.2.43–45, 2.2.76–77, 2.3.9–16 [Native probes](corpus/m6/native-probes-01/ORACLE.md); [private native comparison](evidence/m6/private-current-compare-01.log) | -| Paragraph formatting | Named styles, alignment, spacing, RTL, style inheritance | 2.2.44, 2.2.80, 2.2.83, 2.3.81–83 [Structure](corpus/m6/native-structure-01/ORACLE.md); [feature report inspection](evidence/m6/features-report-05/qa/review.json) | -| Lists | Bullets, numbering, restarts, mixed indentation, custom fonts/formats | 2.2.25, 2.2.57, 2.3.20, 2.3.43 [Structure](corpus/m6/native-structure-01/ORACLE.md); [numbering controls](corpus/m6/native-features-01/ORACLE.md) | -| Tables | Multiple rows/columns, nested content, widths, shading, borders, locked columns | 2.2.26–28, 2.2.66, 2.2.70, 2.2.97 [Widths/locks](corpus/m6/native-structure-01/ORACLE.md); [RTL](corpus/m6/native-page-direction-03); [shading limitation](corpus/m6/cell-shading-control-01/ORACLE.md) | -| Links | External/internal links, formatted labels, embedded text-run data | 2.2.78, 2.2.82, 2.2.90, 2.3.75–77 [Native link controls](corpus/m6/native-link-controls-01); [empty-label regression](corpus/m6/native-empty-link-01) | -| Images | Multiple formats, sizes, alt text, filenames, internal/external containers, background/printout images | 2.2.24, 2.2.36, 2.2.59, 2.2.75, 2.2.79 [Native formats/roles](corpus/m6/native-features-01/ORACLE.md); [browser inspection](evidence/m6/features-report-05/qa/review.json) | -| Files and media | Attachments, original filenames, recording identifiers, audio/video payloads and associated text | 2.2.32–33, 2.2.60–61, 2.2.71–72 [Native attachment/recording controls](corpus/m6/native-features-01/ORACLE.md) | -| Ink and embedded objects | Strokes, placement, native payload/export comparison, math and embedded text-run objects | Native ink corpus; 2.3.79–80 [Native ink](corpus/native-ink/cold-ui-ink); [math](corpus/m6/native-math-01/ORACLE.md): payload/run preservation, opaque rendering | -| Tags and tasks | Standard/custom tags, multiple tags, checked/unchecked state, task status/dates | 2.1.9, 2.2.41–42, 2.2.84–88, 2.3.85–96 [Native task controls](corpus/m6/native-features-01/ORACLE.md); [private tags](evidence/m6/private-current-compare-01.log) | -| History and recovery | Version pages/contexts, recycle-bin pages, conflicts and their source relationships | 2.1.1–2, 2.1.17, 2.2.34–40, 2.2.86 [Private history review](evidence/m6/private-current-report-05/qa/review.json); [live conflicts/recovery](corpus/m6/live-collaboration-15/ORACLE.md) | -| Protection and unknowns | Locked section identity/ciphertext, unknown JCIDs/properties, malformed structures | MS-ONESTORE encryption; MS-ONE 2.1.5, 2.1.12 [Existing corpus checks](evidence/m6/corpus-regression-01.log); [document fuzzing](evidence/m6/document-public-fuzz-08.log): ciphertext/raw data preserved | -| Scale and interactions | Rust/native large notebooks, many pages/objects/assets, repeated identities/text, mixed feature histories | All applicable rows [Independent native histories](evidence/m6/native-histories-independent-03.log); [Rust scale](evidence/m6/rust-scale-final-01.log) | - -[M6-ACCEPTANCE.md](M6-ACCEPTANCE.md) records the native builds, comparisons, -report inspections, regression campaigns and interpretation boundaries. `evidence/m6/spec-objects.json` -is the current object-definition inventory extracted from the supplied markdown. diff --git a/M6-ACCEPTANCE.md b/M6-ACCEPTANCE.md deleted file mode 100644 index cf3c8872f1e33d0441f0fbd93c3902ec257e4270..0000000000000000000000000000000000000000 --- a/M6-ACCEPTANCE.md +++ /dev/null @@ -1,102 +0,0 @@ -# Milestone 6 verification - -The deliverable is a read-only document model and reading report. Its acceptance -boundary is the copied personal notebook: automated native comparisons plus a -page-by-page review with no known easily spottable content failures. Native PDF -references preserve access to the original canvas arrangement. - -## Personal notebook - -[The current report](evidence/m6/private-current-report-05/index.html) includes the -user-approved newer `Video.one`. [Its source manifest](corpus/private/current-source-manifest.json) -identifies that snapshot; [the original frozen manifest](corpus/private/source-manifest.json) -remains separate. The original files are never edited by the library tests. - -| Check | Evidence | -| --- | --- | -| 26 current/recycle pages, 109 tags, 188,560 explicit character-format comparisons | [Fresh-cache comparison](evidence/m6/private-current-compare-01.log) | -| 18 historical pages, including all three versions of the updated Video page | [Album](evidence/m6/private-history-final-01.log), [Video](evidence/m6/private-video-current-history-cold-compare-01.log), [deleted page](evidence/m6/private-deleted-history-final-01.log) | -| All 45 report pages reviewed: ordinary, historical, recycle-bin and default template | [Review record](evidence/m6/private-current-report-05/qa/review.json) | -| Frozen copy, approved current copy and live sources match their respective manifests | [Integrity check](evidence/m6/source-integrity-current-01.json) | - -The updated current page was inspected through eight browser viewports and two -native PDF pages. Unchanged page bodies inherit their earlier review only after -an exact rendered-content comparison. All exported image payloads decode. -History labels use America/Los_Angeles. Seventeen historical dates match native -UI evidence; copying the sole deleted-page version changes its native displayed -date, so the report retains the source revision timestamp and records that -normalization separately. - -## Independent and adversarial checks - -| Campaign | Result and evidence | -| --- | --- | -| Native edit histories | [100 histories / 2,000 independently specified operations](evidence/m6/native-histories-independent-03.log); [101 resulting pages compared](evidence/m6/native-histories-final-02.log) | -| Shrinking | [Seed 21](evidence/m6/history-shrink-21/minimal.json) reduced from 20 operations to 3 in 37 fresh-clone replays; [empty-link regression](corpus/m6/native-empty-link-01) | -| Rust scale generation | [128 sections / 2,097,152 Unicode scalars](evidence/m6/rust-scale-final-01.log), including reverse section ordering after native import | -| Document fuzzing | [65,736 runs / 121 seconds](evidence/m6/document-public-fuzz-08.log) after the collapse-field change; prior [100,398-run campaign](evidence/m6/document-public-fuzz-07.log) and [11,423 private-seed runs](evidence/m6/document-private-fuzz-03.log) | -| Rust checks | [All targets](evidence/m6/rust-regression-06.log), [clippy](evidence/m6/clippy-final-04.log) | -| Native/report tooling | [18 tests](evidence/m6/tool-regression-15.log), including deliberate oracle failures and TIFF/native pixel parity | -| Existing writer and fault cases | [Writer](evidence/m6/writer-regression-01.log), [collaboration/FLUSH regression](evidence/m6/collaboration-regression-02.log) | -| Expanded live matrix | [Seven passing cases](corpus/m6/live-collaboration-15/ORACLE.md), [fresh-cache comparison](evidence/m6/live-collaboration-cold-compare-15.log), [native conflict view](corpus/m6/live-collaboration-cold-15/conflict.png) | - -The document fuzzer mutates property streams inside native files and repairs -object checksums, reaching document interpretation beyond header rejection. -It traverses referenced historical revisions and resolves text runs. Native -histories have a separate expected-operation model; matching two views derived -from the Rust decoder is not counted as independent verification. - -The live matrix covers disjoint edits, competing edits, Samba restart, per-client -transport loss, whole-file lock contention, OneNote process termination and abrupt -VM termination. The process/VM cases recover already server-persisted edits. -They do not establish unsynchronized-cache durability or physical power-loss -safety. Earlier stage-5 cases separately cover lost successful SMB FLUSH replies. - -## Interpretation and oracle boundaries - -- Native XML omits partial black highlights. Native PDF rectangles supply a - separate check; the reader retains the stored highlight. -- RTL native XML column order differs from physical left-to-right storage order. - Independent PDF coordinates establish the conversion. Page-origin translation, - locked/unlocked column widths and printout borders have isolated controls. -- A saved paragraph collapse default differs from a client's transient expanded - view. [The native UI control](corpus/m6/native-expanded-control-ui-01/ORACLE.md) - and [Rust-authored control](corpus/m6/rust-collapse-control-01/ORACLE.md) distinguish them. -- Ink, structured equations, encrypted content and unknown properties retain - payloads or source identities. Preservation is not decryption or interpretation. - Structured equation runs expose a native-reference placeholder; ordinary inline - math text remains readable. -- TIFF browser previews match OneNote's exported pixels while original TIFF bytes - remain available. Native image conversion is not a license to replace source data. -- [Cell shading](corpus/m6/cell-shading-control-01/ORACLE.md) follows the documented - stored COLORREF value. A fresh OneNote 2010 cache opens the control but omits - shading from XML and renders the cell white; native rendering parity for that - property is not claimed. - -Verification uses OneNote 2010 build 14.0.4763.1000 in disposable Windows 7 clones. -The earlier storage/collaboration corpus used build 14.0.7015.1000 on the physical -machine. Each run records its environment and cleanup. These finite campaigns -support this milestone's review boundary, not a claim of universal compatibility -or absence of bugs. - -## Reproduce - -Use a new output directory for each native capture or report: - -```sh -cargo test --all-targets -cargo clippy --all-targets -- -D warnings -cargo build --example document -PYTHONPATH=tools python3 -m unittest discover -s tools -p 'test_*.py' -python3 tools/native_runner.py COPIED_NOTEBOOK NEW_CAPTURE --pdf -python3 tools/verify-document.py COPIED_NOTEBOOK NEW_CAPTURE/read -python3 tools/notebook_report.py COPIED_NOTEBOOK NEW_REPORT \ - --native NEW_CAPTURE/read --timezone America/Los_Angeles -cargo +nightly fuzz run document -- -max_total_time=120 -max_len=512 -rss_limit_mb=2048 -python3 tools/native_collaboration.py NEW_CAPTURE --linux OWNED_LAB_NAME -``` - -Python native comparison/report tests require Pillow and pdfplumber. Historical -report references use repeated `--versions CAPTURE_DIRECTORY` arguments after -`verify-document.py --versions CAPTURE_DIRECTORY/version-ui.json` has checked -source hashes and associated the native copies with stored revisions. diff --git a/MILESTONE6.md b/MILESTONE6.md deleted file mode 100644 index cf1959a5a02eb53d867a1d974cb44143aa2ef1d5..0000000000000000000000000000000000000000 --- a/MILESTONE6.md +++ /dev/null @@ -1,183 +0,0 @@ -# Milestone 6: a reviewable OneNote document model - -Completed milestone; verification and review artifacts are recorded in -[M6-ACCEPTANCE.md](M6-ACCEPTANCE.md). Clover chose -extracting a real notebook into a readable document model as the first hands-on -result. This milestone combines document-format work with repeatable native -verification; completion requires no known easily spottable bugs in her notebook -after automated native comparisons and an independent page-by-page inspection. -Her review should find subtle design issues, not serve as the basic QA pass. - -## What Clover receives - -A local, read-only review report generated from the copied personal notebook, -with notebook/section/page navigation, readable content, extracted media, and a -document-structure view. The report exposes paragraph nesting and ordering, -text-run styles, outline coordinates, tables, links, tags, and source identities. -It accompanies a machine-readable model and an asset directory, so the result is -useful to future applications as well as inspectable by a person. - -The report presents a readable interpretation of the document. It does not claim -to reproduce OneNote's layout engine. Coordinate values and native reference -captures let Clover assess spatial information without depending on a new canvas -renderer. Ink, recording data, encrypted sections, and unrecognized structures -must remain explicitly accounted for, with retained payloads or source references -as appropriate. Unsupported interpretation must never turn into silent omission. - -Feedback should be attached to identifiable pages/objects and answer: - -- Does the section/page hierarchy match the notebook, including subpages and order? -- Does the representation preserve meaningful paragraph/list/table structure? -- Are text, formatting, links, images, attachments, and tags correctly associated? -- Where does spatial arrangement communicate something the readable view loses? -- Which opaque content prevents this model from supporting a useful reader? - -The review ends with concrete corrections or priorities for the document model. -Editing API design follows that feedback; no GUI editor is part of this milestone. - -## Sequence and gates - -| Step | Deliverable | Gate before proceeding | -| --- | --- | --- | -| 1. Repeatable native oracle | Automated fresh-clone runs using the existing VM controllers, named disposable notebooks, bounded captures, and unconditional teardown | Existing reader/writer native cases replay successfully; intentional content corruption causes a failed comparison; a failed run retains a reproducible artifact bundle and leaves no owned VM running | -| 2. Document semantics | A typed view over the resolved store graph, independent of COM and filesystem transport | Every interpreted feature has a native fixture and assertions for its semantic invariants; unknown and encrypted content is represented explicitly | -| 3. Real-notebook review | Machine-readable export, assets, and a human-readable report derived from the same model | All copied sections/pages are accounted for, source hashes remain unchanged, and supported content matches newly captured native evidence | -| 4. Adversarial validation | Seeded native edit histories, parser fuzzing, and replay of existing commit/concurrency cases | The feature/failure matrix passes, discrepancies have regression cases, and the acceptance report distinguishes exact matches, allowed native normalization, and opaque content | - -## Document scope - -Start with notebook and section structure, ordered pages/subpages, titles, outlines, -paragraphs, and Unicode text runs. Add character/paragraph formatting, list and -indentation semantics, tables, hyperlinks, images, attachments, and note tags. -Expose conflict pages and recycle-bin membership separately from ordinary pages; -retain their relationship to their source page/section. Account for observed -unrecognized page-like metadata instead of filtering it out of the report. - -MS-ONE text-run boundaries use character positions with rules distinct from Rust -UTF-8 byte indices. Fixtures must cover surrogate pairs, combining marks, RTL text, -mixed formatting, empty paragraphs, repeated text, and embedded objects. Titles, -cached title strings, styles, and layout metadata need explicit interpretation; -generic scalar decoding alone cannot establish those relationships. - -Keep document interpretation above `onestore`'s revision/property graph. Retain -access to raw identities and values for unknown properties. Add types only where -they express document semantics or prevent invalid interpretation; keep the initial -export/API provisional until Clover has reviewed actual notebook content. - -## Independent verification - -```text -Authored operation history ──→ expected document semantics - │ - └─→ real OneNote edit/save → captured .one + native XML/payloads - │ - └─→ Rust document model - │ - compare all three ─┘ - -Existing Rust scalar edit → fresh native read/edit/save → Rust reread -``` - -Native capture runs use a fresh clone/cache for each independent acceptance case. -Tests of a continuing collaboration session deliberately retain that session's -cache, then use a separate fresh verifier after synchronization. Completion must -be observed through file/content state, not inferred from an arbitrary sleep or a -successful asynchronous COM call. COM IDs are cache identities, not persistent -notebook identities. - -Compare page relationships/order, paragraph boundaries, text runs/styles, table -cells, positions, links, and payload bytes. Hashes establish artifact integrity; -they do not establish semantic correctness. Native XML supplies an independent -view of supported content. Selected UI captures cover facts COM omits, especially -conflicts and visual interpretation. Whole-file byte equality after a native save -is not the oracle because OneNote rewrites metadata and representation. - -Build native-only control cases for observed normalization, including the previous -table-width and ink z-order changes. Comparison exceptions require specific -evidence and a narrow assertion; broad removal of timestamps, IDs, or geometry -must not conceal meaningful changes. - -Use a small independent test model for generated operations; expected values must -not be reconstructed by the same Rust decoder being checked. Shrink a failed -history by removing operations and simplifying their parameters, replaying each -candidate from a clean starting state. Preserve seed, operation history, input and -output files, model diff, native XML/payloads, app/server versions, and relevant -trace excerpts. A regression must reproduce the observed discrepancy before its -fix can be accepted. - -Build a specification-derived feature matrix, including features absent from the -personal and stock notebooks. Produce large synthetic notebooks both through Rust -and through real OneNote; compare independent generation paths and exercise size, -ordering, repetition, and mixed-feature interactions. Any unsupported feature must -be explicitly accounted for and investigated, not silently omitted from coverage. - -Initial randomized target: 100 reproducible native histories of roughly 20 -operations, spanning the supported feature matrix. This is a campaign budget, -not a statistical safety claim. Native-generated files then seed fast local -reader fuzzing. Continue the existing stateful short-I/O/partial-persistence -commit tests; run the existing collaboration scenarios with two Windows clients -and the disposable Linux server. Process termination, Samba restart, transport -loss, and abrupt VM termination are distinct fault cases and must be reported as -such. VM termination does not simulate physical host power loss. - -## Tooling findings and setup work - -The implementation is in `tools/w7`. Windows instances use sparse qcow2 overlays -on a sealed base, unique hostnames/MACs/control targets, and an authenticated agent. -Linux instances also use overlays. A VDE network connects Windows clients to one -Linux Samba appliance at `192.168.77.1`; the Mac has forwarded SSH and SMB ports. -The MCP's up/down/status operations are enough for ordinary fresh-clone tests. -Reuse those controllers from the test runner rather than implementing another -QEMU lifecycle. Add a narrow controlled-crash facility only when fault cases need -it, with ownership checks and restart/recovery of the same overlay. - -Start with two Windows clients and one Linux server. Windows uses x86 CPU -emulation on this host; the suggested 25-client capacity has not been measured. -Increase concurrency from measured test throughput and resource use. One Linux -lab address means server restart/configuration tests require exclusive ownership -of that server; ordinary cases can use separate notebook directories. - -The scout confirmed: - -- Windows 7 clone `m6scout`, PowerShell 5.1.14409.1005, OneNote 14.0.4763.1000. - The earlier physical-machine corpus used OneNote 14.0.7015.1000. Record both - builds' evidence separately; success on one must not be silently attributed to - the other. -- The clone desktop was 800×600, so existing 1280×720 AHK coordinates cannot be - reused without setting and verifying display configuration. -- Z: mapped to `\\10.0.0.1\agent` (zenith). The disposable Linux share is a - different destination, `\\192.168.77.1\agent`. Parameterize shared paths rather - than reusing the old A: convention or remapping Z:. -- Linux reported Samba 4.22.10 and an ext4 data filesystem. Windows read a - Linux-created marker and Linux read a Windows-created marker through that share. -- A fresh native cache opened the template-free Rust notebook and returned its - expected paragraph through COM. The captured XML is under - `evidence/milestone6-scout/page.xml`. Cache reset remains necessary when reusing - file identities within a clone. -- Procmon started through `win7_spawn` and wrote a PML capture. The unfiltered - short capture grew to about 126 MB; filter by test process/path and bound capture - duration before using it in a campaign. Trace contents were not analyzed in this - scout, and the transient PML was removed with the clone. -- All six VM-tooling tests passed with the scout running. One test assumed the - first control port was free; it now checks that the registered target points to - the allocated port, while retaining the separate uniqueness assertion. -- Both scout VMs were stopped and deleted after the smoke checks. Neither the - sealed images nor the personal notebook were edited. - -Existing native scripts assume a parked personal Windows profile, fixed A: paths, -and specific display coordinates. Adapt their orchestration for disposable clones -while retaining the guards used for the physical laptop. Windows base identity, -Linux package/server versions, and relevant configuration must accompany every -run; cloud-init currently installs packages at clone creation time. - -## Completion and review boundary - -The milestone is ready after the copied notebook passes automated comparison and -an independent page-by-page inspection with no known easily spottable bugs. -Corrections found during that inspection need reproductions and regression checks -before handoff. Clover's subsequent review addresses subtle representation and -design decisions. Deliver the report, model, -assets, acceptance matrix, and reproduction command together. Preserve originals -and export evidence before automatically deleting all machines created by the -run. The next decision is which document operations to expose for editing based -on that review, rather than another general approval to continue infrastructure. diff --git a/MILESTONE7.md b/MILESTONE7.md deleted file mode 100644 index 3a705c278ee5d9c52da39ff096df1333da96a3fb..0000000000000000000000000000000000000000 --- a/MILESTONE7.md +++ /dev/null @@ -1,429 +0,0 @@ -# Milestone 7: concurrent editing - -Acceptance gates passed for the tested macOS Rust, OneNote 2010 and Linux Samba -configuration. The edited notebook has fresh native references and an HTML -review; independent operation histories verify overlapping clients, retained -conflicts and the final notebook after native application closure. - -## Gates - -1. Document edits: publish text and dependent run boundaries atomically; check - Unicode boundaries, preserve untouched objects and opaque properties, reject - unsupported edits before writing. Validate native round trips before racing. -2. Random-edit CLI: select a page and seed, record the exact intended operation, - source identity and commit outcome. Operate on disposable notebook copies. -3. Local concurrency: independent reader/writer processes, synchronized starts, - stale snapshots, lock contention, randomized delays and interrupted commits. - Verify every successful observation and explain every accepted edit. -4. Native concurrency: grow from three native clients plus multiple Rust - processes to larger measured workloads. Mix disjoint and same-paragraph - changes, ordinary reads, synchronization, disconnects and restarts. Prove - operation overlap from recorded intervals; do not count open idle clients. -5. Adversarial replay: retain seeds, operation logs and snapshots; shrink failures - and add regressions. Compare the independent operation model with current - native content and explicitly reachable conflicts, then cold-reopen results. -6. Review and cleanup: inspect the edited report and native references, rerun - prior gates, verify original source hashes, delete owned machines, and record - observed client counts, operations, faults and the limits of the evidence. - -Disjoint acknowledged changes must survive. Competing changes must be accounted -for by their observed ordering or accessible conflict content; historical bytes -alone do not prove recovery. Unknown commit outcomes require rereading before a -retry. Native COM page snapshots must not replay untouched stale containers. - -Crash tests distinguish application cache state, server-persisted state and -storage durability. No finite campaign proves all schedules or hardware -power-loss behavior. - -## Replay - -Use fresh output directories and Linux VM names; the native harness deletes its -owned machines on exit. The Linux lab address allows one server run at a time. - -```sh -python3 tools/native_collaboration.py evidence/m7/new-stress --linux new-stress --stress-clients 3 --stress-operations 150 --sync-every 0 --rust-writers 4 --rust-readers 3 --edit --seed 913 -python3 tools/native_runner.py evidence/m7/new-stress/stress-closed/notebook evidence/m7/new-cold --expected-pages 1 -python3 tools/verify-document.py evidence/m7/new-stress/stress-closed/notebook evidence/m7/new-cold/read -python3 tools/native_stress.py evidence/m7/new-stress evidence/m7/new-cold/read -python3 tools/native_collaboration.py evidence/m7/new-conflict --linux new-conflict --conflict-clients 3 --rust-writers 3 --rust-readers 2 --stress-operations 5 --seed 917 -cargo +nightly fuzz run edit_text -- -max_total_time=300 -max_len=128 -rss_limit_mb=2048 -``` - -## Evidence - -- `evidence/m7/text-edit-native-02`: a length-changing Unicode edit reopened in - a fresh OneNote VM; 297 explicit formatting comparisons passed on one page. -- `evidence/m7/local-concurrency-02`: ten Rust writers and six readers; - 600 acknowledged commits and 8,101 checked observations. The final notebook - reopened in a fresh native cache (`local-concurrency-native-02`). -- `evidence/m7/edit-text-multi-fuzz-03.log`: 31,394 stateful executions with - six independently stale writer snapshots and interrupted publication. This - simulates interleavings; native concurrency is a separate gate. -- `evidence/m7/native-concurrency-01` and `native-concurrency-02`: three native - clients, four Rust writers and three Rust readers exposed a macOS SMB lock - failure. Native COM acknowledgments were not durable synchronization: the - stalled server copy did not contain those cached native edits. These runs - failed acceptance and all of their machines were deleted. -- `evidence/m7/lock-race-default`: a notebook-free, four-process reproducer - detected overlapping exclusive holders using an independent local marker, - plus stranded locks when switching read-only and read/write opens. Explicit - unlock alone did not fix it. Uniform access modes still violated exclusion. -- `evidence/m7/lock-race-atomic`: atomic open-and-lock passed all three access - modes without overlap. The macOS file adapter now uses that operation. -- `evidence/m7/smb-rust-only-04`: the formerly stalled workload completed all - 120 commits, 940 verified reads and 203 overlapping writer calls after that - change. Broader acceptance remains active; this does not establish a native - multi-client pass. - -- `evidence/m7/lock-race-atomic-16`: sixteen processes, 4,516 acquired locks - across read-only, read/write and alternating access; no marker overlap. -- `evidence/m7/smb-rust-only-05`: ten writers and six readers completed 1,000 - commits and 14,913 verified observations. The saved result subsequently - failed fresh-cache native acceptance (`smb-concurrency-native-05`): OneNote - returned zero pages after five minutes. The dependency-checkpoint fix and subsequent native checks below explain - and resolve this failure; Rust parsing alone was insufficient acceptance. -- `evidence/m7/native-concurrency-03`: all 90 native edits and 120 Rust edits - converged, but the overlap gate correctly rejected the run. Native calls - began 4.5–5.4 seconds after the first Rust commit; Rust finished at 2.7 - seconds. The shared-file start marker was delayed. The harness now waits - for each native client's first-edit acknowledgment before releasing Rust. -- `evidence/m7/local-stateful-01`: eight writers and five readers, 800 random - length-changing Unicode replacements, 9,820 verified observations. The - independent oracle applies recorded UTF-16 edit intents to each committed - state and checks all reader observations against the resulting history. -- `evidence/m7/edit-text-multi-fuzz-04.log`: 26,377 renewed stateful fuzz - executions completed without a failure. -- `evidence/m7/native-concurrency-04`: all 90 native edits and 120 Rust edits - converged, but overlap remained zero. Rust's first successful commit occurred - 10.21 seconds after its start signal, following thousands of busy reads while - native clients synchronized after every edit. Atomic open requests deny-all - sharing on this mount; native open handles can postpone Rust admission. -- `evidence/m7/native-threshold-02` and `native-threshold-03`: a fresh section - containing the same final text opened normally. The long-history section - opened after shortening revision dependencies while preserving its resolved - graph; coalescing file-node fragments alone did not help. Tested histories - through 676 transactions opened; 701 and longer failed. This bounds an - observed compatibility failure, not a documented format limit. -- The writer now appends an independent current-object snapshot before dependency - depth would exceed 512, retaining all older revisions. Section snapshots reuse - immutable property and attachment declarations; TOC snapshots remap CompactIDs - into a complete table. `checkpoint-boundaries-01.log` covers two boundaries in - both native section and TOC fixtures, with every historical revision checked. - `checkpoint-publication-01.log` checks interrupted text publication at the - boundary against complete old/new text and formatting states. -- `evidence/m7/local-checkpoint-01`: ten writers and six readers completed - 1,500 randomized Unicode replacements, 22,995 checked observations and 11,966 - overlapping writer calls. `checkpoint-native-01` cold-opened its saved result; - native text exactly matched the independently replayed edit history. -- `evidence/m7/edit-text-checkpoint-fuzz-05.log`: 4,558 stateful executions in - 302 seconds, including cached snapshots immediately before a full checkpoint, - with interrupted publication and character/formatting oracles. No failure. - `checkpoint-regression-02.log`, `checkpoint-clippy-01.log`, and - `tool-regression-03.log` record passing Rust checks and 22 Python tests. -- `evidence/m7/checkpoint-attachment-native-01`: a native section retained its - text, images, attachment and formatting through 2,052 scalar edits and repeated - snapshots. Fresh-cache reading passed 483 explicit character-format checks; - all native page XML matched the earlier reference except page modification - time, recorded separately in `retention.json`. -- `evidence/m7/checkpoint-toc-native-01`: the native two-page notebook reopened - after 1,026 TOC edits crossing two dependency checkpoints. The generator and - input/output hashes are retained in `evidence/m7/checkpoint-fixtures`. -- `checkpoint-toc-native-02` repeated the TOC check with native PDFs: two pages, - two tags and 3,562 explicit character-format comparisons passed. PDF geometry - verified the existing black-highlight case that native XML omits. -- `native-concurrency-05` established 633 clock-bounded native/Rust call overlaps - under background synchronization. Its original oracle incorrectly treated - transaction numbers as permanent across native renumbering. Replaying intent - content validated all 800 Rust commits and 6,370 reads through one counter - decrease. The run stopped before explicit native convergence; after cleanup, - one native paragraph lacked its final three cache-acknowledged edits. It is - not an acceptance pass. The content-history oracle now rejects branches, - missing acknowledgements, partial observations and reads inconsistent with - recorded real-time bounds; its regressions are in `test_native_stress.py`. -- `native-concurrency-06`: three native writers, four Rust writers and three - Rust readers passed the overlapping background-sync gate. All 600 native edits - and 800 Rust commits converged in every native client and the shared file; - 7,252 Rust reads matched the intent history, with 609 conservatively - clock-bounded native/Rust call overlaps. A separate cold-cache capture follows - in `native-concurrency-cold-06`. -- `random-edit-smb-01.json` and `random-edit-smb-01.one`: the CLI created a - separate edited file on Samba, with byte equality verified directly at the - server. Examples share the commit adapter's standard-fsync fallback when - macOS full-sync is unsupported. Rust commit/edit regressions and clippy passed - (`flush-regression-01.log`, `flush-clippy-01.log`). -- `native-concurrency-cold-06`: a separate fresh OneNote cache retained all - 1,400 intended edits from the successful mixed run. Native comparison passed - 54,330 explicit character-format checks, and exact paragraph text matched - the independently replayed operations. All associated machines were deleted. - -- `native-concurrency-07`: five native writers, eight Rust writers and five Rust - readers completed 1,000 native edits and 1,600 Rust commits, with 21,152 checked - reads and 296 clock-bounded native/Rust call overlaps. All six paragraphs - converged. `native-concurrency-cold-07` independently retained all 2,600 intended - edits and passed 95,034 explicit format comparisons. Every owned VM was deleted. -- `edit-text-checkpoint-fuzz-06.log`: 10,596 stateful executions in 901 seconds, - without failure. `edit-text-insertion-fuzz-07.log`: 4,639 executions in 301 - seconds including legacy and empty text properties, without failure. -- Random edits on the private corpus exposed unsupported legacy Unicode - promotion and absent initial text properties. The writer now adds Unicode - text while preserving the original legacy property, as MS-ONE 2.2.23 permits. - `text-insertion-boundaries-02.log` checks native fixtures, unrelated objects, - historical revisions, short writes and interrupted publication. -- A title edit previously left navigation caches stale. Text and cached titles - now publish in one revision and transaction; `title-atomicity-04.log` and - `title-regression-02.log` check rename/clear, history and interrupted short - writes. `private-random-native-01/title-verification.log` demonstrates that - the strengthened independent verifier rejects the earlier faulty copy. - CachedTitleStringFromPage's mandatory empty value applies to nonempty Unicode - titles; original legacy titles may retain it. The writer's modified-title - checks enforce that condition without rejecting untouched legacy content. -- `edit-text-title-fuzz-08.log`: 7,383 executions in 301 seconds, adding a native - title fixture and assertions relating title text, metadata and alternate title - after every persisted observation. No failure. The later single-title-object - admission guard was separately covered by `title-regression-02.log`. -- `title-empty-native-01`: a cleared title cold-opened successfully; two pages, - two tags and 3,394 explicit format checks passed, with four native-PDF highlight - checks. OneNote regenerated its empty page name from body text during opening; - this check establishes text/format retention, not stored automatic-title parity. -- `private-random-campaign-04`: five seeded edits on each of 26 current pages, - 130 total. An independent UTF-16 splice model checks all resolved styles, - unrelated nodes, metadata, contexts, historical revisions and payload hashes - after every edit. All ten frozen source files remained byte-identical. - `private-random-native-02/verification-02.log` records a fresh-cache pass on - all 26 pages, 109 tags and 186,325 explicit format comparisons, with five native - PDF highlight checks. The capture VM was deleted. -- `title-all-targets-01.log` records passing Rust all-target tests; - `title-clippy-01.log` has no warnings. `random-native-oracles-01.log` records - 30 passing Python tests, including content-chain replacement histories and - deliberate title-cache damage. -- `private-random-report-04`: the edited copy is rendered with all 45 stored - pages and 26 current native references. Browser inspection covered the edited - Video page; all 2,357 local links resolve (`link-check.json`). The original - source remains untouched. -- `native-random-08`: randomized replacements and bold/italic changes exposed - a test-driver error. On its second edit, Win7's framework left OneNote's - `🦀` entity undecoded, so the script treated entity spelling as text. - The independent intent oracle rejected that history. This run is not a pass; - all four machines were deleted. `tools/native/text.ps1` now decodes supplementary - numeric entities before the framework decoder, preserving escaped literals. - `native-text-test-01/failure-artifacts/text-result.json` records six passing - Windows decoder regressions. Its unrelated native-capture phase was correctly - stopped by the profile-isolation guard because this text-only author script - had not initialized a test profile; the VM was deleted. -- `automatic-title-native-01`: 18 application-authored automatic-title cases - captured trimming, empty paragraphs, formatting, entities and long text. - Input and stored metadata are retained with the native page names. Additional - Unicode-boundary and outline-order cases are being measured before extending - automatic-title updates. -- `automatic-title-native-02` captured ten more application-authored cases. - Automatic names select an outline by position (top before bottom, then left), - use the first text line, and trim whitespace. The 255-UTF-16-unit boundary - includes a complete supplementary character when it starts at unit 254, - yielding 256 units. Both fixture sets pass native comparison: 19/11 pages and - 11,680/5,304 explicit format checks. The first set also establishes OneNote's - XML omission of an otherwise empty outline containing only ASCII spaces; - the oracle permits that case while still rejecting omitted text, lists or tags. - These are measured compatibility rules, not a completed automatic-title writer. -- `title-checkpoint-atomicity-02.log` passes interrupted multi-object title - publication both on the native fixture and at a full dependency checkpoint, - using 17/257-byte short writes respectively. -- `native-random-09` replayed seed 912 after the native decoder fix: three - original OneNote writers, four Rust writers and three Rust readers completed - 450 native replacements with bold/italic changes and 600 Rust Unicode - replacements. All four paragraphs converged; 8,564 reads matched the - independently chained replacement intents, with 260 clock-bounded native/Rust - call overlaps. The final native formatting matched each actor's last recorded - intent. All three Windows VMs and the Linux server were deleted. -- `native-random-cold-09`: a separate fresh cache retained the exact final - text produced by all 1,050 replacement intents. Native comparison passed - 1,974 explicit format checks; an independent replay also checked 456 character - bold/italic values against the recorded native editing intent. The cold VM - was deleted. -- `random-noop-01` records seed 301 replacing ` café ` with itself and producing - identical output. The CLI now turns an identical replacement into an insertion. - `test_random_edit_campaign.py` verifies both new-file and in-place operation; - `tool-regression-06.log` records 31 passing Python tests and - `final-clippy-04.log` is clean. - -- `automatic-title-native-03`: twelve additional native cases distinguish - explicit titles (full length, leading whitespace removed, trailing whitespace - retained, first line) from automatic body summaries (trimmed and bounded). - Empty lines/paragraphs/outlines fall through to later text; table cells supply - automatic titles. All 13 captured pages pass 9,744 native format comparisons. -- The writer now publishes automatic navigation metadata when body text changes - and when an explicit title is cleared, choosing body outlines by position. - New sections use the same bounded automatic-title encoding. No public API was - added. `automatic-title-regression-03.log` passes edit/writer regressions; - `automatic-title-edit-tests-01.log` covers native line/UTF-16 boundaries and - multi-object interrupted publication. Historical objects and all non-title - metadata fields remain checked independently. -- `automatic-title-edits-01` applies 26 independently checked edits to the 13-page - native fixture. `automatic-title-edits-native-01` then passes fresh-cache text, - cached navigation title and 6,992 character-format comparisons. Its VM was - deleted. `title-empty-native-02` independently confirms the corrected cleared - title's navigation label, two pages, two tags and 3,394 format checks, with four - PDF black-highlight checks; its VM was deleted. -- `edit-text-automatic-title-fuzz-09.log`: 6,550 stateful executions in 301 seconds - after automatic-title changes, with no failure. `automatic-title-all-targets-01.log` - passes all Rust targets; `automatic-title-clippy-01.log` has no warnings. -- `private-random-campaign-05` passes another 130 edits against the whole-document - and payload-preservation oracle. All ten frozen source files remained unchanged. -- `private-random-native-03` cold-opens that edited copy: 26 pages, 109 tags, - 186,325 format checks, five PDF highlight checks, and cached navigation titles - pass. Its VM was deleted. `private-random-report-05` contains the edited pages - and fresh native references; all 2,357 local links in 46 HTML files resolve. -- `automatic-title-native-04/05` establish RTL outline ordering by descending - x anchor (width does not affect it), reversed RTL table-cell order, skipped - attachments, and whitespace trimming after UTF-16 truncation. The two public - fixtures in `corpus/m7/automatic-titles` retain native provenance. Ten direct - body-edit regressions cover the RTL/attachment selection rules. -- `automatic-title-edits-02/03` apply 21/18 independently checked edits to these - fixtures. Fresh native captures pass seven/six pages, cached navigation labels, - and 2,064/6,288 format checks. Both VMs were deleted. All Rust targets and - clippy pass in `automatic-title-all-targets-02.log` and - `automatic-title-clippy-03.log`. -- `native-random-10` repeats the ten-client replacement workload with seed 913: - 450 native edits, 600 Rust commits, 5,595 reads, and 466 clock-bounded native/Rust - call overlaps. All four paragraphs converge. `native-random-cold-10` passes - 1,740 format comparisons; intent replay independently checks all 1,050 edits - and 392 native bold/italic character values. All associated VMs were deleted. -- `tools/native_stress.py RUN CAPTURE/read` now replays saved editing intents - directly against fresh native XML, including exact paragraph multiplicity and - native formatting intent. `tool-regression-08.log` records 33 passing tests, - including rejection of missing/extra content, incorrect formatting, lost edits - and split surrogate pairs. -- `recovery-01` repeats all seven shared-notebook scenarios after the writer - changes: disjoint edits, reachable competing edits, server restart, transport - reconnect, lock contention, application restart and VM restart. Both Windows - clients and the Linux server were deleted. `recovery-cold-01` opens the final - saved notebook in a new cache and passes text, navigation and 641 character - format checks; its VM was deleted. -- The conflict oracle follows current page-manifest references only. - `conflict-oracle-regressions-01.log` checks the native offline-edit fixture and - rejects treating unreferenced history or detached object spaces as recovery. - `tool-regression-09.log` records 34 passing Python tests. -- `edit-text-rtl-fuzz-10.log` adds the native RTL title and non-title body - candidates to the stateful interrupted-edit workload: 4,531 executions in - 301 seconds, no failure. -- `mixed-conflict-01` FAILS acceptance. Three native clients edited the same - paragraph offline while three Rust writers committed 15 appends and two Rust - readers observed them. After reconnection, all three native alternatives were - reachable, but the final Rust text was absent from the current page and its - conflict pages. The pre-reconnect notebook and every convergence snapshot are - retained. All three Windows VMs and the Linux server were deleted. Rust edits - changed the text object's modification time while ancestor paragraph/outline/ - page times stayed unchanged; native edits changed those ancestors too. That - difference is a hypothesis for investigation, not an established cause. -- `mixed-conflict-02` reduces the workload to one native client, two Rust writers, - one reader and two Rust commits. Both competing results remain reachable; this - reduction does not reproduce the failure. Its Windows and Linux VMs were deleted. -- `mixed-conflict-cold-01` independently opens the failed final notebook in a - fresh cache. The native conflict UI shows only the three native alternatives; - clipboard captures recover the exact Unicode text of both conflict pages. - The Rust result is also absent from all exported stored revision objects in - the last convergence snapshot (`mixed-conflict-01/lost-edit.json`). Native - comparison passes 200 format checks on the surviving main page; that reader - agreement does not validate retention of the missing edits. The VM was deleted. -- `mixed-conflict-03` tests ancestor modification timestamps with the original - three-native/three-Rust-writer workload, using the separate - `evidence/m7/ancestor-timestamp-probe.py`. The first reconnect capture referenced - three conflict spaces without default revisions, so the strict reader stopped - the run. After client teardown, two references remained unresolved. All VMs - were deleted. `mixed-conflict-cold-03` nevertheless opens the saved file in a - fresh OneNote cache with the full Rust text; its VM was deleted. This is an - inconclusive experiment, not acceptance of a timestamp fix. -- The native checkpoint observer now preserves each distinct incomplete snapshot - and retries missing document contexts for at most two minutes. Other parse - errors still fail immediately, and retention still requires every competing - result to be reachable. This allows the next experiment to distinguish an - intermediate cross-space save from a persistent missing conflict. -- `mixed-conflict-04` repeats the timestamp experiment with that observer. One - incomplete snapshot resolves, then all three native alternatives and the full - 15-commit Rust result are reachable. All Windows and Linux VMs were deleted. -- Text edits now publish existing ancestor modification timestamps in the same - transaction as text, run boundaries and navigation caches. Preservation tests - independently follow raw object references and compare every unrelated field; - interrupted title/checkpoint publication also checks all modification times. - `ancestor-edit-regressions-04.log`, `ancestor-all-targets-01.log`, - `ancestor-clippy-01.log` and `ancestor-tool-regressions-01.log` pass. - Native acceptance of the atomic implementation is recorded below, separately - from the timestamp experiment. -- `mixed-conflict-05` stopped before any Rust commit: disconnecting native NICs - stranded an existing SMB handle and correctly excluded Rust. The setup now - holds the file's exclusive lock while disconnecting clients. Rust readers - back off during contention. All owned VMs were deleted; the final stopped - clone's removal is recorded in `cleanup-confirmed.json`. -- `private-random-campaign-06` applies 130 edits with seed 918 and verifies all - ten frozen source files unchanged. Fresh native capture - `private-random-native-04/verification-02.log` passes 26 pages, 109 tags, - 188,956 character-format comparisons and five black-highlight paragraphs. - Its PDF maps a rendered combining-accent glyph to a space. The PDF oracle - checks the uniquely located paragraph and the unhighlighted glyph's inline - region; missing ordinary text, ambiguous matches and black rectangles in that - region fail. `tool-regression-14.log` passes all 35 Python tests. - `private-random-report-06` associates all 26 current pages with the fresh - native references; all 2,357 local links resolve. Its VM was deleted. -- `edit-text-ancestor-fuzz-11.log` exercises stateful edits and interrupted - publication after ancestor timestamp propagation: 4,236 runs in 301 seconds, - no failure. -- `mixed-conflict-06` passes with the atomic implementation: three native - alternatives and the full 15-commit Rust result remain reachable after all - three native clients close. A transient incomplete save resolves before - acceptance. All Windows and Linux VMs were deleted. `mixed-conflict-cold-06` - opens the closed result in a fresh cache; native conflict UI clipboard - captures independently match all four intended results exactly. That VM was - deleted too. The ordinary page comparison passes separately; its zero - explicit format checks do not substitute for the four intent comparisons. -- The native clone cleanup now handles the VM helper's `SystemExit` on shutdown - timeout, terminates the owned VM and deletes its overlay. The regression - checks deletion and the teardown record; `tool-regression-17.log` passes all - 36 Python tests, and the private native comparison still passes. -- `native-random-11` never reached notebook editing: YAML parsed its all-digit - WAN MAC address as a sexagesimal integer, so cloud-init could not configure - the interface. The failed boot log is preserved and its VM was deleted. - Quoting both MAC addresses fixes the seed. `linux-lab-regression-01.log` - passes the two lifecycle tests; `native-random-12` reuses the same VM name - and deterministic MAC after deletion and reaches ready Windows clients. -- `native-random-12` passes seed 918 with the ancestor fix: three native - writers, four Rust writers and three Rust readers; 450 native edits, 600 - Rust commits, 2,471 checked reads and 428 clock-bounded overlapping native/Rust - calls. All four intended paragraphs converge and native formatting matches - the recorded operations. All task-owned Windows and Linux VMs were deleted. -- `native-random-cold-12` independently reopens that result and compares every - recorded native/Rust edit with the native XML. Its VM was deleted. -- `same-second-build` is an isolated controlled-clock build with one recorded - source substitution: a text edit uses its baseline element timestamp. - `same-second-equivalence/result.json` confirms that its resolved revision - exactly matches a production-library edit completed in the same actual - second. The production clock and library are unchanged. `mixed-conflict-07` - uses that build to test baseline-equal timestamps with three native writers, - three Rust writers and two Rust readers. -- `mixed-conflict-07` passes: the four page-content element timestamps remain - exactly equal to the cached baseline across all 15 Rust commits, and all - four competing results survive reconnect and application closure. The three - Windows VMs and Linux server were deleted. This distinguishes valid equal - timestamps from the inconsistent descendant/ancestor times in the original - failing writer; it does not require inventing future timestamps. -- `mixed-conflict-cold-07` independently opens the same-second result and - captures exact Unicode clipboard text from the main page and all three - native conflict pages. All four results match the recorded intent, and the - VM was deleted. Conflict retention now counts duplicate text instead of - collapsing it into a set: `exact-retention.json` rechecks both successful - closed notebooks against the exact four-result multiset. -- `native-random-13` passes seed 919 with twelve clients: four native writers, - five Rust writers and three Rust readers; 600 native edits, 750 Rust commits, - 1,903 checked reads and 456 clock-bounded overlapping native/Rust calls. - All five intended paragraphs converge. The harness now captures - `stress-closed` after every native client closes; fresh-cache acceptance uses - this snapshot instead of the earlier live checkpoint. -- `native-random-cold-13` opens that closed snapshot in a fresh native cache: - all 1,350 recorded edits match exactly across five paragraphs, with 1,672 - explicit character-format comparisons and 390 checks against the native - writers' formatting intents. Its VM and all campaign VMs were deleted. -- Final cleanup records are `final-teardown-check.json` and - `final-source-check.json`: 21 owned Windows clone identities and five Linux - VM identities are absent, and all ten frozen private source files retain - their hashes. `tool-regression-19.log` passes 36 Python tests. The HTML review - is `private-random-report-06/index.html`; its 26 current pages link to the - verified native references. Physical power-loss and unsynchronized native - cache durability remain outside these guarantees. diff --git a/MILESTONE8.md b/MILESTONE8.md deleted file mode 100644 index 846b90a819d3b37ea0ba37d5f3c440304d3b5f26..0000000000000000000000000000000000000000 --- a/MILESTONE8.md +++ /dev/null @@ -1,84 +0,0 @@ -# Workspace, crash recovery and diagnostic editing - -Goal resumed after the user accepted the workspace split and public API audit. -API boundary changes remain authorized when justified by implementation needs. - -## Acceptance gates - -1. **Workspace:** move the existing library, tests and examples into - `crates/onestore`, preserve the root corpus and example executable paths, - and pass Rust, Python and fuzz build checks. A sibling crate can consume - the library without reaching into its source directory. -2. **Public API audit:** review exported types, ownership, validation, edit - contracts and consumer ergonomics; verify a separate crate can read and edit - through public APIs; present [API-AUDIT.md](API-AUDIT.md) before continuing. -3. **Storage interruption:** verify durable images after every write/flush - boundary, including dropped unflushed writes, partial persistence, counter - rollover and revision checkpoints. Reopen the persisted image, continue - editing it, retain reproducible failures, and independently validate a - representative image matrix with OneNote. -4. **Abrupt VM stops:** stop disposable Windows clients and the Samba server - without guest shutdown during normal concurrent editing. Preserve disks, - restart the same machines, compare acknowledged operations and reachable - conflicts, and cold-open the recovered server notebook in fresh OneNote. - Track native cache acceptance separately from server durability. -5. **Diagnostic editor:** extend the HTML reading report with supported text - edits on a task-owned notebook copy. Edits use the Rust library, reject stale - snapshots, preserve unrelated content and expose ambiguous commit outcomes - without automatic replay. Verify browser interaction, two-reader stale-edit - handling, Unicode and native round trips. This is a diagnostic interface; - canvas rendering and product UI remain separate work. -6. **Closure:** rerun affected checks, review the implementation for unnecessary - state and abstractions, verify source hashes, retain evidence and replay - commands, delete owned VMs, and provide the running diagnostic tool. - -Abrupt VM stops discard guest memory while the host remains powered. Simulated -storage loss exercises the library's ordered-flush contract; neither establishes -the physical drive's behavior during actual host power loss. - -## Evidence - -Evidence belongs under `evidence/m8/`. Original notebooks are never edited. - -### Workspace and API review checkpoint - -The workspace gate passed: 56 Rust integration tests (one intentional local -fuzz-seed generator ignored), 36 Python tests, all example builds, all eight fuzz -target builds, Clippy, formatting, rustdoc and its compiled example. The external -API consumer also passes. Logs are listed in [API-AUDIT.md](API-AUDIT.md). - -The user accepted the public API audit and authorized continuation. The checkpoint -changed source locations and documented existing contracts, preserving signatures -and runtime behavior. No VMs were started for that checkpoint. - -### Storage interruption campaign - -`power-loss-02` passed 2,508 persisted-image checks and subsequent edits across -native Unicode text, 255→256 and 65535→65536 counter rollover, an attachment page, -and a pending 512-revision checkpoint. Each write/flush boundary is exercised with -six persistence policies, including complete loss of unflushed writes and partial, -reordered persistence. Comparisons cover every resolved current object, including -raw properties and payloads. Acknowledged publication must select the complete -new state; earlier cuts may select only the complete old or new state. - -`power-loss-03` repeats the same checks with exact source and failure-image -retention added to the harness. `edit-text-fuzz-01.log` records 4,327 stateful -inputs over 302 seconds without failure. The 22 retained boundary images from -`power-loss-02` are undergoing fresh OneNote captures in `power-native-01`. - -Replay: - -```sh -cargo run --release -p onestore --example power_loss -- /new/matrix-directory -python3 tools/power_loss_native.py /new/matrix-directory /new/native-directory -cargo +nightly fuzz run edit_text -- -max_total_time=300 -timeout=60 -max_len=128 -``` - -The TOC extension (`power-toc-01`) passes another 918 persisted images and -subsequent color edits, including counter rollover and a pending checkpoint. -`power-loss-04` passes the combined 3,426-image matrix with exact source/failure -retention after the harness refactor. The native gate uses the 22 retained images -from `power-loss-02` and 13 from `power-toc-01`; captures verify those exact source -hashes. All 22 section captures in `power-native-01` passed. The TOC captures in -`power-toc-native-01` also compare OneNote's notebook color to the persisted value -and are still running at this checkpoint. diff --git a/PROGRESS.md b/PROGRESS.md deleted file mode 100644 index 066a5d4df0cec5e6db10f8fb44734f8d2815eb72..0000000000000000000000000000000000000000 --- a/PROGRESS.md +++ /dev/null @@ -1,227 +0,0 @@ -# OneNote interoperability - -Target: an embeddable Rust library that reads and writes revision stores, -preserves unedited content, and collaborates with OneNote 2010 over SMB. - -| Stage | Acceptance evidence | State | -| --- | --- | --- | -| 1. Corpus | Native single-operation fixtures, private corpus integrity, independent cold reopen, semantic assertions, reproducible provenance | Passed current corpus gate | -| 2. Storage | Committed revision/object resolution, opaque preservation, malformed-input tests and fuzzing against native fixtures | Passed current corpus gate | -| 3. Writer | Create and edit through the native open/edit/save/read loop without collateral changes | Passed current corpus gate | -| 4. Durability | Injected write/flush failures, interrupted commits, acknowledged persistence, native recovery | Passed implemented scalar-commit gate | -| 5. Collaboration | Observed locks and I/O, deliberate contention, competing edits and reconnect convergence | Passed tested scalar collaboration gate | - -The personal source is `/Volumes/clover/Documents/OneNote`. It is read only. -`corpus/private/original` holds the copied baseline; `source-manifest.json` records -SHA-256 hashes and source modification times. Native automation must operate on -disposable copies, with the personal Windows registry and cache parked first. - -Wayback already contained `C:\one-tests\profile-original.reg` and -`C:\one-tests\profile-original-cache` at task start. Native tests used an isolated -profile with UnfiledNotesSection at `C:\one-tests\Loose.one`. After the stage-5 -captures, Restore returned the original registry and cache to their active paths; -Status confirmed restoration, with test backups parked and OneNote closed. -The primary native run is `corpus/native/20260905-05`, generated by OneNote -14.0.7015.1000. Its nine snapshots were independently opened from fresh caches -under `cold-05-*`. Text, styles, outline position, table cells, image bytes, and -attachment bytes pass `python3 tools/verify-corpus.py`. Transactions in the -synthetic section grow from 4 to 28 across the captured operations. - -`corpus/native-ink` records two native mouse-drawn strokes and their cold-open -binary XML payloads. `evidence/stage1/native-ink.png` shows the page. -`corpus/native-delete` records a page before deletion and its cold-open recovery -from the notebook recycle bin. The AHK ink recipe requires the displayed -1280x720 maximized OneNote layout; semantic XML verifies that strokes survived. - -The private corpus has 26 pages including two recycle-bin pages. A fresh read -can return partial page content while loading; the reader now checks the final -hierarchy against all captured page IDs. `corpus/private/exact-native` passed an -independent cold read with 26 expected pages. It includes images, ink, tags, -lists, OCR, and media. All ten original source hashes and modification times -were rechecked unchanged after native testing. COM IDs change across cache -resets and must not serve as the file-identity oracle. - -`corpus/native-encrypted` records native password protection and a fresh-cache -unlock with password `fictitious-only`. The cold XML and attachment bytes pass -the corpus verifier. OneNote's UI introduced one leading empty paragraph; -that paragraph is retained in the fixture oracle. `read.ps1 -UseCurrentCache` -captures the manually unlocked test session; the caller closes its UI afterward. - -The corpus is a starting interoperability gate, not a complete feature matrix. -Shared-file conflicts now have native evidence in the stage-5 corpus; ordinary -user-visible page version history is outside this corpus gate. - -The Rust reader follows committed transactions, resolves revision dependencies, -global IDs, roles, contexts, roots, reference-count overrides, and file payloads. -Checks cover object and object-space cycles, missing targets, immutable data, -MD5 hashes, override CRCs, and reference counts. All revisions in the ten private -files pass the current unencrypted checks. The encrypted fixture retains opaque -ciphertext and explicitly refuses property traversal. - -Native compatibility findings: -- Transaction CRCs accumulate across prior sentinel entries and fragment links - are excluded. Native transaction fragments can leave four bytes after the link. -- Reference counts include repeated references from the same source object. -- Table-of-contents override CRCs include preceding object declaration counts, - although that format does not use object groups. -- Encryption-key containers can have zero padding after the footer when their - compressed chunk references round the size to eight bytes. - -Coverage-guided runs recorded under `evidence/stage2` completed 5,529,107 storage -inputs, 6,537,029 property inputs, and 2,998,472 revision inputs without crashes. -These are bounded initial runs, not durability or interoperability proofs. -An independent 100,000-level property nesting test checks parse and drop safety. -The extended revision run completed 20,718,981 inputs in 601 seconds with a -262,144-byte input limit and no crashes. `tools/verify-reader.py` independently -matches 26 private pages and 581 nonempty text objects by page, three hyperlink -targets, 18 image payloads byte for byte, and three native GIF-to-PNG conversions -pixel for pixel. It requires Pillow. Whitespace-only paragraphs and boilerplate -date/time text remain in the raw graph but are outside this text comparison. -One page-like object space has undocumented metadata JCID 0x0002003E; it remains -in the raw graph and is excluded from the ordinary-page inventory. - -Specs in `resources/md` are the implementation references. File-format conformance, -actual OneNote acceptance, and SMB durability are separate checks. - -The first stage-3 encoding of `replace_property_bytes` appended new chunks, -replaced the affected list path with fresh list identities, copied committed -transaction entries, and published new header references. Every original byte -outside the header remained unchanged. The stage-4 append protocol below -superseded that encoding; the stage-3 fixtures retain its native acceptance. - -`corpus/writer` records the first native cycle: Rust replaces the plain-text -fixture with `Portable plain text...`; OneNote reads that text from a fresh cache; -OneNote changes it back to `Fictitious plain text.` and saves; Rust reads and edits -that file again; a second fresh native read returns `Portable plain text...`. -The native test uses equal-length text. Storage tests also replace the scalar -with lengths 0 through 40 and re-resolve every prior revision unchanged. - -Stage 3 now includes template-free `create_section` and `create_table_of_contents`. -The first creates one page with one plain-text paragraph and an explicit author; -the second records ordered section filenames and their file identities. Native -fresh-cache reads verify Unicode including a surrogate pair, both newly created -notebook files, native editing/saving of the created section, and a subsequent -longer Rust text replacement. `replace_property_bytes` also edits `.onetoc2` -scalars; OneNote independently confirms the requested notebook color. -`tools/verify-writer.py` verifies the captured native semantics and artifact hashes. - -The writer preserves unknown data and every prior revision. Native XML comparison -of the ink fixture retains formatting, positions, tables, image data, two ink -strokes, and attachment bytes; the changed outline height is allowed to reflow. -Creation initially exposed two malformed sequences: an object group needs its -following dependency-override node, and a root-space selector must follow that -space's declaration. The reader now rejects both. An isolated byte-order probe -changed a rejected empty section into a native-readable one. Failed creation -experiments live under evidence rather than the accepted writer corpus. - -The native capture script now filters exact `.one` extensions: Windows wildcard -matching had also matched `.onetoc2` and created an extra empty section. Old -captures retain that provenance; newer reads verify one section and one page. -Writer fuzzing completed 2,386,509 scalar inputs and 2,900,483 template-free -section inputs without crashes. A subsequent combined section/TOC run covers -variable section counts and reference graphs (statistics in evidence/stage3). -The corpus gates describe these tested operations, not every MS-ONE feature. - -Stage 4 begins with a concrete failure: directly copying the standalone writer's -header over a live file leaves 82 of 1025 prefix-tear states invalid on the small -native text fixture. `examples/header_faults.rs` reproduces this. Standalone -serialization must not be mistaken for an in-place commit protocol. - -The stage-4 writer now extends the existing revision-manifest list and transaction -log instead of replacing their header pointers. Original committed nodes, property -blobs, and prior revisions remain unchanged; unused fragment tails and log capacity -are populated. Restoring the previous header resolves the previous graph. This -encoding now passes independent native reads; the earlier stage-3 evidence used -the standalone encoding. - -`commit_property_bytes` accepts a storage implementation through `CommitIo`. -Its caller must supply OneNote-compatible exclusion and durable ordered flushes. -A locked snapshot comparison precedes writes. Data and metadata flush before a -one-byte publication write. At counter-byte rollover the log includes empty -transactions up to the largest intermediate count; a flush of the highest changed -counter byte precedes cleanup of lower bytes. An interrupted cleanup still resolves -the new revision. Failures distinguish NotCommitted, Unknown, and Committed. -The in-memory crash model exercises short reads/writes, every I/O failure point, -and arbitrary subsets of unflushed byte changes for ordinary commits and 255→256. -It passes, including log-fragment rollover; this is not yet a native crash test. -Updated append scalar fuzzing completed 465,817 inputs in 181 seconds, no crashes. -The combined section/TOC creation fuzz run completed 350,293 inputs in 181 seconds. - -The Windows laptop became reachable again with no Procmon or OneNote process; -the user confirmed its battery had run out. The personal profile and cache -remained isolated. Procmon 4.1 produced no trace, and -the native agent is not elevated. Shared testing continued without that driver. -`corpus/append/round-01` records independent cold native reads of plain and complex -appended edits, TOC color, counter 255→256 and 65535→65536, their interrupted -cleanup states, and all eight combinations of the tested numeric metadata tears. -OneNote reads the expected old/new text, preserves the complex fixture's full XML -and attachment bytes, and can edit/save after an interrupted counter cleanup. -The manifest records artifact hashes; `tools/verify-writer.py` checks the captures. - -The crash model now includes read failures and has a shared implementation used by -`fuzz/fuzz_targets/commit.rs`. Stateful fuzzing completed 21,506 inputs in 302 -seconds with no crash, retrying from partially persisted files at ordinary and -255→256 commits. Full Rust tests, clippy, original corpus integrity, the private -reader oracle, and native writer comparisons pass. - -`commit_file_property` acquires a whole-file filesystem lock and serializes calls -inside the process because macOS SMB flock is reentrant within a process. POSIX -byte-range locking returned ENOTSUP on this mount. Flock produced a server-visible -exclusive 0+UINT64_MAX lock, blocked a Windows read, and rejected a second Mac -process's lock. Rust sync_all uses F_FULLFSYNC on macOS; when that extension is -unsupported, the adapter uses standard fsync. Apple's SMB source routes it through -the ordinary SMB flush path. A real committed edit through the mount passed a -subsequent independent cold OneNote read. This depends on the filesystem/server -honoring flush and exclusion; it does not claim physical server power-loss testing. -Native shared activity exposed read locks at 0xFFFFFFFB and write locks at -0xFFFFFFFD in `evidence/stage4/shared-native-locks-02.log`. - -Stage 5 acceptance is captured in `corpus/collaboration/round-01`, with artifact -hashes and `tools/verify-collaboration.py`. Each final notebook was independently -reopened from a fresh native cache on OneNote 2010 14.0.7015.1000. - -| Scenario | Observed result | -| --- | --- | -| Whole-file lock contention | The native edit/sync calls completed while the file remained byte-identical under the lock; the saved edit appeared after release and survived cold reopen | -| Different paragraphs | The main page contained both Rust's Unicode edit and the native outline edit | -| Same paragraph | Rust text remained on the main page; native competing text survived in a conflict object space | -| Native offline edit | With A: disconnected, the server retained the old text; another SMB session committed Rust text; native reconnect preserved both versions | -| Lost preparation FLUSH reply | `NotCommitted`; native cold read retained the old paragraph | -| Lost publication FLUSH reply | `Unknown`; native cold read recovered the new paragraph | -| Lost counter-cleanup FLUSH reply | `Committed`; native cold read recovered the new paragraph after 255→511→256 publication | - -The dedicated loopback SMB session recorded server-visible whole-file locks, -actual write ranges/counter values, and successful FLUSH replies withheld before -client delivery. The trace verifies a successful flush of counter 511 before -cleanup to 256. macOS revoked the disconnected file handles; the adapter returned -errors rather than treating reconnection as acknowledgement. The proxy was stopped -and its temporary mount directory removed after capture. A: was restored to its -existing `\\zenith.miku-sun.ts.net\agent` mapping; the unrelated Z: mapping and -personal macOS mounts were left in place. - -OneNote's COM hierarchy omits conflict pages. The reader resolves their metadata -JCID 0x20038 and retained text; the offline fixture additionally includes a native -UI screenshot showing the conflicting edit. The full-page COM update harness -produced a redundant conflict copy even for different paragraphs. Its first merge -also changed one table column from 39.14614105224609 to 38.61000061035156 points -and one ink z-order from 7 to 6. The verifier binds those exact changes and checks -retained image/ink payloads, table contents/styles, attachment bytes, quick styles, -and outline positions; it does not claim pixel-identical native layout. - -An unlocked snapshot caught during native saving referenced an object space that -had not yet been written. It is retained as `corpus/malformed/native-inflight.one`. -`read_file` now shares the commit exclusion, and writer preflight validates the -whole current graph. A regression verifies that even a no-op request against this -intermediate graph fails before any storage I/O. Diagnostic inventory and edit -selection also validate the graph before traversing it. - -Final coverage-guided revision fuzzing, seeded with native collaboration and -transport-recovery files, completed 1,453,528 inputs in 122 seconds without a crash -(608 MB peak RSS). The final stateful commit run completed 4,992 inputs in 91 -seconds without a crash (483 MB peak RSS), following the earlier 21,506-input run. -All Rust targets, clippy, the corpus/private reader oracles, writer comparisons, -and collaboration verification pass. The ten personal source files still match -their original hashes, sizes, and modification times. This completes the bounded -stage-5 scalar collaboration gate on the tested Windows/macOS/Samba combination; -physical power-loss durability and other client/server implementations are not -established by these protocol-failure captures. diff --git a/crates/onestore-diagnostic/Cargo.toml b/crates/onestore-diagnostic/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..adfac86624601961da50aa32be0a0a653f2bfd2a --- /dev/null +++ b/crates/onestore-diagnostic/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "onestore-diagnostic" +version = "0.1.0" +edition = "2024" +publish = false + +[dependencies] +onestore = { path = "../onestore" } +serde = { version = "1.0.229", features = ["derive"] } +serde_json = "1.0.151" diff --git a/crates/onestore-diagnostic/src/main.rs b/crates/onestore-diagnostic/src/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..6352ff5035a89cc69b0154056fc60c9aea774789 --- /dev/null +++ b/crates/onestore-diagnostic/src/main.rs @@ -0,0 +1,107 @@ +use onestore::{ExGuid, Insertion, PreparedEdit, TextAttribute}; +use serde::Deserialize; +use serde_json::{Value, json}; +use std::{ + env, fs, + io::{self, Write}, +}; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Edit { + space: ExGuid, + object: ExGuid, + action: Action, +} + +#[derive(Deserialize)] +#[serde(tag = "type", deny_unknown_fields)] +enum Action { + Text { + start: u32, + end: u32, + replacement: String, + }, + Format { + start: u32, + end: u32, + attributes: Vec, + }, + Paragraph { + before: Option, + text: String, + author: String, + }, + Outline { + x: f32, + y: f32, + text: String, + author: String, + }, +} + +fn run() -> Result> { + let args: Vec<_> = env::args_os().skip(1).collect(); + if args.len() != 3 + || !["snapshot", "check", "commit"] + .iter() + .any(|mode| args[0] == *mode) + { + return Err("Usage: onestore-diagnostic snapshot FILE NEW_SNAPSHOT | check SNAPSHOT - | commit FILE SNAPSHOT; edits arrive as JSON on stdin".into()); + } + if args[0] == "snapshot" { + let bytes = onestore::read_file(&args[1])?; + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&args[2])?; + file.write_all(&bytes)?; + file.sync_all()?; + return Ok(json!({"ok": true, "bytes": bytes.len()})); + } + let check = args[0] == "check"; + if check && args[2] != "-" { + return Err("The check command requires '-' as its final argument".into()); + } + let bytes = fs::read(if check { &args[1] } else { &args[2] })?; + let edit: Edit = serde_json::from_reader(io::stdin().lock())?; + let sid = edit.space; + let oid = edit.object; + let prepared = match edit.action { + Action::Text { + start, + end, + replacement, + } => PreparedEdit::text(&bytes, sid, oid, start..end, &replacement)?, + Action::Format { + start, + end, + attributes, + } => PreparedEdit::format(&bytes, sid, oid, start..end, &attributes)?, + Action::Paragraph { + before, + text, + author, + } => PreparedEdit::insert( + &bytes, + sid, + &Insertion::paragraph(oid, before, &text, &author)?, + )?, + Action::Outline { x, y, text, author } => { + PreparedEdit::insert(&bytes, sid, &Insertion::outline(oid, x, y, &text, &author)?)? + } + }; + if check { + return Ok(json!({"ok": true})); + } + Ok(match prepared.commit_file(&args[1]) { + Ok(()) => json!({"ok": true, "state": "Committed"}), + Err(error) => json!({"ok": false, "state": format!("{:?}", error.state), + "kind": format!("{:?}", error.error.kind()), "error": error.error.to_string()}), + }) +} + +fn main() { + let result = run().unwrap_or_else(|error| json!({"ok": false, "state": "NotCommitted", "kind": "Input", "error": error.to_string()})); + println!("{result}"); +} diff --git a/crates/onestore-offline/Cargo.toml b/crates/onestore-offline/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..a9859360d60b3db96106c14ce3d5c147411de05f --- /dev/null +++ b/crates/onestore-offline/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "onestore-offline" +version = "0.1.0" +edition = "2024" +publish = false + +[features] +smb = ["dep:onestore-smb"] + +[dependencies] +onestore = { path = "../onestore" } +onestore-smb = { path = "../onestore-smb", optional = true } +rusqlite = { version = "=0.40.2", features = ["bundled"] } +thiserror = "2" +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[dev-dependencies] +tempfile = "3" + +[[example]] +name = "smb_offline_client" +required-features = ["smb"] diff --git a/crates/onestore-offline/README.md b/crates/onestore-offline/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4084707e6ed4de780b69a75032f94757e1dbc221 --- /dev/null +++ b/crates/onestore-offline/README.md @@ -0,0 +1,191 @@ +# onestore-offline + +Durable local editing for a OneNote file, in an optional sibling crate. The current +foundation stores a complete working image and typed text, insertion and formatting intents in a local +SQLite database. `sync_once` provides a reconciliation step and `start_sync` owns +automatic polling and reconnects. Local success does not +acknowledge publication to a shared notebook. + +```no_run +use onestore::ExGuid; +use onestore_offline::Replica; +# fn example(path: &std::path::Path, source: &[u8], space: ExGuid, text: ExGuid) +# -> Result<(), Box> { +// `space` and `text` are identities from the supplied section's document model. +let cache = Replica::create(path, source)?; +let snapshot = cache.snapshot()?; +let local_id = cache.edit_text(&snapshot, space, text, 0..0, "Offline edit ")?; +drop(cache); + +let reopened = Replica::open(path)?; +let current = reopened.snapshot()?; +let pending = reopened.pending()?; +assert_eq!(pending.last().map(|edit| edit.id), local_id); +# Ok(()) +# } +``` + +`insert` accepts the core library's `Insertion` value and durably retains its +object identities. Keep that value across retries; its `text_object()` identifies +the new text for subsequent offline edits. Pending entries expose +`Operation::Text(TextEdit)`, `Operation::Insert(Insertion)` or +`Operation::Format(FormatEdit)` through their +`operation` field. Synchronization applies these in queue order, so an inserted +outline can precede its paragraphs and their later edits. Missing anchors or +existing insertion identities preserve a conflict and the complete local image. + +`rebase_paragraph_conflict(id, local, remote, parent, before)` and +`rebase_outline_conflict(id, local, remote, page, x, y)` accept reviewed replacement +placements for the oldest insertion conflict. They preserve creation time, text, +author and object identities, keeping later edits attached to their original targets. +The images must still match the cache; the new placement must be valid in the remote +image. Paragraph intents cannot become outlines or vice versa. Pending edits and +uncertain publication attempts cannot be repositioned through conflict review. + +```no_run +use onestore::{ExGuid, Insertion, TextAttribute}; +use onestore_offline::{EditStatus, Replica}; +# fn add_outline(cache: &Replica, space: ExGuid, page: ExGuid) +# -> Result<(), Box> { +let outline = Insertion::outline(page, 144.0, 216.0, "Offline outline", "Author")?; +let id = cache.insert(&cache.snapshot()?, space, &outline)?; +cache.format( + &cache.snapshot()?, space, outline.text_object(), 0..7, + &[TextAttribute::Bold(true)], +)?; +if let Some(id) = id { + // A running worker may already have advanced this state. + match cache.status(id)? { + Some(EditStatus::Published { revision }) => println!("{revision}"), + state => println!("{state:?}"), + } +} +# Ok(()) +# } +``` + +`format` accepts the core `TextAttribute` slice and a UTF-16 range. Its durable intent +retains the observed text and selected attribute values. Remote text changes must +leave an unambiguous range; independent remote attributes merge, while competing +values preserve `FormattingChanged`. A remote value that already matches the +requested value is accepted. Enabling superscript or subscript also checks the +opposite attribute that the operation clears. If the remote image already satisfies +the whole operation, guarded confirmation still precedes a durable receipt. + +Recognized version-one through version-three caches migrate transactionally to the typed +queue. The migration retains images, local IDs, publication attempts, conflicts, +receipts and the autoincrement sequence; it does not reuse acknowledged IDs when +the pending queue is empty. + +Share one `Replica` between application threads. Each edit compares its supplied +snapshot under the cache transaction; stale snapshots return `Io(ResourceBusy)`. +The intent and its resulting image commit together. No-op edits return `None`. +Keep the cache on a local filesystem: the connection holds exclusive ownership +between transactions, and a second open fails busy. No network wait occurs in a +local edit. After a database error, reopen and inspect the durable state before +retrying. + +`sync_once(&mut remote)` processes the oldest pending edit through a `Remote` +implementation, returning its ID and `EditStatus`. A durable `Published` receipt +survives reopening. Publication attempts are recorded before network I/O; a retained +attempted revision requires comparison, flushing and refreshed header version +metadata before acknowledgement. If a formatting attempt's revision is missing, +the complete requested effect can instead be confirmed on a uniquely aligned +range; its receipt identifies that confirmed current revision. Otherwise the +missing attempt remains `AwaitingConfirmation`. Neither path replays an uncertain +publication. Overlapping or ambiguous edits retain `Conflict` status, their complete +local image and the last observed remote image returned by `remote_snapshot`. +Transport errors return `Error::Remote` or `Error::RemoteIo`; inspect `status(id)` +after the error to distinguish a retained attempt from a pending edit or receipt. +The error return does not roll back a locally acknowledged intent. + +Rebasing accepts only character mappings shared by every minimum insertion/deletion +alignment. UTF-16 ranges must preserve Unicode scalar boundaries. A bounded +alignment search also leaves a conflict when it cannot establish a unique mapping. +The current operation processes one queue head; while edits remain, `snapshot` +preserves the complete local working image. An empty queue can refresh from the +remote image. Synchronization holds a separate owner lock, so local edits can +continue during network waits; competing synchronization calls return `WouldBlock`. + +`rebase_conflict(id, local, remote, range)` lets a caller explicitly place the +oldest conflicting text or formatting intent at a reviewed UTF-16 range in the remote image. +The supplied images must still match `snapshot()` and `remote_snapshot()`. +It preserves the original replacement or requested attributes, intent ID, complete local working image +and every later intent; the selected range and remote paragraph become the +intent's new comparison base in one local transaction. Formatting also captures +the reviewed attribute values as its new precondition. This operation performs +no network I/O, clears the conflict to `Pending`, and wakes the worker. +Publication still reads the latest remote image and uses exact guarded comparison; +another overlapping remote edit can produce a new conflict. An uncertain attempt +cannot be rebased, and selecting text that already equals the replacement does +not create a publication acknowledgement. + +With the optional `smb` feature, `SmbRemote::new(client, path, limit)` binds an +`onestore_smb::Client` to one share-relative file and snapshot limit. Remote identity uses the logical root +object space, which survives the tested native compaction that replaces the file ID. + +An `Arc` can own one background worker. Supply a connection factory, poll +interval and observer; successful publications drain immediately, durable local +edits wake the worker, and `wake()` requests an immediate reachability retry. +Transport failures discard the old connection and retry through the factory; +Read contention and `NotCommitted` operations with `WouldBlock` or `ResourceBusy` +reuse the connection. Contended `NotCommitted` operations use randomized backoff, +capped at one second, to separate competing retry cycles. Cancellation interrupts this delay; local wake notifications +remain coalesced until its end. +`RemoteIo` distinguishes connection/read failures from +local `Io` errors. Cache/document errors stop the worker. Observers receive every +attempt's result on the worker thread, including unchanged conflict/uncertain +statuses; durable edit state remains available through `status`. + +```no_run +# #[cfg(feature = "smb")] +# fn example(cache: std::sync::Arc, username: String, password: String) +# -> Result<(), Box> { +use onestore_offline::SmbRemote; +use onestore_smb::{Client, Credentials}; +use std::time::Duration; + +let worker = cache.start_sync( + Duration::from_secs(2), + move || { + Client::connect( + "server:445", "notes", + Credentials { username: &username, password: &password, domain: "" }, + Duration::from_secs(5), + ).map(|client| SmbRemote::new(client, "Personal/Video.one", 64 * 1024 * 1024)) + }, + |result| { + if let Err(error) = result { eprintln!("{error}"); } + }, +)?; +// Retain `worker` while synchronization should run; local edits wake it automatically. +worker.stop()?; +# Ok(()) +# } +``` + +`stop()` cancels future steps and joins the worker, returning a fatal cache error +or worker panic. Dropping it requests cancellation without waiting. An in-flight +step completes before releasing ownership; a replacement worker cannot start +while the old one still owns the replica. Remote operations and callbacks must +have bounded execution times if shutdown latency matters. A dropped worker can +briefly retain the cache; use `stop()` before requiring an immediate reopen. +Cancellation and application restart retain pending edits and publication attempts. +Credentials belong to the factory, not the cache database. + +Creation refuses existing paths. Opening recognizes the application identity and +schema version, validates database integrity and both notebook images, and rejects +unsupported journal modes without converting them. Failed initialization preserves +the file for inspection. SQLite uses DELETE journaling, EXTRA synchronization and +fullfsync; each required setting is queried back. + +The cache and its pending intents contain notebook content. The core `onestore` +crate stays independent of SQLite and network runtimes. Device and simulator +examples compile and link for iOS; recorded process-interruption tests do not +establish physical power-loss durability. Evidence is tracked in [Milestone 9](../../evidence/MILESTONE9.md). + +The SMB-enabled `smb_offline_client` example is an owned-lab workload for +`tools/native_collaboration.py --offline --embedded-smb`. It separates local +acknowledgements, remote publication attempts, persisted receipts and cache reopen +checks. Its append-specific conflict review policy lives in the test client; +the library continues to preserve conflicts requiring an explicit decision. diff --git a/crates/onestore-offline/examples/cache_probe.rs b/crates/onestore-offline/examples/cache_probe.rs new file mode 100644 index 0000000000000000000000000000000000000000..abd7ff2cf78c670c46635acf394fdd001aba39ff --- /dev/null +++ b/crates/onestore-offline/examples/cache_probe.rs @@ -0,0 +1,230 @@ +use onestore::{ + ExGuid, Insertion, RevisionIndex, Store, TextAttribute, + document::{Document, Kind}, +}; +use onestore_offline::Replica; +use std::{ + io::{self, BufRead, Write}, + path::Path, +}; + +fn content(bytes: &[u8]) -> (ExGuid, ExGuid, String) { + let store = Store::parse(bytes).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + let document = Document::parse(&index).unwrap(); + document + .spaces + .iter() + .find_map(|(sid, space)| { + let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; + revision + .nodes + .iter() + .find_map(|(oid, node)| match &node.kind { + Kind::RichText { text, .. } => Some((*sid, *oid, text.clone())), + _ => None, + }) + }) + .unwrap() +} + +fn payload(operation: u64, size: usize) -> String { + format!("{operation}:🦀{}", "x".repeat(size)) +} + +fn main() -> Result<(), Box> { + let args: Vec<_> = std::env::args().collect(); + let mode = &args[1]; + let path = Path::new(&args[2]); + let operation_kind = + std::env::var("ONESTORE_CACHE_PROBE_OPERATION").unwrap_or_else(|_| "text".into()); + assert!(matches!( + operation_kind.as_str(), + "text" | "insert" | "format" + )); + let size = match std::env::var("ONESTORE_CACHE_PROBE_BYTES") { + Ok(value) => value.parse::()?, + Err(std::env::VarError::NotPresent) => 2 * 1024 * 1024, + Err(error) => return Err(error.into()), + }; + assert!(size > 0 && size <= 2 * 1024 * 1024); + let seed = std::env::var_os("ONESTORE_CACHE_PROBE_SOURCE") + .map(std::fs::read) + .transpose()?; + if mode == "init" { + let source = match seed { + Some(source) => source, + None => onestore::create_section( + "cache.one", + &if operation_kind == "format" { + payload(0, size) + } else { + "Base".into() + }, + "Fixture", + )?, + }; + Replica::create(path, &source)?; + return Ok(()); + } + let cache = Replica::open(path)?; + if mode == "read" { + let snapshot = cache.snapshot()?; + let base = cache.remote_snapshot()?; + let (sid, target, mut expected) = content(&base); + let store = Store::parse(&snapshot)?; + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store)?; + index.validate_current()?; + let document = Document::parse(&index)?; + let space = &document.spaces[&sid]; + let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; + let mut operations = Vec::new(); + let mut ids = Vec::new(); + let mut font_size = None; + for pending in cache.pending()? { + let operation = match pending.operation { + onestore_offline::Operation::Text(edit) => { + assert_eq!(operation_kind, "text"); + assert_eq!(edit.object, target); + assert_eq!(edit.before, expected); + assert_eq!( + edit.range, + 0..u32::try_from(expected.encode_utf16().count())? + ); + let operation: u64 = edit.replacement.split_once(':').unwrap().0.parse()?; + expected = payload(operation, size); + assert_eq!(edit.replacement, expected); + operation + } + onestore_offline::Operation::Insert(insertion) => { + assert_eq!(operation_kind, "insert"); + let Kind::RichText { text, .. } = + &revision.nodes[&insertion.text_object()].kind + else { + panic!("Missing inserted text") + }; + let operation: u64 = text.split_once(':').unwrap().0.parse()?; + assert_eq!(*text, payload(operation, size)); + assert_eq!(serde_json::to_value(&insertion)?["text"], *text); + let outline = &revision.nodes[&insertion.object()]; + assert!(matches!(outline.kind, Kind::Outline { .. })); + assert_eq!( + (outline.layout.x, outline.layout.y), + (Some(144.0), Some(operation as f32 * 72.0)) + ); + let (_, page) = document + .pages()? + .into_iter() + .find(|(space, _)| *space == sid) + .unwrap(); + assert!(revision.nodes[&page].children.contains(&insertion.object())); + operation + } + onestore_offline::Operation::Format(edit) => { + assert_eq!(operation_kind, "format"); + assert_eq!(edit.object, target); + assert_eq!(edit.before, expected); + assert_eq!( + edit.range, + 0..u32::try_from(expected.encode_utf16().count())? + ); + let [TextAttribute::FontSize(value)] = edit.attributes.as_slice() else { + panic!("Unexpected formatting intent") + }; + assert!((7.0..=130.0).contains(value) && value.fract() == 0.0); + font_size = Some(*value); + *value as u64 - 6 + } + }; + assert!(operations.last().is_none_or(|last| *last < operation)); + assert!(ids.last().is_none_or(|last| *last < pending.id)); + operations.push(operation); + ids.push(pending.id); + } + assert!(matches!(&revision.nodes[&target].kind,Kind::RichText{text,..} if *text==expected)); + if let Some(font_size) = font_size { + assert!( + revision + .text_runs(target)? + .iter() + .all(|run| run.format.font_size == Some(font_size)) + ); + } + if operations.is_empty() { + assert!( + snapshot == base, + "An empty local queue changed its working image" + ); + } + if let Some(output) = args.get(3) { + std::fs::write(output, &snapshot)?; + } + println!( + "{}", + serde_json::json!({"operations": operations, "ids": ids, "section_bytes": snapshot.len(), "complete_payloads": true}) + ); + return Ok(()); + } + assert_eq!(mode, "edit"); + assert!( + matches!(Replica::open(path), Err(onestore_offline::Error::Database(error)) if error.sqlite_error_code() == Some(rusqlite::ErrorCode::DatabaseBusy)) + ); + println!("ready"); + io::stdout().flush()?; + let input = io::stdin(); + let mut lines = input.lock().lines(); + let instruction = lines.next().unwrap()?; + let (operation, acknowledgement) = instruction.split_once(' ').unwrap(); + let operation: u64 = operation.parse()?; + let source = cache.snapshot()?; + let (sid, oid, text) = content(&source); + let replacement = payload(operation, size); + println!("editing {operation}"); + io::stdout().flush()?; + let id = match operation_kind.as_str() { + "text" => cache.edit_text( + &source, + sid, + oid, + 0..text.encode_utf16().count().try_into()?, + &replacement, + )?, + "insert" => { + let store = Store::parse(&source)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + let (sid, page) = document.pages()?[0]; + let insertion = Insertion::outline( + page, + 144.0, + operation as f32 * 72.0, + &replacement, + "Fixture", + )?; + cache.insert(&source, sid, &insertion)? + } + "format" => { + assert!((1..=124).contains(&operation)); + cache.format( + &source, + sid, + oid, + 0..text.encode_utf16().count().try_into()?, + &[TextAttribute::FontSize(6.0 + operation as f32)], + )? + } + _ => unreachable!(), + } + .unwrap(); + if acknowledgement == "unack" { + println!("durable {operation} {id}"); + } else { + println!("ack {operation} {id}"); + } + io::stdout().flush()?; + lines.next().transpose()?; + Ok(()) +} diff --git a/crates/onestore-offline/examples/recovery_probe.rs b/crates/onestore-offline/examples/recovery_probe.rs new file mode 100644 index 0000000000000000000000000000000000000000..60fc68a028e5a3a72bf6b2cf9ac861960a861556 --- /dev/null +++ b/crates/onestore-offline/examples/recovery_probe.rs @@ -0,0 +1,160 @@ +mod support { + pub mod view; +} +use support::view::view; + +use onestore::{CommitError, CommitIo, PreparedEdit}; +use onestore_offline::{EditStatus, Remote, Replica}; +use serde_json::json; +use std::{ + env, + fs::{self, File, OpenOptions}, + io::{self, Write}, + os::unix::fs::FileExt, + path::Path, +}; + +fn phase(name: &str) { + println!("{}", json!({"event":"phase", "name":name})); + io::stdout().flush().unwrap(); + if env::var("ONESTORE_RECOVERY_PAUSE").ok().as_deref() == Some(name) { + let mut line = String::new(); + assert!( + io::stdin().read_line(&mut line).unwrap() > 0, + "Controller closed a paused operation" + ); + } +} + +struct Disk { + file: File, + writes: usize, + flushes: usize, +} + +impl CommitIo for Disk { + fn read_at(&mut self, offset: u64, output: &mut [u8]) -> io::Result { + self.file.read_at(output, offset) + } + fn write_at(&mut self, offset: u64, data: &[u8]) -> io::Result { + self.writes += 1; + println!( + "{}", + json!({"event":"write", "number":self.writes, "offset":offset, "bytes":data.len()}) + ); + phase(&format!("write-{}-before", self.writes)); + let result = self.file.write_at(data, offset); + phase(&format!("write-{}-after", self.writes)); + result + } + fn flush(&mut self) -> io::Result<()> { + self.flushes += 1; + phase(&format!("flush-{}-before", self.flushes)); + let result = self.file.sync_all(); + phase(&format!("flush-{}-after", self.flushes)); + result + } +} + +impl Remote for Disk { + fn read(&mut self) -> io::Result> { + phase("read-before"); + let result = onestore::read_snapshot( + |offset, output| self.file.read_at(output, offset), + 256 * 1024 * 1024, + ) + .and_then(|snapshot| snapshot.ok_or_else(|| io::ErrorKind::WouldBlock.into())); + phase("read-after"); + result + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + phase("publish-before"); + let result = edit.commit(self); + phase("publish-after"); + result + } + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + phase("confirm-before"); + let result = onestore::confirm_snapshot(self, snapshot); + phase("confirm-after"); + result + } +} + +fn report(cache: &Replica, root: &Path) -> Result<(), Box> { + let local = view(&cache.snapshot()?)?; + let remote = view(&fs::read(root.join("remote.one"))?)?; + let (status, revision) = match cache.status(1)? { + Some(EditStatus::Pending) => ("pending", None), + Some(EditStatus::AwaitingConfirmation { revision }) => { + ("uncertain", Some(revision.to_string())) + } + Some(EditStatus::Published { revision }) => ("published", Some(revision.to_string())), + Some(EditStatus::Conflict(_)) => ("conflict", None), + None => ("missing", None), + }; + println!( + "{}", + json!({"event":"state", "status":status, "revision":revision, + "local_text":local.text, "remote_text":remote.text, "remote_revision":remote.revision.to_string(), + "pending":cache.pending()?.iter().map(|pending|match &pending.operation { + onestore_offline::Operation::Text(edit) => json!({"id":pending.id,"before":edit.before,"replacement":edit.replacement,"range":[edit.range.start,edit.range.end]}), + operation => json!({"id":pending.id,"operation":operation}), + }).collect::>() }) + ); + Ok(()) +} + +fn main() -> Result<(), Box> { + let args: Vec<_> = env::args().skip(1).collect(); + let mode = args + .first() + .ok_or("Expected init|inspect|sync DIRECTORY [SOURCE]")?; + let root = Path::new(args.get(1).ok_or("Missing owned directory")?); + if mode == "init" && args.len() == 3 { + let source = fs::read(&args[2])?; + let target = view(&source)?; + fs::create_dir(root)?; + let mut remote = OpenOptions::new() + .write(true) + .create_new(true) + .open(root.join("remote.one"))?; + remote.write_all(&source)?; + remote.sync_all()?; + drop(remote); + let cache = Replica::create(root.join("cache.sqlite"), &source)?; + let at = u32::try_from(target.text.encode_utf16().count())?; + assert_eq!( + cache.edit_text( + &source, + target.space, + target.object, + at..at, + " [offline-recovery]" + )?, + Some(1) + ); + phase("local-after"); + report(&cache, root)?; + } else if args.len() == 2 && ["inspect", "sync"].contains(&mode.as_str()) { + let cache = Replica::open(root.join("cache.sqlite"))?; + if mode == "sync" { + let mut disk = Disk { + file: OpenOptions::new() + .read(true) + .write(true) + .open(root.join("remote.one"))?, + writes: 0, + flushes: 0, + }; + phase("sync-before"); + let result = cache.sync_once(&mut disk); + phase("sync-after"); + result?; + } + report(&cache, root)?; + } else { + return Err("Expected init|inspect|sync DIRECTORY [SOURCE]".into()); + } + Ok(()) +} diff --git a/crates/onestore-offline/examples/smb_offline_client.rs b/crates/onestore-offline/examples/smb_offline_client.rs new file mode 100644 index 0000000000000000000000000000000000000000..db8bfbcfd7a648c3e465600572ba36c6bbe2ff9c --- /dev/null +++ b/crates/onestore-offline/examples/smb_offline_client.rs @@ -0,0 +1,593 @@ +#[path = "../../onestore/examples/support/concurrent.rs"] +mod concurrent; + +use onestore::{ + CommitError, CommitState, ExGuid, Insertion, PreparedEdit, RevisionIndex, Store, TextAttribute, + document::Document, +}; +use onestore_offline::{EditStatus, Error, Remote, Replica, SmbRemote}; +use onestore_smb::{Client, Credentials}; +use serde_json::json; +use std::{ + env, + io::{self, Write}, + path::{Path, PathBuf}, + sync::Arc, + thread, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +mod support { + pub mod view; +} +use concurrent::document_view; +use support::view::view; + +fn now() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_micros() +} + +#[derive(Clone)] +enum Pause { + Outage(PathBuf), + FormatReply(PathBuf), +} + +struct Traced { + remote: SmbRemote, + before: Option<(String, Option)>, + pause: Option, + documents: bool, +} + +impl Remote for Traced { + fn read(&mut self) -> io::Result> { + let started = now(); + let bytes = self.remote.read()?; + let observed = view(&bytes).map_err(|error| io::Error::other(error.to_string()))?; + let documents = if self.documents { + Some(document_view(&bytes).map_err(io::Error::other)?) + } else { + None + }; + println!( + "{}", + json!({"event":"read", "started_us":started, "finished_us":now(), "text":observed.text, "documents":documents}) + ); + self.before = Some((observed.text, documents)); + Ok(bytes) + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + let after = view(edit.as_bytes()).map_err(|error| CommitError { + state: CommitState::NotCommitted, + error: io::Error::other(error.to_string()), + })?; + let documents = if self.documents { + Some(document_view(edit.as_bytes()).map_err(|error| CommitError { + state: CommitState::NotCommitted, + error: io::Error::other(error), + })?) + } else { + None + }; + let changes = if let Some(after) = &documents { + let before = self + .before + .as_ref() + .and_then(|(_, documents)| documents.as_ref()) + .ok_or_else(|| CommitError { + state: CommitState::NotCommitted, + error: io::Error::other("Document publication has no observed source"), + })?; + Some( + after + .as_object() + .unwrap() + .iter() + .filter(|(id, value)| before.get(*id) != Some(*value)) + .map(|(id, value)| (id.clone(), value.clone())) + .collect::>(), + ) + } else { + None + }; + let pause = match &self.pause { + Some(Pause::Outage(marker)) => Some(( + marker, + marker.parent().unwrap().join("offline-outage-resumed"), + "outage", + )), + Some(Pause::FormatReply(marker)) + if changes.as_ref().is_some_and(|changes| { + changes.len() == 1 + && self + .before + .as_ref() + .and_then(|(_, before)| before.as_ref()) + .is_some_and(|before| changes.keys().all(|id| before.get(id).is_some())) + }) => + { + Some((marker, marker.with_extension("resume"), "format")) + } + _ => None, + }; + if let Some((marker, resumed, kind)) = pause + && !resumed.exists() + { + std::fs::write(marker, after.revision.to_string()).map_err(|error| CommitError { + state: CommitState::NotCommitted, + error, + })?; + println!( + "{}", + json!({"event":"publication_paused", "revision":after.revision.to_string(), "kind":kind, "at_us":now()}) + ); + let deadline = Instant::now() + Duration::from_secs(120); + while !resumed.exists() { + if Instant::now() >= deadline { + return Err(CommitError { + state: CommitState::NotCommitted, + error: io::Error::new( + io::ErrorKind::TimedOut, + "Offline publication barrier timed out", + ), + }); + } + thread::sleep(Duration::from_millis(10)); + } + } + let started = now(); + let result = self.remote.publish(edit); + let finished = now(); + println!( + "{}", + json!({"event":"remote_attempt", "started_us":started, "finished_us":finished, + "revision":after.revision.to_string(), "space":after.space.to_string(), "object":after.object.to_string(), "before":self.before.as_ref().map(|(text,_)|text), "after":after.text, + "state":format!("{:?}", result.as_ref().map_or_else(|error| error.state, |_| CommitState::Committed)), + "documents":documents, "document_changes":changes}) + ); + if result + .as_ref() + .is_err_and(|error| error.state == CommitState::Unknown) + && let Some(Pause::FormatReply(marker)) = &self.pause + && marker.with_extension("isolate").exists() + { + std::fs::write(marker.with_extension("isolate"), serde_json::to_vec(&json!({"space":after.space.to_string(), "revision":after.revision.to_string(), "after_us":finished})).unwrap()) + .map_err(|error| CommitError { state:CommitState::Unknown, error })?; + println!( + "{}", + json!({"event":"confirmation_paused", "revision":after.revision.to_string(), "at_us":now()}) + ); + let deadline = Instant::now() + Duration::from_secs(120); + while !marker.with_extension("confirmation-resume").exists() { + if Instant::now() >= deadline { + return Err(CommitError { + state: CommitState::Unknown, + error: io::Error::new( + io::ErrorKind::TimedOut, + "Offline confirmation barrier timed out", + ), + }); + } + thread::sleep(Duration::from_millis(10)); + } + } + result + } + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + let captured = (|| -> Result<_, Box> { + let store = Store::parse(snapshot)?; + let index = RevisionIndex::parse(&store)?; + let revisions = index + .spaces + .iter() + .map(|(id, space)| { + ( + id.to_string(), + space + .revisions + .keys() + .map(ToString::to_string) + .collect::>(), + ) + }) + .collect::>(); + let current = index + .spaces + .iter() + .filter_map(|(id, space)| { + space + .labels + .get(&(ExGuid::default(), 1)) + .map(|revision| (id.to_string(), revision.to_string())) + }) + .collect::>(); + let path = env::var_os("ONESTORE_OFFLINE_CONFIRM_DIR").map(|directory| { + PathBuf::from(directory).join(format!("{}-{}.one", std::process::id(), now())) + }); + if let Some(path) = &path { + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path)? + .write_all(snapshot)?; + } + Ok((revisions, current, path)) + })(); + let (revisions, current, capture) = captured.map_err(|error| CommitError { + state: CommitState::NotCommitted, + error: io::Error::other(error.to_string()), + })?; + let started = now(); + let result = self.remote.confirm(snapshot); + println!( + "{}", + json!({"event":"remote_confirm", "started_us":started, "finished_us":now(), "revisions":revisions, "current_revisions":current, "capture":capture.as_ref().and_then(|path| path.file_name()).map(|name| name.to_string_lossy()), "text":self.before.as_ref().map(|(text,_)|text), + "state":format!("{:?}", result.as_ref().map_or_else(|error| error.state, |_| CommitState::Committed)), "error":result.as_ref().err().map(|error|error.error.to_string())}) + ); + result + } +} + +fn tokens(text: &str) -> Option> { + let mut remaining = text.strip_prefix("Concurrent edits:")?; + let mut tokens = Vec::new(); + while !remaining.is_empty() { + let body = remaining.strip_prefix(" [w")?; + let (token, tail) = body.split_once(']')?; + let (actor, operation) = token.split_once(':')?; + if actor.is_empty() + || operation.is_empty() + || !actor + .bytes() + .chain(operation.bytes()) + .all(|b| b.is_ascii_digit()) + || tokens.contains(&token) + { + return None; + } + tokens.push(token); + remaining = tail; + } + Some(tokens) +} + +// This owned workload explicitly resolves append conflicts after all retained tokens. +fn append_position(before: &str, current: &str, token: &str) -> Option { + let original = tokens(before)?; + let present = tokens(current)?; + let mut retained = present.iter(); + for token in original { + retained.find(|&&candidate| candidate == token)?; + } + if current.contains(token) { + return None; + } + u32::try_from(current.encode_utf16().count()).ok() +} + +fn queue_document( + cache: &Replica, + actor: &str, + operation: usize, + parent: Option, + deadline: Instant, +) -> Result<(ExGuid, [u64; 2]), Box> { + let source = cache.snapshot()?; + let store = Store::parse(&source)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + let (space, page) = document.pages()?[0]; + let text = format!("Document {actor}:{operation} 🦀"); + let insertion = if operation.is_multiple_of(2) { + let column: u32 = actor + .strip_prefix('w') + .ok_or("Missing writer number")? + .parse()?; + Insertion::outline( + page, + 144.0 + column as f32 * 240.0, + 144.0 + operation as f32 * 72.0, + &text, + "Offline document writer", + )? + } else { + Insertion::paragraph( + parent.ok_or("Missing prior outline")?, + None, + &text, + "Offline document writer", + )? + }; + let parent = if operation.is_multiple_of(2) { + insertion.object() + } else { + parent.unwrap() + }; + let range = 1..u32::try_from(text.encode_utf16().count())? - 2; + let attributes = [ + TextAttribute::Bold(true), + TextAttribute::FontSize(18.0 + (operation % 9) as f32), + TextAttribute::Color(Some([0x12, 0x34, 0x56])), + ]; + let mut ids = [0; 2]; + for (step, id) in ids.iter_mut().enumerate() { + loop { + if Instant::now() >= deadline { + return Err("Document queue timed out; cache retained".into()); + } + let source = cache.snapshot()?; + let started = now(); + let result = if step == 0 { + cache.insert(&source, space, &insertion) + } else { + cache.format( + &source, + space, + insertion.text_object(), + range.clone(), + &attributes, + ) + }; + match result { + Ok(Some(acknowledged)) => { + *id = acknowledged; + println!( + "{}", + json!({"event":"local_document_commit","id":acknowledged,"operation":operation,"kind":if step==0 {"insert"} else {"format"},"space":space.to_string(),"object":insertion.text_object().to_string(),"text":text,"insertion":if step==0 {Some(&insertion)} else {None},"range":[range.start,range.end],"attributes":attributes,"started_us":started,"finished_us":now()}) + ); + break; + } + Err(Error::Io(error)) if error.kind() == io::ErrorKind::ResourceBusy => {} + other => return Err(format!("Unexpected document queue result: {other:?}").into()), + } + } + } + Ok((parent, ids)) +} + +fn main() -> Result<(), Box> { + let args: Vec<_> = env::args().skip(1).collect(); + if args.len() != 7 + || !["read", "write"].contains(&args[0].as_str()) + || !args[2].bytes().all(|b| b.is_ascii_alphanumeric()) + { + return Err("Expected read|write FILE ACTOR OPERATIONS START_FILE STOP_FILE SEED".into()); + } + let documents = env::var_os("ONESTORE_OFFLINE_DOCUMENTS").is_some(); + let address = env::var("ONESTORE_SMB_LAB")?; + let share = env::var("ONESTORE_SMB_SHARE")?; + let initial = Client::connect( + &address, + &share, + Credentials::default(), + Duration::from_secs(5), + )?; + if args[0] == "read" { + return concurrent::run( + &args, + |path| initial.read(path, 256 * 1024 * 1024), + |_, _, _, _, _, _| unreachable!(), + ); + } + let timeout: u64 = env::var("ONESTORE_CLIENT_TIMEOUT_MS") + .unwrap_or_else(|_| "600000".into()) + .parse()?; + let deadline = Instant::now() + .checked_add(Duration::from_millis(timeout)) + .ok_or("Invalid timeout")?; + let operations: usize = args[3].parse()?; + let mut seed: u64 = args[6].parse()?; + if operations == 0 { + return Err("Expected positive operations".into()); + } + let source = initial.read(&args[1], 256 * 1024 * 1024)?; + drop(initial); + let cache_path = Path::new(&args[4]) + .parent() + .ok_or("Missing workload directory")? + .join(format!("{}.sqlite", args[2])); + let cache = Arc::new(Replica::create(&cache_path, &source)?); + println!( + "{}", + json!({"event":"ready", "pid":std::process::id(), "actor":args[2], "offline":true, "document_operations":documents}) + ); + while !Path::new(&args[4]).exists() { + if Instant::now() >= deadline { + return Err("Start barrier timed out".into()); + } + thread::sleep(Duration::from_millis(5)); + } + let path = args[1].clone(); + let outage = env::var_os("ONESTORE_OFFLINE_OUTAGE_DIR").map(PathBuf::from); + let pause = outage + .as_ref() + .map(|directory| Pause::Outage(directory.join(format!("offline-paused-{}", args[2])))) + .or_else(|| { + env::var_os("ONESTORE_OFFLINE_FORMAT_REPLY_DIR").map(|directory| { + Pause::FormatReply( + PathBuf::from(directory).join(format!("offline-paused-{}", args[2])), + ) + }) + }); + let (fatal_tx, fatal_rx) = std::sync::mpsc::channel(); + let worker = cache.start_sync(Duration::from_millis(50), move || { + let client = Client::connect(&address, &share, Credentials::default(), Duration::from_secs(5))?; + println!("{}", json!({"event":"transport_connected", "at_us":now()})); + Ok(Traced { remote: SmbRemote::new(client, &path, 256 * 1024 * 1024), before:None, pause:pause.clone(), documents }) + }, move |result| { + if let Err(error) = result { + println!("{}", json!({"event":"sync_error", "error":error.to_string(), "at_us":now()})); + if !matches!(error, Error::RemoteIo(_) | Error::Remote(_)) && !matches!(error, Error::Io(error) if error.kind() == io::ErrorKind::WouldBlock) { + let _ = fatal_tx.send(error.to_string()); + } + } + })?; + let mut ids = Vec::new(); + let mut document_ids = std::collections::BTreeSet::new(); + let mut document_parent = None; + let result = (|| -> Result<(), Box> { + let mut generated = 0; + let mut received = 0; + loop { + match fatal_rx.try_recv() { + Ok(error) => return Err(error.into()), + Err(std::sync::mpsc::TryRecvError::Disconnected) => { + return Err("Worker exited before completion".into()); + } + Err(std::sync::mpsc::TryRecvError::Empty) => {} + } + while received < ids.len() { + match cache.status(ids[received])? { + Some(EditStatus::Published { revision }) => { + println!( + "{}", + json!({"event":if document_ids.contains(&ids[received]) {"document_receipt"} else {"remote_receipt"}, "id":ids[received], "revision":revision.to_string(), "at_us":now()}) + ); + received += 1; + } + Some(_) => break, + None => return Err("Local intent disappeared".into()), + } + } + if Instant::now() >= deadline { + return Err("Offline workload timed out; cache retained".into()); + } + let pending = cache.pending()?; + let capacity = if outage + .as_ref() + .is_some_and(|directory| !directory.join("offline-outage-down").exists()) + { + 1 + } else if documents && outage.is_some() { + 24 + } else { + 8 + }; + if generated < operations && pending.len() < capacity { + let source = cache.snapshot()?; + let target = view(&source)?; + let at = u32::try_from(target.text.encode_utf16().count())?; + let token = format!(" [{}:{}]", args[2], generated); + let started = now(); + match cache.edit_text(&source, target.space, target.object, at..at, &token) { + Ok(Some(id)) => { + println!( + "{}", + json!({"event":"local_commit", "id":id, "operation":generated, "space":target.space.to_string(), "object":target.object.to_string(), "before":target.text, "token":token, "started_us":started, "finished_us":now()}) + ); + ids.push(id); + if documents { + let (parent, added) = queue_document( + &cache, + &args[2], + generated, + document_parent, + deadline, + )?; + document_parent = Some(parent); + document_ids.extend(added); + ids.extend(added); + } + generated += 1; + } + Err(Error::Io(error)) if error.kind() == io::ErrorKind::ResourceBusy => {} + other => return Err(format!("Unexpected local result: {other:?}").into()), + } + } + if let Some(intent) = pending.first() + && matches!(cache.status(intent.id)?, Some(EditStatus::Conflict(_))) + { + let local = cache.snapshot()?; + let remote = cache.remote_snapshot()?; + let current = view(&remote)?; + let onestore_offline::Operation::Text(edit) = &intent.operation else { + return Err("Expected text probe intents".into()); + }; + let at = append_position(&edit.before, ¤t.text, &edit.replacement) + .ok_or("Append model disagrees with retained history")?; + match cache.rebase_conflict(intent.id, &local, &remote, at..at) { + Ok(()) => println!( + "{}", + json!({"event":"reviewed_append", "id":intent.id, "before":edit.before, "remote":current.text, "token":edit.replacement, "at_us":now()}) + ), + Err(Error::Io(error)) + if [ + io::ErrorKind::ResourceBusy, + io::ErrorKind::WouldBlock, + io::ErrorKind::InvalidInput, + ] + .contains(&error.kind()) => {} + Err(error) => return Err(error.into()), + } + } + if generated == operations && received == ids.len() { + if !cache.pending()?.is_empty() { + return Err("Acknowledged queue did not drain".into()); + } + break; + } + seed = seed + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + thread::sleep(Duration::from_millis(1 + (seed >> 32) % 7)); + } + Ok(()) + })(); + let stopped = worker.stop(); + result?; + stopped?; + drop(cache); + let reopened = Replica::open(&cache_path)?; + if !reopened.pending()?.is_empty() { + return Err("Pending edits reappeared after reopen".into()); + } + for id in ids { + let Some(EditStatus::Published { revision }) = reopened.status(id)? else { + return Err("Receipt did not survive reopen".into()); + }; + println!( + "{}", + json!({"event":if document_ids.contains(&id) {"reopened_document_receipt"} else {"reopened_receipt"}, "id":id, "revision":revision.to_string()}) + ); + } + println!( + "{}", + json!({"event":"done", "operations":operations, "at_us":now()}) + ); + Ok(()) +} + +#[test] +fn append_review_requires_a_unique_ordered_history_and_an_absent_new_token() { + assert_eq!( + append_position( + "Concurrent edits: [w0:0]", + "Concurrent edits: [w1:0] [w0:0]", + " [w0:1]" + ), + Some(31) + ); + for current in [ + "Concurrent edits:", + "Concurrent edits: [w0:0] [w0:0]", + "Concurrent edits: [w0:1] [w0:0]", + "Concurrent edits: changed [w0:0]", + ] { + assert_eq!( + append_position("Concurrent edits: [w0:0]", current, " [w0:1]"), + None + ); + } + assert_eq!( + append_position( + "Concurrent edits: [w0:0] [w1:0]", + "Concurrent edits: [w1:0] [w0:0]", + " [w0:1]" + ), + None + ); +} diff --git a/crates/onestore-offline/examples/support/view.rs b/crates/onestore-offline/examples/support/view.rs new file mode 100644 index 0000000000000000000000000000000000000000..280be0443d90acca9f8db8dd3d7f0dc63d1b5ecc --- /dev/null +++ b/crates/onestore-offline/examples/support/view.rs @@ -0,0 +1,57 @@ +use onestore::{ + ExGuid, RevisionIndex, Store, + document::{Document, Kind}, +}; + +pub struct View { + pub space: ExGuid, + pub object: ExGuid, + pub revision: ExGuid, + pub text: String, +} + +pub fn view(bytes: &[u8]) -> Result> { + let store = Store::parse(bytes)?; + if !store.checksum_mismatches.is_empty() { + return Err("Transaction checksum damage".into()); + } + let index = RevisionIndex::parse(&store)?; + index.validate_current()?; + let document = Document::parse(&index)?; + let mut found = Vec::new(); + for (sid, page) in document.pages()? { + let space = &document.spaces[&sid]; + let rid = space.contexts[&ExGuid::default()]; + let revision = &space.revisions[&rid]; + let mut pending = vec![page]; + let mut seen = std::collections::BTreeSet::new(); + while let Some(oid) = pending.pop() { + if !seen.insert(oid) { + continue; + } + let node = &revision.nodes[&oid]; + pending.extend( + node.children + .iter() + .chain(&node.content) + .chain(&node.structure) + .copied(), + ); + if let Kind::RichText { text, .. } = &node.kind + && text.starts_with("Concurrent edits:") + { + revision.text_runs(oid)?; + found.push(View { + space: sid, + object: oid, + revision: rid, + text: text.clone(), + }); + } + } + } + if found.len() != 1 { + return Err("Expected one concurrent-edit paragraph".into()); + } + Ok(found.pop().unwrap()) +} diff --git a/crates/onestore-offline/src/formatting.rs b/crates/onestore-offline/src/formatting.rs new file mode 100644 index 0000000000000000000000000000000000000000..8b0dc1f4db0ec5b1220f0dc79b0234431116d706 --- /dev/null +++ b/crates/onestore-offline/src/formatting.rs @@ -0,0 +1,204 @@ +use super::*; +use onestore::TextAttribute; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::BTreeMap; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Span { + end: u32, + values: Vec, +} + +/// A formatting intent and the text/attribute values observed before local publication. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FormatEdit { + pub object: ExGuid, + pub before: String, + pub range: Range, + pub attributes: Vec, + observed: Vec, +} + +fn fields(attributes: &[TextAttribute]) -> BTreeMap<&'static str, Value> { + let mut fields = BTreeMap::new(); + for attribute in attributes { + let (name, value) = match attribute { + TextAttribute::Bold(v) => ("bold", json!(v)), + TextAttribute::Italic(v) => ("italic", json!(v)), + TextAttribute::Underline(v) => ("underline", json!(v)), + TextAttribute::Strike(v) => ("strike", json!(v)), + TextAttribute::Superscript(v) => { + if *v { + fields.insert("subscript", json!(false)); + } + ("superscript", json!(v)) + } + TextAttribute::Subscript(v) => { + if *v { + fields.insert("superscript", json!(false)); + } + ("subscript", json!(v)) + } + TextAttribute::Font(v) => ("font", json!(v)), + TextAttribute::FontSize(v) => ("font_size", json!(v)), + TextAttribute::Color(v) | TextAttribute::Highlight(v) => ( + if matches!(attribute, TextAttribute::Color(_)) { + "color" + } else { + "highlight" + }, + json!(v.map_or(0xff000000, |[r, g, b]| u32::from_le_bytes([r, g, b, 0]))), + ), + }; + fields.insert(name, value); + } + fields +} + +fn observe( + source: &[u8], + space: ExGuid, + object: ExGuid, + range: Range, + fields: &BTreeMap<&str, Value>, +) -> Result> { + let store = Store::parse(source)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + let space = &document.spaces[&space]; + let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; + let mut spans: Vec = Vec::new(); + let mut start = 0; + for run in revision.text_runs(object)? { + let end = + start + u32::try_from(run.text.encode_utf16().count()).map_err(io::Error::other)?; + if (start < range.end && range.start < end) || (start == 0 && end == 0 && range == (0..0)) { + let format = serde_json::to_value(run.format).map_err(io::Error::other)?; + let values = fields + .iter() + .map(|(name, desired)| { + let value = &format[*name]; + if value.is_null() && desired.is_boolean() { + json!(false) + } else if value.is_null() && matches!(*name, "color" | "highlight") { + json!(0xff000000_u32) + } else { + value.clone() + } + }) + .collect::>(); + let end = end.min(range.end) - range.start; + if let Some(last) = spans.last_mut().filter(|s| s.values == values) { + last.end = end; + } else { + spans.push(Span { end, values }); + } + } + start = end; + } + Ok(spans) +} + +impl Replica { + /// Durably records a visual-formatting change and its observed attribute values. + /// Independent remote attributes can merge; competing values preserve a conflict. + pub fn format( + &self, + source: &[u8], + space: ExGuid, + object: ExGuid, + range: Range, + attributes: &[TextAttribute], + ) -> Result> { + let (edit, prepared) = FormatEdit::capture(source, space, object, range, attributes)?; + self.record(source, space, Operation::Format(edit), &prepared) + } +} + +impl FormatEdit { + pub(crate) fn capture<'a>( + source: &'a [u8], + space: ExGuid, + object: ExGuid, + range: Range, + attributes: &[TextAttribute], + ) -> Result<(Self, PreparedEdit<'a>)> { + let prepared = PreparedEdit::format(source, space, object, range.clone(), attributes)?; + let before = paragraph(source, space, object)?.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "Prepared formatting has no text target", + ) + })?; + let observed = observe(source, space, object, range.clone(), &fields(attributes))?; + let edit = Self { + object, + before, + range, + attributes: attributes.to_vec(), + observed, + }; + Ok((edit, prepared)) + } + + pub(crate) fn prepare<'a>( + &self, + snapshot: &'a [u8], + space: ExGuid, + ) -> Result, ConflictKind>> { + let Some(text) = paragraph(snapshot, space, self.object)? else { + return Ok(Err(ConflictKind::TargetUnavailable)); + }; + let Some(range) = rebase::rebase(&self.before, &text, self.range.clone()) else { + return Ok(Err(ConflictKind::TextChanged)); + }; + let prepared = match PreparedEdit::format( + snapshot, + space, + self.object, + range.clone(), + &self.attributes, + ) { + Ok(prepared) => prepared, + Err(_) => return Ok(Err(ConflictKind::UnsupportedEdit)), + }; + let desired = fields(&self.attributes); + let observed = observe(snapshot, space, self.object, range.clone(), &desired)?; + let wanted: Vec<_> = desired.values().collect(); + for spans in [&self.observed, &observed] { + if spans.is_empty() + || (!range.is_empty() && spans[0].end == 0) + || spans.last().unwrap().end != range.end - range.start + || spans.windows(2).any(|s| s[0].end >= s[1].end) + || spans.iter().any(|s| s.values.len() != wanted.len()) + { + return Ok(Err(ConflictKind::UnsupportedEdit)); + } + } + let (mut before, mut after) = (0, 0); + while before < self.observed.len() && after < observed.len() { + let old = &self.observed[before]; + let current = &observed[after]; + if old + .values + .iter() + .zip(¤t.values) + .zip(&wanted) + .any(|((old, new), wanted)| new != old && new != *wanted) + { + return Ok(Err(ConflictKind::FormattingChanged)); + } + let end = old.end.min(current.end); + if old.end == end { + before += 1; + } + if current.end == end { + after += 1; + } + } + Ok(Ok(prepared)) + } +} diff --git a/crates/onestore-offline/src/lib.rs b/crates/onestore-offline/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..f96a06aea852e09c5ff0e753a41075bddcc7d8cd --- /dev/null +++ b/crates/onestore-offline/src/lib.rs @@ -0,0 +1,362 @@ +#![forbid(unsafe_code)] +#![doc = include_str!("../README.md")] + +use onestore::{ + ExGuid, Insertion, PreparedEdit, RevisionIndex, Store, + document::{Document, Kind}, +}; +use rusqlite::{Connection, OpenFlags, TransactionBehavior, params}; +use std::{fs::OpenOptions, io, ops::Range, path::Path, sync::Mutex, time::Duration}; + +mod formatting; +mod rebase; +mod schema; +pub use formatting::FormatEdit; +mod sync; +pub use sync::{ConflictKind, EditStatus, Remote}; +mod worker; +pub use worker::SyncWorker; +#[cfg(feature = "smb")] +mod smb; +#[cfg(feature = "smb")] +pub use smb::SmbRemote; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error(transparent)] + Database(#[from] rusqlite::Error), + #[error(transparent)] + Io(#[from] io::Error), + #[error(transparent)] + Document(#[from] onestore::Error), + #[error(transparent)] + Remote(#[from] onestore::CommitError), + #[error(transparent)] + RemoteIo(io::Error), +} + +type Result = std::result::Result; + +const APPLICATION_ID: u32 = 0x4f4e454f; +const SCHEMA_VERSION: u32 = 4; + +/// Text and its observed precondition, retained across cache reopen and rebasing. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TextEdit { + pub object: ExGuid, + pub before: String, + pub range: Range, + pub replacement: String, +} + +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub enum Operation { + Text(TextEdit), + Insert(Insertion), + Format(FormatEdit), +} + +/// A locally acknowledged intent; its ID remains stable across cache reopen. +#[derive(Debug, Clone, PartialEq)] +pub struct PendingEdit { + pub id: u64, + pub space: ExGuid, + pub operation: Operation, +} + +/// Owns one local cache. Share this handle between threads; a second open fails busy. +/// SQLite's exclusive connection retains ownership between local transactions. +pub struct Replica { + connection: Mutex, + synchronization: Mutex<()>, + worker: Mutex>, +} + +impl Replica { + /// Seeds a new cache from a validated notebook image, refusing any existing path. + /// An initialization error preserves the created file for inspection. + pub fn create(path: impl AsRef, source: &[u8]) -> Result { + validate(source)?; + let mut options = OpenOptions::new(); + options.read(true).write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + drop(options.open(path.as_ref())?); + Self::connect(path.as_ref(), Some(source)) + } + + /// Reopens an existing cache and its durable pending edits without network access. + /// Unrecognized databases and unsupported journal modes are rejected without conversion. + pub fn open(path: impl AsRef) -> Result { + Self::connect(path.as_ref(), None) + } + + fn connect(path: &Path, source: Option<&[u8]>) -> Result { + let mut connection = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_WRITE)?; + connection.busy_timeout(Duration::ZERO)?; + connection.execute_batch( + "PRAGMA locking_mode=EXCLUSIVE; PRAGMA synchronous=EXTRA; PRAGMA fullfsync=ON; PRAGMA foreign_keys=ON;", + )?; + for (name, expected) in [("locking_mode", "exclusive"), ("journal_mode", "delete")] { + let actual: String = connection.pragma_query_value(None, name, |row| row.get(0))?; + if actual != expected { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Unsupported cache locking or journal mode", + ) + .into()); + } + } + for (name, expected) in [("synchronous", 3), ("fullfsync", 1), ("foreign_keys", 1)] { + let actual: i64 = connection.pragma_query_value(None, name, |row| row.get(0))?; + if actual != expected { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "Required cache synchronization is unavailable", + ) + .into()); + } + } + let transaction = connection.transaction_with_behavior(TransactionBehavior::Exclusive)?; + let application: u32 = + transaction.pragma_query_value(None, "application_id", |row| row.get(0))?; + let version: u32 = + transaction.pragma_query_value(None, "user_version", |row| row.get(0))?; + if let Some(source) = source { + let tables: i64 = + transaction + .query_row("SELECT count(*) FROM sqlite_schema", [], |row| row.get(0))?; + if application != 0 || version != 0 || tables != 0 { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "Cache initialization found an existing database", + ) + .into()); + } + transaction.pragma_update(None, "application_id", APPLICATION_ID)?; + transaction.pragma_update(None, "user_version", SCHEMA_VERSION)?; + transaction.execute_batch( + " + CREATE TABLE replica ( + id INTEGER PRIMARY KEY CHECK(id=1), + base BLOB NOT NULL, + working BLOB NOT NULL + ) STRICT; + ", + )?; + schema::create(&transaction)?; + transaction.execute("INSERT INTO replica VALUES (1, ?1, ?1)", [source])?; + } else { + if application != APPLICATION_ID || !(1..=SCHEMA_VERSION).contains(&version) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Unrecognized cache or unsupported schema version", + ) + .into()); + } + let integrity: String = + transaction.query_row("PRAGMA quick_check", [], |row| row.get(0))?; + if integrity != "ok" { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Cache integrity check failed", + ) + .into()); + } + let (base, working): (Vec, Vec) = transaction.query_row( + "SELECT base, working FROM replica WHERE id=1", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + if validate(&base)? != validate(&working)? { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Cache images belong to different documents", + ) + .into()); + } + if version < SCHEMA_VERSION { + schema::migrate(&transaction, version)?; + transaction.pragma_update(None, "user_version", SCHEMA_VERSION)?; + } + pending(&transaction)?; + } + transaction.commit()?; + Ok(Self { + connection: Mutex::new(connection), + synchronization: Mutex::new(()), + worker: Mutex::new(std::sync::Weak::new()), + }) + } + + /// Returns the latest complete locally committed image, including pending edits. + pub fn snapshot(&self) -> Result> { + let connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + Ok( + connection.query_row("SELECT working FROM replica WHERE id=1", [], |row| { + row.get(0) + })?, + ) + } + + pub fn pending(&self) -> Result> { + let connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + pending(&connection) + } + + /// Atomically records an intent and its resulting local image; returns its durable ID. + /// Unchanged text returns `None`. A stale image returns `Io(ResourceBusy)`. + /// After a database error, reopen and inspect the cache before retrying the edit. + pub fn edit_text( + &self, + source: &[u8], + space: ExGuid, + object: ExGuid, + range: Range, + replacement: &str, + ) -> Result> { + let edit = PreparedEdit::text(source, space, object, range.clone(), replacement)?; + let before = paragraph(source, space, object)?.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "Prepared edit has no text target", + ) + })?; + self.record( + source, + space, + Operation::Text(TextEdit { + object, + before, + range, + replacement: replacement.to_owned(), + }), + &edit, + ) + } + + /// Durably queues a validated insertion with its stable object identities. + /// Uses the same snapshot and local-acknowledgement contract as `edit_text`. + pub fn insert( + &self, + source: &[u8], + space: ExGuid, + insertion: &Insertion, + ) -> Result> { + let edit = PreparedEdit::insert(source, space, insertion)?; + self.record(source, space, Operation::Insert(insertion.clone()), &edit) + } + + fn record( + &self, + source: &[u8], + space: ExGuid, + operation: Operation, + edit: &PreparedEdit<'_>, + ) -> Result> { + let mut connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let current: Vec = + transaction.query_row("SELECT working FROM replica WHERE id=1", [], |row| { + row.get(0) + })?; + if current != source { + return Err(io::Error::new( + io::ErrorKind::ResourceBusy, + "The local snapshot changed before this edit", + ) + .into()); + } + if edit.as_bytes() == source { + return Ok(None); + } + transaction.execute( + "INSERT INTO edits(space, operation) VALUES (?1, ?2)", + params![ + space.to_string(), + serde_json::to_string(&operation).map_err(io::Error::other)? + ], + )?; + let id = u64::try_from(transaction.last_insert_rowid()).map_err(io::Error::other)?; + transaction.execute( + "UPDATE replica SET working=?1 WHERE id=1", + [edit.as_bytes()], + )?; + transaction.commit()?; + drop(connection); + self.wake_sync(); + Ok(Some(id)) + } + + fn wake_sync(&self) { + if let Ok(worker) = self.worker.lock() + && let Some(worker) = worker.upgrade() + { + worker.wake(); + } + } +} + +fn paragraph(source: &[u8], space: ExGuid, object: ExGuid) -> Result> { + let store = Store::parse(source)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + let node = document + .spaces + .get(&space) + .and_then(|space| { + space + .contexts + .get(&ExGuid::default()) + .and_then(|revision| space.revisions.get(revision)) + }) + .and_then(|revision| revision.nodes.get(&object)); + Ok(match node.map(|node| &node.kind) { + Some(Kind::RichText { text, .. }) => Some(text.clone()), + _ => None, + }) +} + +fn validate(source: &[u8]) -> Result { + let store = Store::parse(source)?; + if !store.checksum_mismatches.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Notebook transaction checksum damage", + ) + .into()); + } + let index = RevisionIndex::parse(&store)?; + index.validate_current()?; + Document::parse(&index)?; + Ok(index.root) +} + +fn pending(connection: &Connection) -> Result> { + let mut query = connection.prepare("SELECT id, space, operation FROM edits ORDER BY id")?; + let mut rows = query.query([])?; + let mut edits = Vec::new(); + while let Some(row) = rows.next()? { + edits.push(PendingEdit { + id: u64::try_from(row.get::<_, i64>(0)?).map_err(io::Error::other)?, + space: row.get::<_, String>(1)?.parse()?, + operation: serde_json::from_str(&row.get::<_, String>(2)?) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?, + }); + } + Ok(edits) +} diff --git a/crates/onestore-offline/src/rebase.rs b/crates/onestore-offline/src/rebase.rs new file mode 100644 index 0000000000000000000000000000000000000000..9a6dce0a8b16201774a6482925ad04659b6ab1b6 --- /dev/null +++ b/crates/onestore-offline/src/rebase.rs @@ -0,0 +1,297 @@ +use std::ops::Range; + +const INFINITY: u32 = 1 << 30; +const MAX_CELLS: usize = 4_000_000; + +struct Matrix { + band: usize, + values: Vec, +} + +impl Matrix { + fn get(&self, row: usize, column: usize) -> u32 { + let Some(column) = column + .checked_add(self.band) + .and_then(|at| at.checked_sub(row)) + else { + return INFINITY; + }; + let width = self.band * 2 + 1; + if column >= width { + return INFINITY; + } + self.values + .get(row * width + column) + .copied() + .unwrap_or(INFINITY) + } + + fn build(before: &[char], after: &[char], band: usize) -> Self { + let width = band * 2 + 1; + let mut matrix = Self { + band, + values: vec![INFINITY; (before.len() + 1) * width], + }; + for row in 0..=before.len() { + for column in row.saturating_sub(band)..=after.len().min(row + band) { + let mut cost = if row == 0 && column == 0 { 0 } else { INFINITY }; + if row > 0 { + cost = cost.min(matrix.get(row - 1, column) + 1); + } + if column > 0 { + cost = cost.min(matrix.get(row, column - 1) + 1); + } + if row > 0 && column > 0 && before[row - 1] == after[column - 1] { + cost = cost.min(matrix.get(row - 1, column - 1)); + } + matrix.values[row * width + column + band - row] = cost; + } + } + matrix + } +} + +/// Requires the same mapping in every minimum insertion/deletion alignment. +pub(crate) fn rebase(before: &str, after: &str, range: Range) -> Option> { + if range.start > range.end { + return None; + } + let mut a: Vec<_> = before.chars().collect(); + let offsets: Vec<_> = std::iter::once(0) + .chain(a.iter().scan(0_u32, |at, character| { + *at = at.checked_add(character.len_utf16() as u32)?; + Some(*at) + })) + .collect(); + let local = + offsets.binary_search(&range.start).ok()?..offsets.binary_search(&range.end).ok()?; + if before == after { + return Some(range); + } + let mut b: Vec<_> = after.chars().collect(); + let (n, m) = (a.len(), b.len()); + let mut band = n.abs_diff(m).max(1); + let forward = loop { + let width = band.checked_mul(2)?.checked_add(1)?; + if (n + 1).checked_mul(width)? > MAX_CELLS { + return None; + } + let matrix = Matrix::build(&a, &b, band); + if matrix.get(n, m) <= band as u32 { + break matrix; + } + band *= 2; + }; + let distance = forward.get(n, m); + a.reverse(); + b.reverse(); + let backward = Matrix::build(&a, &b, band); + a.reverse(); + b.reverse(); + let position = |index: usize| { + let mut matched = None; + for column in index.saturating_sub(band)..=m.min(index + band) { + let cost = forward.get(index, column); + if cost + 1 + backward.get(n - index - 1, m - column) == distance { + return None; + } + if b.get(column) == Some(&a[index]) + && cost + backward.get(n - index - 1, m - column - 1) == distance + { + if matched.is_some() { + return None; + } + matched = Some(column); + } + } + matched + }; + let mapped = if local.is_empty() { + if local.start > 0 { + position(local.start - 1)?; + } + if local.start < n { + position(local.start)?; + } + let mut boundary = None; + for column in local.start.saturating_sub(band)..=m.min(local.start + band) { + if forward.get(local.start, column) + backward.get(n - local.start, m - column) + == distance + { + if boundary.is_some() { + return None; + } + boundary = Some(column); + } + } + let at = boundary?; + at..at + } else { + let start = position(local.start)?; + for index in local.start + 1..local.end { + if position(index)? != start + index - local.start { + return None; + } + } + start..start + local.len() + }; + let start = b[..mapped.start] + .iter() + .map(|character| character.len_utf16()) + .sum::(); + let end = start + + b[mapped] + .iter() + .map(|character| character.len_utf16()) + .sum::(); + Some(u32::try_from(start).ok()?..u32::try_from(end).ok()?) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn optimal_paths(a: &[char], b: &[char]) -> Vec> { + fn visit( + a: &[char], + b: &[char], + path: &mut Vec<(usize, usize)>, + cost: usize, + best: &mut usize, + found: &mut Vec>, + ) { + if cost > *best { + return; + } + let (i, j) = *path.last().unwrap(); + if (i, j) == (a.len(), b.len()) { + if cost < *best { + found.clear(); + *best = cost; + } + found.push(path.clone()); + return; + } + for (next_i, next_j, charge) in [(i + 1, j + 1, 0), (i + 1, j, 1), (i, j + 1, 1)] { + if next_i > a.len() || next_j > b.len() || (charge == 0 && a[i] != b[j]) { + continue; + } + path.push((next_i, next_j)); + visit(a, b, path, cost + charge, best, found); + path.pop(); + } + } + let mut found = Vec::new(); + let mut best = usize::MAX; + visit(a, b, &mut vec![(0, 0)], 0, &mut best, &mut found); + found + } + + #[test] + fn bounded_alignment_agrees_with_exhaustive_paths_for_every_small_unicode_edit() { + let mut words = vec![String::new()]; + for length in 1..=4 { + for bits in 0..1 << length { + words.push( + (0..length) + .map(|bit| if bits & (1 << bit) == 0 { 'a' } else { '🦀' }) + .collect(), + ); + } + } + let mut cases = 0; + for before in &words { + let a: Vec<_> = before.chars().collect(); + for after in &words { + let b: Vec<_> = after.chars().collect(); + let paths = optimal_paths(&a, &b); + for start in 0..=a.len() { + for end in start..=a.len() { + let mut expected = None; + let mut valid = true; + for path in &paths { + let mapping: Vec<_> = (0..a.len()) + .map(|i| { + path.windows(2).find_map(|edge| { + (edge[0].0 == i && edge[1] == (i + 1, edge[0].1 + 1)) + .then_some(edge[0].1) + }) + }) + .collect(); + let candidate = if start == end { + let vertices: Vec<_> = path + .iter() + .filter(|(i, _)| *i == start) + .map(|(_, j)| *j) + .collect(); + if (start > 0 && mapping[start - 1].is_none()) + || (start < a.len() && mapping[start].is_none()) + || vertices.len() != 1 + { + None + } else { + Some(vertices[0]..vertices[0]) + } + } else if let Some(first) = mapping[start] { + (start..end) + .all(|i| mapping[i] == Some(first + i - start)) + .then_some(first..first + end - start) + } else { + None + }; + let Some(candidate) = candidate else { + valid = false; + break; + }; + if expected.as_ref().is_some_and(|old| *old != candidate) { + valid = false; + break; + } + expected = Some(candidate); + } + let expected = if valid { + expected.map(|range| { + b[..range.start].iter().map(|c| c.len_utf16() as u32).sum() + ..b[..range.end].iter().map(|c| c.len_utf16() as u32).sum() + }) + } else { + None + }; + let range = a[..start].iter().map(|c| c.len_utf16() as u32).sum() + ..a[..end].iter().map(|c| c.len_utf16() as u32).sum(); + assert_eq!( + rebase(before, after, range), + expected, + "{before:?} -> {after:?}, characters {start}..{end}" + ); + cases += 1; + } + } + } + } + assert_eq!(cases, 10881); + } + + #[test] + fn maps_disjoint_unicode_edits_and_rejects_ambiguous_or_overlapping_changes() { + assert_eq!(rebase("ab🦀cd", "Xab🦀cYd", 2..4), Some(3..5)); + assert_eq!(rebase("abc", "XabcY", 1..2), Some(2..3)); + assert_eq!(rebase("abc", "XabcY", 1..1), Some(2..2)); + assert_eq!(rebase("abc", "abcX", 3..3), None); + assert_eq!(rebase("abc", "ac", 1..2), None); + assert_eq!(rebase("abc", "", 1..1), None); + assert_eq!(rebase("aaaa", "aaaaa", 1..2), None); + assert_eq!(rebase("ab🦀cd", "ab🦀cd", 3..4), None); + assert_eq!(rebase("abc", "abc", 4..4), None); + } + + #[test] + fn long_paragraphs_with_small_remote_changes_use_a_narrow_band() { + let text = format!("{}🦀{}", "a".repeat(8192), "b".repeat(8192)); + assert_eq!( + rebase(&text, &format!("X{text}Y"), 8192..8194), + Some(8193..8195) + ); + assert_eq!(rebase(&"a".repeat(8192), &"b".repeat(8192), 1..2), None); + } +} diff --git a/crates/onestore-offline/src/schema.rs b/crates/onestore-offline/src/schema.rs new file mode 100644 index 0000000000000000000000000000000000000000..a65c19446f13f76bfac63adaed10061da7bfbed8 --- /dev/null +++ b/crates/onestore-offline/src/schema.rs @@ -0,0 +1,119 @@ +use super::*; +use rusqlite::{OptionalExtension, Transaction}; + +const CONFLICTS: &str = "CREATE TABLE conflicts ( + edit_id INTEGER PRIMARY KEY REFERENCES edits(id) ON DELETE CASCADE, + kind INTEGER NOT NULL CHECK(kind BETWEEN 0 AND 3) +) STRICT;"; + +pub(crate) fn create(transaction: &Transaction<'_>) -> Result<()> { + transaction.execute_batch( + "CREATE TABLE edits ( + id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(id>0), + space TEXT NOT NULL, + operation TEXT NOT NULL + ) STRICT; + CREATE TABLE attempt ( + id INTEGER PRIMARY KEY CHECK(id=1), + edit_id INTEGER NOT NULL UNIQUE REFERENCES edits(id) ON DELETE CASCADE, + revision TEXT NOT NULL + ) STRICT; + CREATE TABLE receipts ( + edit_id INTEGER PRIMARY KEY CHECK(edit_id>0), + revision TEXT NOT NULL + ) STRICT;", + )?; + transaction.execute_batch(CONFLICTS)?; + Ok(()) +} + +pub(crate) fn migrate(transaction: &Transaction<'_>, version: u32) -> Result<()> { + if version == 3 { + transaction.execute_batch("ALTER TABLE conflicts RENAME TO old_conflicts;")?; + transaction.execute_batch(CONFLICTS)?; + transaction.execute_batch( + "INSERT INTO conflicts SELECT * FROM old_conflicts; DROP TABLE old_conflicts;", + )?; + return Ok(()); + } + + let mut query = transaction.prepare( + "SELECT id, space, object, before_text, start, end, replacement FROM edits ORDER BY id", + )?; + let mut rows = query.query([])?; + let mut edits = Vec::new(); + while let Some(row) = rows.next()? { + edits.push(PendingEdit { + id: u64::try_from(row.get::<_, i64>(0)?).map_err(io::Error::other)?, + space: row.get::<_, String>(1)?.parse()?, + operation: Operation::Text(TextEdit { + object: row.get::<_, String>(2)?.parse()?, + before: row.get(3)?, + range: row.get(4)?..row.get(5)?, + replacement: row.get(6)?, + }), + }); + } + drop(rows); + drop(query); + let sequence: i64 = transaction + .query_row( + "SELECT seq FROM sqlite_sequence WHERE name='edits'", + [], + |row| row.get(0), + ) + .optional()? + .unwrap_or(0); + let (mut attempts, mut conflicts, mut receipts) = (Vec::new(), Vec::new(), Vec::new()); + if version == 2 { + let mut query = transaction.prepare("SELECT edit_id, revision FROM attempt")?; + attempts = query + .query_map([], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) + })? + .collect::>()?; + let mut query = transaction.prepare("SELECT edit_id, kind FROM conflicts")?; + conflicts = query + .query_map([], |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)))? + .collect::>()?; + let mut query = transaction.prepare("SELECT edit_id, revision FROM receipts")?; + receipts = query + .query_map([], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) + })? + .collect::>()?; + transaction + .execute_batch("DROP TABLE attempt; DROP TABLE conflicts; DROP TABLE receipts;")?; + } + transaction.execute_batch("DROP TABLE edits;")?; + create(transaction)?; + for edit in edits { + transaction.execute( + "INSERT INTO edits(id,space,operation) VALUES (?1,?2,?3)", + params![ + i64::try_from(edit.id).map_err(io::Error::other)?, + edit.space.to_string(), + serde_json::to_string(&edit.operation).map_err(io::Error::other)? + ], + )?; + } + // A drained queue must not reuse IDs belonging to existing durable receipts. + transaction.execute("DELETE FROM sqlite_sequence WHERE name='edits'", [])?; + transaction.execute( + "INSERT INTO sqlite_sequence(name,seq) VALUES ('edits',?1)", + [sequence], + )?; + for (id, revision) in attempts { + transaction.execute( + "INSERT INTO attempt VALUES (1,?1,?2)", + params![id, revision], + )?; + } + for (id, kind) in conflicts { + transaction.execute("INSERT INTO conflicts VALUES (?1,?2)", params![id, kind])?; + } + for (id, revision) in receipts { + transaction.execute("INSERT INTO receipts VALUES (?1,?2)", params![id, revision])?; + } + Ok(()) +} diff --git a/crates/onestore-offline/src/smb.rs b/crates/onestore-offline/src/smb.rs new file mode 100644 index 0000000000000000000000000000000000000000..6076ee03937af7bce342e751daa9e65cd447dad7 --- /dev/null +++ b/crates/onestore-offline/src/smb.rs @@ -0,0 +1,36 @@ +use crate::Remote; +use onestore::{CommitError, PreparedEdit}; +use onestore_smb::Client; +use std::io; + +/// Binds every reconciliation operation to one share-relative file and read limit. +/// Connection loss retires the client; reconnect before subsequent sync attempts. +pub struct SmbRemote { + client: Client, + path: String, + limit: usize, +} + +impl SmbRemote { + pub fn new(client: Client, path: impl Into, limit: usize) -> Self { + Self { + client, + path: path.into(), + limit, + } + } +} + +impl Remote for SmbRemote { + fn read(&mut self) -> io::Result> { + self.client.read(&self.path, self.limit) + } + + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + self.client.commit_prepared(&self.path, edit) + } + + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + self.client.confirm_snapshot(&self.path, snapshot) + } +} diff --git a/crates/onestore-offline/src/sync.rs b/crates/onestore-offline/src/sync.rs new file mode 100644 index 0000000000000000000000000000000000000000..f1bfc013ad8c2a959204ff8dde9f702e487dc9d9 --- /dev/null +++ b/crates/onestore-offline/src/sync.rs @@ -0,0 +1,458 @@ +use super::*; +use onestore::{CommitError, CommitState}; +use rusqlite::OptionalExtension; +use std::sync::{MutexGuard, TryLockError}; + +/// A single remote file with fresh reads and native-compatible guarded publication. +/// Errors retain publication state; confirmation compares, flushes, and notifies cached readers. +pub trait Remote { + fn read(&mut self) -> io::Result>; + fn publish(&mut self, edit: &PreparedEdit<'_>) -> std::result::Result<(), CommitError>; + fn confirm(&mut self, snapshot: &[u8]) -> std::result::Result<(), CommitError>; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i64)] +pub enum ConflictKind { + TextChanged = 0, + TargetUnavailable = 1, + UnsupportedEdit = 2, + FormattingChanged = 3, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EditStatus { + Pending, + AwaitingConfirmation { + revision: ExGuid, + }, + Conflict(ConflictKind), + /// Revision containing the confirmed effect; it can differ from a retired attempted revision. + Published { + revision: ExGuid, + }, +} + +impl Replica { + /// Returns a durable receipt or the persisted state of a locally acknowledged edit. + pub fn status(&self, id: u64) -> Result> { + let id = i64::try_from(id).map_err(io::Error::other)?; + let connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + if let Some(revision) = connection + .query_row( + "SELECT revision FROM receipts WHERE edit_id=?1", + [id], + |row| row.get::<_, String>(0), + ) + .optional()? + { + return Ok(Some(EditStatus::Published { + revision: revision.parse()?, + })); + } + let record: Option<(Option, Option)> = connection.query_row( + "SELECT attempt.revision, conflicts.kind FROM edits LEFT JOIN attempt ON attempt.edit_id=edits.id LEFT JOIN conflicts ON conflicts.edit_id=edits.id WHERE edits.id=?1", [id], |row| Ok((row.get(0)?,row.get(1)?))).optional()?; + Ok(match record { + None => None, + Some((Some(revision), _)) => Some(EditStatus::AwaitingConfirmation { + revision: revision.parse()?, + }), + Some((None, Some(kind))) => Some(EditStatus::Conflict(match kind { + 0 => ConflictKind::TextChanged, + 1 => ConflictKind::TargetUnavailable, + 2 => ConflictKind::UnsupportedEdit, + 3 => ConflictKind::FormattingChanged, + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Unknown cached conflict kind", + ) + .into()); + } + })), + Some((None, None)) => Some(EditStatus::Pending), + }) + } + + /// The last observed remote image, retained alongside the complete local working image. + /// Observation alone does not acknowledge any pending edit's remote durability. + pub fn remote_snapshot(&self) -> Result> { + let connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + Ok(connection.query_row("SELECT base FROM replica WHERE id=1", [], |row| row.get(0))?) + } + + /// Reconciles one pending edit, or refreshes the working image when the queue is empty. + /// Network I/O holds synchronization ownership without holding the cache mutex. + /// Uncertain edits are never replayed; retired formatting may confirm its complete observed effect. + pub fn sync_once(&self, remote: &mut impl Remote) -> Result> { + let _owner = self.sync_owner()?; + let snapshot = remote.read().map_err(Error::RemoteIo)?; + let identity = validate(&snapshot)?; + let (intent, attempted) = { + let mut connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + let transaction = + connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let base: Vec = + transaction + .query_row("SELECT base FROM replica WHERE id=1", [], |row| row.get(0))?; + let base_store = Store::parse(&base)?; + if RevisionIndex::parse(&base_store)?.root != identity { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Remote snapshot belongs to another document", + ) + .into()); + } + let Some(intent) = pending(&transaction)?.into_iter().next() else { + transaction.execute( + "UPDATE replica SET base=?1, working=?1 WHERE id=1", + [&snapshot], + )?; + transaction.commit()?; + return Ok(None); + }; + let attempted = transaction + .query_row( + "SELECT revision FROM attempt WHERE edit_id=?1", + [i64::try_from(intent.id).map_err(io::Error::other)?], + |row| row.get::<_, String>(0), + ) + .optional()?; + transaction.execute("UPDATE replica SET base=?1 WHERE id=1", [&snapshot])?; + transaction.commit()?; + (intent, attempted) + }; + if let Some(revision) = attempted { + let mut revision = revision.parse::()?; + let store = Store::parse(&snapshot)?; + let index = RevisionIndex::parse(&store)?; + if !index + .spaces + .get(&intent.space) + .is_some_and(|space| space.revisions.contains_key(&revision)) + { + let satisfied = match &intent.operation { + Operation::Format(edit) => edit + .prepare(&snapshot, intent.space)? + .is_ok_and(|prepared| prepared.as_bytes() == snapshot), + _ => false, + }; + if !satisfied { + return Ok(Some(( + intent.id, + EditStatus::AwaitingConfirmation { revision }, + ))); + } + revision = index.spaces[&intent.space].labels[&(ExGuid::default(), 1)]; + } + if let Err(error) = remote.confirm(&snapshot) { + if error.state == CommitState::Committed { + self.acknowledge(intent.id, revision, &snapshot)?; + } + return Err(error.into()); + } + self.acknowledge(intent.id, revision, &snapshot)?; + return Ok(Some((intent.id, EditStatus::Published { revision }))); + } + let candidate = match &intent.operation { + Operation::Format(edit) => edit.prepare(&snapshot, intent.space)?, + Operation::Insert(insertion) => { + PreparedEdit::insert(&snapshot, intent.space, insertion) + .map_err(|_| ConflictKind::UnsupportedEdit) + } + Operation::Text(edit) => paragraph(&snapshot, intent.space, edit.object)? + .ok_or(ConflictKind::TargetUnavailable) + .and_then(|text| { + crate::rebase::rebase(&edit.before, &text, edit.range.clone()) + .ok_or(ConflictKind::TextChanged) + }) + .and_then(|range| { + PreparedEdit::text( + &snapshot, + intent.space, + edit.object, + range, + &edit.replacement, + ) + .map_err(|_| ConflictKind::UnsupportedEdit) + }), + }; + let prepared = match candidate { + Ok(prepared) => prepared, + Err(kind) => { + let connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + connection.execute("INSERT INTO conflicts(edit_id, kind) VALUES (?1, ?2) ON CONFLICT(edit_id) DO UPDATE SET kind=excluded.kind", params![i64::try_from(intent.id).map_err(io::Error::other)?, kind as i64])?; + return Ok(Some((intent.id, EditStatus::Conflict(kind)))); + } + }; + if prepared.as_bytes() == snapshot { + let store = Store::parse(&snapshot)?; + let index = RevisionIndex::parse(&store)?; + let revision = index.spaces[&intent.space].labels[&(ExGuid::default(), 1)]; + if let Err(error) = remote.confirm(&snapshot) { + if error.state == CommitState::Committed { + self.acknowledge(intent.id, revision, &snapshot)?; + } + return Err(error.into()); + } + self.acknowledge(intent.id, revision, &snapshot)?; + return Ok(Some((intent.id, EditStatus::Published { revision }))); + } + let store = Store::parse(prepared.as_bytes())?; + let index = RevisionIndex::parse(&store)?; + let revision = index.spaces[&intent.space].labels[&(ExGuid::default(), 1)]; + { + let mut connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + let transaction = + connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + transaction.execute( + "DELETE FROM conflicts WHERE edit_id=?1", + [i64::try_from(intent.id).map_err(io::Error::other)?], + )?; + transaction.execute( + "INSERT INTO attempt(id, edit_id, revision) VALUES (1, ?1, ?2)", + params![ + i64::try_from(intent.id).map_err(io::Error::other)?, + revision.to_string() + ], + )?; + transaction.commit()?; + } + match remote.publish(&prepared) { + Ok(()) => {} + Err(error) if error.state == CommitState::NotCommitted => { + let connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + connection.execute( + "DELETE FROM attempt WHERE edit_id=?1", + [i64::try_from(intent.id).map_err(io::Error::other)?], + )?; + return Err(error.into()); + } + Err(error) if error.state == CommitState::Committed => { + self.acknowledge(intent.id, revision, prepared.as_bytes())?; + return Err(error.into()); + } + Err(error) => return Err(error.into()), + } + self.acknowledge(intent.id, revision, prepared.as_bytes())?; + Ok(Some((intent.id, EditStatus::Published { revision }))) + } + + /// Places the oldest text or formatting conflict at a reviewed remote UTF-16 range. + /// The requested replacement/attributes, local image, intent ID and later edits are preserved. + /// Both supplied images must match `snapshot` and `remote_snapshot`; stale review + /// returns `Io(ResourceBusy)`. Uncertain publication attempts cannot be rebased. + pub fn rebase_conflict( + &self, + id: u64, + local: &[u8], + remote: &[u8], + range: Range, + ) -> Result<()> { + self.resolve_conflict(id, local, remote, |intent| { + Ok(match intent.operation { + Operation::Text(mut edit) => { + let prepared = PreparedEdit::text( + remote, + intent.space, + edit.object, + range.clone(), + &edit.replacement, + )?; + if prepared.as_bytes() == remote { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "The selected range already contains the replacement", + ) + .into()); + } + edit.before = + paragraph(remote, intent.space, edit.object)?.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "The remote text target is unavailable", + ) + })?; + edit.range = range; + Operation::Text(edit) + } + Operation::Format(edit) => Operation::Format( + FormatEdit::capture( + remote, + intent.space, + edit.object, + range, + &edit.attributes, + )? + .0, + ), + Operation::Insert(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Insertion conflicts require a placement, not a text range", + ) + .into()); + } + }) + }) + } + + /// Repositions the oldest paragraph insertion conflict against reviewed cache images. + /// Object identities and dependent edits are retained; uncertain attempts cannot be moved. + pub fn rebase_paragraph_conflict( + &self, + id: u64, + local: &[u8], + remote: &[u8], + parent: ExGuid, + before: Option, + ) -> Result<()> { + self.resolve_conflict(id, local, remote, |intent| { + let Operation::Insert(insertion) = intent.operation else { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Select a paragraph insertion conflict", + ) + .into()); + }; + let insertion = insertion.reposition_paragraph(parent, before)?; + PreparedEdit::insert(remote, intent.space, &insertion)?; + Ok(Operation::Insert(insertion)) + }) + } + + /// Repositions the oldest outline insertion conflict against reviewed cache images. + /// Object identities and dependent edits are retained; uncertain attempts cannot be moved. + pub fn rebase_outline_conflict( + &self, + id: u64, + local: &[u8], + remote: &[u8], + page: ExGuid, + x: f32, + y: f32, + ) -> Result<()> { + self.resolve_conflict(id, local, remote, |intent| { + let Operation::Insert(insertion) = intent.operation else { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Select an outline insertion conflict", + ) + .into()); + }; + let insertion = insertion.reposition_outline(page, x, y)?; + PreparedEdit::insert(remote, intent.space, &insertion)?; + Ok(Operation::Insert(insertion)) + }) + } + + fn resolve_conflict( + &self, + id: u64, + local: &[u8], + remote: &[u8], + update: impl FnOnce(PendingEdit) -> Result, + ) -> Result<()> { + let owner = self.sync_owner()?; + let intent = self + .pending()? + .into_iter() + .next() + .filter(|intent| intent.id == id) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "Only the oldest conflict can be rebased", + ) + })?; + let operation = update(intent)?; + let mut connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let (base, working): (Vec, Vec) = + transaction.query_row("SELECT base, working FROM replica WHERE id=1", [], |row| { + Ok((row.get(0)?, row.get(1)?)) + })?; + if working != local || base != remote { + return Err(io::Error::new( + io::ErrorKind::ResourceBusy, + "The reviewed cache images changed", + ) + .into()); + } + let id = i64::try_from(id).map_err(io::Error::other)?; + let eligible: bool = transaction.query_row( + "SELECT EXISTS(SELECT 1 FROM conflicts WHERE edit_id=?1) AND NOT EXISTS(SELECT 1 FROM attempt)", + [id], |row| row.get(0), + )?; + if !eligible { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "The edit is not an unattempted conflict", + ) + .into()); + } + transaction.execute( + "UPDATE edits SET operation=?1 WHERE id=?2", + params![ + serde_json::to_string(&operation).map_err(io::Error::other)?, + id + ], + )?; + transaction.execute("DELETE FROM conflicts WHERE edit_id=?1", [id])?; + transaction.commit()?; + drop(connection); + drop(owner); + self.wake_sync(); + Ok(()) + } + + fn sync_owner(&self) -> Result> { + Ok(self + .synchronization + .try_lock() + .map_err(|error| match error { + TryLockError::WouldBlock => io::Error::from(io::ErrorKind::WouldBlock), + TryLockError::Poisoned(_) => { + io::Error::other("Synchronization owner panicked; reopen the cache") + } + })?) + } + + fn acknowledge(&self, id: u64, revision: ExGuid, snapshot: &[u8]) -> Result<()> { + let id = i64::try_from(id).map_err(io::Error::other)?; + let mut connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + transaction.execute( + "INSERT INTO receipts(edit_id, revision) VALUES (?1, ?2)", + params![id, revision.to_string()], + )?; + transaction.execute("DELETE FROM edits WHERE id=?1", [id])?; + transaction.execute("UPDATE replica SET base=?1, working=CASE WHEN EXISTS(SELECT 1 FROM edits) THEN working ELSE ?1 END WHERE id=1", [snapshot])?; + transaction.commit()?; + Ok(()) + } +} diff --git a/crates/onestore-offline/src/worker.rs b/crates/onestore-offline/src/worker.rs new file mode 100644 index 0000000000000000000000000000000000000000..d506a60361e4adc11ead3d22c4b73a052ea07268 --- /dev/null +++ b/crates/onestore-offline/src/worker.rs @@ -0,0 +1,167 @@ +use super::*; +use std::{ + collections::hash_map::RandomState, + hash::BuildHasher, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + mpsc::{self, SyncSender}, + }, + thread::{self, JoinHandle}, + time::Instant, +}; + +pub(super) struct Signal { + stopped: AtomicBool, + sender: SyncSender<()>, +} + +impl Signal { + pub(super) fn wake(&self) { + // One retained notification covers edits that arrive during network I/O. + let _ = self.sender.try_send(()); + } +} + +/// Owns automatic reconciliation. Dropping requests cancellation without blocking. +/// The in-flight sync step finishes before ownership is released; `stop` waits for it. +pub struct SyncWorker { + signal: Arc, + thread: Option>>, +} + +impl SyncWorker { + /// Requests a retry, for example after a network reachability change. + /// A pending contention backoff finishes before processing the notification. + pub fn wake(&self) { + self.signal.wake(); + } + + /// Cancels future steps and waits for the current step and callback to finish. + /// A stopped worker leaves pending edits and uncertain attempts in the cache. + /// Call outside the worker's own callback, which cannot join its calling thread. + pub fn stop(mut self) -> Result<()> { + self.signal.stopped.store(true, Ordering::Release); + self.signal.wake(); + self.thread + .take() + .expect("Worker owns its thread") + .join() + .map_err(|_| io::Error::other("Synchronization worker panicked"))? + } +} + +impl Drop for SyncWorker { + fn drop(&mut self) { + self.signal.stopped.store(true, Ordering::Release); + self.signal.wake(); + } +} + +impl Replica { + /// Starts one worker, reconnecting through `connect` after transport failures. + /// Local edits wake it; `interval` controls idle polling and transport retries. + /// Contended operations returning `NotCommitted` also back off by up to one second. + /// `observe` runs on the worker after each attempt, including connection errors. + /// Cache/document errors stop the worker; inspect them through `observe` or `stop`. + /// Remote calls and callbacks must be bounded for `stop` to have bounded latency. + pub fn start_sync( + self: &Arc, + interval: Duration, + mut connect: F, + mut observe: O, + ) -> io::Result + where + R: Remote + 'static, + F: FnMut() -> io::Result + Send + 'static, + O: FnMut(&Result>) + Send + 'static, + { + if interval.is_zero() || Instant::now().checked_add(interval).is_none() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Synchronization interval must be positive and representable", + )); + } + let mut owner = self + .worker + .lock() + .map_err(|_| io::Error::other("Synchronization worker registration panicked"))?; + if owner.upgrade().is_some() { + return Err(io::ErrorKind::WouldBlock.into()); + } + let (sender, receiver) = mpsc::sync_channel(1); + let signal = Arc::new(Signal { + stopped: AtomicBool::new(false), + sender, + }); + let replica = Arc::clone(self); + let worker_signal = Arc::clone(&signal); + let thread = thread::Builder::new() + .name("onestore-sync".into()) + .spawn(move || { + let mut remote = None; + let jitter = RandomState::new(); + let mut contention = 0_u32; + while !worker_signal.stopped.load(Ordering::Acquire) { + let result = match remote.as_mut() { + Some(remote) => replica.sync_once(remote), + None => match connect() { + Ok(connected) => { + remote = Some(connected); + continue; + } + Err(error) => Err(Error::RemoteIo(error)), + }, + }; + observe(&result); + match result { + Ok(Some((_, EditStatus::Published { .. }))) => { + contention = 0; + continue; + } + Ok(None) => contention = 0, + Ok(_) => {} + Err(Error::Remote(onestore::CommitError { + state: onestore::CommitState::NotCommitted, + ref error, + })) if matches!( + error.kind(), + io::ErrorKind::WouldBlock | io::ErrorKind::ResourceBusy + ) => + { + contention = contention.saturating_add(1); + let ceiling = (50_u64 << contention.min(5)).min(1000); + let until = Instant::now() + + Duration::from_millis(jitter.hash_one(contention) % ceiling); + // Local wakes must not keep competing writers in the same retry phase. + while !worker_signal.stopped.load(Ordering::Acquire) { + let Some(remaining) = until.checked_duration_since(Instant::now()) + else { + break; + }; + let _ = receiver.recv_timeout(remaining); + } + continue; + } + Err(Error::RemoteIo(ref error)) + if matches!( + error.kind(), + io::ErrorKind::WouldBlock | io::ErrorKind::ResourceBusy + ) => {} + Err(Error::RemoteIo(_) | Error::Remote(_)) => remote = None, + Err(Error::Io(ref error)) if error.kind() == io::ErrorKind::WouldBlock => {} + Err(error) => return Err(error), + } + if !worker_signal.stopped.load(Ordering::Acquire) { + let _ = receiver.recv_timeout(interval); + } + } + Ok(()) + })?; + *owner = Arc::downgrade(&signal); + Ok(SyncWorker { + signal, + thread: Some(thread), + }) + } +} diff --git a/crates/onestore-offline/tests/cache.rs b/crates/onestore-offline/tests/cache.rs new file mode 100644 index 0000000000000000000000000000000000000000..0623378170bb8b45c642c54fb9692d840c5254cc --- /dev/null +++ b/crates/onestore-offline/tests/cache.rs @@ -0,0 +1,685 @@ +use onestore::{ + ExGuid, RevisionIndex, Store, + document::{Document, Kind}, +}; +use onestore_offline::{Error, Replica}; +use std::{ + collections::BTreeSet, + fs, + io::ErrorKind, + sync::Barrier, + time::{Duration, Instant}, +}; + +fn target(source: &[u8]) -> (ExGuid, ExGuid, String) { + let store = Store::parse(source).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + let doc = Document::parse(&index).unwrap(); + doc.spaces + .iter() + .find_map(|(sid, space)| { + let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; + revision + .nodes + .iter() + .find_map(|(oid, node)| match &node.kind { + Kind::RichText { text, .. } => Some((*sid, *oid, text.clone())), + _ => None, + }) + }) + .unwrap() +} + +#[test] +fn cache_reopen_preserves_exact_images_and_intents() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("section.sqlite"); + let source = onestore::create_section("section.one", "café 🦀", "Fixture").unwrap(); + let replica = Replica::create(&path, &source).unwrap(); + assert_eq!(replica.snapshot().unwrap(), source); + assert!(replica.pending().unwrap().is_empty()); + let (sid, oid, _) = target(&source); + assert_eq!( + replica.edit_text(&source, sid, oid, 0..0, "").unwrap(), + None + ); + assert_eq!( + replica.edit_text(&source, sid, oid, 0..4, "café").unwrap(), + None + ); + let first = replica + .edit_text(&source, sid, oid, 5..7, "🐈 日本語") + .unwrap() + .unwrap(); + let edited = replica.snapshot().unwrap(); + assert_eq!(target(&edited).2, "café 🐈 日本語"); + let intents = replica.pending().unwrap(); + assert_eq!(intents.len(), 1); + assert_eq!(intents[0].id, first); + let onestore_offline::Operation::Text(first_edit) = &intents[0].operation else { + panic!() + }; + assert_eq!(first_edit.before, "café 🦀"); + assert_eq!(first_edit.range, 5..7); + assert_eq!(first_edit.replacement, "🐈 日本語"); + assert_eq!((intents[0].space, first_edit.object), (sid, oid)); + drop(replica); + let replica = Replica::open(&path).unwrap(); + assert_eq!(replica.snapshot().unwrap(), edited); + assert_eq!(replica.pending().unwrap(), intents); + let second = replica + .edit_text(&edited, sid, oid, 0..0, "Recovered ") + .unwrap() + .unwrap(); + assert!(second > first); + let onestore_offline::Operation::Text(second_edit) = &replica.pending().unwrap()[1].operation + else { + panic!() + }; + assert_eq!(second_edit.before, "café 🐈 日本語"); +} + +#[test] +fn failed_edits_preserve_both_intent_queue_and_working_image() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("section.sqlite"); + let source = onestore::create_section("section.one", "café 🦀", "Fixture").unwrap(); + let replica = Replica::create(&path, &source).unwrap(); + let (sid, oid, _) = target(&source); + for (range, replacement) in [(6..7, "X"), (0..u32::MAX, "X"), (0..1, "\n")] { + assert!( + replica + .edit_text(&source, sid, oid, range, replacement) + .is_err() + ); + assert_eq!(replica.snapshot().unwrap(), source); + assert!(replica.pending().unwrap().is_empty()); + } + drop(replica); + let connection = rusqlite::Connection::open(&path).unwrap(); + connection.execute_batch("CREATE TRIGGER fail_image BEFORE UPDATE ON replica BEGIN SELECT RAISE(ABORT, 'Injected image update failure'); END;").unwrap(); + drop(connection); + let replica = Replica::open(&path).unwrap(); + assert!(matches!( + replica.edit_text(&source, sid, oid, 0..0, "lost? "), + Err(Error::Database(_)) + )); + drop(replica); + let replica = Replica::open(&path).unwrap(); + assert_eq!(replica.snapshot().unwrap(), source); + assert!(replica.pending().unwrap().is_empty()); +} + +#[test] +fn ownership_and_foreign_file_rejection_preserve_existing_data() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("section.sqlite"); + assert!(Replica::open(&path).is_err()); + assert!(!path.exists()); + assert!(Replica::create(&path, b"invalid").is_err()); + assert!(!path.exists()); + let source = onestore::create_section("section.one", "Owned", "Fixture").unwrap(); + let replica = Replica::create(&path, &source).unwrap(); + for _ in 0..3 { + assert!( + matches!(Replica::open(&path), Err(Error::Database(error)) if error.sqlite_error_code() == Some(rusqlite::ErrorCode::DatabaseBusy)) + ); + assert!( + matches!(Replica::create(&path, &source), Err(Error::Io(error)) if error.kind() == ErrorKind::AlreadyExists) + ); + assert_eq!(replica.snapshot().unwrap(), source); + } + drop(replica); + for sql in [ + "PRAGMA application_id=0", + "PRAGMA application_id=1330529615; PRAGMA user_version=99", + ] { + let connection = rusqlite::Connection::open(&path).unwrap(); + connection.execute_batch(sql).unwrap(); + drop(connection); + let before = fs::read(&path).unwrap(); + assert!(Replica::open(&path).is_err()); + assert_eq!(fs::read(&path).unwrap(), before); + } + let foreign = dir.path().join("foreign.sqlite"); + let connection = rusqlite::Connection::open(&foreign).unwrap(); + connection + .execute_batch( + "CREATE TABLE unrelated (value TEXT); INSERT INTO unrelated VALUES ('preserve');", + ) + .unwrap(); + drop(connection); + let before = fs::read(&foreign).unwrap(); + assert!(Replica::open(&foreign).is_err()); + assert_eq!(fs::read(&foreign).unwrap(), before); + let incomplete = dir.path().join("incomplete.sqlite"); + fs::write(&incomplete, []).unwrap(); + assert!(Replica::open(&incomplete).is_err()); + assert_eq!(fs::read(&incomplete).unwrap(), b""); +} + +#[test] +fn twelve_local_editors_reject_stale_ranges_and_preserve_every_acknowledgement() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("section.sqlite"); + let source = onestore::create_section("section.one", "Shared café 🦀", "Fixture").unwrap(); + let replica = Replica::create(&path, &source).unwrap(); + let (sid, oid, _) = target(&source); + let barrier = Barrier::new(12); + let deadline = Instant::now() + Duration::from_secs(60); + let outcomes = std::thread::scope(|scope| { + let handles: Vec<_> = (0..12) + .map(|writer| { + let (replica, source, barrier) = (&replica, &source, &barrier); + scope.spawn(move || { + barrier.wait(); + let first = + replica.edit_text(source, sid, oid, 0..0, &format!("[initial-{writer}] ")); + let mut ids = Vec::new(); + let first_won = match first { + Ok(Some(id)) => { + ids.push(id); + true + } + Err(Error::Io(error)) if error.kind() == ErrorKind::ResourceBusy => false, + other => panic!("Unexpected first edit: {other:?}"), + }; + for edit in 0..20 { + loop { + assert!( + Instant::now() < deadline, + "Writer {writer} stopped progressing at {edit}" + ); + let source = replica.snapshot().unwrap(); + match replica.edit_text( + &source, + sid, + oid, + 0..0, + &format!("[{writer}-{edit}] "), + ) { + Ok(Some(id)) => { + ids.push(id); + break; + } + Err(Error::Io(error)) + if error.kind() == ErrorKind::ResourceBusy => {} + other => panic!("Unexpected edit: {other:?}"), + } + } + } + (first_won, ids) + }) + }) + .collect(); + handles + .into_iter() + .map(|handle| handle.join().unwrap()) + .collect::>() + }); + assert_eq!(outcomes.iter().filter(|(won, _)| *won).count(), 1); + let ids: BTreeSet<_> = outcomes.into_iter().flat_map(|(_, ids)| ids).collect(); + assert_eq!(ids.len(), 241); + let final_bytes = replica.snapshot().unwrap(); + let content = target(&final_bytes).2; + let mut expected = "Shared café 🦀".to_owned(); + for pending in replica.pending().unwrap() { + let onestore_offline::Operation::Text(edit) = pending.operation else { + panic!() + }; + assert_eq!(edit.before, expected); + assert_eq!(edit.range, 0..0); + expected.insert_str(0, &edit.replacement); + } + assert_eq!(content, expected); + for writer in 0..12 { + for edit in 0..20 { + assert_eq!(content.matches(&format!("[{writer}-{edit}] ")).count(), 1); + } + } + assert_eq!( + replica + .pending() + .unwrap() + .iter() + .map(|edit| edit.id) + .collect::>(), + ids + ); + assert!( + matches!(replica.edit_text(&source, sid, oid, 0..0, ""), Err(Error::Io(error)) if error.kind() == ErrorKind::ResourceBusy) + ); + drop(replica); + let reopened = Replica::open(&path).unwrap(); + assert_eq!(reopened.snapshot().unwrap(), final_bytes); + assert_eq!(reopened.pending().unwrap().len(), 241); +} + +#[test] +fn seeded_unicode_edits_and_restarts_match_an_independent_text_model() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("model.sqlite"); + let mut text = "ab🚀ab🦀 é repeated repeated".to_owned(); + let mut source = onestore::create_section("model.one", &text, "Fixture").unwrap(); + let mut replica = Replica::create(&path, &source).unwrap(); + let (space, object, _) = target(&source); + let mut random = 911_u64; + let mut next = || { + random ^= random << 13; + random ^= random >> 7; + random ^= random << 17; + random + }; + let mut intents = Vec::new(); + for step in 0..1024 { + let boundaries: Vec<_> = text + .char_indices() + .map(|(at, _)| at) + .chain([text.len()]) + .collect(); + let first = boundaries[next() as usize % boundaries.len()]; + let last = boundaries[next() as usize % boundaries.len()]; + let bytes = first.min(last)..first.max(last); + let range = u32::try_from(text[..bytes.start].encode_utf16().count()).unwrap() + ..u32::try_from(text[..bytes.end].encode_utf16().count()).unwrap(); + let replacement = ["", "🐈", "日本語", "repeated", "é", "ab🦀ab"][next() as usize % 6]; + let mut expected = text.clone(); + expected.replace_range(bytes, replacement); + let acknowledgement = replica + .edit_text(&source, space, object, range.clone(), replacement) + .unwrap(); + if let Some(id) = acknowledgement { + assert_ne!(text, expected); + assert!( + intents + .last() + .is_none_or(|edit: &onestore_offline::PendingEdit| edit.id < id) + ); + intents.push(onestore_offline::PendingEdit { + id, + space, + operation: onestore_offline::Operation::Text(onestore_offline::TextEdit { + object, + before: text, + range, + replacement: replacement.into(), + }), + }); + assert!( + matches!(replica.edit_text(&source, space, object, 0..0, "stale"), Err(Error::Io(error)) if error.kind() == ErrorKind::ResourceBusy) + ); + } else { + assert_eq!(text, expected); + } + text = expected; + source = replica.snapshot().unwrap(); + assert_eq!(target(&source).2, text, "seed 911, step {step}"); + if step % 37 == 0 { + drop(replica); + replica = Replica::open(&path).unwrap(); + assert_eq!(replica.snapshot().unwrap(), source); + assert_eq!(replica.pending().unwrap(), intents); + } + } + assert!( + intents.len() > 512, + "The history checkpoint boundary was not exercised" + ); + drop(replica); + let replica = Replica::open(&path).unwrap(); + assert_eq!(replica.pending().unwrap(), intents); + assert_eq!(replica.snapshot().unwrap(), source); +} + +#[test] +#[ignore = "migrates a fresh copy of a retained version-two or version-three cache"] +fn migrate_retained_cache_copy() { + use onestore_offline::{EditStatus, Operation, PendingEdit, TextEdit}; + let source = std::path::PathBuf::from(std::env::var_os("ONESTORE_MIGRATION_SOURCE").unwrap()); + let output = std::path::PathBuf::from(std::env::var_os("ONESTORE_MIGRATION_OUTPUT").unwrap()); + assert!(source.is_absolute() && output.is_absolute()); + let mut original = fs::File::open(&source).unwrap(); + let mut destination = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&output) + .unwrap(); + std::io::copy(&mut original, &mut destination).unwrap(); + destination.sync_all().unwrap(); + drop(destination); + let db = rusqlite::Connection::open(&output).unwrap(); + let version = db + .pragma_query_value(None, "user_version", |r| r.get::<_, u32>(0)) + .unwrap(); + assert!(matches!(version, 2 | 3)); + if std::env::var_os("ONESTORE_MIGRATION_ROLLBACK").is_some() { + assert_eq!(version, 2); + db.pragma_update(None, "foreign_keys", false).unwrap(); + db.execute( + "INSERT INTO attempt VALUES (1,999999,'{00000001-0000-0000-0000-000000000000},1')", + [], + ) + .unwrap(); + drop(db); + let before = fs::read(&output).unwrap(); + assert!( + matches!(Replica::open(&output), Err(Error::Database(rusqlite::Error::SqliteFailure(error, _))) if error.extended_code == 787) + ); + assert_eq!(fs::read(&output).unwrap(), before); + let db = rusqlite::Connection::open(&output).unwrap(); + assert_eq!( + db.pragma_query_value(None, "user_version", |r| r.get::<_, u32>(0)) + .unwrap(), + 2 + ); + println!("migration: rollback preserved {} bytes", before.len()); + return; + } + if std::env::var_os("ONESTORE_MIGRATION_CONFLICT").is_some() { + db.execute("INSERT INTO conflicts SELECT min(id),0 FROM edits", []) + .unwrap(); + } + let (base, working): (Vec, Vec) = db + .query_row("SELECT base,working FROM replica", [], |r| { + Ok((r.get(0)?, r.get(1)?)) + }) + .unwrap(); + let sequence: i64 = db + .query_row( + "SELECT seq FROM sqlite_sequence WHERE name='edits'", + [], + |r| r.get(0), + ) + .unwrap(); + let pending: Vec = + if version == 2 { + let mut query = db + .prepare("SELECT id,space,object,before_text,start,end,replacement FROM edits ORDER BY id") + .unwrap(); + let pending: Vec = query + .query_map([], |r| { + Ok(PendingEdit { + id: u64::try_from(r.get::<_, i64>(0)?).unwrap(), + space: r.get::<_, String>(1)?.parse().unwrap(), + operation: Operation::Text(TextEdit { + object: r.get::<_, String>(2)?.parse().unwrap(), + before: r.get(3)?, + range: r.get(4)?..r.get(5)?, + replacement: r.get(6)?, + }), + }) + }) + .unwrap() + .collect::>() + .unwrap(); + drop(query); + pending + } else { + let mut query = db + .prepare("SELECT id,space,operation FROM edits ORDER BY id") + .unwrap(); + query + .query_map([], |r| { + Ok(PendingEdit { + id: u64::try_from(r.get::<_, i64>(0)?).unwrap(), + space: r.get::<_, String>(1)?.parse().unwrap(), + operation: serde_json::from_str(&r.get::<_, String>(2)?).unwrap(), + }) + }) + .unwrap() + .collect::>() + .unwrap() + }; + let mut states = std::collections::BTreeMap::new(); + for edit in &pending { + states.insert(edit.id, EditStatus::Pending); + } + for table in ["receipts", "attempt"] { + let mut query = db + .prepare(&format!("SELECT edit_id,revision FROM {table}")) + .unwrap(); + for row in query + .query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?))) + .unwrap() + { + let (id, rid) = row.unwrap(); + let revision = rid.parse().unwrap(); + states.insert( + u64::try_from(id).unwrap(), + if table == "receipts" { + EditStatus::Published { revision } + } else { + EditStatus::AwaitingConfirmation { revision } + }, + ); + } + } + let mut query = db.prepare("SELECT edit_id,kind FROM conflicts").unwrap(); + for row in query + .query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, u32>(1)?))) + .unwrap() + { + let (id, kind) = row.unwrap(); + assert_eq!(kind, 0); + states.insert( + u64::try_from(id).unwrap(), + EditStatus::Conflict(onestore_offline::ConflictKind::TextChanged), + ); + } + drop(query); + drop(db); + let cache = Replica::open(&output).unwrap(); + assert_eq!(cache.snapshot().unwrap(), working); + assert_eq!(cache.remote_snapshot().unwrap(), base); + assert_eq!(cache.pending().unwrap(), pending); + for (id, status) in &states { + assert_eq!(cache.status(*id).unwrap(), Some(*status)); + } + drop(cache); + let cache = Replica::open(&output).unwrap(); + assert_eq!(cache.pending().unwrap(), pending); + let store = Store::parse(&working).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let (sid, page) = doc.pages().unwrap()[0]; + let insertion = onestore::Insertion::outline( + page, + 144.0, + 720.0, + "After cache migration", + "Migration author", + ) + .unwrap(); + let next = cache.insert(&working, sid, &insertion).unwrap().unwrap(); + assert_eq!(next, u64::try_from(sequence + 1).unwrap()); + let updated = cache.snapshot().unwrap(); + drop(cache); + let cache = Replica::open(&output).unwrap(); + assert_eq!(cache.snapshot().unwrap(), updated); + assert_eq!( + cache.pending().unwrap().last().unwrap().operation, + Operation::Insert(insertion) + ); + println!( + "migration: {}", + serde_json::json!({"pending":pending.len(),"retained_statuses":states.len(),"next_id":next,"base_bytes":base.len(),"working_bytes":working.len()}) + ); +} + +#[test] +fn twelve_local_clients_preserve_inserted_identities_and_dependent_edits() { + use onestore::Insertion; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("parallel.sqlite"); + let source = onestore::create_section("parallel.one", "Original", "Author").unwrap(); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let (sid, page) = doc.pages().unwrap()[0]; + let cache = Replica::create(&path, &source).unwrap(); + let barrier = Barrier::new(12); + let deadline = Instant::now() + Duration::from_secs(90); + let all = std::thread::scope(|scope| { + let handles: Vec<_> = (0..12) + .map(|client| { + let (cache, barrier) = (&cache, &barrier); + scope.spawn(move || { + let outline = Insertion::outline( + page, + 72.0, + 144.0 + client as f32 * 72.0, + &format!("Client {client}"), + "Author", + ) + .unwrap(); + let mut ids = Vec::new(); + let mut objects = Vec::new(); + barrier.wait(); + for sequence in 0..4 { + let insertion = if sequence == 0 { + outline.clone() + } else { + Insertion::paragraph( + outline.object(), + None, + &format!("Paragraph {client}:{sequence}"), + "Author", + ) + .unwrap() + }; + loop { + assert!( + Instant::now() < deadline, + "Client {client} stopped at insertion {sequence}" + ); + let snapshot = cache.snapshot().unwrap(); + match cache.insert(&snapshot, sid, &insertion) { + Ok(Some(id)) => { + ids.push(id); + objects.push(insertion.text_object()); + break; + } + Err(Error::Io(e)) if e.kind() == ErrorKind::ResourceBusy => {} + other => panic!("{other:?}"), + } + } + if sequence == 0 { + continue; + } + loop { + assert!( + Instant::now() < deadline, + "Client {client} stopped at text {sequence}" + ); + let snapshot = cache.snapshot().unwrap(); + match cache.edit_text( + &snapshot, + sid, + insertion.text_object(), + 0..0, + "Edited ", + ) { + Ok(Some(id)) => { + ids.push(id); + break; + } + Err(Error::Io(e)) if e.kind() == ErrorKind::ResourceBusy => {} + other => panic!("{other:?}"), + } + } + } + (ids, objects) + }) + }) + .collect(); + handles + .into_iter() + .map(|h| h.join().unwrap()) + .collect::>() + }); + let ids: BTreeSet<_> = all + .iter() + .flat_map(|(ids, _)| ids.iter().copied()) + .collect(); + assert_eq!(ids.len(), 84); + let snapshot = cache.snapshot().unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.snapshot().unwrap(), snapshot); + assert_eq!( + cache + .pending() + .unwrap() + .iter() + .map(|e| e.id) + .collect::>(), + ids + ); + let store = Store::parse(&snapshot).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let s = &doc.spaces[&sid]; + let v = &s.revisions[&s.contexts[&ExGuid::default()]]; + for (client, (_, objects)) in all.iter().enumerate() { + for (sequence, id) in objects.iter().enumerate() { + let wanted = if sequence == 0 { + format!("Client {client}") + } else { + format!("Edited Paragraph {client}:{sequence}") + }; + assert!(matches!(&v.nodes[id].kind,Kind::RichText{text,..} if *text==wanted)); + } + } +} + +#[test] +fn unrecognized_persisted_operations_are_rejected_without_dropping_fields() { + for operation in ["Text", "Insert", "Format"] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("unknown.sqlite"); + let source = onestore::create_section("unknown.one", "Original", "Author").unwrap(); + let cache = Replica::create(&path, &source).unwrap(); + let (sid, oid, _) = target(&source); + if operation == "Insert" { + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let (_, page) = doc.pages().unwrap()[0]; + let insertion = + onestore::Insertion::outline(page, 144.0, 144.0, "Inserted", "Author").unwrap(); + cache.insert(&source, sid, &insertion).unwrap(); + } else if operation == "Text" { + cache.edit_text(&source, sid, oid, 0..0, "New ").unwrap(); + } else { + cache + .format( + &source, + sid, + oid, + 0..4, + &[onestore::TextAttribute::Bold(true)], + ) + .unwrap(); + } + drop(cache); + let db = rusqlite::Connection::open(&path).unwrap(); + let encoded: String = db + .query_row("SELECT operation FROM edits", [], |r| r.get(0)) + .unwrap(); + let mut value: serde_json::Value = serde_json::from_str(&encoded).unwrap(); + value[operation]["future_option"] = true.into(); + db.execute("UPDATE edits SET operation=?1", [value.to_string()]) + .unwrap(); + if operation != "Format" { + db.execute_batch("DROP TABLE conflicts; CREATE TABLE conflicts (edit_id INTEGER PRIMARY KEY REFERENCES edits(id) ON DELETE CASCADE, kind INTEGER NOT NULL CHECK(kind BETWEEN 0 AND 2)) STRICT; PRAGMA user_version=3;").unwrap(); + } + drop(db); + let before = fs::read(&path).unwrap(); + assert!( + matches!(Replica::open(&path),Err(Error::Io(error))if error.kind()==ErrorKind::InvalidData) + ); + assert_eq!(fs::read(&path).unwrap(), before); + } +} diff --git a/crates/onestore-offline/tests/sync.rs b/crates/onestore-offline/tests/sync.rs new file mode 100644 index 0000000000000000000000000000000000000000..16cc13ed761c55efb76e6b6736511abcb7fd4a69 --- /dev/null +++ b/crates/onestore-offline/tests/sync.rs @@ -0,0 +1,2618 @@ +use onestore::{ + CommitError, CommitIo, CommitState, ExGuid, PreparedEdit, RevisionIndex, Store, + document::{Document, Kind}, +}; +use onestore_offline::{ConflictKind, EditStatus, Error, Remote, Replica}; +use std::io; + +#[derive(Clone, Copy, Default)] +enum Fault { + #[default] + None, + Before, + UnknownBefore, + UnknownAfter, + Committed, + PanicBefore, + PanicAfter, + Confirm, + ConfirmCommitted, +} + +struct Server { + visible: Vec, + durable: Vec, + fault: Fault, + publications: usize, + confirmations: usize, +} + +impl Server { + fn new(source: &[u8]) -> Self { + Self { + visible: source.to_vec(), + durable: source.to_vec(), + fault: Fault::None, + publications: 0, + confirmations: 0, + } + } +} + +fn failure(state: CommitState) -> CommitError { + CommitError { + state, + error: io::Error::from(io::ErrorKind::ConnectionAborted), + } +} + +impl CommitIo for Server { + fn read_at(&mut self, offset: u64, output: &mut [u8]) -> io::Result { + let offset = usize::try_from(offset).unwrap(); + let size = output.len().min(self.visible.len().saturating_sub(offset)); + if size > 0 { + output[..size].copy_from_slice(&self.visible[offset..offset + size]); + } + Ok(size) + } + fn write_at(&mut self, offset: u64, bytes: &[u8]) -> io::Result { + let offset = usize::try_from(offset).unwrap(); + self.visible + .resize(self.visible.len().max(offset + bytes.len()), 0); + self.visible[offset..offset + bytes.len()].copy_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + self.durable.clone_from(&self.visible); + Ok(()) + } +} + +impl Remote for Server { + fn read(&mut self) -> io::Result> { + Ok(self.visible.clone()) + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + self.publications += 1; + let fault = std::mem::take(&mut self.fault); + match fault { + Fault::Before => return Err(failure(CommitState::NotCommitted)), + Fault::UnknownBefore => return Err(failure(CommitState::Unknown)), + Fault::PanicBefore => panic!("Terminated before remote I/O"), + _ => {} + } + let old = self.durable.clone(); + edit.commit(self)?; + match fault { + Fault::UnknownAfter => { + self.durable = old; + Err(failure(CommitState::Unknown)) + } + Fault::Committed => Err(failure(CommitState::Committed)), + Fault::PanicAfter => panic!("Terminated after remote publication"), + _ => Ok(()), + } + } + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + self.confirmations += 1; + if matches!(self.fault, Fault::Confirm) { + self.fault = Fault::None; + return Err(failure(CommitState::Unknown)); + } + onestore::confirm_snapshot(self, snapshot)?; + if matches!(self.fault, Fault::ConfirmCommitted) { + self.fault = Fault::None; + return Err(failure(CommitState::Committed)); + } + Ok(()) + } +} + +fn text(source: &[u8]) -> (ExGuid, ExGuid, String) { + let store = Store::parse(source).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + let doc = Document::parse(&index).unwrap(); + doc.spaces + .iter() + .find_map(|(sid, space)| { + space.revisions[&space.contexts[&ExGuid::default()]] + .nodes + .iter() + .find_map(|(oid, node)| match &node.kind { + Kind::RichText { text, .. } => Some((*sid, *oid, text.clone())), + _ => None, + }) + }) + .unwrap() +} + +#[test] +fn rebases_multiple_disjoint_remote_changes_and_persists_the_remote_receipt() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("sync.one", "ab🦀cd", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 2..4, "🐈") + .unwrap() + .unwrap(); + let remote = onestore::replace_text(&source, sid, oid, 0..6, "Xab🦀cYd").unwrap(); + let mut server = Server::new(&remote); + let outcome = cache.sync_once(&mut server).unwrap().unwrap(); + assert_eq!(outcome.0, id); + assert!(matches!(outcome.1, EditStatus::Published { .. })); + assert_eq!(text(&server.durable).2, "Xab🐈cYd"); + assert_eq!(cache.snapshot().unwrap(), server.durable); + assert!(cache.pending().unwrap().is_empty()); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.status(id).unwrap(), Some(outcome.1)); + assert_eq!(cache.sync_once(&mut server).unwrap(), None); + assert_eq!(server.publications, 1); + assert_eq!(cache.status(id + 1).unwrap(), None); + let source = cache.snapshot().unwrap(); + let next = cache + .edit_text(&source, sid, oid, 0..0, "Later ") + .unwrap() + .unwrap(); + assert!(next > id); +} + +#[test] +fn overlapping_changes_preserve_both_images_and_survive_restart() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 1..2, "L") + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let remote = onestore::replace_text(&source, sid, oid, 1..2, "R").unwrap(); + let mut server = Server::new(&remote); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, EditStatus::Conflict(ConflictKind::TextChanged))) + ); + assert_eq!(server.publications, 0); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.remote_snapshot().unwrap(), remote); + assert_eq!(cache.pending().unwrap().len(), 1); + assert_eq!( + cache.status(id).unwrap(), + Some(EditStatus::Conflict(ConflictKind::TextChanged)) + ); +} + +#[test] +fn lost_replies_and_process_termination_never_blindly_replay_an_attempt() { + for fault in [ + Fault::UnknownBefore, + Fault::UnknownAfter, + Fault::PanicBefore, + Fault::PanicAfter, + ] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 0..0, "Once ") + .unwrap() + .unwrap(); + let mut server = Server::new(&source); + server.fault = fault; + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + cache.sync_once(&mut server) + })); + let attempted = cache.status(id).unwrap().unwrap(); + assert!(matches!(attempted, EditStatus::AwaitingConfirmation { .. })); + drop(cache); + let cache = Replica::open(&path).unwrap(); + let result = cache.sync_once(&mut server).unwrap().unwrap(); + if matches!(fault, Fault::UnknownAfter | Fault::PanicAfter) { + assert!(matches!(result.1, EditStatus::Published { .. })); + assert_eq!(text(&server.durable).2, "Once abc"); + assert_eq!(server.confirmations, 1); + assert!(cache.pending().unwrap().is_empty()); + } else { + assert_eq!(result.1, attempted); + assert_eq!(cache.pending().unwrap().len(), 1); + assert_eq!(server.confirmations, 0); + assert_eq!(server.durable, source); + } + assert_eq!(server.publications, 1); + } +} + +#[test] +fn failed_confirmation_does_not_promote_visible_bytes_to_a_receipt() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 0..0, "Once ") + .unwrap() + .unwrap(); + let mut server = Server::new(&source); + server.fault = Fault::UnknownAfter; + assert!(cache.sync_once(&mut server).is_err()); + server.fault = Fault::Confirm; + assert!(cache.sync_once(&mut server).is_err()); + assert_eq!(server.durable, source); + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::AwaitingConfirmation { .. }) + )); + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Published { .. })) + )); + assert_eq!(server.publications, 1); + assert_eq!(server.confirmations, 2); + assert_eq!(server.visible, server.durable); +} + +#[test] +fn confirmation_cleanup_failure_still_records_a_durable_receipt() { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(dir.path().join("cache.sqlite"), &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 0..0, "Once ") + .unwrap() + .unwrap(); + let mut server = Server::new(&source); + server.fault = Fault::UnknownAfter; + assert!(cache.sync_once(&mut server).is_err()); + server.fault = Fault::ConfirmCommitted; + assert!( + matches!(cache.sync_once(&mut server), Err(Error::Remote(error)) if error.state == CommitState::Committed) + ); + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::Published { .. }) + )); + assert!(cache.pending().unwrap().is_empty()); + assert_eq!(server.visible, server.durable); + assert_eq!(server.publications, 1); + assert_eq!(server.confirmations, 1); +} + +#[test] +fn proven_unpublished_attempts_retry_and_committed_cleanup_errors_keep_receipts() { + for fault in [Fault::Before, Fault::Committed] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 0..0, "Once ") + .unwrap() + .unwrap(); + let mut server = Server::new(&source); + server.fault = fault; + assert!(matches!( + cache.sync_once(&mut server), + Err(Error::Remote(_)) + )); + if matches!(fault, Fault::Before) { + assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Published { .. })) + )); + assert_eq!(server.publications, 2); + } else { + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::Published { .. }) + )); + assert_eq!(cache.sync_once(&mut server).unwrap(), None); + assert_eq!(server.publications, 1); + } + assert_eq!(text(&server.durable).2, "Once abc"); + } +} + +#[test] +fn database_failures_before_and_after_publication_preserve_recovery_state() { + for table in ["attempt", "receipts"] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 0..0, "Once ") + .unwrap() + .unwrap(); + drop(cache); + let db = rusqlite::Connection::open(&path).unwrap(); + db.execute_batch(&format!("CREATE TRIGGER interrupted BEFORE INSERT ON {table} BEGIN SELECT RAISE(ABORT,'Injected cache failure'); END;")).unwrap(); + drop(db); + let cache = Replica::open(&path).unwrap(); + let mut server = Server::new(&source); + assert!(matches!( + cache.sync_once(&mut server), + Err(Error::Database(_)) + )); + assert_eq!(server.publications, usize::from(table == "receipts")); + assert_eq!(cache.pending().unwrap().len(), 1); + drop(cache); + let db = rusqlite::Connection::open(&path).unwrap(); + db.execute_batch("DROP TRIGGER interrupted").unwrap(); + drop(db); + let cache = Replica::open(&path).unwrap(); + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Published { .. })) + )); + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::Published { .. }) + )); + assert_eq!(server.publications, 1); + assert_eq!(server.confirmations, usize::from(table == "receipts")); + assert_eq!(text(&server.durable).2, "Once abc"); + } +} + +#[test] +fn twelve_local_editors_progress_during_remote_reads_publication_and_confirmation() { + struct Paused { + server: Server, + phase: &'static str, + entered: std::sync::mpsc::Sender<()>, + resume: std::sync::mpsc::Receiver<()>, + } + impl Paused { + fn wait(&self, phase: &str) { + if self.phase == phase { + self.entered.send(()).unwrap(); + self.resume + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + } + } + } + impl Remote for Paused { + fn read(&mut self) -> io::Result> { + self.wait("read"); + self.server.read() + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + self.wait("publish"); + self.server.publish(edit) + } + fn confirm(&mut self, source: &[u8]) -> Result<(), CommitError> { + self.wait("confirm"); + self.server.confirm(source) + } + } + for phase in ["read", "publish", "confirm"] { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(dir.path().join("cache.sqlite"), &source).unwrap(); + let first = cache + .edit_text(&source, sid, oid, 0..0, "First ") + .unwrap() + .unwrap(); + let mut server = Server::new(&source); + if phase == "confirm" { + server.fault = Fault::UnknownAfter; + assert!(cache.sync_once(&mut server).is_err()); + } + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (resume_tx, resume_rx) = std::sync::mpsc::channel(); + let mut paused = Paused { + server, + phase, + entered: entered_tx, + resume: resume_rx, + }; + let mut server = std::thread::scope(|scope| { + let running = scope.spawn(|| { + let result = cache.sync_once(&mut paused); + assert!( + matches!(result, Ok(Some((id, EditStatus::Published { .. }))) if id == first) + ); + paused.server + }); + entered_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + assert!( + matches!(cache.sync_once(&mut Server::new(&source)),Err(Error::Io(error)) if error.kind()==io::ErrorKind::WouldBlock) + ); + assert!( + matches!(cache.rebase_conflict(first, &[], &[], 0..0), Err(Error::Io(error)) if error.kind() == io::ErrorKind::WouldBlock) + ); + let started = std::time::Instant::now(); + let handles: Vec<_> = (0..12) + .map(|writer| { + let cache = &cache; + scope.spawn(move || { + loop { + let snapshot = cache.snapshot().unwrap(); + match cache.edit_text( + &snapshot, + sid, + oid, + 0..0, + &format!("[{writer}] "), + ) { + Ok(Some(id)) => break id, + Err(Error::Io(error)) + if error.kind() == io::ErrorKind::ResourceBusy => {} + other => panic!("Unexpected local outcome {other:?}"), + } + } + }) + }) + .collect(); + let ids: std::collections::BTreeSet<_> = handles + .into_iter() + .map(|handle| handle.join().unwrap()) + .collect(); + assert_eq!(ids.len(), 12); + assert!( + started.elapsed() < std::time::Duration::from_secs(2), + "Local edits waited for remote {phase}" + ); + resume_tx.send(()).unwrap(); + running.join().unwrap() + }); + assert_eq!(cache.pending().unwrap().len(), 12); + let expected = text(&cache.snapshot().unwrap()).2; + for _ in 0..12 { + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Published { .. })) + )); + } + assert!(cache.pending().unwrap().is_empty()); + assert_eq!(server.publications, 13); + assert_eq!(text(&server.durable).2, expected); + assert_eq!(cache.snapshot().unwrap(), server.durable); + } +} + +#[test] +fn unrelated_remote_files_never_replace_a_local_cache() { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(dir.path().join("cache.sqlite"), &source).unwrap(); + let other = onestore::create_section("other.one", "abc", "Fixture").unwrap(); + let mut server = Server::new(&other); + for pending in [false, true] { + if pending { + cache.edit_text(&source, sid, oid, 0..0, "Local ").unwrap(); + } + let before = cache.snapshot().unwrap(); + assert!( + matches!(cache.sync_once(&mut server),Err(Error::Io(error)) if error.kind()==io::ErrorKind::InvalidInput) + ); + assert_eq!(cache.snapshot().unwrap(), before); + assert_eq!(cache.remote_snapshot().unwrap(), source); + assert_eq!(server.publications, 0); + } +} + +#[test] +fn version_one_cache_migration_preserves_images_intents_and_local_ids() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 0..0, "Local ") + .unwrap() + .unwrap(); + let snapshot = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + drop(cache); + let db = rusqlite::Connection::open(&path).unwrap(); + db.execute_batch( + "DROP TABLE attempt; DROP TABLE conflicts; DROP TABLE receipts; DROP TABLE edits; + CREATE TABLE edits ( + id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(id>0), space TEXT NOT NULL, + object TEXT NOT NULL, before_text TEXT NOT NULL, + start INTEGER NOT NULL CHECK(start BETWEEN 0 AND 4294967295), + end INTEGER NOT NULL CHECK(end BETWEEN start AND 4294967295), replacement TEXT NOT NULL + ) STRICT; PRAGMA user_version=1;", + ) + .unwrap(); + db.execute("INSERT INTO edits(id,space,object,before_text,start,end,replacement) VALUES (?1,?2,?3,'abc',0,0,'Local ')",rusqlite::params![i64::try_from(id).unwrap(),sid.to_string(),oid.to_string()]).unwrap(); + drop(db); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.snapshot().unwrap(), snapshot); + assert_eq!(cache.pending().unwrap(), pending); + assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); + let mut server = Server::new(&source); + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Published { .. })) + )); + drop(cache); + let db = rusqlite::Connection::open(&path).unwrap(); + assert_eq!( + db.pragma_query_value(None, "user_version", |row| row.get::<_, u32>(0)) + .unwrap(), + 4 + ); +} + +#[test] +fn reviewed_conflict_rebase_preserves_twelve_dependent_edits_and_survives_reopen() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("resolve.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let first = cache + .edit_text(&source, sid, oid, 1..2, "L") + .unwrap() + .unwrap(); + for n in 0..12 { + cache + .edit_text( + &cache.snapshot().unwrap(), + sid, + oid, + 0..0, + &format!("[{n}] "), + ) + .unwrap(); + } + let local = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + let remote = onestore::replace_text(&source, sid, oid, 0..3, "aRcZ").unwrap(); + let mut server = Server::new(&remote); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((first, EditStatus::Conflict(ConflictKind::TextChanged))) + ); + cache.rebase_conflict(first, &local, &remote, 1..2).unwrap(); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.remote_snapshot().unwrap(), remote); + let rebased = cache.pending().unwrap(); + assert_eq!(&rebased[1..], &pending[1..]); + assert_eq!(rebased[0].id, first); + let onestore_offline::Operation::Text(updated) = &rebased[0].operation else { + panic!() + }; + let onestore_offline::Operation::Text(previous) = &pending[0].operation else { + panic!() + }; + assert_eq!(updated.replacement, previous.replacement); + assert_eq!(updated.before, "aRcZ"); + assert_eq!(cache.status(first).unwrap(), Some(EditStatus::Pending)); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), rebased); + assert_eq!(cache.snapshot().unwrap(), local); + for intent in &pending { + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((id, EditStatus::Published { .. })) if id == intent.id) + ); + } + assert_eq!(server.publications, 13); + assert_eq!(text(&server.durable).2, text(&local).2 + "Z"); + assert_eq!(cache.snapshot().unwrap(), server.durable); + assert!(cache.pending().unwrap().is_empty()); +} + +#[test] +fn conflict_review_rejects_stale_images_invalid_ranges_and_nonconflicting_states() { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("resolve.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(dir.path().join("cache.sqlite"), &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 1..2, "L") + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + assert!( + matches!(cache.rebase_conflict(id, &local, &source, 1..2), Err(Error::Io(error)) if error.kind() == io::ErrorKind::InvalidInput) + ); + let remote = onestore::replace_text(&source, sid, oid, 0..3, "🦀Rc").unwrap(); + let mut server = Server::new(&remote); + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Conflict(_))) + )); + let pending = cache.pending().unwrap(); + for range in [1..2, 100..101] { + assert!(matches!( + cache.rebase_conflict(id, &local, &remote, range), + Err(Error::Document(_)) + )); + assert_eq!(cache.pending().unwrap(), pending); + } + assert!( + matches!(cache.rebase_conflict(id + 1, &local, &remote, 2..3), Err(Error::Io(error)) if error.kind() == io::ErrorKind::InvalidInput) + ); + cache.edit_text(&local, sid, oid, 0..0, "Later ").unwrap(); + let changed = cache.snapshot().unwrap(); + assert!( + matches!(cache.rebase_conflict(id, &local, &remote, 2..3), Err(Error::Io(error)) if error.kind() == io::ErrorKind::ResourceBusy) + ); + let new_remote = onestore::replace_text(&remote, sid, oid, 2..3, "Q").unwrap(); + server.visible = new_remote.clone(); + server.durable = new_remote; + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Conflict(_))) + )); + assert!( + matches!(cache.rebase_conflict(id, &changed, &remote, 2..3), Err(Error::Io(error)) if error.kind() == io::ErrorKind::ResourceBusy) + ); + assert_eq!(cache.snapshot().unwrap(), changed); + assert_eq!(cache.pending().unwrap()[0], pending[0]); + assert_eq!(server.publications, 0); + + let current = cache.remote_snapshot().unwrap(); + cache.rebase_conflict(id, &changed, ¤t, 2..3).unwrap(); + server.fault = Fault::UnknownBefore; + assert!( + matches!(cache.sync_once(&mut server), Err(Error::Remote(error)) if error.state == CommitState::Unknown) + ); + let attempted = cache.status(id).unwrap(); + let pending = cache.pending().unwrap(); + assert!( + matches!(cache.rebase_conflict(id, &changed, ¤t, 2..3), Err(Error::Io(error)) if error.kind() == io::ErrorKind::InvalidInput) + ); + assert_eq!(cache.status(id).unwrap(), attempted); + assert_eq!(cache.pending().unwrap(), pending); + assert_eq!(server.publications, 1); +} + +#[test] +fn failure_between_rebase_and_conflict_clear_rolls_back_the_entire_resolution() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("resolve.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 1..2, "L") + .unwrap() + .unwrap(); + let remote = onestore::replace_text(&source, sid, oid, 0..3, "XaRc").unwrap(); + let mut server = Server::new(&remote); + cache.sync_once(&mut server).unwrap(); + let local = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + let status = cache.status(id).unwrap(); + drop(cache); + let connection = rusqlite::Connection::open(&path).unwrap(); + connection.execute_batch("CREATE TRIGGER fail_clear BEFORE DELETE ON conflicts BEGIN SELECT RAISE(ABORT, 'test conflict clear failure'); END;").unwrap(); + drop(connection); + let cache = Replica::open(&path).unwrap(); + assert!(matches!( + cache.rebase_conflict(id, &local, &remote, 2..3), + Err(Error::Database(_)) + )); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), pending); + assert_eq!(cache.status(id).unwrap(), status); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.remote_snapshot().unwrap(), remote); + assert_eq!(server.publications, 0); +} + +#[test] +fn seeded_reviewed_ranges_preserve_unicode_and_edits_inside_the_original_replacement() { + let dir = tempfile::tempdir().unwrap(); + let original: Vec<_> = "abcdefghij🦀klmnop".chars().collect(); + let mut seed = 911_u64; + for case in 0..64 { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + let at = seed as usize % original.len(); + let prefix = "[".repeat((seed >> 8) as usize % 5); + let suffix = "]".repeat((seed >> 16) as usize % 5); + let start: u32 = original[..at].iter().map(|ch| ch.len_utf16() as u32).sum(); + let end = start + original[at].len_utf16() as u32; + let source = onestore::create_section( + "resolve.one", + &original.iter().collect::(), + "Fixture", + ) + .unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(dir.path().join(format!("{case}.sqlite")), &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, start..end, "λ🦊μ") + .unwrap() + .unwrap(); + let dependent = cache + .edit_text( + &cache.snapshot().unwrap(), + sid, + oid, + start + 1..start + 3, + "🐕", + ) + .unwrap() + .unwrap(); + let mut remote_text = original.clone(); + remote_text.splice(at..at + 1, "Ω🐈π".chars()); + let remote_text = prefix.clone() + &remote_text.iter().collect::() + &suffix; + let remote = onestore::replace_text( + &source, + sid, + oid, + 0..original.iter().map(|ch| ch.len_utf16() as u32).sum(), + &remote_text, + ) + .unwrap(); + let mut server = Server::new(&remote); + assert!( + matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Conflict(_))) + ), + "case {case}, seed {seed}" + ); + let at_remote = start + prefix.len() as u32; + cache + .rebase_conflict( + id, + &cache.snapshot().unwrap(), + &remote, + at_remote..at_remote + 4, + ) + .unwrap(); + for expected in [id, dependent] { + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == expected), + "case {case}, seed {seed}" + ); + } + let mut expected = original.clone(); + expected.splice(at..at + 1, "λ🐕μ".chars()); + let expected = prefix + &expected.iter().collect::() + &suffix; + assert_eq!( + text(&server.durable).2, + expected, + "case {case}, seed {seed}" + ); + assert_eq!(server.publications, 2); + assert!(cache.pending().unwrap().is_empty()); + } +} + +#[test] +fn remote_changes_after_review_cannot_be_overwritten_by_the_reviewed_placement() { + struct ChangedAfterRead { + server: Server, + change: Option>, + } + impl Remote for ChangedAfterRead { + fn read(&mut self) -> io::Result> { + let snapshot = self.server.read()?; + if let Some(changed) = self.change.take() { + self.server.visible = changed.clone(); + self.server.durable = changed; + } + Ok(snapshot) + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + self.server.publish(edit) + } + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + self.server.confirm(snapshot) + } + } + for after_read in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("resolve.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(dir.path().join("cache.sqlite"), &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 1..2, "L") + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let remote = onestore::replace_text(&source, sid, oid, 1..2, "R").unwrap(); + let mut remote = ChangedAfterRead { + server: Server::new(&remote), + change: None, + }; + assert!(matches!( + cache.sync_once(&mut remote).unwrap(), + Some((_, EditStatus::Conflict(_))) + )); + cache + .rebase_conflict(id, &local, &cache.remote_snapshot().unwrap(), 1..2) + .unwrap(); + let changed = onestore::replace_text(&remote.server.visible, sid, oid, 1..2, "Q").unwrap(); + if after_read { + remote.change = Some(changed.clone()); + } else { + remote.server.visible = changed.clone(); + remote.server.durable = changed.clone(); + } + if after_read { + assert!( + matches!(cache.sync_once(&mut remote), Err(Error::Remote(error)) if error.state == CommitState::NotCommitted) + ); + assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); + } + assert_eq!( + cache.sync_once(&mut remote).unwrap(), + Some((id, EditStatus::Conflict(ConflictKind::TextChanged))) + ); + assert_eq!(remote.server.durable, changed); + assert_eq!(remote.server.visible, changed); + assert_eq!(remote.server.publications, usize::from(after_read)); + assert_eq!(cache.snapshot().unwrap(), local); + let onestore_offline::Operation::Text(edit) = &cache.pending().unwrap()[0].operation else { + panic!() + }; + assert_eq!(edit.replacement, "L"); + } +} + +mod worker { + use super::*; + use std::{ + sync::{Arc, Mutex, mpsc}, + time::{Duration, Instant}, + }; + + #[derive(Clone)] + struct Shared(Arc>); + + impl Remote for Shared { + fn read(&mut self) -> io::Result> { + self.0.lock().unwrap().read() + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + self.0.lock().unwrap().publish(edit) + } + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + self.0.lock().unwrap().confirm(snapshot) + } + } + + #[test] + fn reconnects_after_connect_read_and_uncertain_publish_without_replaying() { + struct Session { + shared: Shared, + fail_read: bool, + } + impl Remote for Session { + fn read(&mut self) -> io::Result> { + if self.fail_read { + return Err(io::ErrorKind::ConnectionReset.into()); + } + self.shared.read() + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + self.shared.publish(edit) + } + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + self.shared.confirm(snapshot) + } + } + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Arc::new(Replica::create(&path, &source).unwrap()); + let id = cache + .edit_text(&source, sid, oid, 1..2, "🦀") + .unwrap() + .unwrap(); + let mut server = Server::new(&source); + server.fault = Fault::UnknownAfter; + let server = Arc::new(Mutex::new(server)); + let shared = Shared(Arc::clone(&server)); + let (connected_tx, connected_rx) = mpsc::channel(); + let (observed_tx, observed_rx) = mpsc::channel(); + let mut connections = 0; + let worker = cache + .start_sync( + Duration::from_millis(10), + move || { + connections += 1; + connected_tx.send(connections).unwrap(); + if connections <= 2 { + return Err(io::ErrorKind::ConnectionRefused.into()); + } + Ok(Session { + shared: shared.clone(), + fail_read: connections == 3, + }) + }, + move |result| { + observed_tx + .send(result.as_ref().copied().map_err(|error| error.to_string())) + .unwrap(); + }, + ) + .unwrap(); + let mut errors = 0; + let published = loop { + match observed_rx.recv_timeout(Duration::from_secs(5)).unwrap() { + Err(_) => errors += 1, + Ok(Some((actual, status @ EditStatus::Published { .. }))) => { + assert_eq!(actual, id); + break status; + } + other => panic!("Unexpected result: {other:?}"), + } + }; + worker.stop().unwrap(); + assert_eq!(errors, 4); + assert_eq!(connected_rx.try_iter().collect::>(), [1, 2, 3, 4, 5]); + let server = server.lock().unwrap(); + assert_eq!(server.publications, 1); + assert_eq!(server.confirmations, 1); + assert_eq!(text(&server.durable).2, "a🦀c"); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.status(id).unwrap(), Some(published)); + assert_eq!(cache.snapshot().unwrap(), server.durable); + } + + #[test] + fn local_edits_wake_an_idle_worker_and_coalesced_notifications_drain_twelve_writers() { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); + let server = Arc::new(Mutex::new(Server::new(&source))); + let shared = Shared(Arc::clone(&server)); + let (observed_tx, observed_rx) = mpsc::channel(); + let (resume_tx, resume_rx) = mpsc::channel(); + let mut first = true; + let worker = cache + .start_sync( + Duration::from_secs(3600), + move || Ok(shared.clone()), + move |result| { + observed_tx.send(*result.as_ref().unwrap()).unwrap(); + if first { + first = false; + resume_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + } + }, + ) + .unwrap(); + assert_eq!( + observed_rx.recv_timeout(Duration::from_secs(5)).unwrap(), + None + ); + let started = Instant::now(); + let edits = std::thread::scope(|scope| { + (0..12) + .map(|writer| { + let cache = &cache; + scope.spawn(move || { + let replacement = format!("[{writer}] "); + loop { + let snapshot = cache.snapshot().unwrap(); + match cache.edit_text(&snapshot, sid, oid, 0..0, &replacement) { + Ok(Some(id)) => break (id, replacement), + Err(Error::Io(error)) + if error.kind() == io::ErrorKind::ResourceBusy => {} + other => panic!("Unexpected local outcome: {other:?}"), + } + } + }) + }) + .collect::>() + .into_iter() + .map(|join| join.join().unwrap()) + .collect::>() + }); + resume_tx.send(()).unwrap(); + let mut published = std::collections::BTreeSet::new(); + while published.len() < 12 { + if let Some((id, status)) = observed_rx.recv_timeout(Duration::from_secs(5)).unwrap() { + assert!(matches!(status, EditStatus::Published { .. }), "{status:?}"); + assert!(published.insert(id)); + } + } + assert!( + started.elapsed() < Duration::from_secs(5), + "Edits waited for the hourly poll" + ); + worker.stop().unwrap(); + assert_eq!(published, edits.keys().copied().collect()); + let expected = edits.values().rev().cloned().collect::() + "abc"; + let server = server.lock().unwrap(); + assert_eq!(text(&server.durable).2, expected); + assert_eq!(server.publications, 12); + assert_eq!(cache.snapshot().unwrap(), server.durable); + assert!(cache.pending().unwrap().is_empty()); + } + + #[test] + fn dropping_during_publication_is_nonblocking_and_retains_ownership_until_recovery_is_recorded() + { + struct Paused { + shared: Shared, + entered: mpsc::Sender<()>, + resume: mpsc::Receiver<()>, + } + impl Remote for Paused { + fn read(&mut self) -> io::Result> { + self.shared.read() + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + self.entered.send(()).unwrap(); + self.resume.recv_timeout(Duration::from_secs(5)).unwrap(); + self.shared.publish(edit) + } + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + self.shared.confirm(snapshot) + } + } + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Arc::new(Replica::create(&path, &source).unwrap()); + let id = cache + .edit_text(&source, sid, oid, 0..0, "L ") + .unwrap() + .unwrap(); + let mut server = Server::new(&source); + server.fault = Fault::UnknownAfter; + let server = Arc::new(Mutex::new(server)); + let shared = Shared(Arc::clone(&server)); + let (entered_tx, entered_rx) = mpsc::channel(); + let (resume_tx, resume_rx) = mpsc::channel(); + let mut session = Some(Paused { + shared, + entered: entered_tx, + resume: resume_rx, + }); + let worker = cache + .start_sync( + Duration::from_secs(3600), + move || Ok(session.take().unwrap()), + |_| {}, + ) + .unwrap(); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + let stopped = Instant::now(); + drop(worker); + assert!(stopped.elapsed() < Duration::from_millis(500)); + let shared = Shared(Arc::clone(&server)); + let start = cache.start_sync( + Duration::from_secs(3600), + move || Ok(shared.clone()), + |_| {}, + ); + assert!(matches!(start, Err(error) if error.kind() == io::ErrorKind::WouldBlock)); + let weak = Arc::downgrade(&cache); + drop(cache); + resume_tx.send(()).unwrap(); + while weak.upgrade().is_some() { + assert!(stopped.elapsed() < Duration::from_secs(5)); + std::thread::sleep(Duration::from_millis(1)); + } + let cache = Arc::new(Replica::open(&path).unwrap()); + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::AwaitingConfirmation { .. }) + )); + assert_eq!(server.lock().unwrap().publications, 1); + let shared = Shared(Arc::clone(&server)); + let (tx, rx) = mpsc::channel(); + let worker = cache + .start_sync( + Duration::from_secs(3600), + move || Ok(shared.clone()), + move |result| { + tx.send(*result.as_ref().unwrap()).unwrap(); + }, + ) + .unwrap(); + assert!( + matches!(rx.recv_timeout(Duration::from_secs(5)).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) + ); + worker.stop().unwrap(); + let server = server.lock().unwrap(); + assert_eq!(server.publications, 1); + assert_eq!(server.confirmations, 1); + assert_eq!(text(&server.durable).2, "L abc"); + } + + #[test] + fn cache_failures_stop_retries_and_return_the_error_without_remote_publication() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 0..0, "L ") + .unwrap() + .unwrap(); + drop(cache); + let connection = rusqlite::Connection::open(&path).unwrap(); + connection.execute_batch("CREATE TRIGGER fail_attempt BEFORE INSERT ON attempt BEGIN SELECT RAISE(ABORT, 'test cache write failure'); END;").unwrap(); + drop(connection); + let cache = Arc::new(Replica::open(&path).unwrap()); + let server = Arc::new(Mutex::new(Server::new(&source))); + let shared = Shared(Arc::clone(&server)); + let (tx, rx) = mpsc::channel(); + let worker = cache + .start_sync( + Duration::from_millis(1), + move || Ok(shared.clone()), + move |result| { + tx.send(matches!(result, Err(Error::Database(_)))).unwrap(); + }, + ) + .unwrap(); + assert!(rx.recv_timeout(Duration::from_secs(5)).unwrap()); + assert!(matches!(worker.stop(), Err(Error::Database(_)))); + assert!(rx.try_iter().next().is_none()); + assert_eq!(server.lock().unwrap().publications, 0); + assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); + assert_eq!(text(&cache.snapshot().unwrap()).2, "L abc"); + } + + #[test] + fn polling_preserves_conflicts_and_absent_uncertain_revisions_without_replay() { + for uncertain in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = + Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); + let id = cache + .edit_text(&source, sid, oid, 1..2, "L") + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let mut server = if uncertain { + Server::new(&source) + } else { + Server::new(&onestore::replace_text(&source, sid, oid, 1..2, "R").unwrap()) + }; + if uncertain { + server.fault = Fault::UnknownBefore; + } + let server = Arc::new(Mutex::new(server)); + let shared = Shared(Arc::clone(&server)); + let (tx, rx) = mpsc::channel(); + let worker = cache + .start_sync( + Duration::from_millis(10), + move || Ok(shared.clone()), + move |result| { + tx.send(result.as_ref().copied().map_err(|_| ())).unwrap(); + }, + ) + .unwrap(); + if uncertain { + assert!(rx.recv_timeout(Duration::from_secs(5)).unwrap().is_err()); + } + let mut previous = None; + let started = Instant::now(); + for _ in 0..5 { + let (actual, status) = rx + .recv_timeout(Duration::from_secs(5)) + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(actual, id); + if uncertain { + assert!(matches!(status, EditStatus::AwaitingConfirmation { .. })); + } else { + assert_eq!(status, EditStatus::Conflict(ConflictKind::TextChanged)); + } + if let Some(previous) = previous { + assert_eq!(previous, status); + } + previous = Some(status); + } + assert!( + started.elapsed() >= Duration::from_millis(30), + "Worker spun instead of waiting between retries" + ); + worker.stop().unwrap(); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.pending().unwrap().len(), 1); + let server = server.lock().unwrap(); + assert_eq!(server.publications, usize::from(uncertain)); + assert_eq!(server.confirmations, 0); + } + } + + #[test] + fn reachability_notification_retries_without_waiting_for_the_poll() { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); + let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); + let (tx, rx) = mpsc::channel(); + let worker = cache + .start_sync( + Duration::from_secs(3600), + || -> io::Result { Err(io::ErrorKind::NotConnected.into()) }, + move |result| { + tx.send(matches!(result, Err(Error::RemoteIo(_)))).unwrap(); + }, + ) + .unwrap(); + assert!(rx.recv_timeout(Duration::from_secs(5)).unwrap()); + worker.wake(); + assert!(rx.recv_timeout(Duration::from_secs(5)).unwrap()); + worker.stop().unwrap(); + assert!(rx.try_iter().next().is_none()); + } + + #[test] + fn cancellation_during_connect_does_not_read_or_report_a_false_refresh() { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); + let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); + let (entered_tx, entered_rx) = mpsc::channel(); + let (resume_tx, resume_rx) = mpsc::channel(); + let (tx, rx) = mpsc::channel(); + // An invalid image makes any unexpected read observable as an error callback. + let shared = Shared(Arc::new(Mutex::new(Server::new(&[])))); + let worker = cache + .start_sync( + Duration::from_secs(3600), + move || { + entered_tx.send(()).unwrap(); + resume_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + Ok(shared.clone()) + }, + move |_| { + tx.send(()).unwrap(); + }, + ) + .unwrap(); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + drop(worker); + let weak = Arc::downgrade(&cache); + drop(cache); + resume_tx.send(()).unwrap(); + let started = Instant::now(); + while weak.upgrade().is_some() { + assert!(started.elapsed() < Duration::from_secs(5)); + std::thread::sleep(Duration::from_millis(1)); + } + assert_eq!( + rx.recv_timeout(Duration::from_secs(5)), + Err(mpsc::RecvTimeoutError::Disconnected) + ); + } + + #[test] + fn invalid_intervals_and_callback_panics_leave_worker_ownership_recoverable() { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); + let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); + for interval in [Duration::ZERO, Duration::MAX] { + assert!( + matches!(cache.start_sync(interval, || -> io::Result { panic!("Unexpected connection") }, |_| {}), Err(error) if error.kind() == io::ErrorKind::InvalidInput) + ); + } + let shared = Shared(Arc::new(Mutex::new(Server::new(&source)))); + let first = shared.clone(); + let (tx, rx) = mpsc::channel(); + let worker = cache + .start_sync( + Duration::from_secs(3600), + move || Ok(first.clone()), + move |_| { + tx.send(()).unwrap(); + panic!("Test observer panic"); + }, + ) + .unwrap(); + rx.recv_timeout(Duration::from_secs(5)).unwrap(); + assert!(matches!(worker.stop(), Err(Error::Io(_)))); + let (tx, rx) = mpsc::channel(); + let worker = cache + .start_sync( + Duration::from_secs(3600), + move || Ok(shared.clone()), + move |result| { + tx.send(*result.as_ref().unwrap()).unwrap(); + }, + ) + .unwrap(); + assert_eq!(rx.recv_timeout(Duration::from_secs(5)).unwrap(), None); + worker.stop().unwrap(); + assert_eq!(cache.snapshot().unwrap(), source); + } + + #[test] + fn reviewed_conflict_wakes_the_worker_and_publishes_the_original_intent_once() { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("resolve.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); + let id = cache + .edit_text(&source, sid, oid, 1..2, "L") + .unwrap() + .unwrap(); + let remote = onestore::replace_text(&source, sid, oid, 1..2, "R").unwrap(); + let server = Arc::new(Mutex::new(Server::new(&remote))); + let shared = Shared(Arc::clone(&server)); + let (tx, rx) = mpsc::channel(); + let (resume_tx, resume_rx) = mpsc::channel(); + let mut first = true; + let worker = cache + .start_sync( + Duration::from_secs(3600), + move || Ok(shared.clone()), + move |result| { + tx.send(*result.as_ref().unwrap()).unwrap(); + if first { + first = false; + resume_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + } + }, + ) + .unwrap(); + assert_eq!( + rx.recv_timeout(Duration::from_secs(5)).unwrap(), + Some((id, EditStatus::Conflict(ConflictKind::TextChanged))) + ); + cache + .rebase_conflict( + id, + &cache.snapshot().unwrap(), + &cache.remote_snapshot().unwrap(), + 1..2, + ) + .unwrap(); + assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); + resume_tx.send(()).unwrap(); + assert!( + matches!(rx.recv_timeout(Duration::from_secs(5)).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) + ); + worker.stop().unwrap(); + assert!(cache.pending().unwrap().is_empty()); + let server = server.lock().unwrap(); + assert_eq!(server.publications, 1); + assert_eq!(text(&server.durable).2, "aLc"); + assert_eq!(cache.snapshot().unwrap(), server.durable); + } + + #[test] + fn ordinary_read_and_unpublished_write_contention_reuse_the_connection() { + struct Busy { + server: Server, + reads: usize, + writes: usize, + } + impl Remote for Busy { + fn read(&mut self) -> io::Result> { + self.reads += 1; + match self.reads { + 1 => Err(io::ErrorKind::WouldBlock.into()), + 2 => Err(io::ErrorKind::ResourceBusy.into()), + _ => self.server.read(), + } + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + self.writes += 1; + match self.writes { + 1 | 2 => Err(CommitError { + state: CommitState::NotCommitted, + error: if self.writes == 1 { + io::ErrorKind::WouldBlock + } else { + io::ErrorKind::ResourceBusy + } + .into(), + }), + _ => self.server.publish(edit), + } + } + fn confirm(&mut self, source: &[u8]) -> Result<(), CommitError> { + self.server.confirm(source) + } + } + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); + let id = cache + .edit_text(&source, sid, oid, 0..0, "L ") + .unwrap() + .unwrap(); + let mut remote = Some(Busy { + server: Server::new(&source), + reads: 0, + writes: 0, + }); + let (tx, rx) = mpsc::channel(); + let worker = cache + .start_sync( + Duration::from_millis(1), + move || Ok(remote.take().expect("Contention caused a reconnect")), + move |result| { + tx.send(result.as_ref().copied().map_err(|error| error.to_string())) + .unwrap(); + }, + ) + .unwrap(); + for _ in 0..4 { + assert!(rx.recv_timeout(Duration::from_secs(5)).unwrap().is_err()); + } + assert!( + matches!(rx.recv_timeout(Duration::from_secs(5)).unwrap().unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) + ); + worker.stop().unwrap(); + assert_eq!(text(&cache.snapshot().unwrap()).2, "L abc"); + assert!(cache.pending().unwrap().is_empty()); + } + #[test] + fn publication_backoff_drains_local_wakes_without_waiting_for_the_idle_poll() { + struct BusyOnce(Server); + impl Remote for BusyOnce { + fn read(&mut self) -> io::Result> { + self.0.read() + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + let server = &mut self.0; + if server.publications == 0 { + server.publications += 1; + return Err(CommitError { + state: CommitState::NotCommitted, + error: io::ErrorKind::ResourceBusy.into(), + }); + } + server.publish(edit) + } + fn confirm(&mut self, source: &[u8]) -> Result<(), CommitError> { + self.0.confirm(source) + } + } + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); + let first = cache + .edit_text(&source, sid, oid, 0..0, "L ") + .unwrap() + .unwrap(); + let mut remote = Some(BusyOnce(Server::new(&source))); + let observed = Arc::clone(&cache); + let (tx, rx) = mpsc::channel(); + let mut failed = false; + let worker = cache + .start_sync( + Duration::from_secs(3600), + move || Ok(remote.take().expect("Contention must retain the session")), + move |result| match result { + Err(Error::Remote(error)) if error.state == CommitState::NotCommitted => { + assert!(!failed); + failed = true; + let source = observed.snapshot().unwrap(); + let second = observed + .edit_text(&source, sid, oid, 0..0, "Q ") + .unwrap() + .unwrap(); + tx.send((second, None)).unwrap(); + } + Ok(Some((id, status))) => tx.send((*id, Some(*status))).unwrap(), + Ok(None) => {} + other => panic!("Unexpected worker result: {other:?}"), + }, + ) + .unwrap(); + let (second, status) = rx.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!(status, None); + for expected in [first, second] { + let (id, status) = rx.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!(id, expected); + assert!(matches!(status, Some(EditStatus::Published { .. }))); + } + worker.stop().unwrap(); + assert!(cache.pending().unwrap().is_empty()); + assert_eq!(text(&cache.snapshot().unwrap()).2, "Q L abc"); + } +} + +#[test] +fn offline_insertions_survive_reopen_rebase_and_dependent_text_edits() { + use onestore::Insertion; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("insert.sqlite"); + let source = onestore::create_section("insert.one", "Original", "Author").unwrap(); + let (sid, original, _) = text(&source); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let (_, page) = doc.pages().unwrap()[0]; + let cache = Replica::create(&path, &source).unwrap(); + let outline = + Insertion::outline(page, 72.0, 144.0, "Offline outline", "Offline author").unwrap(); + let first = cache.insert(&source, sid, &outline).unwrap().unwrap(); + let snapshot = cache.snapshot().unwrap(); + let paragraph = Insertion::paragraph( + outline.object(), + None, + "Offline paragraph 🦀", + "Offline author", + ) + .unwrap(); + let second = cache.insert(&snapshot, sid, ¶graph).unwrap().unwrap(); + let third = cache + .edit_text( + &cache.snapshot().unwrap(), + sid, + paragraph.text_object(), + 0..0, + "Edited ", + ) + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.pending().unwrap(), pending); + let remote = onestore::replace_text(&source, sid, original, 0..0, "Remote ").unwrap(); + let mut server = Server::new(&remote); + for id in [first, second, third] { + assert!( + matches!(cache.sync_once(&mut server).unwrap(),Some((observed,EditStatus::Published{..})) if observed==id) + ); + } + assert_eq!(server.publications, 3); + assert!(cache.pending().unwrap().is_empty()); + assert_eq!(cache.snapshot().unwrap(), server.durable); + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let s = &doc.spaces[&sid]; + let v = &s.revisions[&s.contexts[&ExGuid::default()]]; + for (id, wanted) in [ + (original, "Remote Original"), + (outline.text_object(), "Offline outline"), + (paragraph.text_object(), "Edited Offline paragraph 🦀"), + ] { + assert!(matches!(&v.nodes[&id].kind,Kind::RichText{text,..} if text==wanted)); + } + assert_eq!( + v.nodes[&outline.object()].children.last(), + Some(¶graph.object()) + ); +} + +#[test] +fn uncertain_insertions_reconcile_the_original_revision_without_duplicate_objects() { + use onestore::Insertion; + for fault in [ + Fault::Before, + Fault::UnknownBefore, + Fault::UnknownAfter, + Fault::PanicBefore, + Fault::PanicAfter, + Fault::Committed, + ] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("uncertain-insert.sqlite"); + let source = onestore::create_section("insert.one", "Original", "Author").unwrap(); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let (sid, page) = doc.pages().unwrap()[0]; + let cache = Replica::create(&path, &source).unwrap(); + let insertion = + Insertion::outline(page, 144.0, 144.0, "Uncertain insertion", "Author").unwrap(); + let id = cache.insert(&source, sid, &insertion).unwrap().unwrap(); + let local = cache.snapshot().unwrap(); + let mut server = Server::new(&source); + server.fault = fault; + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + cache.sync_once(&mut server) + })); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(server.publications, 1); + if matches!(fault, Fault::UnknownBefore | Fault::PanicBefore) { + let status = cache.status(id).unwrap(); + assert!(matches!( + status, + Some(EditStatus::AwaitingConfirmation { .. }) + )); + for _ in 0..5 { + assert_eq!( + cache.sync_once(&mut server).unwrap(), + status.map(|state| (id, state)) + ); + } + assert_eq!(server.publications, 1); + assert_eq!(server.durable, source); + assert_eq!(cache.snapshot().unwrap(), local); + } else { + if !matches!(fault, Fault::Committed) { + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Published { .. })) + )); + } + assert_eq!( + server.publications, + if matches!(fault, Fault::Before) { 2 } else { 1 } + ); + assert_eq!( + server.confirmations, + usize::from(matches!(fault, Fault::UnknownAfter | Fault::PanicAfter)) + ); + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let s = &doc.spaces[&sid]; + let v = &s.revisions[&s.contexts[&ExGuid::default()]]; + assert_eq!(v.nodes.values().filter(|node|matches!(&node.kind,Kind::RichText{text,..} if text=="Uncertain insertion")).count(),1); + assert!(v.nodes.contains_key(&insertion.object())); + assert!(cache.pending().unwrap().is_empty()); + } + } +} + +#[test] +fn offline_formatting_rebases_text_and_merges_independent_attributes() { + use onestore::TextAttribute as A; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("format.sqlite"); + let source = onestore::create_section("format.one", "ab🦀cd", "Author").unwrap(); + let (sid, id, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let local = cache + .format(&source, sid, id, 2..4, &[A::Bold(true)]) + .unwrap() + .unwrap(); + let pending = cache.pending().unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), pending); + let moved = PreparedEdit::text(&source, sid, id, 0..0, "Prefix ").unwrap(); + let styled = + PreparedEdit::format(moved.as_bytes(), sid, id, 9..11, &[A::Italic(true)]).unwrap(); + let mut server = Server::new(styled.as_bytes()); + assert!( + matches!(cache.sync_once(&mut server).unwrap(),Some((observed,EditStatus::Published{..}))if observed==local) + ); + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let space = &doc.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let runs = view.text_runs(id).unwrap(); + let selected = runs.iter().find(|r| r.text == "🦀").unwrap(); + assert_eq!(selected.format.bold, Some(true)); + assert_eq!(selected.format.italic, Some(true)); + assert_eq!( + runs.iter().map(|r| r.text).collect::(), + "Prefix ab🦀cd" + ); +} + +#[test] +fn competing_font_changes_remain_preserved_conflicts() { + use onestore::TextAttribute as A; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("conflict.sqlite"); + let source = onestore::create_section("format.one", "abcdef", "Author").unwrap(); + let (sid, id, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let local_id = cache + .format(&source, sid, id, 1..5, &[A::FontSize(14.0)]) + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + let remote = PreparedEdit::format(&source, sid, id, 2..4, &[A::FontSize(18.0)]).unwrap(); + let mut server = Server::new(remote.as_bytes()); + for _ in 0..3 { + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some(( + local_id, + EditStatus::Conflict(ConflictKind::FormattingChanged) + )) + ); + } + assert_eq!(server.publications, 0); + assert_eq!(server.confirmations, 0); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.pending().unwrap(), pending); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!( + cache.status(local_id).unwrap(), + Some(EditStatus::Conflict(ConflictKind::FormattingChanged)) + ); + assert_eq!(cache.snapshot().unwrap(), local); +} + +#[test] +fn independently_satisfied_formatting_requires_confirmation_before_a_receipt() { + use onestore::TextAttribute as A; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("satisfied.sqlite"); + let source = onestore::create_section("format.one", "abcdef", "Author").unwrap(); + let (sid, id, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let local_id = cache + .format(&source, sid, id, 1..5, &[A::Bold(true)]) + .unwrap() + .unwrap(); + let remote = PreparedEdit::format(&source, sid, id, 1..5, &[A::Bold(true)]).unwrap(); + let mut server = Server::new(remote.as_bytes()); + server.durable = source.clone(); + server.fault = Fault::Confirm; + assert!( + matches!(cache.sync_once(&mut server),Err(Error::Remote(e))if e.state==CommitState::Unknown) + ); + assert_eq!(cache.status(local_id).unwrap(), Some(EditStatus::Pending)); + assert_eq!(server.publications, 0); + assert_eq!(server.durable, source); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert!( + matches!(cache.sync_once(&mut server).unwrap(),Some((id,EditStatus::Published{..}))if id==local_id) + ); + assert_eq!(server.publications, 0); + assert_eq!(server.confirmations, 2); + assert_ne!(server.durable, source); + assert!(cache.pending().unwrap().is_empty()); +} + +#[test] +fn retired_format_attempts_require_the_complete_durable_effect_without_replay() { + use onestore::TextAttribute as A; + for (complete, fault) in [ + (true, Fault::None), + (true, Fault::Confirm), + (true, Fault::ConfirmCommitted), + (false, Fault::None), + ] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("retired-format.sqlite"); + let source = onestore::create_section("format.one", "abcdef", "Author").unwrap(); + let (sid, object, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .format( + &source, + sid, + object, + 1..5, + &[A::Bold(true), A::FontSize(18.0)], + ) + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let mut server = Server::new(&source); + server.fault = Fault::UnknownAfter; + assert!(cache.sync_once(&mut server).is_err()); + let attempted = cache.status(id).unwrap(); + let Some(EditStatus::AwaitingConfirmation { revision: retired }) = attempted else { + panic!() + }; + drop(cache); + let prefix = PreparedEdit::text(&source, sid, object, 0..0, "prefix ").unwrap(); + let mut attributes = vec![A::Bold(true), A::Italic(true)]; + if complete { + attributes.push(A::FontSize(18.0)); + } + server.visible = PreparedEdit::format(prefix.as_bytes(), sid, object, 8..12, &attributes) + .unwrap() + .as_bytes() + .to_vec(); + let store = Store::parse(&server.visible).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let current = index.spaces[&sid].labels[&(ExGuid::default(), 1)]; + assert!(!index.spaces[&sid].revisions.contains_key(&retired)); + let cache = Replica::open(&path).unwrap(); + server.fault = fault; + let result = cache.sync_once(&mut server); + if !complete { + assert_eq!(result.unwrap(), attempted.map(|s| (id, s))); + assert_eq!(server.confirmations, 0); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(server.durable, source); + } else { + if matches!(fault, Fault::Confirm) { + assert!(matches!(result, Err(Error::Remote(e)) if e.state == CommitState::Unknown)); + assert_eq!(cache.status(id).unwrap(), attempted); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(server.durable, source); + let complete = server.visible.clone(); + server.visible = + PreparedEdit::format(&complete, sid, object, 8..12, &[A::Bold(false)]) + .unwrap() + .as_bytes() + .to_vec(); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + attempted.map(|s| (id, s)) + ); + assert_eq!(server.confirmations, 1); + assert_eq!(cache.snapshot().unwrap(), local); + server.visible = complete; + cache.sync_once(&mut server).unwrap(); + } else if matches!(fault, Fault::ConfirmCommitted) { + assert!( + matches!(result, Err(Error::Remote(e)) if e.state == CommitState::Committed) + ); + } else { + assert_eq!( + result.unwrap(), + Some((id, EditStatus::Published { revision: current })) + ); + } + assert_ne!(current, retired); + assert_eq!( + cache.status(id).unwrap(), + Some(EditStatus::Published { revision: current }) + ); + assert!(cache.pending().unwrap().is_empty()); + assert_eq!(text(&server.durable).2, "prefix abcdef"); + assert_ne!(server.durable, source); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!( + cache.status(id).unwrap(), + Some(EditStatus::Published { revision: current }) + ); + } + assert_eq!(server.publications, 1); + } +} + +#[test] +fn uncertain_formatting_keeps_the_original_attempt_and_never_replays() { + use onestore::TextAttribute as A; + for fault in [Fault::UnknownBefore, Fault::UnknownAfter] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("unknown-format.sqlite"); + let source = onestore::create_section("format.one", "abcdef", "Author").unwrap(); + let (sid, id, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let local_id = cache + .format(&source, sid, id, 1..5, &[A::Bold(true)]) + .unwrap() + .unwrap(); + let mut server = Server::new(&source); + server.fault = fault; + assert!( + matches!(cache.sync_once(&mut server),Err(Error::Remote(e))if e.state==CommitState::Unknown) + ); + let status = cache.status(local_id).unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + if matches!(fault, Fault::UnknownBefore) { + for _ in 0..4 { + assert_eq!( + cache.sync_once(&mut server).unwrap(), + status.map(|s| (local_id, s)) + ); + } + assert_eq!(server.confirmations, 0); + } else { + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Published { .. })) + )); + assert_eq!(server.confirmations, 1); + } + assert_eq!(server.publications, 1); + } +} + +#[test] +fn reviewed_format_conflicts_preserve_dependent_edits_and_recheck_later_remote_changes() { + use onestore::TextAttribute as A; + for changed_again in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("review-format.sqlite"); + let source = onestore::create_section("format.one", "abcdef", "Author").unwrap(); + let (sid, id, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let first = cache + .format(&source, sid, id, 1..5, &[A::FontSize(14.0)]) + .unwrap() + .unwrap(); + let second = cache + .edit_text(&cache.snapshot().unwrap(), sid, id, 2..2, "X") + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + let remote = PreparedEdit::format(&source, sid, id, 1..5, &[A::FontSize(18.0)]).unwrap(); + let mut server = Server::new(remote.as_bytes()); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((first, EditStatus::Conflict(ConflictKind::FormattingChanged))) + ); + assert!( + matches!(cache.rebase_conflict(first,&source,remote.as_bytes(),1..5),Err(Error::Io(e))if e.kind()==io::ErrorKind::ResourceBusy) + ); + cache + .rebase_conflict(first, &local, remote.as_bytes(), 1..5) + .unwrap(); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.pending().unwrap()[1], pending[1]); + drop(cache); + let cache = Replica::open(&path).unwrap(); + if changed_again { + let newer = + PreparedEdit::format(&server.visible, sid, id, 1..5, &[A::FontSize(20.0)]).unwrap(); + server = Server::new(newer.as_bytes()); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((first, EditStatus::Conflict(ConflictKind::FormattingChanged))) + ); + assert_eq!(server.publications, 0); + assert_eq!(cache.snapshot().unwrap(), local); + } else { + for expected in [first, second] { + assert!( + matches!(cache.sync_once(&mut server).unwrap(),Some((id,EditStatus::Published{..}))if id==expected) + ); + } + assert_eq!(text(&server.durable).2, "abXcdef"); + assert!(cache.pending().unwrap().is_empty()); + } + } +} + +#[test] +fn superscript_reconciliation_checks_the_implicit_subscript_change() { + use onestore::TextAttribute as A; + let directory = tempfile::tempdir().unwrap(); + let source = onestore::create_section("script.one", "abc", "Author").unwrap(); + let (sid, id, _) = text(&source); + let cache = Replica::create(directory.path().join("script.sqlite"), &source).unwrap(); + let local = cache + .format(&source, sid, id, 0..3, &[A::Superscript(true)]) + .unwrap() + .unwrap(); + let remote = PreparedEdit::format(&source, sid, id, 0..3, &[A::Subscript(true)]).unwrap(); + let mut server = Server::new(remote.as_bytes()); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((local, EditStatus::Conflict(ConflictKind::FormattingChanged))) + ); + assert_eq!(server.publications, 0); +} + +#[test] +fn seeded_formatting_reconciliation_matches_a_character_model() { + use onestore::TextAttribute as A; + #[derive(Clone, Debug, PartialEq)] + struct Style { + size: f32, + color: u32, + bold: bool, + italic: bool, + } + let mut conflicts = 0; + let mut published = 0; + let mut satisfied_parts = 0; + for seed in 1_u64..=128 { + let mut random = seed; + let mut next = || { + random = random + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + random >> 32 + }; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("model.sqlite"); + let mut source = + onestore::create_section("model.one", "abcdefghijklmnopqrstuvwxyz012345", "Author") + .unwrap(); + let (sid, object, original_text) = text(&source); + let mut baseline = vec![ + Style { + size: 11.0, + color: 0xff000000, + bold: false, + italic: false + }; + 32 + ]; + for _ in 0..6 { + let start = next() as usize % 32; + let end = start + 1 + next() as usize % (32 - start); + let size = [12.0, 14.0, 18.0][next() as usize % 3]; + let color: u32 = [0xabcdef, 0x987654, 0xff000000][next() as usize % 3]; + let bold = next() % 2 == 0; + source = PreparedEdit::format( + &source, + sid, + object, + start as u32..end as u32, + &[ + A::FontSize(size), + A::Color((color != 0xff000000).then(|| { + let b = color.to_le_bytes(); + [b[0], b[1], b[2]] + })), + A::Bold(bold), + ], + ) + .unwrap() + .as_bytes() + .to_vec(); + for style in &mut baseline[start..end] { + style.size = size; + style.color = color; + style.bold = bold; + } + } + let start = next() as usize % 24; + let end = start + 2 + next() as usize % (31 - start); + let attribute = match seed % 3 { + 0 => A::FontSize(21.0), + 1 => A::Color(Some([0x12, 0x34, 0x56])), + _ => A::Bold(!baseline[start].bold), + }; + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .format( + &source, + sid, + object, + start as u32..end as u32, + std::slice::from_ref(&attribute), + ) + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let mut remote = source.clone(); + let mut expected = baseline.clone(); + for step in 0..8 { + let left = next() as usize % 32; + let right = left + 1 + next() as usize % (32 - left); + let update = if step % 3 == 0 { + attribute.clone() + } else { + match next() % 4 { + 0 => A::FontSize([11.0, 14.0, 18.0, 21.0][next() as usize % 4]), + 1 => A::Color(Some([0x99, 0x88, 0x77])), + 2 => A::Bold(next() % 2 == 0), + _ => A::Italic(true), + } + }; + remote = PreparedEdit::format( + &remote, + sid, + object, + left as u32..right as u32, + std::slice::from_ref(&update), + ) + .unwrap() + .as_bytes() + .to_vec(); + for style in &mut expected[left..right] { + match update { + A::FontSize(value) => style.size = value, + A::Color(Some([r, g, b])) => style.color = u32::from_le_bytes([r, g, b, 0]), + A::Bold(value) => style.bold = value, + A::Italic(value) => style.italic = value, + _ => unreachable!(), + } + } + } + let conflict = (start..end).any(|i| match attribute { + A::FontSize(value) => expected[i].size != baseline[i].size && expected[i].size != value, + A::Color(_) => expected[i].color != baseline[i].color && expected[i].color != 0x563412, + A::Bold(value) => expected[i].bold != baseline[i].bold && expected[i].bold != value, + _ => unreachable!(), + }); + drop(cache); + let cache = Replica::open(&path).unwrap(); + let mut server = Server::new(&remote); + let result = cache.sync_once(&mut server).unwrap().unwrap(); + assert_eq!(result.0, id, "seed {seed}"); + if conflict { + conflicts += 1; + assert_eq!( + result.1, + EditStatus::Conflict(ConflictKind::FormattingChanged), + "seed {seed}" + ); + assert_eq!(server.publications, 0, "seed {seed}"); + assert_eq!(cache.snapshot().unwrap(), local, "seed {seed}"); + continue; + } + published += 1; + assert!( + matches!(result.1, EditStatus::Published { .. }), + "seed {seed}: {result:?}" + ); + for style in &mut expected[start..end] { + match attribute { + A::FontSize(value) => { + satisfied_parts += usize::from(style.size == value); + style.size = value; + } + A::Color(_) => { + satisfied_parts += usize::from(style.color == 0x563412); + style.color = 0x563412; + } + A::Bold(value) => { + satisfied_parts += usize::from(style.bold == value); + style.bold = value; + } + _ => unreachable!(), + } + } + assert_eq!(text(&server.durable).2, original_text, "seed {seed}"); + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let space = &doc.spaces[&sid]; + let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; + let actual: Vec<_> = revision + .text_runs(object) + .unwrap() + .iter() + .flat_map(|run| { + std::iter::repeat_n( + Style { + size: run.format.font_size.unwrap(), + color: run.format.color.unwrap_or(0xff000000), + bold: run.format.bold.unwrap_or(false), + italic: run.format.italic.unwrap_or(false), + }, + run.text.chars().count(), + ) + }) + .collect(); + assert_eq!(actual, expected, "seed {seed}"); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.status(id).unwrap(), Some(result.1)); + let snapshot = cache.snapshot().unwrap(); + assert_eq!(snapshot.len(), server.durable.len(), "seed {seed}"); + assert!( + snapshot + .iter() + .zip(&server.durable) + .enumerate() + .all(|(offset, (a, b))| a == b || (212..252).contains(&offset)), + "seed {seed}: only confirmation version metadata may change" + ); + assert_eq!(cache.sync_once(&mut server).unwrap(), None); + assert!( + cache.snapshot().unwrap() == server.durable, + "seed {seed}: refreshed snapshot" + ); + } + assert!( + conflicts >= 16 && published >= 32 && satisfied_parts >= 128, + "conflicts={conflicts}, published={published}, already desired characters={satisfied_parts}" + ); + println!( + "128 seeds: {conflicts} conflicts, {published} publications, {satisfied_parts} already desired characters" + ); +} + +#[test] +fn inserted_empty_text_retains_formatting_and_dependent_text_across_sync() { + use onestore::{Insertion, TextAttribute as A}; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("empty.sqlite"); + let source = onestore::create_section("empty.one", "Original", "Author").unwrap(); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let (sid, page) = doc.pages().unwrap()[0]; + let insertion = Insertion::outline(page, 144.0, 144.0, "", "Author").unwrap(); + let cache = Replica::create(&path, &source).unwrap(); + let first = cache.insert(&source, sid, &insertion).unwrap().unwrap(); + let second = cache + .format( + &cache.snapshot().unwrap(), + sid, + insertion.text_object(), + 0..0, + &[A::Bold(true), A::FontSize(18.0)], + ) + .unwrap() + .unwrap(); + let third = cache + .edit_text( + &cache.snapshot().unwrap(), + sid, + insertion.text_object(), + 0..0, + "Typed 🦀", + ) + .unwrap() + .unwrap(); + let pending = cache.pending().unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), pending); + let mut server = Server::new(&source); + for id in [first, second, third] { + let result = cache.sync_once(&mut server).unwrap(); + assert!( + matches!(result,Some((actual,EditStatus::Published{..}))if actual==id), + "{result:?}" + ); + } + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let space = &doc.spaces[&sid]; + let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; + let runs = revision.text_runs(insertion.text_object()).unwrap(); + assert_eq!(runs.iter().map(|r| r.text).collect::(), "Typed 🦀"); + assert!( + runs.iter() + .all(|r| r.format.bold == Some(true) && r.format.font_size == Some(18.0)) + ); + assert_eq!(cache.snapshot().unwrap(), server.durable); +} + +#[test] +fn every_visual_attribute_rebases_with_an_independent_remote_attribute() { + use onestore::TextAttribute as A; + use serde_json::json; + let mut cases = Vec::new(); + for value in [false, true] { + cases.extend([ + (A::Bold(!value), A::Bold(value), "bold", json!(value)), + (A::Italic(!value), A::Italic(value), "italic", json!(value)), + ( + A::Underline(!value), + A::Underline(value), + "underline", + json!(value), + ), + (A::Strike(!value), A::Strike(value), "strike", json!(value)), + ( + A::Superscript(!value), + A::Superscript(value), + "superscript", + json!(value), + ), + ( + A::Subscript(!value), + A::Subscript(value), + "subscript", + json!(value), + ), + ]); + } + cases.extend([ + ( + A::Font("Georgia".into()), + A::Font("Arial".into()), + "font", + json!("Arial"), + ), + ( + A::FontSize(11.0), + A::FontSize(18.0), + "font_size", + json!(18.0), + ), + ( + A::Color(None), + A::Color(Some([1, 2, 3])), + "color", + json!(0x030201), + ), + ( + A::Color(Some([1, 2, 3])), + A::Color(None), + "color", + json!(0xff000000_u32), + ), + ( + A::Highlight(None), + A::Highlight(Some([4, 5, 6])), + "highlight", + json!(0x060504), + ), + ( + A::Highlight(Some([4, 5, 6])), + A::Highlight(None), + "highlight", + json!(0xff000000_u32), + ), + ]); + for (baseline, desired, field, value) in cases { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("attribute.sqlite"); + let source = onestore::create_section("attribute.one", "abc", "Author").unwrap(); + let (sid, object, _) = text(&source); + let source = PreparedEdit::format(&source, sid, object, 0..3, &[baseline]) + .unwrap() + .as_bytes() + .to_vec(); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .format(&source, sid, object, 0..3, &[desired]) + .unwrap() + .unwrap(); + let moved = PreparedEdit::text(&source, sid, object, 0..0, "Z").unwrap(); + let (other, other_field, other_value) = if field == "font_size" { + (A::Italic(true), "italic", json!(true)) + } else { + (A::FontSize(22.0), "font_size", json!(22.0)) + }; + let remote = PreparedEdit::format(moved.as_bytes(), sid, object, 1..4, &[other]).unwrap(); + let mut server = Server::new(remote.as_bytes()); + drop(cache); + let cache = Replica::open(&path).unwrap(); + let result = cache.sync_once(&mut server).unwrap(); + assert!( + matches!(result,Some((observed,EditStatus::Published{..}))if observed==id), + "{field}={value}: {result:?}" + ); + assert_eq!(text(&server.durable).2, "Zabc"); + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let space = &doc.spaces[&sid]; + let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; + let mut position = 0; + for run in revision.text_runs(object).unwrap() { + let format = serde_json::to_value(run.format).unwrap(); + for _ in run.text.chars() { + if position > 0 { + assert_eq!(format[field], value, "{field}, character {position}"); + assert_eq!( + format[other_field], other_value, + "{field}, character {position}" + ); + } + position += 1; + } + } + assert_eq!(position, 4); + } +} + +#[test] +fn reviewed_insertion_placements_preserve_identity_and_dependent_operations() { + use onestore::{Insertion, TextAttribute as A}; + use onestore_offline::Operation; + for outline_case in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("placement.sqlite"); + let base = onestore::create_section("placement.one", "Original", "Author").unwrap(); + let (sid, _, _) = text(&base); + let store = Store::parse(&base).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let (_, page) = doc.pages().unwrap()[0]; + let space = &doc.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let parent = *view.nodes[&page] + .children + .iter() + .find(|id| matches!(view.nodes[id].kind, Kind::Outline { .. })) + .unwrap(); + let anchor = Insertion::paragraph(parent, None, "Temporary anchor", "Author").unwrap(); + let source = PreparedEdit::insert(&base, sid, &anchor) + .unwrap() + .as_bytes() + .to_vec(); + let insertion = if outline_case { + Insertion::outline(page, 144.0, 144.0, "Offline", "Author").unwrap() + } else { + Insertion::paragraph(parent, Some(anchor.object()), "Offline", "Author").unwrap() + }; + let cache = Replica::create(&path, &source).unwrap(); + let first = cache.insert(&source, sid, &insertion).unwrap().unwrap(); + let second = cache + .format( + &cache.snapshot().unwrap(), + sid, + insertion.text_object(), + 0..7, + &[A::Bold(true)], + ) + .unwrap() + .unwrap(); + let third = cache + .edit_text( + &cache.snapshot().unwrap(), + sid, + insertion.text_object(), + 7..7, + " 🦀", + ) + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + let mut server = Server::new(&base); + if outline_case { + drop(cache); + let db = rusqlite::Connection::open(&path).unwrap(); + db.execute( + "INSERT INTO conflicts VALUES (?1,2)", + [i64::try_from(first).unwrap()], + ) + .unwrap(); + db.execute("UPDATE replica SET base=?1", [&base]).unwrap(); + drop(db); + } else { + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((first, EditStatus::Conflict(ConflictKind::UnsupportedEdit))) + ); + drop(cache); + } + let cache = Replica::open(&path).unwrap(); + if outline_case { + assert!( + cache + .rebase_paragraph_conflict(first, &local, &base, parent, None) + .is_err() + ); + assert!( + cache + .rebase_outline_conflict(first, &local, &base, page, f32::NAN, 288.0) + .is_err() + ); + assert!( + matches!(cache.rebase_outline_conflict(first,&source,&base,page,288.0,360.0),Err(Error::Io(e))if e.kind()==io::ErrorKind::ResourceBusy) + ); + cache + .rebase_outline_conflict(first, &local, &base, page, 288.0, 360.0) + .unwrap(); + } else { + assert!( + cache + .rebase_outline_conflict(first, &local, &base, page, 288.0, 360.0) + .is_err() + ); + assert!( + cache + .rebase_paragraph_conflict(first, &local, &base, parent, Some(anchor.object())) + .is_err() + ); + assert!( + matches!(cache.rebase_paragraph_conflict(first,&source,&base,parent,None),Err(Error::Io(e))if e.kind()==io::ErrorKind::ResourceBusy) + ); + cache + .rebase_paragraph_conflict(first, &local, &base, parent, None) + .unwrap(); + } + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.pending().unwrap()[1..], pending[1..]); + let Operation::Insert(rebased) = cache.pending().unwrap().remove(0).operation else { + panic!() + }; + assert_eq!(rebased.object(), insertion.object()); + assert_eq!(rebased.text_object(), insertion.text_object()); + assert_eq!(cache.status(first).unwrap(), Some(EditStatus::Pending)); + drop(cache); + let cache = Replica::open(&path).unwrap(); + for id in [first, second, third] { + let result = cache.sync_once(&mut server).unwrap(); + assert!( + matches!(result,Some((actual,EditStatus::Published{..}))if actual==id), + "{result:?}" + ); + } + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let space = &doc.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let runs = view.text_runs(insertion.text_object()).unwrap(); + assert_eq!( + runs.iter().map(|r| r.text).collect::(), + "Offline 🦀" + ); + assert!(runs.iter().all(|r| r.format.bold == Some(true))); + if outline_case { + assert_eq!( + ( + view.nodes[&insertion.object()].layout.x, + view.nodes[&insertion.object()].layout.y + ), + (Some(288.0), Some(360.0)) + ); + } else { + assert_eq!( + view.nodes[&parent].children.last(), + Some(&insertion.object()) + ); + } + assert!(!view.nodes.contains_key(&anchor.object())); + assert_eq!(server.publications, 3); + assert!(cache.pending().unwrap().is_empty()); + assert_eq!(cache.snapshot().unwrap(), server.durable); + } +} + +#[test] +fn placement_reviews_cannot_replace_pending_or_uncertain_attempts() { + use onestore::Insertion; + for outline_case in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("attempt-placement.sqlite"); + let source = onestore::create_section("placement.one", "Original", "Author").unwrap(); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let (sid, page) = doc.pages().unwrap()[0]; + let space = &doc.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let parent = *view.nodes[&page] + .children + .iter() + .find(|id| matches!(view.nodes[id].kind, Kind::Outline { .. })) + .unwrap(); + let insertion = if outline_case { + Insertion::outline(page, 144.0, 144.0, "Offline", "Author").unwrap() + } else { + Insertion::paragraph(parent, None, "Offline", "Author").unwrap() + }; + let cache = Replica::create(&path, &source).unwrap(); + let id = cache.insert(&source, sid, &insertion).unwrap().unwrap(); + let local = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + let mut server = Server::new(&source); + for attempted in [false, true] { + if attempted { + server.fault = Fault::UnknownBefore; + assert!( + matches!(cache.sync_once(&mut server),Err(Error::Remote(e))if e.state==CommitState::Unknown) + ); + } + let state = cache.status(id).unwrap(); + let result = if outline_case { + cache.rebase_outline_conflict(id, &local, &source, page, 288.0, 360.0) + } else { + cache.rebase_paragraph_conflict(id, &local, &source, parent, None) + }; + assert!(matches!(result,Err(Error::Io(e))if e.kind()==io::ErrorKind::InvalidInput)); + assert_eq!(cache.pending().unwrap(), pending); + assert_eq!(cache.status(id).unwrap(), state); + assert_eq!(cache.snapshot().unwrap(), local); + } + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), pending); + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::AwaitingConfirmation { .. })) + )); + assert_eq!(server.publications, 1); + } +} diff --git a/crates/onestore-smb/Cargo.toml b/crates/onestore-smb/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..65a3eb6a84d2f313991f0ce6247ee7d1399d2c0c --- /dev/null +++ b/crates/onestore-smb/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "onestore-smb" +version = "0.1.0" +edition = "2024" +publish = false + +[dependencies] +onestore = { path = "../onestore" } +smb2 = "=0.21.0" +tokio = { version = "1", features = ["rt-multi-thread", "time"] } + +[dev-dependencies] +serde_json = "1" diff --git a/crates/onestore-smb/README.md b/crates/onestore-smb/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e379d79821866a87061eb418e82151b006c42cf5 --- /dev/null +++ b/crates/onestore-smb/README.md @@ -0,0 +1,43 @@ +# onestore-smb + +Optional blocking SMB access for OneNote sections and table-of-contents files. +The core `onestore` crate remains independent of network runtimes. This is an +experimental Rust API with native interoperability evidence in the repository's +[Milestone 9](../../evidence/MILESTONE9.md). + +```no_run +use onestore_smb::{Client, Credentials}; +use std::time::Duration; + +let client = Client::connect( + "server:445", + "notes", + Credentials { username: "user", password: "password", domain: "" }, + Duration::from_secs(5), +)?; +let snapshot = client.read("Personal/Video.one", 64 * 1024 * 1024)?; +let store = onestore::Store::parse(&snapshot)?; +let revisions = onestore::RevisionIndex::parse(&store)?; +let document = onestore::document::Document::parse(&revisions)?; +# Ok::<(), Box>(()) +``` + +Paths are relative to the share. The read limit bounds the complete physical +snapshot. Call from a background thread outside a Tokio runtime. Use identities +from the document and the same snapshot with `Client::commit_text` or +`Client::commit_property_bytes`; their errors retain `onestore::CommitState`. +`PreparedEdit::{text,insert,format}` separate preparation from I/O: inspect the immutable image +and persist the intended revision identity before `Client::commit_prepared`. +`Client::confirm_snapshot` compares and flushes an observed image, then refreshes +its header version metadata without adding a revision. The caller must first +establish which intents that image contains and reread before another commit. + +Readers use shared native guards while writers publish under native write-open +and byte-lock exclusion. Maintenance is excluded during each operation; pathname +identity is checked after acquiring the guards. Connection loss retires the +client. Reconnect for subsequent operations, and reconcile an `Unknown` edit +before retrying it. The transport does not automatically replay requests. + +Device and simulator builds link for iOS. Native acceptance uses disposable +OneNote 2010 clients and Samba; it does not establish on-device execution or +physical power-loss durability. diff --git a/crates/onestore-smb/examples/smb_concurrent_client.rs b/crates/onestore-smb/examples/smb_concurrent_client.rs new file mode 100644 index 0000000000000000000000000000000000000000..c54cc91159f7125667d07638bc77ebc4f148f9ab --- /dev/null +++ b/crates/onestore-smb/examples/smb_concurrent_client.rs @@ -0,0 +1,22 @@ +#[path = "../../onestore/examples/support/concurrent.rs"] +mod concurrent; + +use onestore_smb::{Client, Credentials}; +use std::{env, time::Duration}; + +fn main() -> Result<(), Box> { + let client = Client::connect( + &env::var("ONESTORE_SMB_LAB")?, + &env::var("ONESTORE_SMB_SHARE")?, + Credentials::default(), + Duration::from_secs(10), + )?; + let args: Vec<_> = env::args().skip(1).collect(); + concurrent::run( + &args, + |path| client.read(path, 256 * 1024 * 1024), + |path, source, space, object, range, replacement| { + client.commit_text(path, source, space, object, range, replacement) + }, + ) +} diff --git a/crates/onestore-smb/examples/smb_reconnect_client.rs b/crates/onestore-smb/examples/smb_reconnect_client.rs new file mode 100644 index 0000000000000000000000000000000000000000..1c88874791fe610c2b3fb6f7221014bbd0de45ae --- /dev/null +++ b/crates/onestore-smb/examples/smb_reconnect_client.rs @@ -0,0 +1,267 @@ +#[path = "../../onestore/examples/support/concurrent.rs"] +mod concurrent; + +use onestore::{ + CommitError, CommitState, ExGuid, RevisionIndex, Store, + document::{Document, Kind}, +}; +use onestore_smb::{Client, Credentials}; +use serde_json::json; +use std::{ + cell::RefCell, + env, io, thread, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +fn paragraph( + bytes: &[u8], + space: ExGuid, + object: ExGuid, +) -> Result> { + let store = Store::parse(bytes)?; + let index = RevisionIndex::parse(&store)?; + index.validate_current()?; + let document = Document::parse(&index)?; + let space = &document.spaces[&space]; + let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; + let Kind::RichText { text, .. } = &revision.nodes[&object].kind else { + return Err("The append target is no longer text.".into()); + }; + Ok(text.clone()) +} + +// Only the owned append workload guarantees unique tokens that no writer removes. +fn retained(before: &str, token: &str, current: &str, state: CommitState) -> io::Result { + let count = current.matches(token).count(); + if count == 0 && state != CommitState::Committed && current.starts_with(before) { + return Ok(false); + } + if count == 1 + && state != CommitState::NotCommitted + && current.starts_with(&format!("{before}{token}")) + { + return Ok(true); + } + Err(io::Error::other( + "The append history contradicts the commit outcome.", + )) +} + +fn main() -> Result<(), Box> { + let args: Vec<_> = env::args().skip(1).collect(); + if args + .first() + .is_none_or(|mode| !["read", "write"].contains(&mode.as_str())) + { + return Err("Reconnect testing requires the append-only workload.".into()); + } + let address = env::var("ONESTORE_SMB_LAB")?; + let share = env::var("ONESTORE_SMB_SHARE")?; + let client = RefCell::new(Some(Client::connect( + &address, + &share, + Credentials::default(), + Duration::from_secs(5), + )?)); + let read = |path: &str| { + let mut session = client.borrow_mut(); + if session.is_none() { + match Client::connect( + &address, + &share, + Credentials::default(), + Duration::from_secs(5), + ) { + Ok(fresh) => { + *session = Some(fresh); + println!( + "{}", + json!({"event":"transport_connected", "at_us":SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_micros()}) + ); + } + Err(error) => { + println!( + "{}", + json!({"event":"transport_connect_error","error":error.to_string()}) + ); + return Err(io::ErrorKind::WouldBlock.into()); + } + } + } + let started = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_micros(); + match session.as_ref().unwrap().read(path, 256 * 1024 * 1024) { + Ok(bytes) => { + if args.get(2).is_some_and(|actor| actor == "r0") + && let Some(folder) = env::var_os("ONESTORE_OFFLINE_FORMAT_REPLY_DIR") + { + let folder = std::path::PathBuf::from(folder); + let captured = folder.join("offline-retired.one"); + if !captured.exists() + && let Ok(marker) = std::fs::read(folder.join("offline-paused-w0.isolate")) + && let Ok(watched) = serde_json::from_slice::(&marker) + && watched["after_us"] + .as_u64() + .is_some_and(|after| started > u128::from(after)) + { + let sid: ExGuid = watched["space"] + .as_str() + .ok_or_else(|| io::Error::other("Missing watched space"))? + .parse() + .map_err(io::Error::other)?; + let rid: ExGuid = watched["revision"] + .as_str() + .ok_or_else(|| io::Error::other("Missing watched revision"))? + .parse() + .map_err(io::Error::other)?; + let store = Store::parse(&bytes).map_err(io::Error::other)?; + let index = RevisionIndex::parse(&store).map_err(io::Error::other)?; + if index + .spaces + .get(&sid) + .is_some_and(|space| !space.revisions.contains_key(&rid)) + { + index.validate_current().map_err(io::Error::other)?; + std::fs::write(captured.with_extension("tmp"), &bytes)?; + std::fs::rename(captured.with_extension("tmp"), captured)?; + println!( + "{}", + json!({"event":"revision_retired", "space":sid.to_string(), "revision":rid.to_string(), "started_us":started, "finished_us":SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_micros()}) + ); + } + } + } + Ok(bytes) + } + Err(error) + if matches!( + error.kind(), + io::ErrorKind::Other | io::ErrorKind::TimedOut | io::ErrorKind::NotConnected + ) => + { + println!( + "{}", + json!({"event":"transport_read_error","error":error.to_string()}) + ); + *session = None; + Err(io::ErrorKind::WouldBlock.into()) + } + result => result, + } + }; + concurrent::run( + &args, + read, + |path, source, space, object, range, replacement| { + let outcome = client.borrow().as_ref().unwrap().commit_text( + path, + source, + space, + object, + range, + replacement, + ); + let Err(error) = outcome else { + return Ok(()); + }; + if error.state == CommitState::NotCommitted + && matches!( + error.error.kind(), + io::ErrorKind::WouldBlock + | io::ErrorKind::ResourceBusy + | io::ErrorKind::PermissionDenied + | io::ErrorKind::NotFound + ) + { + return Err(error); + } + println!( + "{}", + json!({"event":"transport_commit_error","state":format!("{:?}",error.state),"token":replacement,"error":error.error.to_string()}) + ); + *client.borrow_mut() = None; + let deadline = Instant::now() + Duration::from_secs(60); + loop { + if Instant::now() >= deadline { + return Err(error); + } + let current = match read(path) { + Ok(bytes) => bytes, + Err(retry) if retry.kind() == io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(100)); + continue; + } + Err(_) => return Err(error), + }; + let published = paragraph(source, space, object) + .and_then(|before| { + Ok(retained( + &before, + replacement, + ¶graph(¤t, space, object)?, + error.state, + )?) + }) + .map_err(|failure| CommitError { + state: error.state, + error: io::Error::other(failure.to_string()), + })?; + if published { + let confirmation = client.borrow().as_ref().unwrap().commit_text( + path, + ¤t, + space, + object, + 0..0, + "", + ); + if let Err(failure) = confirmation + && failure.state != CommitState::Committed + { + println!( + "{}", + json!({"event":"transport_confirmation_error","state":format!("{:?}", failure.state),"error":failure.error.to_string()}) + ); + *client.borrow_mut() = None; + thread::sleep(Duration::from_millis(100)); + continue; + } + println!( + "{}", + json!({"event":"transport_reconciled","token":replacement,"published":true,"flush_confirmed":true}) + ); + return Ok(()); + } + println!( + "{}", + json!({"event":"transport_reconciled","token":replacement,"published":false}) + ); + return Err(CommitError { + state: CommitState::NotCommitted, + error: io::ErrorKind::ResourceBusy.into(), + }); + } + }, + ) +} + +#[test] +fn uncertain_append_requires_one_retained_token_and_its_predecessor() { + for state in [CommitState::Unknown, CommitState::Committed] { + assert!(retained("before", " [w0:0]", "before [w0:0] [w1:0]", state).unwrap()); + } + for state in [CommitState::Unknown, CommitState::NotCommitted] { + assert!(!retained("before", " [w0:0]", "before [w1:0]", state).unwrap()); + } + for (current, state) in [ + ("before [w0:0] [w0:0]", CommitState::Unknown), + ("changed [w0:0]", CommitState::Unknown), + ("befor", CommitState::Unknown), + ("before", CommitState::Committed), + ("before [w0:0]", CommitState::NotCommitted), + ] { + assert!(retained("before", " [w0:0]", current, state).is_err()); + } +} diff --git a/crates/onestore-smb/src/lib.rs b/crates/onestore-smb/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..f8bf1c9155a4c0c4fbea52a43b6ca5e6b4b1720e --- /dev/null +++ b/crates/onestore-smb/src/lib.rs @@ -0,0 +1,466 @@ +#![forbid(unsafe_code)] +#![doc = include_str!("../README.md")] + +use onestore::{CommitError, CommitIo, CommitState, ExGuid}; +use smb2::{ + Session, Tree, + client::connection::{Connection, NegotiatedParams}, + msg::{ + close::{CloseRequest, CloseResponse}, + create::{ + CreateDisposition, CreateRequest, CreateResponse, ImpersonationLevel, ShareAccess, + }, + flush::{FlushRequest, FlushResponse}, + lock::{LockElement, LockRequest, LockResponse}, + query_info::{InfoType, QueryInfoRequest, QueryInfoResponse}, + read::{ReadRequest, ReadResponse}, + write::{WriteRequest, WriteResponse}, + }, + pack::{Pack, ReadCursor, Unpack}, + types::{ + Command, CreditCharge, Dialect, FileId, OplockLevel, + flags::{Capabilities, FileAccessMask}, + }, +}; +use std::{io, ops::Range, sync::Mutex, time::Duration}; +use tokio::runtime::{Handle, Runtime}; + +#[derive(Default)] +pub struct Credentials<'a> { + pub username: &'a str, + pub password: &'a str, + pub domain: &'a str, +} + +/// Blocking connection with no automatic request replay or cached file contents. +/// Call from a background thread outside a Tokio runtime. +/// Paths are relative to the share; both `/` and `\` are separators. +pub struct Client { + connection: Mutex>, + tree: Tree, + timeout: Duration, + runtime: Mutex>, +} + +impl Client { + pub fn connect( + address: &str, + share: &str, + credentials: Credentials<'_>, + timeout: Duration, + ) -> io::Result { + if Handle::try_current().is_ok() || timeout.is_zero() { + return Err(io::ErrorKind::InvalidInput.into()); + } + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build()?; + let (connection, tree) = runtime + .block_on(async { + tokio::time::timeout(timeout, async { + let mut connection = Connection::connect(address, timeout).await?; + connection.set_compression_requested(false); + connection.negotiate().await?; + Session::setup( + &mut connection, + credentials.username, + credentials.password, + credentials.domain, + ) + .await?; + let tree = Tree::connect(&mut connection, share).await?; + Ok::<_, smb2::Error>((connection, tree)) + }) + .await + }) + .map_err(|_| io::Error::from(io::ErrorKind::TimedOut))? + .map_err(io::Error::other)?; + Ok(Self { + connection: Mutex::new(Some(connection)), + tree, + timeout, + runtime: Mutex::new(Some(runtime)), + }) + } + + fn retire(&self) { + if let Some(connection) = self + .connection + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take() + { + connection.mark_dead(); + } + if let Some(runtime) = self + .runtime + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take() + { + runtime.shutdown_background(); + } + } + + fn request(&self, command: Command, body: impl Pack) -> io::Result { + self.request_with(command, |_| (body, CreditCharge(1))) + } + + fn request_with( + &self, + command: Command, + prepare: impl FnOnce(&Connection) -> (B, CreditCharge), + ) -> io::Result { + if Handle::try_current().is_ok() { + return Err(io::ErrorKind::InvalidInput.into()); + } + let connection = self + .connection + .lock() + .map_err(|_| io::ErrorKind::Other)? + .clone() + .ok_or(io::ErrorKind::NotConnected)?; + let frame = { + let runtime = self.runtime.lock().map_err(|_| io::ErrorKind::Other)?; + let (body, charge) = prepare(&connection); + runtime + .as_ref() + .ok_or(io::ErrorKind::NotConnected)? + .block_on(async { + tokio::time::timeout( + self.timeout, + connection.execute_with_credits( + command, + &body, + Some(self.tree.tree_id), + charge, + ), + ) + .await + }) + }; + let frame = frame + .map_err(|_| io::Error::from(io::ErrorKind::TimedOut)) + .and_then(|result| result.map_err(io::Error::other)) + .inspect_err(|_| self.retire())?; + if frame.header.command != command { + self.retire(); + return Err(io::ErrorKind::InvalidData.into()); + } + if frame.header.status.0 != 0 { + let kind = match frame.header.status.0 { + 0xc0000043 | 0xc0000054 | 0xc0000055 => io::ErrorKind::WouldBlock, + 0xc0000011 => io::ErrorKind::UnexpectedEof, + 0xc0000034 | 0xc000003a => io::ErrorKind::NotFound, + _ => io::ErrorKind::Other, + }; + return Err(io::Error::new( + kind, + smb2::Error::Protocol { + status: frame.header.status, + command, + }, + )); + } + T::unpack(&mut ReadCursor::new(&frame.body)).map_err(|error| { + self.retire(); + io::Error::new(io::ErrorKind::InvalidData, error) + }) + } + + fn open(&self, path: &str, write: bool) -> io::Result> { + if path.is_empty() || path.contains('\0') || path.encode_utf16().count() > 32767 { + return Err(io::ErrorKind::InvalidInput.into()); + } + let response: CreateResponse = self.request( + Command::Create, + CreateRequest { + requested_oplock_level: OplockLevel::None, + impersonation_level: ImpersonationLevel::Impersonation, + desired_access: FileAccessMask::new(if write { 0xc0000000 } else { 0x80000000 }), + file_attributes: 0, + share_access: ShareAccess(if write { 5 } else { 7 }), + create_disposition: CreateDisposition::FileOpen, + create_options: 0x42, + name: smb2::encode_path(&path.replace('\\', "/")), + create_contexts: Vec::new(), + }, + )?; + Ok(File { + client: self, + id: Some(response.file_id), + }) + } + + /// Reads one bounded, consistent snapshot; contention returns WouldBlock. + pub fn read(&self, path: &str, limit: usize) -> io::Result> { + let mut file = self.open(path, false)?.coordinate(path, false)?; + let result = onestore::read_snapshot(|offset, output| file.read_at(offset, output), limit) + .and_then(|snapshot| snapshot.ok_or_else(|| io::ErrorKind::WouldBlock.into())); + let closed = file.close(); + let snapshot = result?; + closed?; + Ok(snapshot) + } + + pub fn commit_text( + &self, + path: &str, + source: &[u8], + space: ExGuid, + object: ExGuid, + range: Range, + replacement: &str, + ) -> Result<(), CommitError> { + self.commit(path, |file| { + onestore::commit_text(file, source, space, object, range, replacement) + }) + } + + pub fn commit_property_bytes( + &self, + path: &str, + source: &[u8], + space: ExGuid, + object: ExGuid, + property: u32, + value: &[u8], + ) -> Result<(), CommitError> { + self.commit(path, |file| { + onestore::commit_property_bytes(file, source, space, object, property, value) + }) + } + + /// Publishes a prepared edit using the same native writer coordination as text commits. + pub fn commit_prepared( + &self, + path: &str, + edit: &onestore::PreparedEdit<'_>, + ) -> Result<(), CommitError> { + self.commit(path, |file| edit.commit(file)) + } + + /// Confirms an observed snapshot's durability under native writer coordination. + pub fn confirm_snapshot(&self, path: &str, source: &[u8]) -> Result<(), CommitError> { + self.commit(path, |file| onestore::confirm_snapshot(file, source)) + } + + fn commit( + &self, + path: &str, + operation: impl FnOnce(&mut File<'_>) -> Result<(), CommitError>, + ) -> Result<(), CommitError> { + let mut file = self + .open(path, true) + .and_then(|file| file.coordinate(path, true)) + .map_err(|error| CommitError { + state: CommitState::NotCommitted, + error, + })?; + let result = operation(&mut file); + let closed = file.close(); + result?; + closed.map_err(|error| CommitError { + state: CommitState::Committed, + error, + }) + } +} + +impl Drop for Client { + fn drop(&mut self) { + self.retire(); + } +} + +struct File<'a> { + client: &'a Client, + id: Option, +} +impl File<'_> { + fn coordinate(self, path: &str, write: bool) -> io::Result { + self.lock(0xfffffffb, 0x11)?; + if write { + self.lock(0xfffffffd, 0x12)?; + } + let current = self.client.open(path, false)?; + let same = self.identity()? == current.identity()?; + current.close()?; + if !same { + return Err(io::ErrorKind::ResourceBusy.into()); + } + Ok(self) + } + + fn lock(&self, offset: u64, flags: u32) -> io::Result<()> { + let _: LockResponse = self.client.request( + Command::Lock, + LockRequest { + file_id: self.id.unwrap(), + lock_sequence: 0, + locks: vec![LockElement { + offset, + length: 1, + flags, + }], + }, + )?; + Ok(()) + } + + fn identity(&self) -> io::Result<(u64, u32)> { + let index: QueryInfoResponse = self.client.request( + Command::QueryInfo, + QueryInfoRequest { + info_type: InfoType::File, + file_info_class: 6, + output_buffer_length: 8, + additional_information: 0, + flags: 0, + file_id: self.id.unwrap(), + input_buffer: Vec::new(), + }, + )?; + let index = u64::from_le_bytes( + index + .output_buffer + .try_into() + .map_err(|_| io::ErrorKind::InvalidData)?, + ); + if index == 0 { + return Err(io::ErrorKind::Unsupported.into()); + } + let volume: QueryInfoResponse = self.client.request( + Command::QueryInfo, + QueryInfoRequest { + info_type: InfoType::Filesystem, + file_info_class: 1, + output_buffer_length: 1024, + additional_information: 0, + flags: 0, + file_id: self.id.unwrap(), + input_buffer: Vec::new(), + }, + )?; + let serial = volume + .output_buffer + .get(8..12) + .ok_or(io::ErrorKind::InvalidData)?; + Ok((index, u32::from_le_bytes(serial.try_into().unwrap()))) + } + + fn close(mut self) -> io::Result<()> { + self.release() + } + + fn release(&mut self) -> io::Result<()> { + if let Some(file_id) = self.id.take() { + let result: io::Result = self + .client + .request(Command::Close, CloseRequest { file_id, flags: 0 }); + if result.is_err() { + self.client.retire(); + } + result?; + } + Ok(()) + } +} +impl Drop for File<'_> { + fn drop(&mut self) { + let _ = self.release(); + } +} +fn read_size(params: &NegotiatedParams, credits: u16, requested: usize) -> usize { + let budget = if params.dialect != Dialect::Smb2_0_2 + && params.capabilities.contains(Capabilities::LARGE_MTU) + { + usize::from(credits.max(1)) * 65536 + } else { + 65536 + }; + requested + .min(params.max_read_size as usize) + .min(budget) + .min(1024 * 1024) +} + +impl CommitIo for File<'_> { + fn read_at(&mut self, offset: u64, output: &mut [u8]) -> io::Result { + if output.is_empty() { + return Ok(0); + } + let mut size = 0; + let response: io::Result = + self.client.request_with(Command::Read, |connection| { + size = read_size( + &connection + .params() + .expect("connected SMB session is negotiated"), + connection.credits(), + output.len(), + ); + ( + ReadRequest { + file_id: self.id.unwrap(), + offset, + length: size as u32, + minimum_count: 1, + flags: 0, + padding: 0, + channel: 0, + remaining_bytes: 0, + read_channel_info: Vec::new(), + }, + CreditCharge(size.div_ceil(65536) as u16), + ) + }); + let response = match response { + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(0), + result => result?, + }; + if response.data.len() > size { + self.client.retire(); + return Err(io::ErrorKind::InvalidData.into()); + } + output[..response.data.len()].copy_from_slice(&response.data); + Ok(response.data.len()) + } + fn write_at(&mut self, offset: u64, bytes: &[u8]) -> io::Result { + if bytes.is_empty() { + return Ok(0); + } + let size = bytes.len().min(65536); + let response: WriteResponse = self.client.request( + Command::Write, + WriteRequest { + file_id: self.id.unwrap(), + offset, + data: bytes[..size].to_vec(), + data_offset: 112, + flags: 1, + channel: 0, + remaining_bytes: 0, + write_channel_info_offset: 0, + write_channel_info_length: 0, + }, + )?; + if response.count as usize > size { + return Err(io::ErrorKind::InvalidData.into()); + } + Ok(response.count as usize) + } + fn flush(&mut self) -> io::Result<()> { + let _: FlushResponse = self.client.request( + Command::Flush, + FlushRequest { + file_id: self.id.unwrap(), + }, + )?; + Ok(()) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/onestore-smb/src/tests.rs b/crates/onestore-smb/src/tests.rs new file mode 100644 index 0000000000000000000000000000000000000000..fb20e9dfc177b2049b494c563f36f741b9d0cf8a --- /dev/null +++ b/crates/onestore-smb/src/tests.rs @@ -0,0 +1,340 @@ +use super::*; +use onestore::{ + RevisionIndex, Store, + document::{Document, Kind}, +}; +use std::{ + fs, + time::{Instant, SystemTime, UNIX_EPOCH}, +}; + +mod faults; + +fn client() -> Client { + Client::connect( + &std::env::var("ONESTORE_SMB_LAB").unwrap(), + "agent", + Credentials::default(), + Duration::from_secs(5), + ) + .unwrap() +} +fn create(client: &Client, path: &str, bytes: &[u8]) { + let response: CreateResponse = client + .request( + Command::Create, + CreateRequest { + requested_oplock_level: OplockLevel::None, + impersonation_level: ImpersonationLevel::Impersonation, + desired_access: FileAccessMask::new(0xc0000000), + file_attributes: 0, + share_access: ShareAccess(7), + create_disposition: CreateDisposition::FileCreate, + create_options: 0x42, + name: path.to_owned(), + create_contexts: Vec::new(), + }, + ) + .unwrap(); + let mut file = File { + client, + id: Some(response.file_id), + }; + let mut offset = 0; + while offset < bytes.len() { + offset += file.write_at(offset as u64, &bytes[offset..]).unwrap(); + } + file.flush().unwrap(); + file.close().unwrap(); +} +fn text(bytes: &[u8]) -> (ExGuid, ExGuid, String) { + let store = Store::parse(bytes).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + let doc = Document::parse(&index).unwrap(); + doc.spaces + .iter() + .find_map(|(sid, space)| { + let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; + revision + .nodes + .iter() + .find_map(|(oid, node)| match &node.kind { + Kind::RichText { text, .. } => Some((*sid, *oid, text.clone())), + _ => None, + }) + }) + .unwrap() +} +#[test] +#[ignore = "requires ONESTORE_SMB_LAB pointing to disposable Samba"] +fn live_coordination() { + let writer = client(); + let path = format!( + "adapter-{}.one", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + let source = onestore::create_section(&path, "Before café 🦀", "Fixture").unwrap(); + create(&writer, &path, &source); + assert_eq!(writer.read(&path, 1 << 20).unwrap(), source); + let readers: Vec<_> = (0..12).map(|_| client()).collect(); + let mut held: Vec<_> = readers + .iter() + .map(|client| { + client + .open(&path, false) + .unwrap() + .coordinate(&path, false) + .unwrap() + }) + .collect(); + let (sid, oid, before) = text(&source); + let replacement = "After café 🦀"; + writer + .commit_text( + &path, + &source, + sid, + oid, + 0..before.encode_utf16().count() as u32, + replacement, + ) + .unwrap(); + let after = writer.read(&path, 1 << 20).unwrap(); + assert_eq!(text(&after).2, replacement); + for file in &mut held { + let snapshot = onestore::read_snapshot(|offset, out| file.read_at(offset, out), 1 << 20) + .unwrap() + .unwrap(); + assert_eq!(snapshot, after); + } + let error = writer + .commit_text(&path, &source, sid, oid, 0..1, "X") + .unwrap_err(); + assert_eq!(error.state, CommitState::NotCommitted); + assert_eq!(error.error.kind(), io::ErrorKind::ResourceBusy); + assert_eq!(writer.read(&path, 1 << 20).unwrap(), after); + drop(held); + + let stale = writer.open(&path, true).unwrap(); + let maintenance = client(); + let guard = maintenance.open(&path, false).unwrap(); + let _: LockResponse = maintenance + .request( + Command::Lock, + LockRequest { + file_id: guard.id.unwrap(), + lock_sequence: 0, + locks: vec![ + LockElement { + offset: 0xfffffffc, + length: 1, + flags: 0x12, + }, + LockElement { + offset: 0xffffeffc, + length: 4096, + flags: 0x12, + }, + ], + }, + ) + .unwrap(); + let replacement_path = format!("{path}.replacement"); + create(&maintenance, &replacement_path, &source); + let mut connection = maintenance + .connection + .lock() + .unwrap() + .as_ref() + .unwrap() + .clone(); + maintenance + .runtime + .lock() + .unwrap() + .as_ref() + .unwrap() + .block_on( + maintenance + .tree + .rename(&mut connection, &path, &format!("{path}.old")), + ) + .unwrap(); + maintenance + .runtime + .lock() + .unwrap() + .as_ref() + .unwrap() + .block_on( + maintenance + .tree + .rename(&mut connection, &replacement_path, &path), + ) + .unwrap(); + drop(connection); + guard.close().unwrap(); + let error = stale.coordinate(&path, true).err().unwrap(); + assert_eq!(error.kind(), io::ErrorKind::ResourceBusy); + assert_eq!(writer.read(&path, 1 << 20).unwrap(), source); + + let retiring = client(); + let file = retiring + .open(&path, true) + .unwrap() + .coordinate(&path, true) + .unwrap(); + retiring.retire(); + assert_eq!( + retiring.read(&path, 1 << 20).unwrap_err().kind(), + io::ErrorKind::NotConnected + ); + drop(file); + let deadline = Instant::now() + Duration::from_secs(5); + loop { + match writer + .open(&path, true) + .and_then(|file| file.coordinate(&path, true)) + { + Ok(file) => { + file.close().unwrap(); + break; + } + Err(error) + if error.kind() == io::ErrorKind::WouldBlock && Instant::now() < deadline => + { + std::thread::sleep(Duration::from_millis(10)) + } + Err(error) => panic!("retired connection retained locks: {error}"), + } + } + + let mut unfinished = writer + .open(&path, true) + .unwrap() + .coordinate(&path, true) + .unwrap(); + assert_eq!( + unfinished + .write_at(source.len() as u64, b"unpublished") + .unwrap(), + 11 + ); + unfinished.flush().unwrap(); + unfinished.close().unwrap(); + let snapshot = writer.read(&path, 1 << 20).unwrap(); + assert_eq!(snapshot.len(), source.len() + 11); + assert_eq!(text(&snapshot).2, before); + writer + .commit_text( + &path, + &snapshot, + sid, + oid, + 0..before.encode_utf16().count() as u32, + "Recovered café 🦀", + ) + .unwrap(); + let recovered = writer.read(&path, 1 << 20).unwrap(); + assert_eq!(text(&recovered).2, "Recovered café 🦀"); + let spare = client(); + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() + .block_on(async { + drop(spare); + }); + let output = std::env::var("ONESTORE_SMB_EVIDENCE").unwrap(); + fs::write(std::path::Path::new(&output).join("source.one"), &source).unwrap(); + fs::write(std::path::Path::new(&output).join("committed.one"), &after).unwrap(); + fs::write( + std::path::Path::new(&output).join("recovered.one"), + &recovered, + ) + .unwrap(); + println!( + "{}", + serde_json::json!({"path":path,"readers":12,"committed_while_readers_held":true,"fresh_reads":12,"stale_snapshot_rejected":true,"replaced_handle_rejected":true,"retirement_releases_locks":true,"unpublished_tail_recovered":true,"drop_inside_runtime":true}) + ); +} + +#[test] +#[ignore = "requires an owned Samba fixture and maintenance controller"] +fn live_reader_hold() { + let client = client(); + let path = std::env::var("ONESTORE_SMB_PATH").unwrap(); + let output = std::path::PathBuf::from(std::env::var("ONESTORE_SMB_HOLD").unwrap()); + assert!(!output.join("ready").exists() && !output.join("release").exists()); + let mut file = client + .open(&path, false) + .unwrap() + .coordinate(&path, false) + .unwrap(); + fs::write(output.join("ready"), b"held").unwrap(); + let deadline = Instant::now() + Duration::from_secs(300); + let mut accepted = 0; + let mut retries = 0; + while !output.join("release").exists() { + assert!( + Instant::now() < deadline, + "maintenance controller timed out" + ); + match onestore::read_snapshot(|offset, out| file.read_at(offset, out), 256 << 20).unwrap() { + Some(_) => accepted += 1, + None => retries += 1, + } + std::thread::sleep(Duration::from_millis(5)); + } + file.close().unwrap(); + assert!(accepted > 0); + fs::write( + output.join("released.json"), + serde_json::to_vec(&serde_json::json!({"accepted": accepted, "retries": retries})).unwrap(), + ) + .unwrap(); +} + +#[test] +fn read_limits_respect_negotiation_and_available_credits() { + for dialect in Dialect::ALL { + for large_mtu in [false, true] { + for max_read_size in [65536, 65537, 131072, 1048576, u32::MAX] { + let params = NegotiatedParams { + dialect: *dialect, + max_read_size, + max_write_size: 65536, + max_transact_size: 65536, + server_guid: Default::default(), + signing_required: false, + capabilities: Capabilities(if large_mtu { + Capabilities::LARGE_MTU + } else { + 0 + }), + gmac_negotiated: false, + cipher: None, + compression_supported: false, + }; + for credits in [0, 1, 2, 3, 15, 16, 17, u16::MAX] { + for requested in [1, 1024, 65535, 65536, 65537, 131072, 1048576, usize::MAX] { + let size = read_size(¶ms, credits, requested); + assert!(size > 0 && size <= requested && size <= max_read_size as usize); + assert!(size <= 1048576); + assert!(size.div_ceil(65536) <= usize::from(credits.max(1))); + if *dialect == Dialect::Smb2_0_2 || !large_mtu { + assert!(size <= 65536); + } else if credits >= 16 && max_read_size >= 1048576 && requested >= 1048576 + { + assert_eq!(size, 1048576); + } + } + } + } + } + } +} diff --git a/crates/onestore-smb/src/tests/faults.rs b/crates/onestore-smb/src/tests/faults.rs new file mode 100644 index 0000000000000000000000000000000000000000..fb4daed377dfb6f4382ec8e8474dcae0b61cab92 --- /dev/null +++ b/crates/onestore-smb/src/tests/faults.rs @@ -0,0 +1,444 @@ +use super::*; +use serde_json::{Value, json}; +use std::{collections::BTreeMap, path::Path, process::Child}; + +#[derive(Clone)] +struct Snapshot { + content: BTreeMap, + modified: BTreeMap<(ExGuid, ExGuid), u32>, +} + +fn snapshot(bytes: &[u8]) -> Snapshot { + let store = Store::parse(bytes).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + Document::parse(&index).unwrap(); + let mut content = BTreeMap::new(); + let mut modified = BTreeMap::new(); + for (sid, space) in &index.spaces { + let revision = index + .resolve(*sid, space.labels[&(ExGuid::default(), 1)]) + .unwrap(); + let mut objects = BTreeMap::new(); + for (oid, object) in &revision.objects { + if let Some(onestore::FileDataReference::Internal(guid)) = + object.file_reference().unwrap() + { + store.file_data(guid).unwrap(); + } + let data = match object.data { + onestore::ObjectData::Properties(bytes) => { + let mut properties = onestore::PropertySets::parse(bytes).unwrap(); + for property in &mut properties.sets[0] { + if property.id == 0x14001d7a { + let onestore::Value::Bytes(value) = property.value else { + panic!() + }; + assert!( + modified + .insert( + (*sid, *oid), + u32::from_le_bytes(value.try_into().unwrap()) + ) + .is_none() + ); + property.value = onestore::Value::Bytes(&[0; 4]); + } + } + format!("{properties:?}") + } + other => format!("{other:?}"), + }; + objects.insert( + *oid, + ( + object.jcid, + object.reference_count, + data, + format!("{:?}", object.references().unwrap()), + ), + ); + } + content.insert(*sid, format!("{:?} {objects:?}", revision.roots)); + } + Snapshot { content, modified } +} + +fn stamp() -> u32 { + (SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() + - 315532800) + .try_into() + .unwrap() +} + +fn published( + actual: &Snapshot, + old: &Snapshot, + new: &Snapshot, + time: std::ops::RangeInclusive, +) -> bool { + if actual.content == old.content { + assert_eq!( + actual.modified, old.modified, + "Unpublished modification time changed" + ); + return false; + } + assert_eq!( + actual.content, new.content, + "Partial or unexpected publication" + ); + assert!(actual.modified.keys().eq(new.modified.keys())); + for (key, value) in &actual.modified { + if old.modified.get(key) != new.modified.get(key) { + assert!( + time.contains(value), + "Modification time outside the commit interval" + ); + } else { + assert_eq!( + *value, new.modified[key], + "Unrelated modification time changed" + ); + } + } + true +} + +#[test] +fn publication_oracle_bounds_changed_timestamps_and_preserves_others() { + let id = ExGuid::default(); + let other = ExGuid { n: 1, ..id }; + let old = Snapshot { + content: BTreeMap::from([(id, "old".into())]), + modified: BTreeMap::from([((id, id), 10), ((id, other), 7)]), + }; + let new = Snapshot { + content: BTreeMap::from([(id, "new".into())]), + modified: BTreeMap::from([((id, id), 20), ((id, other), 7)]), + }; + let mut actual = new.clone(); + actual.modified.insert((id, id), 30); + assert!(published(&actual, &old, &new, 25..=35)); + assert!(!published(&old, &old, &new, 25..=35)); + for (key, value) in [((id, id), 24), ((id, id), 36), ((id, other), 30)] { + let mut changed = actual.clone(); + changed.modified.insert(key, value); + assert!(std::panic::catch_unwind(|| published(&changed, &old, &new, 25..=35)).is_err()); + } + actual.content = old.content.clone(); + assert!(std::panic::catch_unwind(|| published(&actual, &old, &new, 25..=35)).is_err()); +} + +struct Proxy(Child); +impl Drop for Proxy { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +fn records(output: &Path) -> Vec { + let data = fs::read_to_string(output.join("proxy.jsonl")).unwrap(); + data.rsplit_once('\n') + .map_or("", |(complete, _)| complete) + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() +} + +fn configure(output: &Path, state: Value) -> usize { + fs::write(output.join("control.tmp"), state.to_string()).unwrap(); + fs::rename(output.join("control.tmp"), output.join("control.json")).unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let events = records(output); + if let Some(index) = events + .iter() + .rposition(|event| event.get("control") == Some(&state)) + { + return index + 1; + } + assert!( + Instant::now() < deadline, + "proxy did not acknowledge its control state" + ); + std::thread::sleep(Duration::from_millis(5)); + } +} + +#[test] +#[ignore = "requires an owned Samba share and a new ONESTORE_SMB_EVIDENCE directory"] +fn live_message_loss() { + let output = std::path::PathBuf::from(std::env::var("ONESTORE_SMB_EVIDENCE").unwrap()); + fs::create_dir(&output).unwrap(); + fs::create_dir(output.join("interrupted")).unwrap(); + fs::create_dir(output.join("recovered")).unwrap(); + fs::create_dir(output.join("source")).unwrap(); + let address = std::env::var("ONESTORE_SMB_LAB").unwrap(); + let (host, port) = address.rsplit_once(':').unwrap(); + let mut proxy = Proxy( + std::process::Command::new("python3") + .arg(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tools/smb-proxy.py")) + .arg(output.join("control.json")) + .args(["--port", "0", "--server", host, "--server-port", port]) + .stdout(fs::File::create(output.join("proxy.jsonl")).unwrap()) + .stderr(fs::File::create(output.join("proxy.stderr")).unwrap()) + .spawn() + .unwrap(), + ); + let deadline = Instant::now() + Duration::from_secs(5); + let port = loop { + if let Some(port) = records(&output) + .iter() + .find_map(|event| event["listening"].as_u64()) + { + break port; + } + assert!(proxy.0.try_wait().unwrap().is_none()); + assert!(Instant::now() < deadline, "proxy did not start"); + std::thread::sleep(Duration::from_millis(5)); + }; + let proxied = format!("127.0.0.1:{port}"); + let observer = client(); + let prefix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let fixtures = [ + ( + "chunked", + onestore::create_section("fault.one", "Before café 🦀", "Fault test").unwrap(), + ), + ( + "carry", + fs::read( + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../corpus/append/round-01/tx-255/notebook/synthetic.one"), + ) + .unwrap(), + ), + ]; + let mut results = Vec::new(); + for (fixture, source) in fixtures { + fs::write( + output.join("source").join(format!("{fixture}.one")), + &source, + ) + .unwrap(); + let (sid, oid, before) = text(&source); + let after = if fixture == "chunked" { + "After café 🦀 ".repeat(6000) + } else { + "After café 🦀".to_owned() + }; + let suffix = " [reconnected]"; + fs::write( + output.join(format!("{fixture}-intent.json")), + serde_json::to_vec(&json!({"before":before,"after":after,"suffix":suffix})).unwrap(), + ) + .unwrap(); + let old = snapshot(&source); + let deadline = Instant::now() + Duration::from_secs(2); + while old.modified.values().any(|value| *value >= stamp()) { + assert!( + Instant::now() < deadline, + "source timestamps did not precede the test" + ); + std::thread::sleep(Duration::from_millis(5)); + } + let expected = onestore::replace_text( + &source, + sid, + oid, + 0..before.encode_utf16().count() as u32, + &after, + ) + .unwrap(); + let new = snapshot(&expected); + let baseline_path = format!("fault-{prefix}-{fixture}-baseline.one"); + create(&observer, &baseline_path, &source); + configure(&output, json!({"phase":format!("{fixture}-setup")})); + let baseline = Client::connect( + &proxied, + "agent", + Credentials::default(), + Duration::from_secs(5), + ) + .unwrap(); + let start = configure(&output, json!({"phase":format!("{fixture}-baseline")})); + let began = stamp(); + baseline + .commit_text( + &baseline_path, + &source, + sid, + oid, + 0..before.encode_utf16().count() as u32, + &after, + ) + .unwrap(); + let events = records(&output); + let mut occurrences = BTreeMap::new(); + let mut cuts = Vec::new(); + for event in &events[start..] { + let Some(direction) = event["direction"].as_str() else { + continue; + }; + if event["status"] == "0x103" { + continue; + } + let command = event["command"].as_u64().unwrap(); + let status = event["status"].as_str().map(str::to_owned); + let count = occurrences + .entry((direction.to_owned(), command, status.clone())) + .or_insert(0); + *count += 1; + cuts.push((direction.to_owned(), command, status, *count)); + } + assert!( + cuts.iter() + .any(|(direction, command, _, count)| direction == "response" + && *command == 7 + && *count == 3) + ); + assert!(published( + &snapshot(&observer.read(&baseline_path, 1 << 20).unwrap()), + &old, + &new, + began..=stamp() + )); + drop(baseline); + for (case, (direction, command, status, occurrence)) in cuts.iter().enumerate() { + let name = format!("{fixture}-{case:03}"); + let path = format!("fault-{prefix}-{name}.one"); + create(&observer, &path, &source); + configure(&output, json!({"phase":format!("{name}-setup")})); + let interrupted = Client::connect( + &proxied, + "agent", + Credentials::default(), + Duration::from_secs(5), + ) + .unwrap(); + let start = configure( + &output, + json!({"phase":name, "direction":direction, "cut":command, "status":status, "occurrence":occurrence}), + ); + let began = stamp(); + let error = interrupted + .commit_text( + &path, + &source, + sid, + oid, + 0..before.encode_utf16().count() as u32, + &after, + ) + .unwrap_err(); + assert_eq!( + interrupted.read(&path, 1 << 20).unwrap_err().kind(), + io::ErrorKind::NotConnected + ); + let events = records(&output); + assert_eq!( + events[start..] + .iter() + .filter(|event| event.get("cut").is_some()) + .count(), + 1 + ); + assert!( + !events + .iter() + .any(|event| event.get("trace_error").is_some()) + ); + let fresh = client(); + let deadline = Instant::now() + Duration::from_secs(5); + loop { + match fresh + .open(&path, true) + .and_then(|file| file.coordinate(&path, true)) + { + Ok(file) => { + file.close().unwrap(); + break; + } + Err(error) + if error.kind() == io::ErrorKind::WouldBlock + && Instant::now() < deadline => + { + std::thread::sleep(Duration::from_millis(5)) + } + Err(error) => panic!("{name}: retired session retained exclusion: {error}"), + } + } + let saved = fresh.read(&path, 1 << 20).unwrap(); + fs::write( + output.join("interrupted").join(format!("{name}.one")), + &saved, + ) + .unwrap(); + let visible = published(&snapshot(&saved), &old, &new, began..=stamp()); + match error.state { + CommitState::NotCommitted => assert!(!visible, "{name}"), + CommitState::Committed => assert!(visible, "{name}"), + CommitState::Unknown => {} + } + if !visible { + fresh + .commit_text( + &path, + &saved, + sid, + oid, + 0..before.encode_utf16().count() as u32, + &after, + ) + .unwrap(); + } + let saved = fresh.read(&path, 1 << 20).unwrap(); + assert!( + published(&snapshot(&saved), &old, &new, began..=stamp()), + "{name}: replay did not publish" + ); + let end = after.encode_utf16().count() as u32; + fresh + .commit_text(&path, &saved, sid, oid, end..end, suffix) + .unwrap(); + let recovered = fresh.read(&path, 1 << 20).unwrap(); + assert_eq!(text(&recovered).2, format!("{after}{suffix}")); + fs::write( + output.join("recovered").join(format!("{name}.one")), + recovered, + ) + .unwrap(); + results.push(json!({"case":name,"path":path,"direction":direction,"command":command,"status":status,"occurrence":occurrence, + "state":format!("{:?}",error.state),"visible":if visible {"after"} else {"before"}, + "retired_session_rejected":true,"exclusion_released":true,"fresh_commit_succeeded":true})); + fs::write( + output.join("results.json"), + serde_json::to_vec_pretty(&results).unwrap(), + ) + .unwrap(); + } + } + for state in ["NotCommitted", "Unknown", "Committed"] { + assert!(results.iter().any(|result| result["state"] == state)); + } + assert!( + results + .iter() + .any(|result| result["state"] == "Unknown" && result["visible"] == "before") + ); + assert!( + results + .iter() + .any(|result| result["state"] == "Unknown" && result["visible"] == "after") + ); + println!("{} message-loss cases passed", results.len()); +} diff --git a/README.md b/crates/onestore/README.md similarity index 69% rename from README.md rename to crates/onestore/README.md index 4294d7dd15e5f71360213b6c8ebfda9663019efb..1652d89d5f8106550d2d19d37b7dd0a1e30a72db 100644 --- a/README.md +++ b/crates/onestore/README.md @@ -2,31 +2,38 @@ An experimental native Rust library for OneNote revision stores (`.one` and `.onetoc2`). It reads committed object graphs, creates a small notebook without -a template, appends scalar-property and text edits with a recoverable commit protocol, -and interprets MS-ONE document structure, formatting, media, and historical pages. -The storage gates are recorded in [PROGRESS.md](PROGRESS.md); document-model -verification is recorded in [M6-ACCEPTANCE.md](M6-ACCEPTANCE.md). Concurrent-editing -verification is recorded in [MILESTONE7.md](MILESTONE7.md). +a template, appends property, text, paragraph, outline and formatting edits with a +recoverable commit protocol, and interprets MS-ONE document structure, formatting, media, and historical pages. +The storage gates are recorded in [PROGRESS.md](../../evidence/PROGRESS.md); document-model +verification is recorded in [M6-ACCEPTANCE.md](../../evidence/M6-ACCEPTANCE.md). Concurrent-editing +verification is recorded in [MILESTONE7.md](../../evidence/MILESTONE7.md). Crash recovery and the +read/write HTML diagnostic editor are recorded in [MILESTONE8.md](../../evidence/MILESTONE8.md). +Embedded SMB coordination, durable offline editing and document-growth acceptance +are recorded in [MILESTONE9.md](../../evidence/MILESTONE9.md#document-writer-and-offline-acceptance). -Text edits publish ancestor modification timestamps with the changed content. -Omitting those timestamps caused acknowledged edits to disappear during native -conflict merging. The repaired eight-client replay retains all four competing -results after reconnection, application closure and fresh-cache native inspection; -see `mixed-conflict-06` in the milestone evidence. Use disposable copies for -notebook editing. +Use disposable copies for notebook editing. Header version notification now +follows durable transaction publication, fixing a native cached-reader race. +The [lost-reply acceptance](../../evidence/MILESTONE9.md#lost-reply-acceptance-with-version-notification-published-last) +records the failure, reduced regression model, twelve-client repeat and cold +OneNote verification of all 3200 editing intents. ## Workspace `crates/onestore` contains the library, examples and integration tests. New Rust prototypes belong in sibling directories under `crates/` and depend on `onestore = { path = "../onestore" }`. The root manifest discovers these crates. -The consumer boundary and API tradeoffs are recorded in [API-AUDIT.md](API-AUDIT.md). +The consumer boundary and API tradeoffs are recorded in [API-AUDIT.md](../../evidence/API-AUDIT.md). +`crates/onestore-diagnostic` backs the [HTML diagnostic editor](../../evidence/DIAGNOSTIC.md). +[`onestore-smb`](../onestore-smb/README.md) provides optional embedded network +access; [`onestore-offline`](../onestore-offline/README.md) provides local +SQLite persistence and reconnect reconciliation for text, insertion and formatting. Shared native fixtures, specifications, evidence and Python/VM tools stay at the repository root; `fuzz/` remains an independent cargo-fuzz workspace. Run Cargo commands from the root. Select `-p onestore` when working only on the library, or `--workspace` for checks across all crates. Example binary paths -remain `target/debug/examples/…` for the native verification tools. +remain `target/debug/examples/…` for the native verification tools. The collaboration +harness also accepts `--client-profile release`. ## Supported surface @@ -40,7 +47,11 @@ remain `target/debug/examples/…` for the native verification tools. | `create_table_of_contents` | Create ordered section entries from filenames and file identities | | `replace_property_bytes` | Append one scalar-property revision; preserve prior revisions and unrelated property values and references | | `replace_text`, `commit_text`, `commit_file_text` | Replace a UTF-16 range within one ordinary text run; publish text, run boundaries and modification time together | +| `Insertion`, `PreparedEdit::insert` | Insert paragraphs into editable containers or positioned outlines into a page, retaining intent identities across rebases | +| `TextAttribute`, `PreparedEdit::format` | Change character formatting over a UTF-16 range while sharing immutable styles; preserve unselected runs | +| `PreparedEdit::commit`, `PreparedEdit::commit_file` | Publish the exact prepared image under caller-held exclusion or the conservative filesystem adapter | | `read_file` | Read a snapshot under whole-file exclusion | +| `read_snapshot` | Read a validated snapshot through fresh positioned I/O while the caller excludes maintenance | | `commit_file_property` | Lock, compare the source snapshot, append and flush, then publish the revision | | `CommitIo`, `commit_property_bytes` | Supply another storage backend with equivalent exclusion and ordered durability | @@ -54,13 +65,24 @@ while retaining historical revisions. TOC snapshots can remap encoded CompactIDs without changing their resolved references. Password-protected sections retain their encrypted structure and payloads; the library does not derive password keys or decrypt their pages. +Insertions update child references, reference counts, modification times and automatic +titles atomically. Paragraphs can be nested or inserted into table cells; outline +coordinates use points. Retain the `Insertion` value for rebasing: creating another +value creates different object identities. Duplicate insertion identities require +reconciliation. Formatting accepts explicit attributes, preserves inherited values, +and gives retired immutable styles zero current references while retaining history. +Generated fields, protected targets and unsupported run-data boundary changes are +rejected before publication. Local caches expose text, insertion and formatting edits; +the [document-writer acceptance](../../evidence/MILESTONE9.md#document-writer-and-offline-acceptance) +includes twelve mixed native/Rust clients, outages, lost replies and native revision retirement. + External `.onebin` references identify payloads for the caller to obtain. Cloud FSSHTTP synchronization and a C ABI are outside the implemented surface. ## Try it Requires Rust 1.97 or later for the verified build. Examples create new destinations -and refuse to overwrite them. The Python report requires Pillow. +and refuse to overwrite them. The Python tools require Python 3.10 or later and Pillow. ```sh cargo run --example create_notebook -- /tmp/one-demo 'Hello from Rust.' 'Example Author' @@ -115,7 +137,8 @@ exclusive lock → exact snapshot comparison → append data → flush → prepare header metadata → flush → publish transaction counter → flush - → finish counter rollover → flush → unlock + → finish counter rollover → flush + → notify cached readers → flush → unlock ``` Stale snapshots fail before writing. Live readers must use equivalent exclusion. @@ -143,12 +166,23 @@ calls allowed overlapping exclusive holders and stranded server locks under multi-process SMB contention. `tools/smb_lock_race.py` reproduces that failure without notebook parsing or writing. On the tested macOS SMB mount, POSIX byte-range locks returned `ENOTSUP`; whole-file locks excluded native -OneNote's lock ranges. `sync_all` falls back to `fsync` on macOS only when -`F_FULLFSYNC` is unsupported. Successful SMB FLUSH replies were observed on the +OneNote's lock ranges. This adapter serializes readers and writers during each +operation; it does not reproduce native reader/writer concurrency. The +[locking audit](../../evidence/LOCKING.md) records native coordination bytes, write-open share +modes, and a working macOS SMB-specific byte-range lock probe. `sync_all` falls +back to `fsync` on macOS only when `F_FULLFSYNC` is unsupported. Successful SMB FLUSH replies were observed on the wire. Correctness requires the backend to honor exclusion and ordered flushes. The evidence covers transport failures, not physical server power loss or every filesystem's lock implementation. +For shared network notebooks, use the optional +[`onestore-smb`](../onestore-smb/README.md) crate. It uses native share modes, +shared reader guards, writer exclusion and fresh pathname identity checks without +an OS-mounted share. Its [coordination acceptance](../../evidence/MILESTONE9.md) covers native +maintenance, mixed readers/writers, reconnects and uncertain publication. The +filesystem adapter retains its conservative locking; mounted-path freshness +across native replacement is not established by that exclusion. + ## Verification ```sh @@ -186,11 +220,11 @@ traverses every retained revision and resolved text run. Its public seeds live i the source target; private seeds are supplied only at runtime. Native edit-history tests compare independently generated operations, OneNote XML and the Rust model; failed histories can be replayed and shrunk in fresh disposable clones. The -document feature matrix is in [FEATURES.md](FEATURES.md), and the milestone's -acceptance contract is in [MILESTONE6.md](MILESTONE6.md). +document feature matrix is in [FEATURES.md](../../evidence/FEATURES.md), and the milestone's +acceptance contract is in [MILESTONE6.md](../../evidence/MILESTONE6.md). The stage-5 gate uses OneNote 2010 build 14.0.7015.1000 on Windows 7, a macOS SMB -mount, and Samba on zenith. [The collaboration corpus](corpus/collaboration/round-01) +mount, and Samba on zenith. [The collaboration corpus](../../corpus/collaboration/round-01) captures native lock contention, different-paragraph merging, same-paragraph conflicts, offline editing/reconnection, and lost successful FLUSH replies at preparation, publication, and counter cleanup. Each final notebook was reopened diff --git a/crates/onestore/examples/concurrent_client.rs b/crates/onestore/examples/concurrent_client.rs index f3ed8bcba59731740c85847f347e20536f5ea8da..c1c4f7797c92304505d3e1fb9a3a73fa1cc3be6d 100644 --- a/crates/onestore/examples/concurrent_client.rs +++ b/crates/onestore/examples/concurrent_client.rs @@ -1,18 +1,9 @@ +#[path = "support/concurrent.rs"] +mod concurrent; #[path = "../src/flush.rs"] mod flush; -use onestore::{ - CommitState, ExGuid, RevisionIndex, Store, - document::{Document, Kind}, -}; -use serde_json::json; -use std::{ - env, fs, - io::{self, Write}, - path::Path, - thread, - time::{Duration, Instant, SystemTime, UNIX_EPOCH}, -}; +use std::{env, fs, io::Write}; fn main() -> Result<(), Box> { let args: Vec<_> = env::args().skip(1).collect(); @@ -27,185 +18,11 @@ fn main() -> Result<(), Box> { flush::flush(&file)?; return Ok(()); } - if args.len() != 7 || !["read", "write", "edit"].contains(&args[0].as_str()) { - return Err("Usage: concurrent_client init FILE | read|write|edit FILE ACTOR OPERATIONS START_FILE STOP_FILE SEED".into()); - } - let mut random: u64 = args[6].parse()?; - let operations: usize = args[3].parse()?; - if operations == 0 { - return Err("Choose at least one operation.".into()); - } - let deadline = Instant::now() + Duration::from_secs(600); - let mut output = io::stdout().lock(); - let mut log = |event: serde_json::Value| -> io::Result<()> { - writeln!(output, "{event}")?; - output.flush() - }; - log(json!({"event": "ready", "pid": std::process::id(), "actor": args[2]}))?; - while !Path::new(&args[4]).exists() { - if Instant::now() > deadline { - return Err("Start barrier timed out.".into()); - } - thread::sleep(Duration::from_millis(5)); - } - let mut completed = 0; - let mut attempts = 0; - while completed < operations || (args[0] == "read" && !Path::new(&args[5]).exists()) { - if Instant::now() > deadline { - return Err("Concurrent client timed out.".into()); - } - attempts += 1; - random = random - .wrapping_mul(6364136223846793005) - .wrapping_add(1442695040888963407); - thread::sleep(Duration::from_millis((random >> 32) % 7)); - let started = SystemTime::now().duration_since(UNIX_EPOCH)?.as_micros(); - let source = match onestore::read_file(&args[1]) { - Ok(source) => source, - Err(error) - if [ - io::ErrorKind::WouldBlock, - io::ErrorKind::PermissionDenied, - io::ErrorKind::NotFound, - ] - .contains(&error.kind()) => - { - log( - json!({"event": "read_busy", "attempt": attempts, "kind": format!("{:?}", error.kind())}), - )?; - thread::sleep(Duration::from_millis(100)); - continue; - } - Err(error) => { - log( - json!({"event": "read_error", "attempt": attempts, "kind": format!("{:?}", error.kind())}), - )?; - return Err(error.into()); - } - }; - let preserve = |error: onestore::Error| { - let path = Path::new(&args[4]) - .parent() - .unwrap() - .join(format!("invalid-{}-{attempts}.one", std::process::id())); - if let Err(failure) = fs::write(&path, &source) { - eprintln!( - "Could not save invalid snapshot {}: {failure}", - path.display() - ); - } - error - }; - let store = Store::parse(&source).map_err(preserve)?; - if !store.checksum_mismatches.is_empty() { - return Err(preserve(onestore::Error { - offset: store.checksum_mismatches[0], - message: "A reader observed transaction checksum damage.", - }) - .into()); - } - let index = RevisionIndex::parse(&store).map_err(preserve)?; - index.validate_current().map_err(preserve)?; - let document = Document::parse(&index).map_err(preserve)?; - let mut targets = Vec::new(); - for (sid, page) in document.pages().map_err(preserve)? { - let space = &document.spaces[&sid]; - let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; - let mut pending = vec![page]; - let mut seen = std::collections::BTreeSet::new(); - while let Some(oid) = pending.pop() { - if !seen.insert(oid) { - continue; - } - let node = &revision.nodes[&oid]; - pending.extend( - node.children - .iter() - .chain(&node.content) - .chain(&node.structure) - .copied(), - ); - if let Kind::RichText { text, .. } = &node.kind - && text.starts_with("Concurrent edits:") - { - revision.text_runs(oid).map_err(preserve)?; - targets.push((sid, oid, text)); - } - } - } - let [(sid, oid, text)] = targets.as_slice() else { - return Err(preserve(onestore::Error { - offset: 0, - message: "Expected one concurrent-edit paragraph.", - }) - .into()); - }; - let read_finished = SystemTime::now().duration_since(UNIX_EPOCH)?.as_micros(); - log( - json!({"event": "read", "attempt": attempts, "started_us": started, "finished_us": read_finished, - "transaction": store.header.transaction_count, "text": text}), - )?; - if args[0] == "read" { - completed += 1; - continue; - } - let token = format!(" [{}:{}]", args[2], completed); - let offset = u32::try_from(text.encode_utf16().count())?; - let mut range = offset..offset; - let mut replacement = token.clone(); - if args[0] == "edit" { - let prefix = "Concurrent edits:"; - let mut boundaries = vec![u32::try_from(prefix.encode_utf16().count())?]; - for character in text[prefix.len()..].chars() { - boundaries.push(boundaries.last().unwrap() + character.len_utf16() as u32); - } - let first = ((random >> 16) % boundaries.len() as u64) as usize; - let second = ((random >> 40) % boundaries.len() as u64) as usize; - range = boundaries[first.min(second)]..boundaries[first.max(second)]; - replacement = format!(" café 🦀{token}"); - } - log( - json!({"event": "intent", "attempt": attempts, "operation": completed, - "source_transaction": store.header.transaction_count, "before": text, - "range": [range.start, range.end], "replacement": replacement, "token": token}), - )?; - thread::sleep(Duration::from_millis((random >> 48) % 13)); - let commit_started = SystemTime::now().duration_since(UNIX_EPOCH)?.as_micros(); - let result = onestore::commit_file_text(&args[1], &source, *sid, *oid, range, &replacement); - let finished = SystemTime::now().duration_since(UNIX_EPOCH)?.as_micros(); - match result { - Ok(()) => { - log( - json!({"event": "commit", "attempt": attempts, "operation": completed, "token": token, - "started_us": commit_started, "finished_us": finished, "source_transaction": store.header.transaction_count}), - )?; - completed += 1; - } - Err(error) - if error.state == CommitState::NotCommitted - && [ - io::ErrorKind::WouldBlock, - io::ErrorKind::ResourceBusy, - io::ErrorKind::PermissionDenied, - io::ErrorKind::NotFound, - ] - .contains(&error.error.kind()) => - { - log( - json!({"event": "retry", "attempt": attempts, "started_us": commit_started, - "finished_us": finished, "kind": format!("{:?}", error.error.kind())}), - )?; - } - Err(error) => { - log( - json!({"event": "commit_error", "attempt": attempts, "operation": completed, - "token": token, "state": format!("{:?}", error.state), "kind": format!("{:?}", error.error.kind()), - "started_us": commit_started, "finished_us": finished}), - )?; - return Err(error.into()); - } - } - } - log(json!({"event": "done", "completed": completed, "attempts": attempts}))?; - Ok(()) + concurrent::run( + &args, + |path| onestore::read_file(path), + |path, source, space, object, range, replacement| { + onestore::commit_file_text(path, source, space, object, range, replacement) + }, + ) } diff --git a/crates/onestore/examples/document.rs b/crates/onestore/examples/document.rs index b498706e84f3aba71b831bd62466cc16f57773c7..98c28e84ebdb61a778a9464d2ee86579893a9a1c 100644 --- a/crates/onestore/examples/document.rs +++ b/crates/onestore/examples/document.rs @@ -2,7 +2,22 @@ use onestore::{ FileDataReference, RevisionIndex, Store, document::{Document, Kind}, }; -use std::{collections::BTreeSet, env, fs, io, path::PathBuf}; +use std::{ + collections::BTreeSet, + env, fs, + io::{self, BufWriter, Write}, + path::{Path, PathBuf}, +}; + +fn write_json( + path: impl AsRef, + value: &impl serde::Serialize, +) -> Result<(), Box> { + let mut output = BufWriter::new(fs::File::create(path)?); + serde_json::to_writer(&mut output, value)?; + output.flush()?; + Ok(()) +} fn main() -> Result<(), Box> { let mut args = env::args_os().skip(1); @@ -38,7 +53,7 @@ fn main() -> Result<(), Box> { assets.push(serde_json::json!({"reference": FileDataReference::Internal(*guid), "path": path})); } } - serde_json::to_writer(fs::File::create(destination.join("assets.json"))?, &assets)?; + write_json(destination.join("assets.json"), &assets)?; let mut text = std::collections::BTreeMap::new(); for (sid, space) in &document.spaces { let mut revisions = std::collections::BTreeMap::new(); @@ -53,11 +68,8 @@ fn main() -> Result<(), Box> { } text.insert(*sid, revisions); } - serde_json::to_writer(fs::File::create(destination.join("text.json"))?, &text)?; - serde_json::to_writer( - fs::File::create(destination.join("document.json"))?, - &document, - )?; + write_json(destination.join("text.json"), &text)?; + write_json(destination.join("document.json"), &document)?; } else { serde_json::to_writer(io::stdout().lock(), &document)?; } diff --git a/crates/onestore/examples/insert.rs b/crates/onestore/examples/insert.rs new file mode 100644 index 0000000000000000000000000000000000000000..e443909cfddb89775826197ba05b3a74b3f9b78d --- /dev/null +++ b/crates/onestore/examples/insert.rs @@ -0,0 +1,52 @@ +#[path = "../src/flush.rs"] +mod flush; + +use onestore::{Insertion, PreparedEdit}; +use std::{fs, io::Write}; + +fn main() -> Result<(), Box> { + let args: Vec<_> = std::env::args().skip(1).collect(); + let usage = "Usage: insert paragraph INPUT OUTPUT|--in-place SPACE PARENT BEFORE|- TEXT AUTHOR\n insert outline INPUT OUTPUT|--in-place SPACE PAGE X Y TEXT AUTHOR"; + let insertion = match args.first().map(String::as_str) { + Some("paragraph") if args.len() == 8 => Insertion::paragraph( + args[4].parse()?, + if args[5] == "-" { + None + } else { + Some(args[5].parse()?) + }, + &args[6], + &args[7], + )?, + Some("outline") if args.len() == 9 => Insertion::outline( + args[4].parse()?, + args[5].parse()?, + args[6].parse()?, + &args[7], + &args[8], + )?, + _ => return Err(usage.into()), + }; + let source = onestore::read_file(&args[1])?; + let prepared = PreparedEdit::insert(&source, args[3].parse()?, &insertion)?; + let mut record = serde_json::json!({"intent": insertion, "object": insertion.object(), "text_object": insertion.text_object()}); + if args[2] == "--in-place" { + if let Err(error) = prepared.commit_file(&args[1]) { + record["state"] = format!("{:?}", error.state).into(); + record["error"] = error.error.to_string().into(); + println!("{record}"); + std::process::exit(2); + } + record["state"] = "Committed".into(); + } else { + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&args[2])?; + file.write_all(prepared.as_bytes())?; + flush::flush(&file)?; + record["state"] = "Created".into(); + } + println!("{record}"); + Ok(()) +} diff --git a/crates/onestore/examples/maintenance_fixture.rs b/crates/onestore/examples/maintenance_fixture.rs new file mode 100644 index 0000000000000000000000000000000000000000..ccfab15788a4bc6b940847fa6a127fe89e465851 --- /dev/null +++ b/crates/onestore/examples/maintenance_fixture.rs @@ -0,0 +1,63 @@ +use onestore::{ + ExGuid, RevisionIndex, Store, + document::{Document, Kind}, +}; +use std::{fs, io, path::PathBuf}; + +fn main() -> Result<(), Box> { + let mut args = std::env::args_os().skip(1); + let output = PathBuf::from( + args.next() + .ok_or(io::Error::from(io::ErrorKind::InvalidInput))?, + ); + let current = args + .next() + .map(|value| value.into_string()) + .transpose() + .map_err(|_| io::Error::from(io::ErrorKind::InvalidInput))?; + if args.next().is_some() { + return Err(io::Error::from(io::ErrorKind::InvalidInput).into()); + } + fs::create_dir(&output)?; + let mut bytes = onestore::create_section("synthetic.one", "Maintenance baseline.", "Fixture")?; + for revision in 0..25 { + let store = Store::parse(&bytes)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + let (sid, oid, end) = document + .spaces + .iter() + .find_map(|(sid, space)| { + space.revisions[&space.contexts[&ExGuid::default()]] + .nodes + .iter() + .find_map(|(oid, node)| { + if let Kind::RichText { text, .. } = &node.kind { + return Some((*sid, *oid, text.encode_utf16().count() as u32)); + } + None + }) + }) + .ok_or(io::Error::from(io::ErrorKind::InvalidData))?; + let text = if revision == 24 { + current + .clone() + .unwrap_or_else(|| "Maintenance current.".to_owned()) + } else { + format!("Revision {revision}: {}", "x".repeat(65536)) + }; + bytes = onestore::replace_text(&bytes, sid, oid, 0..end, &text)?; + } + let store = Store::parse(&bytes)?; + let toc = onestore::create_table_of_contents( + "Open Notebook.onetoc2", + &[("synthetic.one", store.header.file_id)], + )?; + fs::write(output.join("synthetic.one"), &bytes)?; + fs::write(output.join("Open Notebook.onetoc2"), toc)?; + println!( + "{}", + serde_json::json!({"bytes":bytes.len(),"transactions":store.header.transaction_count}) + ); + Ok(()) +} diff --git a/crates/onestore/examples/power_loss.rs b/crates/onestore/examples/power_loss.rs index caf3b31fbf90488b00b3a41d1d90502f0dbc633e..274b7fcf3d7a72df55b8c401483c1f111cea2357 100644 --- a/crates/onestore/examples/power_loss.rs +++ b/crates/onestore/examples/power_loss.rs @@ -1,74 +1,27 @@ +#[path = "../tests/support/current.rs"] +mod current; +use current::current; + +#[path = "../tests/support/trace.rs"] +mod trace; +use trace::{Event, Trace}; + #[path = "../tests/support/checkpoint.rs"] mod checkpoint; use onestore::{ - CommitIo, ExGuid, RevisionIndex, Store, + ExGuid, RevisionIndex, Store, document::{Document, Kind}, }; -use std::{collections::BTreeMap, fs, io, path::PathBuf}; - -enum Event { - Write(usize, Vec), - Flush, -} - -struct Trace { - bytes: Vec, - events: Vec, -} - -impl CommitIo for Trace { - fn read_at(&mut self, offset: u64, output: &mut [u8]) -> io::Result { - let offset = offset as usize; - let count = output.len().min(self.bytes.len().saturating_sub(offset)); - output[..count].copy_from_slice(&self.bytes[offset..offset + count]); - Ok(count) - } - fn write_at(&mut self, offset: u64, bytes: &[u8]) -> io::Result { - let offset = offset as usize; - let count = bytes.len().min(4096); - self.bytes.resize(self.bytes.len().max(offset + count), 0); - self.bytes[offset..offset + count].copy_from_slice(&bytes[..count]); - self.events - .push(Event::Write(offset, bytes[..count].to_vec())); - Ok(count) - } - fn flush(&mut self) -> io::Result<()> { - self.events.push(Event::Flush); - Ok(()) - } -} - -fn current(bytes: &[u8]) -> BTreeMap { - let store = Store::parse(bytes).unwrap(); - assert!(store.checksum_mismatches.is_empty()); - let index = RevisionIndex::parse(&store).unwrap(); - index.validate_current().unwrap(); - Document::parse(&index).unwrap(); - index - .spaces - .iter() - .map(|(sid, space)| { - let rid = space.labels[&(ExGuid::default(), 1)]; - let revision = index.resolve(*sid, rid).unwrap(); - for object in revision.objects.values() { - if let Some(onestore::FileDataReference::Internal(guid)) = - object.file_reference().unwrap() - { - store.file_data(guid).unwrap(); - } - } - (*sid, format!("{revision:?}")) - }) - .collect() -} +use std::{collections::BTreeMap, fs, path::PathBuf}; fn main() -> Result<(), Box> { - let destination = PathBuf::from( - std::env::args_os() - .nth(1) - .ok_or("Provide a new evidence directory")?, - ); + let mut args = std::env::args_os().skip(1); + let destination = PathBuf::from(args.next().ok_or("Provide a new evidence directory")?); + let option = args.next(); + if option.as_deref().is_some_and(|flag| flag != "--toc") || args.next().is_some() { + return Err("Usage: power_loss NEW_DIRECTORY [--toc]".into()); + } fs::create_dir(&destination)?; let fixtures = [ ( @@ -107,7 +60,7 @@ fn main() -> Result<(), Box> { let mut records = Vec::new(); let mut saved = BTreeMap::new(); for (name, path) in fixtures { - if std::env::args().nth(2).as_deref() == Some("--toc") && !name.starts_with("toc") { + if option.is_some() && !name.starts_with("toc") { continue; } println!("Checking {name}"); diff --git a/crates/onestore/examples/support/concurrent.rs b/crates/onestore/examples/support/concurrent.rs new file mode 100644 index 0000000000000000000000000000000000000000..bb28c70c106927eb9b71ce0a7be2056a641be230 --- /dev/null +++ b/crates/onestore/examples/support/concurrent.rs @@ -0,0 +1,258 @@ +use onestore::{ + CommitState, ExGuid, RevisionIndex, Store, + document::{Document, Kind}, +}; +use serde_json::json; +use std::{ + fs, + io::{self, Write}, + path::Path, + thread, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +pub fn document_view(bytes: &[u8]) -> Result { + let store = Store::parse(bytes)?; + let index = RevisionIndex::parse(&store)?; + index.validate_current()?; + let document = Document::parse(&index)?; + let mut texts = serde_json::Map::new(); + for (sid, _) in document.pages()? { + let space = &document.spaces[&sid]; + let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; + for (id, node) in &revision.nodes { + if let Kind::RichText { text, .. } = &node.kind + && text.starts_with("Document w") + { + let runs=revision.text_runs(*id)?.into_iter().map(|run|json!({"text":run.text,"bold":run.format.bold.unwrap_or(false),"size":run.format.font_size,"color":run.format.color.unwrap_or(0xff000000)})).collect::>(); + texts.insert(id.to_string(), json!({"text":text,"runs":runs})); + } + } + } + Ok(texts.into()) +} + +pub fn run( + args: &[String], + mut read: impl FnMut(&str) -> io::Result>, + mut commit: impl FnMut( + &str, + &[u8], + ExGuid, + ExGuid, + std::ops::Range, + &str, + ) -> Result<(), onestore::CommitError>, +) -> Result<(), Box> { + if args.len() != 7 || !["read", "write", "edit"].contains(&args[0].as_str()) { + return Err( + "Expected read|write|edit FILE ACTOR OPERATIONS START_FILE STOP_FILE SEED.".into(), + ); + } + let mut random: u64 = args[6].parse()?; + let operations: usize = args[3].parse()?; + if operations == 0 { + return Err("Choose at least one operation.".into()); + } + let timeout = match std::env::var("ONESTORE_CLIENT_TIMEOUT_MS") { + Ok(value) => value.parse::()?, + Err(std::env::VarError::NotPresent) => 600_000, + Err(error) => return Err(error.into()), + }; + if timeout == 0 { + return Err("Choose a positive client timeout.".into()); + } + let deadline = Instant::now() + .checked_add(Duration::from_millis(timeout)) + .ok_or("Client timeout exceeds the clock range.")?; + let mut output = io::stdout().lock(); + let mut log = |event: serde_json::Value| -> io::Result<()> { + writeln!(output, "{event}")?; + output.flush() + }; + log(json!({"event": "ready", "pid": std::process::id(), "actor": args[2]}))?; + while !Path::new(&args[4]).exists() { + if Instant::now() > deadline { + return Err("Start barrier timed out.".into()); + } + thread::sleep(Duration::from_millis(5)); + } + let documents = std::env::var_os("ONESTORE_OFFLINE_DOCUMENTS").is_some(); + let maintenance = std::env::var_os("ONESTORE_MAINTENANCE_DIR").map(std::path::PathBuf::from); + let mut completed = 0; + let mut attempts = 0; + while completed < operations || (args[0] == "read" && !Path::new(&args[5]).exists()) { + if Instant::now() > deadline { + return Err("Concurrent client timed out.".into()); + } + if let Some(control) = &maintenance + && !control.join("resume").exists() + && ((args[0] != "read" && completed == operations / 2) + || (args[0] == "read" && control.join("pause").exists())) + { + fs::write(control.join(format!("paused-{}", args[2])), b"paused")?; + log(json!({"event": "paused", "completed": completed}))?; + while !control.join("resume").exists() { + if Instant::now() > deadline { + return Err("Maintenance pause timed out.".into()); + } + thread::sleep(Duration::from_millis(10)); + } + log(json!({"event": "resumed", "completed": completed}))?; + } + attempts += 1; + random = random + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + thread::sleep(Duration::from_millis((random >> 32) % 7)); + let started = SystemTime::now().duration_since(UNIX_EPOCH)?.as_micros(); + let source = match read(&args[1]) { + Ok(source) => source, + Err(error) + if [ + io::ErrorKind::WouldBlock, + io::ErrorKind::ResourceBusy, + io::ErrorKind::PermissionDenied, + io::ErrorKind::NotFound, + ] + .contains(&error.kind()) => + { + log( + json!({"event": "read_busy", "attempt": attempts, "kind": format!("{:?}", error.kind())}), + )?; + thread::sleep(Duration::from_millis(100)); + continue; + } + Err(error) => { + log( + json!({"event": "read_error", "attempt": attempts, "kind": format!("{:?}", error.kind())}), + )?; + return Err(error.into()); + } + }; + let preserve = |error: onestore::Error| { + let path = Path::new(&args[4]) + .parent() + .unwrap() + .join(format!("invalid-{}-{attempts}.one", std::process::id())); + if let Err(failure) = fs::write(&path, &source) { + eprintln!( + "Could not save invalid snapshot {}: {failure}", + path.display() + ); + } + error + }; + let store = Store::parse(&source).map_err(preserve)?; + if !store.checksum_mismatches.is_empty() { + return Err(preserve(onestore::Error { + offset: store.checksum_mismatches[0], + message: "A reader observed transaction checksum damage.", + }) + .into()); + } + let index = RevisionIndex::parse(&store).map_err(preserve)?; + index.validate_current().map_err(preserve)?; + let document = Document::parse(&index).map_err(preserve)?; + let mut targets = Vec::new(); + for (sid, page) in document.pages().map_err(preserve)? { + let space = &document.spaces[&sid]; + let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; + let mut pending = vec![page]; + let mut seen = std::collections::BTreeSet::new(); + while let Some(oid) = pending.pop() { + if !seen.insert(oid) { + continue; + } + let node = &revision.nodes[&oid]; + pending.extend( + node.children + .iter() + .chain(&node.content) + .chain(&node.structure) + .copied(), + ); + if let Kind::RichText { text, .. } = &node.kind + && text.starts_with("Concurrent edits:") + { + revision.text_runs(oid).map_err(preserve)?; + targets.push((sid, oid, text)); + } + } + } + let [(sid, oid, text)] = targets.as_slice() else { + return Err(preserve(onestore::Error { + offset: 0, + message: "Expected one concurrent-edit paragraph.", + }) + .into()); + }; + let read_finished = SystemTime::now().duration_since(UNIX_EPOCH)?.as_micros(); + log( + json!({"event": "read", "attempt": attempts, "started_us": started, "finished_us": read_finished, + "transaction": store.header.transaction_count, "text": text, "documents":if documents {Some(document_view(&source).map_err(preserve)?)}else{None}}), + )?; + if args[0] == "read" { + completed += 1; + continue; + } + let token = format!(" [{}:{}]", args[2], completed); + let offset = u32::try_from(text.encode_utf16().count())?; + let mut range = offset..offset; + let mut replacement = token.clone(); + if args[0] == "edit" { + let prefix = "Concurrent edits:"; + let mut boundaries = vec![u32::try_from(prefix.encode_utf16().count())?]; + for character in text[prefix.len()..].chars() { + boundaries.push(boundaries.last().unwrap() + character.len_utf16() as u32); + } + let first = ((random >> 16) % boundaries.len() as u64) as usize; + let second = ((random >> 40) % boundaries.len() as u64) as usize; + range = boundaries[first.min(second)]..boundaries[first.max(second)]; + replacement = format!(" café 🦀{token}"); + } + log( + json!({"event": "intent", "attempt": attempts, "operation": completed, + "source_transaction": store.header.transaction_count, "before": text, + "range": [range.start, range.end], "replacement": replacement, "token": token}), + )?; + thread::sleep(Duration::from_millis((random >> 48) % 13)); + let commit_started = SystemTime::now().duration_since(UNIX_EPOCH)?.as_micros(); + let result = commit(&args[1], &source, *sid, *oid, range, &replacement); + let finished = SystemTime::now().duration_since(UNIX_EPOCH)?.as_micros(); + match result { + Ok(()) => { + log( + json!({"event": "commit", "attempt": attempts, "operation": completed, "token": token, + "started_us": commit_started, "finished_us": finished, "source_transaction": store.header.transaction_count}), + )?; + completed += 1; + } + Err(error) + if error.state == CommitState::NotCommitted + && [ + io::ErrorKind::WouldBlock, + io::ErrorKind::ResourceBusy, + io::ErrorKind::PermissionDenied, + io::ErrorKind::NotFound, + ] + .contains(&error.error.kind()) => + { + log( + json!({"event": "retry", "attempt": attempts, "started_us": commit_started, + "finished_us": finished, "kind": format!("{:?}", error.error.kind())}), + )?; + } + Err(error) => { + log( + json!({"event": "commit_error", "attempt": attempts, "operation": completed, + "token": token, "state": format!("{:?}", error.state), "kind": format!("{:?}", error.error.kind()), + "started_us": commit_started, "finished_us": finished}), + )?; + return Err(error.into()); + } + } + } + log(json!({"event": "done", "completed": completed, "attempts": attempts}))?; + Ok(()) +} diff --git a/crates/onestore/src/commit.rs b/crates/onestore/src/commit.rs index ebda8b6cdd38fc09da58e012253ba3a71d44a1d9..64fc73dcdc70bb751e278d18c87a5a6ed8ac5985 100644 --- a/crates/onestore/src/commit.rs +++ b/crates/onestore/src/commit.rs @@ -153,14 +153,113 @@ pub fn commit_text( range: std::ops::Range, replacement: &str, ) -> Result<(), CommitError> { - let written = - crate::replace_text(source, space, object, range, replacement).map_err(|error| { - CommitError { - state: CommitState::NotCommitted, - error: io::Error::new(ErrorKind::InvalidData, error), - } + PreparedEdit::text(source, space, object, range, replacement) + .map_err(|error| CommitError { + state: CommitState::NotCommitted, + error: io::Error::new(ErrorKind::InvalidData, error), + })? + .commit(io) +} + +/// An immutable writer-generated transition tied to its original snapshot. +/// Persist intended revision identities from `as_bytes` before publishing an offline edit. +/// Missing identities after native maintenance do not prove an edit was never published. +pub struct PreparedEdit<'a> { + source: &'a [u8], + written: Vec, +} + +impl<'a> PreparedEdit<'a> { + /// Prepares an insertion and its dependent metadata in one revision, without I/O. + pub fn insert( + source: &'a [u8], + space: ExGuid, + insertion: &crate::Insertion, + ) -> Result { + Ok(Self { + source, + written: insertion.apply(source, space)?, + }) + } + + /// Validates and prepares a text edit without I/O, with `replace_text` semantics. + pub fn text( + source: &'a [u8], + space: ExGuid, + object: ExGuid, + range: std::ops::Range, + replacement: &str, + ) -> Result { + Ok(Self { + source, + written: crate::replace_text(source, space, object, range, replacement)?, + }) + } + + /// Changes character formatting over a UTF-16 range, preserving unselected runs and styles. + /// A zero-length range sets the insertion style only when the paragraph is empty. + /// Fields, associated run objects and boundaries splitting preserved run data are rejected. + pub fn format( + source: &'a [u8], + space: ExGuid, + object: ExGuid, + range: std::ops::Range, + attributes: &[crate::TextAttribute], + ) -> Result { + Ok(Self { + source, + written: crate::formatting::format_text(source, space, object, range, attributes)?, + }) + } + + /// The exact complete image this edit will publish; identities do not regenerate on commit. + /// Do not overwrite a live notebook with this image; use `commit` under exclusion. + pub fn as_bytes(&self) -> &[u8] { + &self.written + } + + /// Publishes these prepared bytes after comparing the entire original snapshot. + /// The caller must retain OneNote-compatible exclusion through the returned outcome. + pub fn commit(&self, io: &mut impl CommitIo) -> Result<(), CommitError> { + commit_bytes(io, self.source, &self.written) + } + + /// Publishes these exact bytes through the conservative whole-file filesystem adapter. + /// A changed source returns ResourceBusy; an uncertain outcome must be reconciled before replay. + #[cfg(any(unix, windows))] + pub fn commit_file(&self, path: impl AsRef) -> Result<(), CommitError> { + let mut io = FileIo::open(path, true).map_err(|error| CommitError { + state: CommitState::NotCommitted, + error, })?; - commit_bytes(io, source, &written) + let result = self.commit(&mut io); + io.finish(result) + } +} + +/// Compares and flushes a snapshot, then refreshes its header version metadata. +/// No revision is added; reread before using the snapshot for another physical commit. +/// The caller must hold OneNote-compatible exclusion and independently establish which +/// intents the snapshot contains. A successful read alone is not a durable acknowledgement. +pub fn confirm_snapshot(io: &mut impl CommitIo, source: &[u8]) -> Result<(), CommitError> { + let mut state = CommitState::NotCommitted; + let result = (|| -> io::Result<()> { + let header = crate::Header::parse(source).map_err(io::Error::other)?; + let generation = header + .generation + .checked_add(1) + .ok_or(ErrorKind::InvalidData)?; + let mut version = [0; 40]; + version[..16].copy_from_slice(&crate::write::fresh_guid().map_err(io::Error::other)?); + version[16..24].copy_from_slice(&generation.to_le_bytes()); + version[24..].copy_from_slice(&crate::write::fresh_guid().map_err(io::Error::other)?); + compare_snapshot(io, source)?; + state = CommitState::Unknown; + io.flush()?; + write_all(io, 212, &version)?; + io.flush() + })(); + result.map_err(|error| CommitError { state, error }) } /// The caller must hold OneNote-compatible exclusion for the entire operation. @@ -221,7 +320,7 @@ fn write_all(io: &mut impl CommitIo, mut offset: usize, mut bytes: &[u8]) -> io: } /// Publishes a scalar revision after checking the locked file against its snapshot. -/// Unknown outcomes require rereading; Committed errors affect counter cleanup or lock release. +/// Unknown outcomes require rereading; Committed errors affect lock release. pub fn commit_property_bytes( io: &mut impl CommitIo, source: &[u8], @@ -240,38 +339,48 @@ pub fn commit_property_bytes( commit_bytes(io, source, &written) } -pub(crate) fn commit_bytes( - io: &mut impl CommitIo, - source: &[u8], - written: &[u8], -) -> Result<(), CommitError> { - let mut state = CommitState::NotCommitted; - let result = (|| -> io::Result<()> { - let mut buffer = [0; 65536]; - let mut offset = 0; - while offset < source.len() { - let size = buffer.len().min(source.len() - offset); - let count = match io.read_at(offset as u64, &mut buffer[..size]) { - Err(error) if error.kind() == ErrorKind::Interrupted => continue, - result => result?, - }; - if count == 0 { - return Err(io::Error::from(ErrorKind::UnexpectedEof)); - } - if count > size || buffer[..count] != source[offset..offset + count] { - return Err(io::Error::new( - ErrorKind::ResourceBusy, - "The locked file differs from the edit snapshot", - )); - } - offset += count; +fn compare_snapshot(io: &mut impl CommitIo, source: &[u8]) -> io::Result<()> { + let capacity = source.len().clamp(1, 1024 * 1024); + let mut buffer = Vec::new(); + buffer + .try_reserve_exact(capacity) + .map_err(io::Error::other)?; + buffer.resize(capacity, 0); + let mut offset = 0; + while offset < source.len() { + let size = buffer.len().min(source.len() - offset); + let count = match io.read_at(offset as u64, &mut buffer[..size]) { + Err(error) if error.kind() == ErrorKind::Interrupted => continue, + result => result?, + }; + if count == 0 { + return Err(io::Error::from(ErrorKind::UnexpectedEof)); } - if io.read_at(source.len() as u64, &mut buffer[..1])? != 0 { + if count > size || buffer[..count] != source[offset..offset + count] { return Err(io::Error::new( ErrorKind::ResourceBusy, - "The locked file grew after the edit snapshot", + "The locked file differs from the edit snapshot", )); } + offset += count; + } + if io.read_at(source.len() as u64, &mut buffer[..1])? != 0 { + return Err(io::Error::new( + ErrorKind::ResourceBusy, + "The locked file grew after the edit snapshot", + )); + } + Ok(()) +} + +pub(crate) fn commit_bytes( + io: &mut impl CommitIo, + source: &[u8], + written: &[u8], +) -> Result<(), CommitError> { + let mut state = CommitState::NotCommitted; + let result = (|| -> io::Result<()> { + compare_snapshot(io, source)?; if written == source { state = CommitState::Unknown; io.flush()?; @@ -291,17 +400,20 @@ pub(crate) fn commit_bytes( write_all(io, start, &written[start..offset])?; } io.flush()?; - write_all(io, 100, &written[100..1024])?; + write_all(io, 100, &written[100..212])?; + write_all(io, 252, &written[252..1024])?; io.flush()?; let highest = (96..100).rfind(|at| source[*at] != written[*at]).unwrap(); state = CommitState::Unknown; write_all(io, highest, &written[highest..highest + 1])?; io.flush()?; - state = CommitState::Committed; if highest > 96 { write_all(io, 96, &written[96..highest])?; io.flush()?; } + // Native readers cache the version GUID without rechecking the transaction count. + write_all(io, 212, &written[212..252])?; + io.flush()?; Ok(()) })(); result.map_err(|error| CommitError { state, error }) diff --git a/crates/onestore/src/create.rs b/crates/onestore/src/create.rs index e84bf7bbccfb4edfa167deca785d39577138cbf3..df06699aafcf8ed0767db5095557602542406a0c 100644 --- a/crates/onestore/src/create.rs +++ b/crates/onestore/src/create.rs @@ -8,7 +8,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; type Result = std::result::Result; -fn string(value: &str) -> Vec { +pub(crate) fn string(value: &str) -> Vec { value .encode_utf16() .chain([0]) @@ -16,7 +16,15 @@ fn string(value: &str) -> Vec { .collect() } -fn properties(values: &[(u32, Vec)]) -> Result> { +pub(crate) fn default_text_style() -> Vec<(u32, Vec)> { + vec![ + (0x14001c3b, 0x409_u32.to_le_bytes().to_vec()), + (0x1c001c0a, string("Calibri")), + (0x10001c0b, 22_u16.to_le_bytes().to_vec()), + ] +} + +pub(crate) fn properties(values: &[(u32, Vec)]) -> Result> { let mut streams: [Vec; 3] = std::array::from_fn(|_| Vec::new()); let mut fields = Vec::new(); for (id, value) in values { @@ -239,11 +247,7 @@ pub fn create_section(file_name: &str, text: &str, author: &str) -> Result Document<'a> { } impl<'a> Element<'a> { - fn parse(object: &Object<'a>, store: &Store<'a>) -> Result { + pub(crate) fn parse(object: &Object<'a>, store: &Store<'a>) -> Result { let empty = |kind| Self { jcid: object.jcid, children: vec![], diff --git a/crates/onestore/src/edit.rs b/crates/onestore/src/edit.rs index bd4f1b958f1694e1484ddc2ef29eb260becbbac6..a33f06a263e69dfd777f497a16ccf53e0a2821ad 100644 --- a/crates/onestore/src/edit.rs +++ b/crates/onestore/src/edit.rs @@ -58,38 +58,7 @@ pub fn replace_text( .into_iter() .filter_map(|(sid, page)| (sid == space).then_some(page)) .collect(); - let mut pending: Vec<_> = pages.iter().map(|page| (*page, false)).collect(); - let mut seen = std::collections::BTreeSet::new(); - let mut parents = std::collections::BTreeMap::<_, Vec<_>>::new(); - let mut editable = false; - while let Some((id, read_only)) = pending.pop() { - if !seen.insert((id, read_only)) { - continue; - } - let element = revision - .nodes - .get(&id) - .ok_or_else(|| invalid("Page content is unavailable"))?; - let read_only = read_only || element.extra[0].iter().any(|field| field.id == 0x88001cde); - if id == object { - if read_only { - return Err(invalid("This page or its content is read-only")); - } - editable = true; - } - for child in element - .children - .iter() - .chain(&element.content) - .chain(&element.structure) - { - parents.entry(*child).or_default().push(id); - pending.push((*child, read_only)); - } - } - if !editable { - return Err(invalid("Select text on an active editable page")); - } + let parents = editable_parents(revision, &pages, object)?; let node = revision .nodes .get(&object) @@ -219,7 +188,7 @@ pub fn replace_text( let mut edits = vec![crate::write::ObjectEdit { object, updates: &updates, - insert, + inserts: insert.as_slice(), }]; // Native conflict merges can discard descendant edits when ancestor timestamps stay stale. let modified_update = [(0x14001d7a, modified.as_slice())]; @@ -233,18 +202,143 @@ pub fn replace_text( edits.push(crate::write::ObjectEdit { object: id, updates: &modified_update, - insert: None, + inserts: &[], }); } pending.extend(parents.get(&id).into_iter().flatten().copied()); } + let Some((page, automatic, title_text)) = + page_title(revision, &pages, Some((object, &changed)))? + else { + return crate::write::replace_objects(source, space, &edits); + }; + let Kind::Page { + alternate_title, .. + } = &revision.nodes[&page].kind + else { + unreachable!() + }; + let metadata = revision + .roots + .get(&2) + .ok_or_else(|| invalid("Page title metadata is unavailable"))?; + let Kind::Metadata { title, .. } = &revision.nodes[metadata].kind else { + return Err(invalid("Page title metadata is unavailable")); + }; + let cached: Vec<_> = title_text + .encode_utf16() + .chain([0]) + .flat_map(u16::to_le_bytes) + .collect(); + let metadata_update = [(0x1c001cf3, cached.as_slice())]; + edits.push(crate::write::ObjectEdit { + object: *metadata, + updates: if title.is_some() { + &metadata_update + } else { + &[] + }, + inserts: if title.is_none() { + &metadata_update + } else { + &[] + }, + }); + let mut alternate_update = vec![( + 0x1c001d3c, + if automatic { + cached.as_slice() + } else { + &[0u8, 0][..] + }, + )]; + if revision.nodes[&page].modified.is_some() { + alternate_update.push(modified_update[0]); + } + edits.retain(|edit| edit.object != page); + edits.push(crate::write::ObjectEdit { + object: page, + updates: if alternate_title.is_some() { + &alternate_update + } else { + &alternate_update[1..] + }, + inserts: if alternate_title.is_none() { + &alternate_update[..1] + } else { + &[] + }, + }); + crate::write::replace_objects(source, space, &edits) +} + +pub(crate) fn editable_parents( + revision: &crate::document::Revision<'_>, + pages: &[ExGuid], + object: ExGuid, +) -> Result>, Error> { + let invalid = |message| Error { offset: 0, message }; + let mut pending: Vec<_> = pages.iter().map(|page| (*page, false)).collect(); + let mut seen = std::collections::BTreeSet::new(); + let mut parents = std::collections::BTreeMap::<_, Vec<_>>::new(); + let mut editable = false; + while let Some((id, read_only)) = pending.pop() { + if !seen.insert((id, read_only)) { + continue; + } + let element = revision + .nodes + .get(&id) + .ok_or_else(|| invalid("Page content is unavailable"))?; + let read_only = read_only || element.extra[0].iter().any(|field| field.id == 0x88001cde); + if id == object { + if read_only { + return Err(invalid("This page or its content is read-only")); + } + editable = true; + } + for child in element + .children + .iter() + .chain(&element.content) + .chain(&element.structure) + { + parents.entry(*child).or_default().push(id); + pending.push((*child, read_only)); + } + } + if !editable { + return Err(invalid("Select content on an active editable page")); + } + Ok(parents) +} + +pub(crate) fn page_title( + revision: &crate::document::Revision<'_>, + pages: &[ExGuid], + text_update: Option<(ExGuid, &str)>, +) -> Result, Error> { + let invalid = |message| Error { offset: 0, message }; + let mut pending = pages.to_vec(); + let mut seen = std::collections::BTreeSet::new(); + while let Some(id) = pending.pop() { + if !seen.insert(id) { + continue; + } + let node = &revision.nodes[&id]; + pending.extend( + node.children + .iter() + .chain(&node.content) + .chain(&node.structure) + .copied(), + ); + } let titles: Vec<_> = revision .nodes .iter() .filter_map(|(id, node)| { - if !(seen.contains(&(*id, false)) || seen.contains(&(*id, true))) - || !node.extra[0].iter().any(|field| field.id == 0x88001cb4) - { + if !seen.contains(id) || !node.extra[0].iter().any(|field| field.id == 0x88001cb4) { return None; } match &node.kind { @@ -260,8 +354,8 @@ pub fn replace_text( let title_text = match titles.as_slice() { [] => "", [(id, text)] => { - if *id == object { - changed.as_str() + if let Some((_, changed)) = text_update.filter(|(object, _)| object == id) { + changed } else { text } @@ -269,18 +363,13 @@ pub fn replace_text( _ => return Err(invalid("Title editing requires a single title text object")), }; let automatic = title_line(title_text).is_empty(); - if !automatic && titles[0].0 != object { - return crate::write::replace_objects(source, space, &edits); + if !automatic && text_update.is_none_or(|(id, _)| titles[0].0 != id) { + return Ok(None); } - let [page] = pages.as_slice() else { + let [page] = pages else { return Err(invalid("Title editing requires a single active page")); }; - let Kind::Page { - alternate_title, - rtl, - .. - } = &revision.nodes[page].kind - else { + let Kind::Page { rtl, .. } = &revision.nodes[page].kind else { unreachable!() }; let mut title_text = title_line(title_text); @@ -312,7 +401,13 @@ pub fn replace_text( .. } = &node.kind { - title_text = automatic_title(if id == object { &changed } else { text }); + title_text = automatic_title( + if let Some((_, changed)) = text_update.filter(|(object, _)| *object == id) { + changed + } else { + text + }, + ); if !title_text.is_empty() { break; } @@ -326,48 +421,5 @@ pub fn replace_text( pending.extend(node.structure.iter().rev().copied()); } } - let metadata = revision - .roots - .get(&2) - .ok_or_else(|| invalid("Page title metadata is unavailable"))?; - let Kind::Metadata { title, .. } = &revision.nodes[metadata].kind else { - return Err(invalid("Page title metadata is unavailable")); - }; - let cached: Vec<_> = title_text - .encode_utf16() - .chain([0]) - .flat_map(u16::to_le_bytes) - .collect(); - let metadata_update = [(0x1c001cf3, cached.as_slice())]; - edits.push(crate::write::ObjectEdit { - object: *metadata, - updates: if title.is_some() { - &metadata_update - } else { - &[] - }, - insert: title.is_none().then_some(metadata_update[0]), - }); - let mut alternate_update = vec![( - 0x1c001d3c, - if automatic { - cached.as_slice() - } else { - &[0u8, 0][..] - }, - )]; - if revision.nodes[page].modified.is_some() { - alternate_update.push(modified_update[0]); - } - edits.retain(|edit| edit.object != *page); - edits.push(crate::write::ObjectEdit { - object: *page, - updates: if alternate_title.is_some() { - &alternate_update - } else { - &alternate_update[1..] - }, - insert: alternate_title.is_none().then_some(alternate_update[0]), - }); - crate::write::replace_objects(source, space, &edits) + Ok(Some((*page, automatic, title_text.to_owned()))) } diff --git a/crates/onestore/src/formatting.rs b/crates/onestore/src/formatting.rs new file mode 100644 index 0000000000000000000000000000000000000000..5d40d1eff4a55274dd75a8889253507376d898d8 --- /dev/null +++ b/crates/onestore/src/formatting.rs @@ -0,0 +1,288 @@ +use crate::{ + Error, ExGuid, PropertySets, RevisionIndex, Store, + create::{current_timestamps, properties, string}, + document::{Document, Kind}, + edit::editable_parents, + write::{PropertyObject, fresh_guid, write_revision}, +}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{BTreeMap, BTreeSet}, + ops::Range, + sync::Arc, +}; + +fn invalid(message: &'static str) -> Error { + Error { offset: 0, message } +} + +/// An explicit character-format change; omitted attributes retain their current values. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum TextAttribute { + Bold(bool), + Italic(bool), + Underline(bool), + Strike(bool), + /// Enabling superscript clears subscript. + Superscript(bool), + /// Enabling subscript clears superscript. + Subscript(bool), + Font(String), + /// Points, from 6 through 130 in half-point increments. + /// OneNote 2010 clamps larger sizes despite the specification allowing 144. + FontSize(f32), + /// RGB, or None for automatic text color. + Color(Option<[u8; 3]>), + /// RGB, or None to clear highlighting. + Highlight(Option<[u8; 3]>), +} + +impl TextAttribute { + fn property(&self) -> Result<(u32, Vec), Error> { + let boolean = |id, value: bool| (id | (u32::from(value) << 31), Vec::new()); + Ok(match self { + Self::Bold(value) => boolean(0x08001c04, *value), + Self::Italic(value) => boolean(0x08001c05, *value), + Self::Underline(value) => boolean(0x08001c06, *value), + Self::Strike(value) => boolean(0x08001c07, *value), + Self::Superscript(value) => boolean(0x08001c08, *value), + Self::Subscript(value) => boolean(0x08001c09, *value), + Self::Font(font) => { + if font.is_empty() || font.contains('\0') { + return Err(invalid("Font names must be nonempty and contain no NUL")); + } + (0x1c001c0a, string(font)) + } + Self::FontSize(points) => { + if !points.is_finite() + || !(6.0..=130.0).contains(points) + || (points * 2.0).fract() != 0.0 + { + return Err(invalid( + "Font size must be 6 to 130 points in half-point increments", + )); + } + (0x10001c0b, ((*points * 2.0) as u16).to_le_bytes().to_vec()) + } + Self::Color(color) | Self::Highlight(color) => ( + if matches!(self, Self::Color(_)) { + 0x14001c0c + } else { + 0x14001c0d + }, + color + .map_or(0xff000000, |[r, g, b]| u32::from_le_bytes([r, g, b, 0])) + .to_le_bytes() + .to_vec(), + ), + }) + } +} + +pub(crate) fn format_text( + source: &[u8], + space: ExGuid, + object: ExGuid, + range: Range, + attributes: &[TextAttribute], +) -> Result, Error> { + if attributes.is_empty() || range.start > range.end { + return Err(invalid( + "Select a text range and at least one formatting attribute", + )); + } + let mut values = Vec::new(); + let mut seen = BTreeSet::new(); + for attribute in attributes { + let (id, value) = attribute.property()?; + if !seen.insert(id & 0x7fffffff) { + return Err(invalid("Specify each formatting attribute once")); + } + values.push((id, value)); + } + if attributes.contains(&TextAttribute::Superscript(true)) + && attributes.contains(&TextAttribute::Subscript(true)) + { + return Err(invalid("Text cannot be both superscript and subscript")); + } + // Setting either script position clears its mutually exclusive counterpart. + for (set, opposite) in [(0x88001c08, 0x08001c09), (0x88001c09, 0x08001c08)] { + if values.iter().any(|(id, _)| *id == set) && !seen.contains(&opposite) { + values.push((opposite, Vec::new())); + } + } + let store = Store::parse(source)?; + let index = RevisionIndex::parse(&store)?; + index.validate_current()?; + let document = Document::parse(&index)?; + let semantic = document + .spaces + .get(&space) + .ok_or_else(|| invalid("The active page is unavailable"))?; + let rid = semantic.contexts[&ExGuid::default()]; + let view = &semantic.revisions[&rid]; + let pages: Vec<_> = document + .pages()? + .into_iter() + .filter_map(|(sid, page)| (sid == space).then_some(page)) + .collect(); + let parents = editable_parents(view, &pages, object)?; + let node = &view.nodes[&object]; + let Kind::RichText { + text, + runs, + boilerplate, + .. + } = &node.kind + else { + return Err(invalid("Select a rich-text object")); + }; + if *boilerplate { + return Err(invalid( + "Generated title fields cannot be formatted as ordinary text", + )); + } + let (mut valid_start, mut valid_end) = (range.start == 0, range.end == 0); + let mut offset = 0; + for character in text.chars() { + offset += character.len_utf16() as u32; + valid_start |= offset == range.start; + valid_end |= offset == range.end; + } + if !valid_start || !valid_end { + return Err(invalid( + "The format range splits a surrogate pair or exceeds the text", + )); + } + if range.is_empty() && !text.is_empty() { + return Err(invalid( + "Select characters, or an empty paragraph's insertion style", + )); + } + let resolved = view.text_runs(object)?; + let mut segments = Vec::new(); + for (i, run) in runs.iter().enumerate() { + let selected = if text.is_empty() { + true + } else { + run.start < range.end && range.start < run.end + }; + if selected { + let format = &resolved[i].format; + if [ + format.hidden, + format.hyperlink, + format.math, + format.embedded_object, + ] + .contains(&Some(true)) + || resolved[i].text.contains(['\u{fffc}', '\u{fddf}']) + { + return Err(invalid( + "This format range contains a field or embedded data", + )); + } + if run.start < range.start { + segments.push((i, range.start, false)); + } + segments.push((i, run.end.min(range.end), true)); + if range.end < run.end { + segments.push((i, run.end, false)); + } + } else { + segments.push((i, run.end, false)); + } + } + let modified = current_timestamps()?.0.to_le_bytes(); + write_revision(source, space, |raw| { + let mut target = PropertyObject::from_object(&raw.objects[&object])?; + let fields = PropertySets::parse(&target.bytes)?; + if fields.sets[0].iter().any(|p| p.id == 0x24003458) { + return Err(invalid("This text object contains associated run objects")); + } + if segments.len() != runs.len() && fields.sets[0].iter().any(|p| p.id == 0x40003499) { + return Err(invalid( + "Formatting boundaries cannot split preserved run data", + )); + } + let mut styles = BTreeMap::new(); + let mut changed = BTreeMap::new(); + let mut references = Vec::new(); + let mut ends = Vec::new(); + let mut updated = false; + let changes: Vec<_> = values + .iter() + .map(|(id, value)| (*id, value.as_slice())) + .collect(); + for &(i, end, selected) in &segments { + let previous = runs[i].format; + let key = (previous, selected); + let id = if let Some(id) = styles.get(&key) { + *id + } else if let Some(id) = previous.filter(|_| !selected) { + id + } else { + let mut style = match previous { + Some(id) => PropertyObject::from_object(&raw.objects[&id])?, + None => PropertyObject { + jcid: 0x12004d, + bytes: properties(&[])?, + global_ids: Arc::new(BTreeMap::new()), + }, + }; + if selected { + style.set(&changes)?; + } + if let Some(id) = previous.filter(|id| { + raw.objects[id].data == crate::ObjectData::Properties(&style.bytes) + }) { + styles.insert(key, id); + id + } else { + if !PropertySets::parse(&style.bytes)?.sets[0] + .iter() + .any(|p| p.id == 0x14001c3b) + { + style.set(&[( + 0x14001c3b, + &resolved[i].format.language.unwrap_or(0x409).to_le_bytes(), + )])?; + } + let id = ExGuid { + guid: fresh_guid()?, + n: 1, + }; + style.reference(id)?; + changed.insert(id, style); + styles.insert(key, id); + id + } + }; + updated |= selected && previous != Some(id); + references.extend_from_slice(&target.reference(id)?); + ends.extend_from_slice(&end.to_le_bytes()); + } + if !updated { + return Ok(BTreeMap::new()); + } + ends.truncate(ends.len() - 4); + target.set(&[ + (0x24001e13, &references), + (0x1c001e12, &ends), + (0x14001d7a, &modified), + ])?; + changed.insert(object, target); + let mut pending = parents.get(&object).cloned().unwrap_or_default(); + let mut ancestors = BTreeSet::new(); + while let Some(id) = pending.pop() { + if !ancestors.insert(id) { + continue; + } + let mut ancestor = PropertyObject::from_object(&raw.objects[&id])?; + ancestor.set(&[(0x14001d7a, &modified)])?; + changed.insert(id, ancestor); + pending.extend(parents.get(&id).into_iter().flatten().copied()); + } + Ok(changed) + }) +} diff --git a/crates/onestore/src/insertion.rs b/crates/onestore/src/insertion.rs new file mode 100644 index 0000000000000000000000000000000000000000..a6589165b4ecf1acadb90afd88f54c1bcd23c3e5 --- /dev/null +++ b/crates/onestore/src/insertion.rs @@ -0,0 +1,343 @@ +use crate::{ + Error, ExGuid, Object, ObjectData, RevisionIndex, Store, + create::{current_timestamps, default_text_style, properties, string}, + document::{Document, Element, Kind}, + edit::{editable_parents, page_title}, + write::{PropertyObject, fresh_guid, write_revision}, +}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::Arc, +}; + +fn invalid(message: &'static str) -> Error { + Error { offset: 0, message } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +enum Placement { + Paragraph { before: Option }, + Outline { x: f32, y: f32 }, +} + +/// A paragraph or outline insertion with stable object identities and creation time. +/// Retain this intent across rebases; constructing another intent allocates different identities. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Insertion { + guid: [u8; 16], + parent: ExGuid, + placement: Placement, + text: String, + author: String, + created: u32, +} + +impl Insertion { + /// Inserts before a direct child, or appends when `before` is None. + /// The parent must be an editable outline, paragraph, outline group or table cell. + /// Carriage returns represent soft line breaks; line feeds and embedded-field markers are rejected. + pub fn paragraph( + parent: ExGuid, + before: Option, + text: &str, + author: &str, + ) -> Result { + Self::new(parent, Placement::Paragraph { before }, text, author) + } + + /// Adds an outline to an editable page at coordinates measured in points. + pub fn outline(page: ExGuid, x: f32, y: f32, text: &str, author: &str) -> Result { + Self::new(page, Placement::Outline { x, y }, text, author) + } + + /// Changes a paragraph intent's placement while retaining its identities, text and author. + pub fn reposition_paragraph( + &self, + parent: ExGuid, + before: Option, + ) -> Result { + if !matches!(self.placement, Placement::Paragraph { .. }) { + return Err(invalid("An outline intent cannot become a paragraph")); + } + let mut intent = self.clone(); + intent.parent = parent; + intent.placement = Placement::Paragraph { before }; + intent.validate()?; + Ok(intent) + } + + /// Changes an outline intent's placement while retaining its identities, text and author. + pub fn reposition_outline(&self, page: ExGuid, x: f32, y: f32) -> Result { + if !matches!(self.placement, Placement::Outline { .. }) { + return Err(invalid("A paragraph intent cannot become an outline")); + } + let mut intent = self.clone(); + intent.parent = page; + intent.placement = Placement::Outline { x, y }; + intent.validate()?; + Ok(intent) + } + + fn new(parent: ExGuid, placement: Placement, text: &str, author: &str) -> Result { + let intent = Self { + guid: fresh_guid()?, + parent, + placement, + text: text.to_owned(), + author: author.to_owned(), + created: current_timestamps()?.0, + }; + intent.validate()?; + Ok(intent) + } + + /// Identity of the new paragraph, or the new outline for an outline insertion. + pub fn object(&self) -> ExGuid { + ExGuid { + guid: self.guid, + n: 1, + } + } + + /// Identity of the insertion's ordinary rich-text object. + pub fn text_object(&self) -> ExGuid { + ExGuid { + guid: self.guid, + n: 2, + } + } + + fn validate(&self) -> Result<(), Error> { + if self.guid == [0; 16] + || self.parent.guid == [0; 16] + || self.text.contains(['\0', '\n', '\u{fffc}', '\u{fddf}']) + || self.author.contains('\0') + { + return Err(invalid( + "Use an editable parent, ordinary paragraph text and a valid author", + )); + } + if let Placement::Outline { x, y } = self.placement + && (!x.is_finite() || !y.is_finite()) + { + return Err(invalid("Outline coordinates must be finite")); + } + Ok(()) + } + + pub(crate) fn apply(&self, source: &[u8], space: ExGuid) -> Result, Error> { + self.validate()?; + let store = Store::parse(source)?; + let index = RevisionIndex::parse(&store)?; + index.validate_current()?; + let mut document = Document::parse(&index)?; + let pages: Vec<_> = document + .pages()? + .into_iter() + .filter_map(|(sid, id)| (sid == space).then_some(id)) + .collect(); + let [page] = pages.as_slice() else { + return Err(invalid("Insertion requires a single active page")); + }; + let semantic_space = document + .spaces + .remove(&space) + .ok_or_else(|| invalid("The active page is unavailable"))?; + let rid = semantic_space.contexts[&ExGuid::default()]; + let mut view = semantic_space + .revisions + .into_iter() + .find_map(|(id, revision)| (id == rid).then_some(revision)) + .unwrap(); + let parents = editable_parents(&view, &pages, self.parent)?; + let parent = &view.nodes[&self.parent]; + let position = match self.placement { + Placement::Paragraph { before } => { + if !matches!( + parent.kind, + Kind::Outline { .. } + | Kind::Paragraph { .. } + | Kind::OutlineGroup + | Kind::Cell { .. } + ) { + return Err(invalid( + "Select an outline, paragraph, outline group or table cell", + )); + } + if let Some(id) = before { + parent + .children + .iter() + .position(|child| *child == id) + .ok_or_else(|| { + invalid("The insertion anchor is no longer a direct child") + })? + } else { + parent.children.len() + } + } + Placement::Outline { .. } => { + if self.parent != *page || !matches!(parent.kind, Kind::Page { .. }) { + return Err(invalid("Select an active page for the new outline")); + } + parent.children.len() + } + }; + let mut ancestors = BTreeSet::new(); + let mut pending = vec![self.parent]; + while let Some(id) = pending.pop() { + if !ancestors.insert(id) { + continue; + } + if matches!(view.nodes[&id].kind, Kind::Title) { + return Err(invalid( + "Title containers do not accept ordinary paragraphs", + )); + } + pending.extend(parents.get(&id).into_iter().flatten().copied()); + } + let modified = current_timestamps()?.0.to_le_bytes(); + let paragraph_n = if matches!(self.placement, Placement::Outline { .. }) { + 3 + } else { + 1 + }; + let table = Arc::new(BTreeMap::from([(0, self.guid)])); + let reference = |n: u32| n.to_le_bytes().to_vec(); + let mut new = BTreeMap::new(); + for (n, jcid, values) in [ + ( + paragraph_n, + 0x6000d, + vec![ + (0x14001d7a, modified.to_vec()), + (0x14001d09, self.created.to_le_bytes().to_vec()), + (0x0c001c03, vec![1]), + (0x24001c1f, reference(2)), + (0x20001d78, reference(4)), + (0x20001d79, reference(4)), + ], + ), + ( + 2, + 0x6000e, + vec![ + (0x14001d7a, modified.to_vec()), + (0x1c001c22, string(&self.text)), + (0x24001e13, reference(5)), + (0x10001cfe, 0x409_u16.to_le_bytes().to_vec()), + ], + ), + (4, 0x120001, vec![(0x1c001d75, string(&self.author))]), + (5, 0x12004d, default_text_style()), + ] { + new.insert( + ExGuid { guid: self.guid, n }, + PropertyObject { + jcid, + bytes: properties(&values)?, + global_ids: Arc::clone(&table), + }, + ); + } + if let Placement::Outline { x, y } = self.placement { + new.insert( + self.object(), + PropertyObject { + jcid: 0x6000c, + global_ids: table, + bytes: properties(&[ + (0x14001d7a, modified.to_vec()), + (0x24001c20, reference(3)), + (0x0c001c03, vec![1]), + (0x1c001c12, vec![1, 0, 0, 0, 0, 0, 0, 0]), + (0x14001c14, (x / 36.0).to_le_bytes().to_vec()), + (0x14001c15, (y / 36.0).to_le_bytes().to_vec()), + (0x14001c1b, 13_f32.to_le_bytes().to_vec()), + (0x14001c1c, 0.6_f32.to_le_bytes().to_vec()), + ])?, + }, + ); + } + let raw = index.resolve(space, rid)?; + if new.keys().any(|id| raw.objects.contains_key(id)) { + return Err(invalid( + "An insertion identity is already present; reconcile the existing edit", + )); + } + // Drop the semantic view before moving the property bytes it borrows. + let title = { + view.nodes + .get_mut(&self.parent) + .unwrap() + .children + .insert(position, self.object()); + for (id, object) in &new { + view.nodes.insert( + *id, + Element::parse( + &Object { + jcid: object.jcid, + reference_count: 0, + data: ObjectData::Properties(&object.bytes), + global_ids: Arc::clone(&object.global_ids), + }, + &store, + )?, + ); + } + let title = page_title(&view, &pages, None)?; + drop(view); + title + }; + write_revision(source, space, |raw| { + let mut changed = new; + for id in &ancestors { + let mut object = PropertyObject::from_object(&raw.objects[id])?; + object.set(&[(0x14001d7a, &modified)])?; + changed.insert(*id, object); + } + let parent = changed.get_mut(&self.parent).unwrap(); + let properties = crate::PropertySets::parse(&parent.bytes)?; + let existing = properties.sets[0].iter().find(|p| p.id == 0x24001c20); + let mut ids = match existing.map(|p| &p.value) { + Some(crate::Value::References { compact_ids, .. }) => compact_ids.to_vec(), + None => Vec::new(), + _ => return Err(invalid("The parent has an invalid child list")), + }; + let child = parent.reference(self.object())?; + if position > ids.len() / 4 { + return Err(invalid("The parent has an invalid child list")); + } + ids.splice(position * 4..position * 4, child); + parent.set(&[(0x24001c20, &ids)])?; + if matches!(self.placement, Placement::Paragraph { .. }) { + let properties = crate::PropertySets::parse(&parent.bytes)?; + if !properties.sets[0].iter().any(|p| p.id == 0x0c001c03) { + parent.set(&[(0x0c001c03, &[1])])?; + } + } + if let Some((page, automatic, title)) = title { + let metadata = raw + .roots + .get(&2) + .ok_or_else(|| invalid("Page title metadata is unavailable"))?; + if raw.objects[metadata].jcid != 0x20030 { + return Err(invalid("Page title metadata is unavailable")); + } + let title = string(&title); + let mut metadata_object = PropertyObject::from_object(&raw.objects[metadata])?; + metadata_object.set(&[(0x1c001cf3, &title)])?; + changed.insert(*metadata, metadata_object); + changed + .get_mut(&page) + .unwrap() + .set(&[(0x1c001d3c, if automatic { &title } else { &[0, 0] })])?; + } + Ok(changed) + }) + } +} diff --git a/crates/onestore/src/lib.rs b/crates/onestore/src/lib.rs index e9e597d571e21db69832710b02844879ddac7ba5..b2d3fba467a0702725a893080fbfda05d37c29cb 100644 --- a/crates/onestore/src/lib.rs +++ b/crates/onestore/src/lib.rs @@ -1,5 +1,5 @@ #![forbid(unsafe_code)] -#![doc = include_str!("../../../README.md")] +#![doc = include_str!("../README.md")] mod bytes; mod commit; @@ -8,20 +8,29 @@ pub mod document; mod edit; mod files; mod flush; +mod formatting; +mod insertion; mod objects; mod properties; mod revisions; +mod snapshot; mod store; mod write; -pub use commit::{CommitError, CommitIo, CommitState, commit_property_bytes, commit_text}; +pub use commit::{ + CommitError, CommitIo, CommitState, PreparedEdit, commit_property_bytes, commit_text, + confirm_snapshot, +}; #[cfg(any(unix, windows))] pub use commit::{commit_file_property, commit_file_text, read_file}; pub use create::{create_section, create_table_of_contents}; pub use edit::replace_text; pub use files::FileDataReference; +pub use formatting::TextAttribute; +pub use insertion::Insertion; pub use objects::{Object, ObjectData, ObjectReferences, ResolvedRevision}; pub use properties::{IdStream, Property, PropertySets, Value}; pub use revisions::{ExGuid, ObjectSpace, Revision, RevisionIndex}; +pub use snapshot::read_snapshot; pub use store::{Chunk, Error, FileType, Header, Node, NodeList, Reference, Store}; pub use write::replace_property_bytes; diff --git a/crates/onestore/src/objects.rs b/crates/onestore/src/objects.rs index b933520337b708ab344e12b9428acd49847504ae..3900af9f53ba01a38cb4f41907d90a43beb1514d 100644 --- a/crates/onestore/src/objects.rs +++ b/crates/onestore/src/objects.rs @@ -81,6 +81,19 @@ impl Object<'_> { impl ResolvedRevision<'_> { pub fn reachable(&self) -> Result> { + let incoming = self.reference_counts()?; + for (id, count) in &incoming { + if self.objects[id].reference_count != *count { + return Err(Error { + offset: 0, + message: "Stored object reference count disagrees with the reachable graph", + }); + } + } + Ok(incoming.into_keys().collect()) + } + + pub(crate) fn reference_counts(&self) -> Result> { let mut pending: Vec<_> = self.roots.values().copied().collect(); let mut incoming = BTreeMap::::new(); for id in &pending { @@ -137,15 +150,7 @@ impl ResolvedRevision<'_> { message: "Object references form a cycle", }); } - for (id, count) in &incoming { - if self.objects[id].reference_count != *count { - return Err(Error { - offset: 0, - message: "Stored object reference count disagrees with the reachable graph", - }); - } - } - Ok(edges.into_keys().collect()) + Ok(incoming) } } diff --git a/crates/onestore/src/revisions.rs b/crates/onestore/src/revisions.rs index c370a666599c0358b57e05d291853ca0d2db30db..1ac8274d5fcd37eb0ffcdc9d80886798b0f44b4e 100644 --- a/crates/onestore/src/revisions.rs +++ b/crates/onestore/src/revisions.rs @@ -5,10 +5,45 @@ type Result = std::result::Result; #[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct ExGuid { + /// GUID bytes in Microsoft's mixed-endian order. pub guid: [u8; 16], pub n: u32, } +impl std::str::FromStr for ExGuid { + type Err = Error; + + /// Parses the display form, accepting either hexadecimal letter case. + fn from_str(value: &str) -> Result { + let invalid = || Error { + offset: 0, + message: "Invalid extended GUID", + }; + let (guid_text, extension) = value.split_once(',').ok_or_else(invalid)?; + if guid_text.len() != 38 || !guid_text.is_ascii() || !(40..=49).contains(&value.len()) { + return Err(invalid()); + } + let mut guid = [0; 16]; + for (byte, at) in guid + .iter_mut() + .zip([1, 3, 5, 7, 10, 12, 15, 17, 20, 22, 25, 27, 29, 31, 33, 35]) + { + *byte = u8::from_str_radix(&guid_text[at..at + 2], 16).map_err(|_| invalid())?; + } + guid[..4].reverse(); + guid[4..6].reverse(); + guid[6..8].reverse(); + let id = Self { + guid, + n: extension.parse().map_err(|_| invalid())?, + }; + if (id.guid == [0; 16] && id.n != 0) || !id.to_string().eq_ignore_ascii_case(value) { + return Err(invalid()); + } + Ok(id) + } +} + impl fmt::Display for ExGuid { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let g = &self.guid; @@ -40,6 +75,15 @@ impl serde::Serialize for ExGuid { } } +impl<'de> serde::Deserialize<'de> for ExGuid { + fn deserialize>( + deserializer: D, + ) -> std::result::Result { + let value = ::deserialize(deserializer)?; + value.parse().map_err(serde::de::Error::custom) + } +} + impl Cursor<'_> { pub(crate) fn exguid(&mut self) -> Result { let id = ExGuid { diff --git a/crates/onestore/src/snapshot.rs b/crates/onestore/src/snapshot.rs new file mode 100644 index 0000000000000000000000000000000000000000..3e9c3266a53459822c14280242d4508d7f896ae3 --- /dev/null +++ b/crates/onestore/src/snapshot.rs @@ -0,0 +1,76 @@ +use crate::{RevisionIndex, Store}; +use std::io; + +fn read_exact( + read: &mut impl FnMut(u64, &mut [u8]) -> io::Result, + mut offset: u64, + mut output: &mut [u8], +) -> io::Result<()> { + while !output.is_empty() { + match read(offset, output) { + Ok(0) => return Err(io::ErrorKind::UnexpectedEof.into()), + Ok(count) if count <= output.len() => { + offset += count as u64; + output = &mut output[count..]; + } + Ok(_) => return Err(io::ErrorKind::InvalidData.into()), + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(error) => return Err(error), + } + } + Ok(()) +} + +/// Reads a bounded snapshot; the caller must provide fresh I/O and exclude in-place maintenance. +/// Includes unpublished trailing bytes so subsequent commits can validate the physical file. +pub fn read_snapshot( + mut read: impl FnMut(u64, &mut [u8]) -> io::Result, + limit: usize, +) -> io::Result>> { + let mut header = [0; 1024]; + read_exact(&mut read, 0, &mut header)?; + let length = u64::from_le_bytes(header[196..204].try_into().unwrap()); + let length = usize::try_from(length) + .ok() + .filter(|length| (1024..=limit).contains(length)) + .ok_or(io::ErrorKind::InvalidData)?; + let mut bytes = Vec::new(); + bytes.try_reserve_exact(length).map_err(io::Error::other)?; + bytes.extend_from_slice(&header); + bytes.resize(length, 0); + if let Err(error) = read_exact(&mut read, 1024, &mut bytes[1024..]) { + // Native writers can shorten unused storage before publishing the new expected length. + return if error.kind() == io::ErrorKind::UnexpectedEof { + Ok(None) + } else { + Err(error) + }; + } + let mut tail = [0; 65536]; + loop { + let size = (limit - bytes.len()).clamp(1, tail.len()); + match read(bytes.len() as u64, &mut tail[..size]) { + Ok(0) => break, + Ok(count) if count <= size && count <= limit - bytes.len() => { + bytes.try_reserve_exact(count).map_err(io::Error::other)?; + bytes.extend_from_slice(&tail[..count]); + } + Ok(_) => return Err(io::ErrorKind::InvalidData.into()), + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(error) => return Err(error), + } + } + let mut after = [0; 1024]; + read_exact(&mut read, 0, &mut after)?; + if header != after { + return Ok(None); + } + let parsed = Store::parse(&bytes).and_then(|store| { + if !store.checksum_mismatches.is_empty() { + return Ok(false); + } + RevisionIndex::parse(&store)?.validate_current()?; + Ok(true) + }); + Ok(matches!(parsed, Ok(true)).then_some(bytes)) +} diff --git a/crates/onestore/src/store.rs b/crates/onestore/src/store.rs index 2ab41dfa64f71dae59a571627ee15e2139d40800..cd8cf84b559d309c61a983d3d3b52f4ea723995a 100644 --- a/crates/onestore/src/store.rs +++ b/crates/onestore/src/store.rs @@ -82,7 +82,7 @@ pub struct Header { } impl Header { - fn parse(data: &[u8]) -> Result { + pub(crate) fn parse(data: &[u8]) -> Result { let mut c = Cursor { bytes: data, offset: 0, diff --git a/crates/onestore/src/write.rs b/crates/onestore/src/write.rs index b0b2a16045e43591e19725064e0bc4694f487b40..19d078326fd9404b759eb1b861369da66a22a77d 100644 --- a/crates/onestore/src/write.rs +++ b/crates/onestore/src/write.rs @@ -4,7 +4,13 @@ use crate::{ store::{crc, transaction_crc}, }; -use std::collections::{BTreeMap, BTreeSet}; +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::Arc, +}; + +#[cfg(test)] +mod tests; type Result = std::result::Result; @@ -98,109 +104,183 @@ fn compact(id: ExGuid, table: &BTreeMap) -> Result<[u8; 4]> { Ok(((index << 8) | id.n).to_le_bytes()) } +fn field_length(property: &crate::Property<'_>, set_lengths: &[usize]) -> usize { + match &property.value { + Value::NoData => 0, + Value::Bytes(bytes) => bytes.len() + usize::from(property.id >> 26 & 31 == 7) * 4, + Value::References { .. } => usize::from(property.id >> 26 & 1 != 0) * 4, + Value::Sets(children) => { + let prefix = if property.id >> 26 & 31 == 16 { + if children.is_empty() { 4 } else { 8 } + } else { + 0 + }; + prefix + children.clone().map(|i| set_lengths[i]).sum::() + } + } +} + fn patch_properties( blob: &[u8], updates: &[(u32, &[u8])], - insert: Option<(u32, &[u8])>, + inserts: &[(u32, &[u8])], ) -> Result> { let properties = PropertySets::parse(blob)?; + let root = &properties.sets[0]; + let ids = properties.root_ids.as_ptr().addr() - blob.as_ptr().addr(); + let ids_end = ids + properties.root_ids.len(); + let body_end = blob.len() - properties.padding.len(); + let mut set_lengths = vec![0; properties.sets.len()]; + for (i, set) in properties.sets.iter().enumerate().rev() { + set_lengths[i] = 2 + + set.len() * 4 + + set + .iter() + .map(|p| field_length(p, &set_lengths)) + .sum::(); + } + let mut offsets = Vec::with_capacity(root.len()); + let mut offset = ids_end; + for property in root { + offsets.push(offset); + offset += field_length(property, &set_lengths); + } let mut patches = Vec::new(); - for (i, &(property, value)) in updates.iter().chain(insert.iter()).enumerate() { - if updates[..i.min(updates.len())] - .iter() - .any(|(id, _)| *id == property) - { + let mut added_ids = Vec::new(); + let mut added_fields = Vec::new(); + let mut added_references = Vec::new(); + let object_header = u32::from_le_bytes(blob[..4].try_into().unwrap()); + let mut object_count = i64::from(object_header & 0xffffff); + let mut seen = BTreeSet::new(); + for (i, &(property, value)) in updates.iter().chain(inserts).enumerate() { + if !seen.insert(property & 0x7fffffff) { return Err(Error { offset: 0, message: "Duplicate property update", }); } - let kind = (property >> 26) & 0x1f; - if !(3..=7).contains(&kind) { + let kind = (property >> 26) & 31; + let valid = match kind { + 2 => value.is_empty(), + 3..=6 => value.len() == 1 << (kind - 3), + 7 => value.len() < 0x40000000, + 8 => value.len() == 4, + 9 => value.len().is_multiple_of(4) && value.len() / 4 <= 0xffffff, + _ => { + return Err(Error { + offset: 0, + message: "Property type cannot be patched", + }); + } + }; + if !valid || (kind != 2 && property & 0x80000000 != 0) { return Err(Error { offset: 0, - message: "Property does not contain scalar bytes", + message: "Replacement has an invalid property value", }); } - if (kind == 7 && value.len() >= 0x40000000) || (kind != 7 && value.len() != 1 << (kind - 3)) - { - return Err(Error { - offset: 0, - message: "Replacement has an invalid property length", - }); - } - let matches: Vec<_> = properties.sets[0] + let matches: Vec<_> = root .iter() - .filter(|candidate| candidate.id == property) + .enumerate() + .filter(|(_, p)| p.id & 0x7fffffff == property & 0x7fffffff) .collect(); - let adding = i == updates.len(); + let adding = i >= updates.len(); if matches.len() != usize::from(!adding) { return Err(Error { offset: 0, message: "Property is missing or duplicated", }); } - let previous = if adding { - None - } else { - let Value::Bytes(previous) = matches[0].value else { - return Err(Error { - offset: 0, - message: "Property does not contain scalar bytes", - }); - }; - if previous == value { - continue; - } - Some(previous) - }; let mut encoded = Vec::new(); - if kind == 7 { - encoded.extend_from_slice(&u32::try_from(value.len()).unwrap().to_le_bytes()); + match kind { + 7 => encoded.extend_from_slice(&(value.len() as u32).to_le_bytes()), + 9 => encoded.extend_from_slice(&(value.len() as u32 / 4).to_le_bytes()), + _ => {} } - encoded.extend_from_slice(value); - if let Some(previous) = previous { - let start = previous.as_ptr().addr() - blob.as_ptr().addr(); - patches.push(( - start - if kind == 7 { 4 } else { 0 }, - start + previous.len(), - encoded, - )); + if (3..=7).contains(&kind) { + encoded.extend_from_slice(value); + } + if adding { + added_ids.extend_from_slice(&property.to_le_bytes()); + added_fields.extend_from_slice(&encoded); + if kind >= 8 { + object_count += (value.len() / 4) as i64; + added_references.extend_from_slice(value); + } } else { - let count = u16::try_from(properties.sets[0].len() + 1).map_err(|_| Error { - offset: 0, - message: "Root property count exceeds the format limit", - })?; - let ids = properties.root_ids.as_ptr().addr() - blob.as_ptr().addr(); - patches.push((ids - 2, ids, count.to_le_bytes().to_vec())); - let end = ids + properties.root_ids.len(); - patches.push((end, end, property.to_le_bytes().to_vec())); - let end = blob.len() - properties.padding.len(); - patches.push((end, end, encoded)); + let (index, previous) = matches[0]; + if kind == 2 && previous.id != property { + patches.push(( + ids + index * 4, + ids + index * 4 + 4, + index, + property.to_le_bytes().to_vec(), + )); + } + let start = offsets[index]; + let end = start + field_length(previous, &set_lengths); + if blob[start..end] != encoded { + patches.push((start, end, index, encoded)); + } + if let Value::References { compact_ids, .. } = previous.value { + object_count += (value.len() / 4) as i64 - (compact_ids.len() / 4) as i64; + if compact_ids != value { + let start = compact_ids.as_ptr().addr() - blob.as_ptr().addr(); + patches.push((start, start + compact_ids.len(), index, value.to_vec())); + } + } } } + if !(0..=0xffffff).contains(&object_count) { + return Err(Error { + offset: 0, + message: "Object reference stream exceeds the format limit", + }); + } + let header = (object_header & 0xff000000) | object_count as u32; + if header != object_header { + patches.push((0, 4, 0, header.to_le_bytes().to_vec())); + } + if !added_references.is_empty() { + let end = 4 + (object_header as usize & 0xffffff) * 4; + patches.push((end, end, usize::MAX, added_references)); + } + if !inserts.is_empty() { + let count = u16::try_from(root.len() + inserts.len()).map_err(|_| Error { + offset: ids - 2, + message: "Root property count exceeds the format limit", + })?; + patches.push((ids - 2, ids, 0, count.to_le_bytes().to_vec())); + patches.push((ids_end, ids_end, usize::MAX - 1, added_ids)); + patches.push((body_end, body_end, usize::MAX, added_fields)); + } if patches.is_empty() { return Ok(blob.to_vec()); } - patches.sort_by_key(|(start, end, _)| (*start, *end)); + patches.sort_by_key(|(start, end, order, _)| (*start, *end, *order)); let mut changed = Vec::new(); let mut cursor = 0; - for (start, end, value) in patches { + for (start, end, _, value) in patches { + if start < cursor { + return Err(Error { + offset: start, + message: "Property patches overlap", + }); + } changed.extend_from_slice(&blob[cursor..start]); changed.extend_from_slice(&value); cursor = end; } - changed.extend_from_slice(&blob[cursor..blob.len() - properties.padding.len()]); + changed.extend_from_slice(&blob[cursor..body_end]); changed.resize(changed.len().next_multiple_of(8), 0); PropertySets::parse(&changed)?; - Ok(changed) } pub(crate) struct ObjectEdit<'a> { pub object: ExGuid, pub updates: &'a [(u32, &'a [u8])], - pub insert: Option<(u32, &'a [u8])>, + pub inserts: &'a [(u32, &'a [u8])], } /// Replaces a root-level scalar byte property in the default active revision. @@ -213,13 +293,19 @@ pub fn replace_property_bytes( property: u32, value: &[u8], ) -> Result> { + if !(3..=7).contains(&((property >> 26) & 31)) { + return Err(Error { + offset: 0, + message: "Property does not contain scalar bytes", + }); + } replace_objects( source, space, &[ObjectEdit { object: object_id, updates: &[(property, value)], - insert: None, + inserts: &[], }], ) } @@ -228,6 +314,96 @@ pub(crate) fn replace_objects( source: &[u8], space: ExGuid, edits: &[ObjectEdit<'_>], +) -> Result> { + write_revision(source, space, |revision| { + let mut changed = BTreeMap::new(); + for edit in edits { + if changed.contains_key(&edit.object) { + return Err(Error { + offset: 0, + message: "Duplicate object edit", + }); + } + let object = revision.objects.get(&edit.object).ok_or(Error { + offset: 0, + message: "Object is absent from the active revision", + })?; + let ObjectData::Properties(blob) = object.data else { + return Err(Error { + offset: 0, + message: "Object does not contain editable properties", + }); + }; + changed.insert( + edit.object, + PropertyObject { + jcid: object.jcid, + bytes: patch_properties(blob, edit.updates, edit.inserts)?, + global_ids: Arc::clone(&object.global_ids), + }, + ); + } + Ok(changed) + }) +} + +pub(crate) struct PropertyObject { + pub jcid: u32, + pub bytes: Vec, + pub global_ids: Arc>, +} + +impl PropertyObject { + pub fn from_object(object: &crate::Object<'_>) -> Result { + let ObjectData::Properties(bytes) = object.data else { + return Err(Error { + offset: 0, + message: "Object does not contain editable properties", + }); + }; + Ok(Self { + jcid: object.jcid, + bytes: bytes.to_vec(), + global_ids: Arc::clone(&object.global_ids), + }) + } + + pub fn set(&mut self, values: &[(u32, &[u8])]) -> Result<()> { + let properties = PropertySets::parse(&self.bytes)?; + let (updates, inserts): (Vec<_>, Vec<_>) = values.iter().copied().partition(|(id, _)| { + properties.sets[0] + .iter() + .any(|p| p.id & 0x7fffffff == id & 0x7fffffff) + }); + self.bytes = patch_properties(&self.bytes, &updates, &inserts)?; + Ok(()) + } + + pub fn reference(&mut self, id: ExGuid) -> Result<[u8; 4]> { + if !self.global_ids.values().any(|guid| *guid == id.guid) { + let mut index = 0; + for key in self.global_ids.keys() { + if *key != index { + break; + } + index += 1; + } + if index >= 0xffffff || id.guid == [0; 16] { + return Err(Error { + offset: 0, + message: "Object identity cannot be added to the global ID table", + }); + } + Arc::make_mut(&mut self.global_ids).insert(index, id.guid); + } + compact(id, &self.global_ids) + } +} + +pub(crate) fn write_revision( + source: &[u8], + space: ExGuid, + edit: impl FnOnce(&crate::ResolvedRevision<'_>) -> Result>, ) -> Result> { let store = Store::parse(source)?; let is_section = store.header.file_type == FileType::Section; @@ -247,46 +423,136 @@ pub(crate) fn replace_objects( offset: 0, message: "Object space has no active default revision", })?; - let revision = index.resolve(space, rid)?; + let mut revision = index.resolve(space, rid)?; let reachable = revision.reachable()?; - let mut changed = BTreeMap::new(); - for (i, edit) in edits.iter().enumerate() { - if edits[..i] - .iter() - .any(|previous| previous.object == edit.object) + let mut replacements = edit(&revision)?; + for (id, replacement) in &replacements { + if let Some(object) = revision.objects.get(id) { + if !reachable.contains(id) { + return Err(Error { + offset: 0, + message: "Object is not reachable in the active revision", + }); + } + if object.jcid & 0x100000 != 0 { + return Err(Error { + offset: 0, + message: "Read-only object requires a new identity", + }); + } + if replacement.jcid != object.jcid || !matches!(object.data, ObjectData::Properties(_)) + { + return Err(Error { + offset: 0, + message: "An existing object's type cannot be changed", + }); + } + } else if !is_section { + return Err(Error { + offset: 0, + message: "New objects require a section file", + }); + } + if replacement.jcid & 0x20000 == 0 || replacement.global_ids.keys().any(|i| *i > 0xffffff) { + return Err(Error { + offset: 0, + message: "Invalid property object declaration", + }); + } + compact(*id, &replacement.global_ids)?; + PropertySets::parse(&replacement.bytes)?; + } + // Native coalescing of duplicate readonly styles can leave dangling references. + let mut aliases = BTreeMap::new(); + for (id, replacement) in &replacements { + if revision.objects.contains_key(id) + || replacement.jcid & 0x100000 == 0 + || PropertySets::parse(&replacement.bytes)? + .sets + .iter() + .flatten() + .any(|p| matches!(p.value, Value::References { .. })) { - return Err(Error { - offset: 0, - message: "Duplicate object edit", - }); + continue; } - if !reachable.contains(&edit.object) { - return Err(Error { - offset: 0, - message: "Object is not reachable in the active revision", - }); + let existing = revision.objects.iter().find_map(|(other, object)| { + (reachable.contains(other) + && object.jcid == replacement.jcid + && object.data == ObjectData::Properties(&replacement.bytes)) + .then_some(*other) + }); + let existing = existing.or_else(|| { + replacements.range(..id).find_map(|(other, object)| { + (object.jcid == replacement.jcid && object.bytes == replacement.bytes) + .then_some(*aliases.get(other).unwrap_or(other)) + }) + }); + if let Some(existing) = existing { + aliases.insert(*id, existing); } - let object = &revision.objects[&edit.object]; - if object.jcid & 0x100000 != 0 { - return Err(Error { - offset: 0, - message: "Read-only object requires a new identity", - }); + } + for id in aliases.keys() { + replacements.remove(id); + } + for object in replacements.values_mut() { + let mut remapped = Vec::new(); + for property in PropertySets::parse(&object.bytes)?.sets.iter().flatten() { + if let Value::References { + stream: crate::IdStream::Objects, + compact_ids, + } = property.value + { + for bytes in compact_ids.chunks_exact(4) { + let offset = bytes.as_ptr().addr() - object.bytes.as_ptr().addr(); + let id = crate::bytes::Cursor { bytes, offset }.compact(&object.global_ids)?; + if let Some(existing) = aliases.get(&id) { + remapped.push((offset, *existing)); + } + } + } } - let ObjectData::Properties(blob) = object.data else { - return Err(Error { - offset: 0, - message: "Object does not contain editable properties", - }); - }; - let bytes = patch_properties(blob, edit.updates, edit.insert)?; - if bytes != blob { - changed.insert(edit.object, bytes); + for (offset, id) in remapped { + let reference = object.reference(id)?; + object.bytes[offset..offset + 4].copy_from_slice(&reference); } } - if changed.is_empty() { + replacements.retain(|id, replacement| { + !revision.objects.get(id).is_some_and(|object| { + object.data == ObjectData::Properties(&replacement.bytes) + && object.global_ids == replacement.global_ids + }) + }); + if replacements.is_empty() { return Ok(source.to_vec()); } + let mut changed: BTreeSet<_> = replacements.keys().copied().collect(); + for (id, replacement) in &replacements { + revision.objects.insert( + *id, + crate::Object { + jcid: replacement.jcid, + reference_count: 0, + data: ObjectData::Properties(&replacement.bytes), + global_ids: Arc::clone(&replacement.global_ids), + }, + ); + } + let incoming = revision.reference_counts()?; + if replacements.keys().any(|id| !incoming.contains_key(id)) { + return Err(Error { + offset: 0, + message: "Edited object is not reachable in the resulting revision", + }); + } + for (id, object) in &mut revision.objects { + if reachable.contains(id) || incoming.contains_key(id) { + let count = incoming.get(id).copied().unwrap_or(0); + if object.reference_count != count { + object.reference_count = count; + changed.insert(*id); + } + } + } // Native cold-open fails on long dependency chains; cap their depth at 512. let checkpoint = std::iter::successors(Some(rid), |id| { @@ -297,7 +563,7 @@ pub(crate) fn replace_objects( let selected: Vec<_> = revision .objects .iter() - .filter(|(id, _)| checkpoint || changed.contains_key(id)) + .filter(|(id, _)| checkpoint || changed.contains(id)) .collect(); let toc_table = if checkpoint && !is_section { if selected.iter().any(|(_, object)| { @@ -398,8 +664,7 @@ pub(crate) fn replace_objects( } group.push(node(0x73, None, &declaration)?); } - ObjectData::Properties(previous) => { - let bytes = changed.get(&id).map(Vec::as_slice).unwrap_or(previous); + ObjectData::Properties(bytes) => { let references = object.references()?; let flags = u8::from(!references.objects.is_empty()) | (u8::from( @@ -422,7 +687,7 @@ pub(crate) fn replace_objects( } } append(&mut output, &mapped)? - } else if changed.contains_key(&id) { + } else if replacements.contains_key(&id) { append(&mut output, bytes)? } else { Chunk { diff --git a/crates/onestore/src/write/tests.rs b/crates/onestore/src/write/tests.rs new file mode 100644 index 0000000000000000000000000000000000000000..f10f53299d01dbb391d47cf15898ec1db91ad6d6 --- /dev/null +++ b/crates/onestore/src/write/tests.rs @@ -0,0 +1,680 @@ +use super::patch_properties; +use crate::{PropertySets, Value, create::properties}; + +#[test] +fn document_insertions_and_formatting_respect_readonly_ancestors() { + use super::{PropertyObject, write_revision}; + use crate::{ExGuid, Insertion, PreparedEdit, RevisionIndex, Store}; + use std::collections::BTreeMap; + let source = crate::create_section("readonly.one", "Original", "Author").unwrap(); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let (sid, page, outline, paragraph) = index + .spaces + .iter() + .find_map(|(sid, s)| { + let raw = index + .resolve(*sid, s.labels[&(ExGuid::default(), 1)]) + .unwrap(); + let by_type = |jcid| { + raw.objects + .iter() + .find_map(|(id, o)| (o.jcid == jcid).then_some(*id)) + }; + Some(( + *sid, + by_type(0x6000b)?, + by_type(0x6000c)?, + by_type(0x6000d)?, + )) + }) + .unwrap(); + for blocked in [page, outline, paragraph] { + let protected = write_revision(&source, sid, |raw| { + let mut object = PropertyObject::from_object(&raw.objects[&blocked])?; + object.set(&[(0x88001cde, &[])])?; + Ok(BTreeMap::from([(blocked, object)])) + }) + .unwrap(); + let child = Insertion::paragraph(paragraph, None, "Nested", "Author").unwrap(); + assert!(PreparedEdit::insert(&protected, sid, &child).is_err()); + let raw = index + .resolve(sid, index.spaces[&sid].labels[&(ExGuid::default(), 1)]) + .unwrap(); + let text = raw + .objects + .iter() + .find_map(|(id, object)| (object.jcid == 0x6000e).then_some(*id)) + .unwrap(); + assert!( + PreparedEdit::format( + &protected, + sid, + text, + 1..3, + &[crate::TextAttribute::Bold(true)] + ) + .is_err() + ); + if blocked == page { + let outline = Insertion::outline(page, 36.0, 36.0, "Outline", "Author").unwrap(); + assert!(PreparedEdit::insert(&protected, sid, &outline).is_err()); + } + } +} + +fn add_paragraph(source: &[u8], number: u32) -> Vec { + use super::{PropertyObject, compact, write_revision}; + use crate::{ExGuid, ObjectData, RevisionIndex, Store}; + use std::{collections::BTreeMap, sync::Arc}; + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let space = index + .spaces + .iter() + .find_map(|(sid, s)| { + let revision = index + .resolve(*sid, s.labels[&(ExGuid::default(), 1)]) + .unwrap(); + revision + .objects + .values() + .any(|o| o.jcid == 0x6000c) + .then_some(*sid) + }) + .unwrap(); + write_revision(source, space, |revision| { + let (&outline_id, outline) = revision + .objects + .iter() + .find(|(_, o)| o.jcid == 0x6000c) + .unwrap(); + let (&author, _) = revision + .objects + .iter() + .find(|(_, o)| o.jcid == 0x120001) + .unwrap(); + let (&style, _) = revision + .objects + .iter() + .find(|(_, o)| o.jcid == 0x12004d) + .unwrap(); + let mut guid = [0x69; 16]; + guid[..4].copy_from_slice(&number.to_le_bytes()); + let paragraph = ExGuid { guid, n: 1 }; + let text = ExGuid { guid, n: 2 }; + let mut table = (*outline.global_ids).clone(); + for guid in [guid, author.guid, style.guid] { + if !table.values().any(|previous| *previous == guid) { + table.insert(table.last_key_value().map_or(0, |(i, _)| i + 1), guid); + } + } + let table = Arc::new(table); + let ObjectData::Properties(blob) = outline.data else { + unreachable!() + }; + let parsed = PropertySets::parse(blob).unwrap(); + let Value::References { compact_ids, .. } = parsed.sets[0] + .iter() + .find(|p| p.id == 0x24001c20) + .unwrap() + .value + else { + unreachable!() + }; + let mut children = compact_ids.to_vec(); + children.extend_from_slice(&compact(paragraph, &table).unwrap()); + let modified = crate::create::current_timestamps()?.0.to_le_bytes(); + let mut changed = BTreeMap::new(); + changed.insert( + outline_id, + PropertyObject { + jcid: outline.jcid, + bytes: patch_properties( + blob, + &[(0x24001c20, &children), (0x14001d7a, &modified)], + &[], + )?, + global_ids: Arc::clone(&table), + }, + ); + changed.insert( + paragraph, + PropertyObject { + jcid: 0x6000d, + bytes: properties(&[ + (0x14001d7a, modified.to_vec()), + (0x14001d09, modified.to_vec()), + (0x0c001c03, vec![1]), + (0x24001c1f, compact(text, &table)?.to_vec()), + (0x20001d78, compact(author, &table)?.to_vec()), + (0x20001d79, compact(author, &table)?.to_vec()), + ])?, + global_ids: Arc::clone(&table), + }, + ); + changed.insert( + text, + PropertyObject { + jcid: 0x6000e, + bytes: properties(&[ + (0x14001d7a, modified.to_vec()), + ( + 0x1c001c22, + format!("Paragraph {number}\0") + .encode_utf16() + .flat_map(u16::to_le_bytes) + .collect(), + ), + (0x24001e13, compact(style, &table)?.to_vec()), + ])?, + global_ids: table, + }, + ); + Ok(changed) + }) + .unwrap() +} + +fn restyle(source: &[u8]) -> Vec { + use super::{PropertyObject, compact, write_revision}; + use crate::{ExGuid, ObjectData, RevisionIndex, Store}; + use std::{collections::BTreeMap, sync::Arc}; + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let space = index + .spaces + .iter() + .find_map(|(sid, s)| { + let revision = index + .resolve(*sid, s.labels[&(ExGuid::default(), 1)]) + .unwrap(); + revision + .objects + .values() + .any(|o| o.jcid == 0x6000c) + .then_some(*sid) + }) + .unwrap(); + let style = ExGuid { + guid: [0x74; 16], + n: 1, + }; + write_revision(source, space, |revision| { + let previous = revision + .objects + .values() + .find(|o| o.jcid == 0x12004d) + .unwrap(); + let ObjectData::Properties(blob) = previous.data else { + unreachable!() + }; + let mut table = (*previous.global_ids).clone(); + table.insert(table.last_key_value().unwrap().0 + 1, style.guid); + let mut changed = BTreeMap::from([( + style, + PropertyObject { + jcid: previous.jcid, + bytes: patch_properties(blob, &[], &[(0x88001c04, &[])])?, + global_ids: Arc::new(table), + }, + )]); + for (id, text) in revision.objects.iter().filter(|(_, o)| o.jcid == 0x6000e) { + let ObjectData::Properties(blob) = text.data else { + unreachable!() + }; + let mut table = (*text.global_ids).clone(); + table.insert(table.last_key_value().unwrap().0 + 1, style.guid); + changed.insert( + *id, + PropertyObject { + jcid: text.jcid, + bytes: patch_properties(blob, &[(0x24001e13, &compact(style, &table)?)], &[])?, + global_ids: Arc::new(table), + }, + ); + } + Ok(changed) + }) + .unwrap() +} + +#[test] +fn replaced_readonly_styles_retain_history_with_zero_current_references() { + use crate::{ExGuid, RevisionIndex, Store}; + let source = add_paragraph( + &crate::create_section("style.one", "Original", "Author").unwrap(), + 1, + ); + let changed = restyle(&source); + let store = Store::parse(&changed).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + let mut styles = Vec::new(); + for (sid, s) in &index.spaces { + let revision = index + .resolve(*sid, s.labels[&(ExGuid::default(), 1)]) + .unwrap(); + for (id, object) in &revision.objects { + if object.jcid == 0x12004d { + styles.push((object.reference_count, id.guid)); + } + } + } + styles.sort(); + assert_eq!(styles.len(), 2); + assert_eq!(styles[0].0, 0); + assert_eq!(styles[1], (2, [0x74; 16])); + let document = crate::document::Document::parse(&index).unwrap(); + let mut count = 0; + for space in document.spaces.values() { + for revision in space.revisions.values() { + for (id, node) in &revision.nodes { + if matches!(node.kind, crate::document::Kind::RichText { .. }) { + for run in revision.text_runs(*id).unwrap() { + assert_eq!(run.format.bold, Some(true)); + } + count += 1; + } + } + } + } + assert_eq!(count, 2); +} + +#[test] +#[ignore = "Writes cold-native candidates to ONESTORE_GROWTH_OUTPUT"] +fn export_native_growth_candidates() { + let output = std::path::PathBuf::from(std::env::var_os("ONESTORE_GROWTH_OUTPUT").unwrap()); + std::fs::create_dir(&output).unwrap(); + let mut source = crate::create_section("growth.one", "Original", "Author").unwrap(); + for number in 1..=24 { + source = add_paragraph(&source, number); + } + std::fs::write(output.join("growth.one"), &source).unwrap(); + std::fs::write(output.join("restyled.one"), restyle(&source)).unwrap(); +} + +#[test] +fn atomic_graph_growth_preserves_history_and_counts_shared_readonly_objects() { + use crate::{ExGuid, RevisionIndex, Store}; + let initial = crate::create_section("growth.one", "Original", "Author").unwrap(); + let mut source = initial.clone(); + for number in 1..=24 { + source = add_paragraph(&source, number); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + assert!(store.checksum_mismatches.is_empty()); + assert_eq!(store.header.transaction_count, number + 1); + let mut text_count = 0; + for (sid, s) in &index.spaces { + let revision = index + .resolve(*sid, s.labels[&(ExGuid::default(), 1)]) + .unwrap(); + let reachable = revision.reachable().unwrap(); + for id in reachable { + let object = &revision.objects[&id]; + match object.jcid { + 0x6000e => text_count += 1, + 0x120001 => assert_eq!(object.reference_count, (number + 1) * 2), + 0x12004d => assert_eq!(object.reference_count, number + 1), + _ => {} + } + } + } + assert_eq!(text_count, number + 1); + } + let initial_store = Store::parse(&initial).unwrap(); + let initial_index = RevisionIndex::parse(&initial_store).unwrap(); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + for (sid, s) in &initial_index.spaces { + let rid = s.labels[&(ExGuid::default(), 1)]; + let before = initial_index.resolve(*sid, rid).unwrap(); + let after = index.resolve(*sid, rid).unwrap(); + assert_eq!(before.roots, after.roots); + for (id, object) in &before.objects { + assert_eq!(object.data, after.objects[id].data); + assert_eq!(object.reference_count, after.objects[id].reference_count); + } + } +} + +#[test] +fn changed_graphs_reject_cycles_dangling_and_unreachable_additions() { + use super::{PropertyObject, compact, write_revision}; + use crate::{ExGuid, ObjectData, RevisionIndex, Store}; + use std::{collections::BTreeMap, sync::Arc}; + let source = crate::create_section("invalid.one", "Original", "Author").unwrap(); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + for (sid, s) in &index.spaces { + let revision = index + .resolve(*sid, s.labels[&(ExGuid::default(), 1)]) + .unwrap(); + let Some((&oid, outline)) = revision.objects.iter().find(|(_, o)| o.jcid == 0x6000c) else { + continue; + }; + let ObjectData::Properties(blob) = outline.data else { + unreachable!() + }; + for target in [ + oid, + ExGuid { + guid: oid.guid, + n: 255, + }, + ] { + assert!( + write_revision(&source, *sid, |_| Ok(BTreeMap::from([( + oid, + PropertyObject { + jcid: outline.jcid, + bytes: patch_properties( + blob, + &[(0x24001c20, &compact(target, &outline.global_ids)?)], + &[] + )?, + global_ids: Arc::clone(&outline.global_ids), + } + )]))) + .is_err() + ); + } + let orphan = ExGuid { + guid: oid.guid, + n: 255, + }; + assert!( + write_revision(&source, *sid, |_| Ok(BTreeMap::from([( + orphan, + PropertyObject { + jcid: 0x6000e, + bytes: properties(&[])?, + global_ids: Arc::clone(&outline.global_ids), + } + )]))) + .is_err() + ); + } +} + +#[test] +fn property_splices_match_independently_encoded_flat_sets() { + let mut seed = 0x749391acb66327d5_u64; + let mut next = || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + for case in 0..20_000 { + let mut original = Vec::new(); + let mut expected = Vec::new(); + let mut updates = Vec::new(); + let mut inserts = Vec::new(); + for index in 0..next() % 30 { + let kind = 2 + (next() % 8) as u32; + let mut id = (kind << 26) | index as u32; + let length = |random: u64| match kind { + 2 => 0, + 3..=6 => 1 << (kind - 3), + 7 => (random % 20) as usize, + 8 => 4, + 9 => (random % 5) as usize * 4, + _ => unreachable!(), + }; + let value = (0..length(next())) + .map(|_| next() as u8) + .collect::>(); + if kind == 2 && next() & 1 != 0 { + id |= 0x80000000; + } + original.push((id, value.clone())); + if next() & 1 != 0 { + if kind == 2 { + id ^= 0x80000000; + } + let value = (0..length(next())) + .map(|_| next() as u8) + .collect::>(); + expected.push((id, value.clone())); + updates.push((id, value)); + } else { + expected.push((id, value)); + } + } + for index in 0..next() % 6 { + let kind = [2, 7, 8, 9][(next() % 4) as usize]; + let mut id = (kind << 26) | (100 + index as u32); + if kind == 2 && next() & 1 != 0 { + id |= 0x80000000; + } + let length = match kind { + 2 => 0, + 7 => next() as usize % 10, + 8 => 4, + 9 => (next() as usize % 4) * 4, + _ => unreachable!(), + }; + let value = (0..length).map(|_| next() as u8).collect::>(); + inserts.push((id, value.clone())); + expected.push((id, value)); + } + updates.reverse(); + let updates: Vec<_> = updates + .iter() + .map(|(id, value)| (*id, value.as_slice())) + .collect(); + let inserts: Vec<_> = inserts + .iter() + .map(|(id, value)| (*id, value.as_slice())) + .collect(); + let original = properties(&original).unwrap(); + let actual = patch_properties(&original, &updates, &inserts).unwrap(); + assert_eq!(actual, properties(&expected).unwrap(), "case {case}"); + } +} + +#[test] +fn nested_fields_and_other_reference_streams_remain_byte_exact() { + let mut original = 0x40000003_u32.to_le_bytes().to_vec(); + original.extend_from_slice(&[1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0]); + original.extend_from_slice(&0x40000001_u32.to_le_bytes()); + original.extend_from_slice(&[4, 0, 0, 0]); + original.extend_from_slice(&1_u32.to_le_bytes()); + original.extend_from_slice(&[5, 0, 0, 0]); + original.extend_from_slice(&4_u16.to_le_bytes()); + for id in [0x24000001_u32, 0x40000002, 0x24000003, 0x1c000004] { + original.extend_from_slice(&id.to_le_bytes()); + } + original.extend_from_slice(&1_u32.to_le_bytes()); + let nested_start = original.len(); + original.extend_from_slice(&1_u32.to_le_bytes()); + original.extend_from_slice(&0x44003456_u32.to_le_bytes()); + original.extend_from_slice(&4_u16.to_le_bytes()); + for id in [0x20000001_u32, 0x28000002, 0x30000003, 0x1c000004] { + original.extend_from_slice(&id.to_le_bytes()); + } + original.extend_from_slice(&3_u32.to_le_bytes()); + original.extend_from_slice(&[91, 92, 93]); + let nested_end = original.len(); + original.extend_from_slice(&1_u32.to_le_bytes()); + original.extend_from_slice(&2_u32.to_le_bytes()); + original.extend_from_slice(&[94, 95]); + original.resize(original.len().next_multiple_of(8), 0); + let changed = patch_properties( + &original, + &[ + (0x24000003, &[]), + (0x1c000004, &[96, 97, 98, 99]), + (0x24000001, &[6, 0, 0, 0, 7, 0, 0, 0]), + ], + &[(0x24000005, &[8, 0, 0, 0]), (0x88000006, &[])], + ) + .unwrap(); + let parsed = PropertySets::parse(&changed).unwrap(); + let previous = PropertySets::parse(&original).unwrap(); + assert_eq!(parsed.sets[1], previous.sets[1]); + let nested = &original[nested_start..nested_end]; + assert_eq!( + changed + .windows(nested.len()) + .filter(|bytes| *bytes == nested) + .count(), + 1 + ); + assert_eq!( + &changed[4..20], + &[6, 0, 0, 0, 7, 0, 0, 0, 2, 0, 0, 0, 8, 0, 0, 0] + ); + assert_eq!(&changed[20..36], &original[16..32]); + assert_eq!(parsed.sets[0][3].value, Value::Bytes(&[96, 97, 98, 99])); +} + +#[test] +fn deep_property_splices_do_not_use_the_call_stack() { + let mut bytes = 0x80000000_u32.to_le_bytes().to_vec(); + for _ in 0..100_000 { + bytes.extend_from_slice(&1_u16.to_le_bytes()); + bytes.extend_from_slice(&0x44000001_u32.to_le_bytes()); + } + bytes.extend_from_slice(&0_u16.to_le_bytes()); + let changed = patch_properties(&bytes, &[], &[(0x88000002, &[])]).unwrap(); + let parsed = PropertySets::parse(&changed).unwrap(); + assert_eq!(parsed.sets.len(), 100_001); + assert_eq!(parsed.sets[0].len(), 2); + assert_eq!( + &changed[14..changed.len() - parsed.padding.len()], + &bytes[10..] + ); +} + +#[test] +fn invalid_property_splices_are_rejected() { + let bytes = properties(&[(0x08000001, vec![]), (0x24000002, vec![])]).unwrap(); + for (updates, inserts) in [ + (vec![(0x88000001, &[][..]), (0x08000001, &[])], vec![]), + (vec![(0x14000003, &[0; 4][..])], vec![]), + (vec![(0x24000002, &[0; 3][..])], vec![]), + (vec![(0x88000001, &[1][..])], vec![]), + (vec![], vec![(0x88000001, &[][..])]), + (vec![], vec![(0x20000003, &[][..])]), + ] { + assert!(patch_properties(&bytes, &updates, &inserts).is_err()); + } +} + +#[test] +fn character_formatting_preserves_inheritance_and_associated_data() { + use super::{PropertyObject, write_revision}; + use crate::{ + ExGuid, PreparedEdit, RevisionIndex, Store, TextAttribute, + document::{Document, Kind}, + }; + use std::collections::BTreeMap; + let source = crate::create_section("inherited.one", "abcdef", "Author").unwrap(); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let (sid, text) = index + .spaces + .iter() + .find_map(|(sid, s)| { + let raw = index + .resolve(*sid, s.labels[&(ExGuid::default(), 1)]) + .unwrap(); + raw.objects + .iter() + .find_map(|(id, o)| (o.jcid == 0x6000e).then_some((*sid, *id))) + }) + .unwrap(); + for variant in 0..7 { + let fixture = write_revision(&source, sid, |raw| { + let mut target = PropertyObject::from_object(&raw.objects[&text])?; + target.bytes = properties(&[ + (0x1c001c22, crate::create::string("abcdef")), + (0x14001d7a, vec![0; 4]), + (0x1c001c0a, crate::create::string("Georgia")), + (0x88001c05, Vec::new()), + ])?; + match variant { + 1 | 4 | 5 | 6 => { + let flag = match variant { + 1 => 0x88001e16, + 4 => 0x88001e14, + 5 => 0x88003401, + _ => 0x88001e22, + }; + target.set(&[(flag, &[])])?; + } + 2 => { + let author = raw + .objects + .iter() + .find_map(|(id, o)| (o.jcid == 0x120001).then_some(*id)) + .unwrap(); + let reference = target.reference(author)?; + target.set(&[(0x24003458, &reference)])?; + } + 3 => { + // An unknown nested run property must survive a style-only edit byte for byte. + let parsed = PropertySets::parse(&target.bytes)?; + let at = parsed.root_ids.as_ptr().addr() - target.bytes.as_ptr().addr(); + let count = parsed.sets[0].len(); + let end = target.bytes.len() - parsed.padding.len(); + let mut bytes = target.bytes[..end].to_vec(); + bytes[at - 2..at].copy_from_slice(&((count + 1) as u16).to_le_bytes()); + bytes.splice(at + count * 4..at + count * 4, 0x40003499_u32.to_le_bytes()); + bytes.extend_from_slice(&1_u32.to_le_bytes()); + bytes.extend_from_slice(&0x44001234_u32.to_le_bytes()); + bytes.extend_from_slice(&1_u16.to_le_bytes()); + bytes.extend_from_slice(&0x14001234_u32.to_le_bytes()); + bytes.extend_from_slice(&0xdeadbeef_u32.to_le_bytes()); + bytes.resize(bytes.len().next_multiple_of(8), 0); + target.bytes = bytes; + } + _ => {} + } + Ok(BTreeMap::from([(text, target)])) + }) + .unwrap(); + let edit = PreparedEdit::format(&fixture, sid, text, 0..6, &[TextAttribute::Bold(true)]); + if matches!(variant, 1 | 2 | 4 | 5 | 6) { + assert!(edit.is_err()); + continue; + } + let edit = edit.unwrap(); + let store = Store::parse(edit.as_bytes()).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let s = &doc.spaces[&sid]; + let view = &s.revisions[&s.contexts[&ExGuid::default()]]; + let runs = view.text_runs(text).unwrap(); + assert_eq!(runs.len(), 1); + assert_eq!(runs[0].format.font.as_deref(), Some("Georgia")); + assert_eq!(runs[0].format.italic, Some(true)); + assert_eq!(runs[0].format.bold, Some(true)); + if variant == 3 { + let Kind::RichText { runs, .. } = &view.nodes[&text].kind else { + panic!() + }; + let data = &view.nodes[&text].extra[runs[0].extra_set.unwrap()]; + assert_eq!(data.len(), 1); + assert_eq!(data[0].id, 0x14001234); + assert!( + PreparedEdit::format(&fixture, sid, text, 1..3, &[TextAttribute::Bold(true)]) + .is_err() + ); + let previous = Store::parse(&fixture).unwrap(); + let previous = RevisionIndex::parse(&previous).unwrap(); + let previous_doc = Document::parse(&previous).unwrap(); + let s = &previous_doc.spaces[&sid]; + let previous_view = &s.revisions[&s.contexts[&ExGuid::default()]]; + assert_eq!( + format!("{:?}", view.nodes[&text].extra), + format!("{:?}", previous_view.nodes[&text].extra) + ); + } + } +} diff --git a/crates/onestore/tests/edit.rs b/crates/onestore/tests/edit.rs index c44d32de2396a1d69c504a415ef7d950b07da855..67294f2abd0d8d34c6335d825f9b5151ecd8af13 100644 --- a/crates/onestore/tests/edit.rs +++ b/crates/onestore/tests/edit.rs @@ -2,6 +2,8 @@ mod checkpoint; #[path = "support/disk.rs"] mod disk; +#[path = "support/trace.rs"] +mod trace; use disk::Disk; use onestore::{ @@ -44,6 +46,201 @@ fn text_runs(source: &[u8], sid: ExGuid, oid: ExGuid) -> serde_json::Value { .unwrap() } +fn assert_refreshed(source: &[u8], confirmed: &[u8]) { + assert_eq!(&source[..212], &confirmed[..212]); + assert_eq!(&source[252..], &confirmed[252..]); + let before = Store::parse(source).unwrap().header; + let after = Store::parse(confirmed).unwrap().header; + assert_ne!(before.version_id, after.version_id); + assert_ne!(before.deny_read_id, after.deny_read_id); + assert_eq!(after.generation, before.generation + 1); +} + +#[test] +fn prepared_publication_preserves_its_identity_through_every_io_failure() { + let source = + onestore::create_section("prepared.one", "Fictitious: café 🦀", "Fixture").unwrap(); + let (sid, oid) = target(&source); + let source = checkpoint::pending(&source, sid, oid, 0x14001d7a); + let edit = onestore::PreparedEdit::text(&source, sid, oid, 0..0, "Prepared 🐈 ").unwrap(); + let store = Store::parse(edit.as_bytes()).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let planned = index.spaces[&sid].labels[&(ExGuid::default(), 1)]; + let persisted: ExGuid = + serde_json::from_str(&serde_json::to_string(&planned).unwrap()).unwrap(); + let before_store = Store::parse(&source).unwrap(); + let before_index = RevisionIndex::parse(&before_store).unwrap(); + assert!(!before_index.spaces[&sid].revisions.contains_key(&persisted)); + let before = text_runs(&source, sid, oid); + let after = text_runs(edit.as_bytes(), sid, oid); + for write_limit in [17, 4096] { + let disk = |fail_at| Disk { + visible: source.clone(), + durable: source.clone(), + operation: 0, + fail_at, + write_limit, + random: 911, + }; + let mut success = disk(None); + edit.commit(&mut success).unwrap(); + assert_eq!(success.durable, edit.as_bytes()); + let operations = success.operation; + let error = edit.commit(&mut success).unwrap_err(); + assert_eq!(error.state, CommitState::NotCommitted); + assert_eq!(error.error.kind(), std::io::ErrorKind::ResourceBusy); + assert_eq!(success.durable, edit.as_bytes()); + for at in 1..=operations { + let mut interrupted = disk(Some(at)); + let error = edit.commit(&mut interrupted).unwrap_err(); + let observed = text_runs(&interrupted.durable, sid, oid); + let store = Store::parse(&interrupted.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let present = index.spaces[&sid].revisions.contains_key(&persisted); + assert_eq!(observed, if present { &after } else { &before }.clone()); + match error.state { + CommitState::NotCommitted => assert!(!present), + CommitState::Committed => assert!(present), + CommitState::Unknown => {} + } + if present { + let snapshot = interrupted.durable.clone(); + interrupted.visible.clone_from(&snapshot); + interrupted.fail_at = None; + interrupted.write_limit = 17; + onestore::confirm_snapshot(&mut interrupted, &snapshot).unwrap(); + assert_refreshed(&snapshot, &interrupted.durable); + } + } + } +} + +#[test] +fn snapshot_confirmation_needs_no_surviving_edit_target() { + let mut disk = Disk { + visible: SOURCE.to_vec(), + durable: Vec::new(), + operation: 0, + fail_at: None, + write_limit: 17, + random: 911, + }; + assert!( + onestore::replace_text(SOURCE, ExGuid::default(), ExGuid::default(), 0..0, "").is_err() + ); + onestore::confirm_snapshot(&mut disk, SOURCE).unwrap(); + assert_refreshed(SOURCE, &disk.durable); + let flush = disk.operation; + for (failure, state) in [ + (1, CommitState::NotCommitted), + (flush, CommitState::Unknown), + ] { + disk.visible = SOURCE.to_vec(); + disk.durable.clear(); + disk.operation = 0; + disk.fail_at = Some(failure); + assert_eq!( + onestore::confirm_snapshot(&mut disk, SOURCE) + .unwrap_err() + .state, + state + ); + } +} + +#[test] +fn confirmation_notifies_cached_readers_after_interrupted_version_publication() { + let source = + onestore::create_section("confirmation.one", "Fictitious: before", "Fixture").unwrap(); + let (sid, oid) = target(&source); + let mut snapshot = onestore::replace_text(&source, sid, oid, 0..0, "Recovered ").unwrap(); + snapshot[212..252].copy_from_slice(&source[212..252]); + let expected = text_runs(&snapshot, sid, oid); + assert_ne!(expected, text_runs(&source, sid, oid)); + for write_limit in [1, 17, 40] { + let disk = |fail_at| Disk { + visible: snapshot.clone(), + durable: snapshot.clone(), + operation: 0, + fail_at, + write_limit, + random: 911, + }; + let mut success = disk(None); + onestore::confirm_snapshot(&mut success, &snapshot).unwrap(); + assert_refreshed(&snapshot, &success.durable); + for failure in 1..=success.operation { + let mut interrupted = disk(Some(failure)); + let error = onestore::confirm_snapshot(&mut interrupted, &snapshot).unwrap_err(); + assert_ne!(error.state, CommitState::Committed); + assert_eq!(text_runs(&interrupted.durable, sid, oid), expected); + assert_eq!(&interrupted.durable[..212], &snapshot[..212]); + assert_eq!(&interrupted.durable[252..], &snapshot[252..]); + } + } +} + +#[test] +fn confirming_visible_text_requires_flush_without_another_revision() { + let (sid, oid) = target(SOURCE); + let visible = onestore::replace_text(SOURCE, sid, oid, 0..0, "Recovered ").unwrap(); + let mut disk = Disk { + visible: visible.clone(), + durable: SOURCE.to_vec(), + operation: 0, + fail_at: None, + write_limit: 0, + random: 1, + }; + onestore::commit_text(&mut disk, &visible, sid, oid, 0..0, "").unwrap(); + assert_eq!(disk.visible, visible); + assert_eq!(disk.durable, visible); + let flush = disk.operation; + for (failure, state) in [ + (1, CommitState::NotCommitted), + (flush, CommitState::Unknown), + ] { + disk.durable = SOURCE.to_vec(); + disk.operation = 0; + disk.fail_at = Some(failure); + let error = onestore::commit_text(&mut disk, &visible, sid, oid, 0..0, "").unwrap_err(); + assert_eq!(error.state, state); + assert_eq!(disk.visible, visible); + } +} + +#[test] +fn confirmation_rejects_changed_or_truncated_physical_tail_before_any_write() { + let (sid, oid) = target(SOURCE); + let mut source = SOURCE.to_vec(); + source.resize(3 * 1024 * 1024 + 131, 0); + let mut disk = trace::Trace { + bytes: source.clone(), + events: Vec::new(), + }; + onestore::commit_text(&mut disk, &source, sid, oid, 0..0, "").unwrap(); + assert_eq!(disk.events.len(), 1); + if let trace::Event::Write(offset, bytes) = &disk.events[0] { + panic!("Confirmation wrote {} bytes at {offset}", bytes.len()); + } + for changed in [65535, 65536, 1048575, 1048576, 2097152, source.len() - 1] { + disk.bytes.clone_from(&source); + disk.bytes[changed] ^= 1; + disk.events.clear(); + let error = onestore::commit_text(&mut disk, &source, sid, oid, 0..0, "").unwrap_err(); + assert_eq!(error.state, CommitState::NotCommitted); + assert_eq!(error.error.kind(), std::io::ErrorKind::ResourceBusy); + assert!(disk.events.is_empty()); + } + for length in [source.len() - 1, source.len() + 1] { + disk.bytes.clone_from(&source); + disk.bytes.resize(length, 0); + let error = onestore::commit_text(&mut disk, &source, sid, oid, 0..0, "").unwrap_err(); + assert_eq!(error.state, CommitState::NotCommitted); + assert!(disk.events.is_empty()); + } +} + fn assert_other_objects_preserved( before: &onestore::ResolvedRevision<'_>, after: &onestore::ResolvedRevision<'_>, @@ -398,6 +595,8 @@ fn title_text_and_navigation_caches_publish_together() { for (source, write_limit) in [(source.as_slice(), 17), (checkpoint.as_slice(), 257)] { let before = state(source); for replacement in ["Renamed 🦀 日本語", ""] { + let edit = + onestore::PreparedEdit::text(source, sid, *oid, 0..end, replacement).unwrap(); let disk = |fail_at| Disk { visible: source.to_vec(), durable: source.to_vec(), @@ -407,7 +606,7 @@ fn title_text_and_navigation_caches_publish_together() { random: 42, }; let mut success = disk(None); - onestore::commit_text(&mut success, source, sid, *oid, 0..end, replacement).unwrap(); + edit.commit(&mut success).unwrap(); let after = state(&success.durable); assert_eq!(after[0]["text"], replacement); assert_eq!( @@ -445,9 +644,7 @@ fn title_text_and_navigation_caches_publish_together() { assert_other_objects_preserved(&old, ¤t, *oid); for at in source.len().div_ceil(193)..=success.operation { let mut interrupted = disk(Some(at)); - let error = - onestore::commit_text(&mut interrupted, source, sid, *oid, 0..end, replacement) - .unwrap_err(); + let error = edit.commit(&mut interrupted).unwrap_err(); let observed = state(&interrupted.durable); match error.state { CommitState::NotCommitted => assert_eq!(observed, before), diff --git a/crates/onestore/tests/formatting.rs b/crates/onestore/tests/formatting.rs new file mode 100644 index 0000000000000000000000000000000000000000..2fbdff3dc7eae65977f8049a722c0d9bef34766a --- /dev/null +++ b/crates/onestore/tests/formatting.rs @@ -0,0 +1,407 @@ +#[path = "support/current.rs"] +mod current; +#[path = "support/disk.rs"] +mod disk; +use onestore::{ + ExGuid, PreparedEdit, RevisionIndex, Store, TextAttribute as A, + document::{Document, Kind}, +}; +use serde_json::{Value, json}; +fn target(source: &[u8]) -> (ExGuid, ExGuid) { + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let (sid, page) = doc.pages().unwrap()[0]; + let space = &doc.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let mut pending = view.nodes[&page].children.clone(); + while let Some(id) = pending.pop() { + let n = &view.nodes[&id]; + if matches!(n.kind, Kind::RichText { .. }) { + return (sid, id); + } + pending.extend(&n.children); + pending.extend(&n.content); + } + panic!("Missing text") +} +fn characters(source: &[u8], sid: ExGuid, id: ExGuid) -> Vec<(char, Value)> { + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + let doc = Document::parse(&index).unwrap(); + let s = &doc.spaces[&sid]; + let view = &s.revisions[&s.contexts[&ExGuid::default()]]; + view.text_runs(id) + .unwrap() + .into_iter() + .flat_map(|r| { + let format = serde_json::to_value(r.format).unwrap(); + r.text.chars().map(move |c| (c, format.clone())) + }) + .collect() +} +#[test] +fn overlapping_unicode_format_edits_match_an_independent_character_model() { + let text = "abcdefgh 東京 🦀 café\rSecond\tline"; + let original = onestore::create_section("format.one", text, "Author").unwrap(); + let (sid, id) = target(&original); + for seed in 1..=16_u64 { + let mut rng = seed; + let mut source = original.clone(); + let mut expected = characters(&source, sid, id); + let offsets: Vec = std::iter::once(0) + .chain(text.chars().scan(0, |n, c| { + *n += c.len_utf16() as u32; + Some(*n) + })) + .collect(); + for step in 0..24 { + let mut next = || { + rng ^= rng << 13; + rng ^= rng >> 7; + rng ^= rng << 17; + rng + }; + let start = next() as usize % expected.len(); + let end = start + 1 + next() as usize % (expected.len() - start); + let enabled = next() & 1 != 0; + let (attribute, key, value) = match step % 9 { + 0 => (A::Bold(enabled), "bold", json!(enabled)), + 1 => (A::Italic(enabled), "italic", json!(enabled)), + 2 => (A::Underline(enabled), "underline", json!(enabled)), + 3 => (A::Strike(enabled), "strike", json!(enabled)), + 4 => (A::Font("Arial".into()), "font", json!("Arial")), + 5 => (A::FontSize(13.5), "font_size", json!(13.5)), + 6 => ( + A::Color(Some([0x24, 0x68, 0xac])), + "color", + json!(0xac6824_u32), + ), + 7 => (A::Highlight(None), "highlight", json!(0xff000000_u32)), + _ => ( + A::Highlight(Some([0, 255, 0])), + "highlight", + json!(0x00ff00_u32), + ), + }; + let edit = PreparedEdit::format( + &source, + sid, + id, + offsets[start]..offsets[end], + std::slice::from_ref(&attribute), + ) + .unwrap(); + for (_, style) in &mut expected[start..end] { + style[key] = value.clone(); + } + assert_eq!( + characters(edit.as_bytes(), sid, id), + expected, + "seed {seed}, step {step}" + ); + assert_eq!( + PreparedEdit::format( + edit.as_bytes(), + sid, + id, + offsets[start]..offsets[end], + &[attribute] + ) + .unwrap() + .as_bytes(), + edit.as_bytes() + ); + let old_store = Store::parse(&source).unwrap(); + let old = RevisionIndex::parse(&old_store).unwrap(); + let new_store = Store::parse(edit.as_bytes()).unwrap(); + let new = RevisionIndex::parse(&new_store).unwrap(); + let rid = old.spaces[&sid].labels[&(ExGuid::default(), 1)]; + assert_eq!( + format!("{:?}", old.resolve(sid, rid).unwrap()), + format!("{:?}", new.resolve(sid, rid).unwrap()) + ); + source = edit.as_bytes().to_vec(); + } + } +} +#[test] +fn script_positions_are_exclusive_and_explicit_false_overrides_true() { + let source = onestore::create_section("script.one", "abc", "Author").unwrap(); + let (sid, id) = target(&source); + let superscript = PreparedEdit::format( + &source, + sid, + id, + 0..3, + &[A::Superscript(true), A::Bold(true)], + ) + .unwrap(); + let subscript = PreparedEdit::format( + superscript.as_bytes(), + sid, + id, + 1..2, + &[A::Subscript(true), A::Bold(false)], + ) + .unwrap(); + let chars = characters(subscript.as_bytes(), sid, id); + assert_eq!(chars[0].1["superscript"], true); + assert_eq!(chars[0].1["subscript"], false); + assert_eq!(chars[0].1["bold"], true); + assert_eq!(chars[1].1["superscript"], false); + assert_eq!(chars[1].1["subscript"], true); + assert_eq!(chars[1].1["bold"], false); + assert_eq!(chars[0].1, chars[2].1); +} +#[test] +fn empty_paragraph_style_is_used_by_later_text_edits() { + let source = onestore::create_section("empty.one", "", "Author").unwrap(); + let (sid, id) = target(&source); + let formatted = PreparedEdit::format( + &source, + sid, + id, + 0..0, + &[A::Italic(true), A::FontSize(18.0)], + ) + .unwrap(); + let filled = PreparedEdit::text(formatted.as_bytes(), sid, id, 0..0, "Added 🦀").unwrap(); + for (_, format) in characters(filled.as_bytes(), sid, id) { + assert_eq!(format["italic"], true); + assert_eq!(format["font_size"], 18.0); + } +} +#[test] +fn invalid_ranges_attributes_and_fields_are_rejected() { + let source = onestore::create_section("invalid.one", "a🦀b", "Author").unwrap(); + let (sid, id) = target(&source); + for (start, end) in [(2, 3), (1, 2), (0, 8), (2, 2), (3, 1), (1, 1)] { + let range = start..end; + assert!(PreparedEdit::format(&source, sid, id, range, &[A::Bold(true)]).is_err()); + } + for attributes in [ + vec![], + vec![A::Bold(true), A::Bold(false)], + vec![A::Superscript(true), A::Subscript(true)], + vec![A::Font("".into())], + vec![A::Font("a\0b".into())], + ] { + assert!(PreparedEdit::format(&source, sid, id, 0..4, &attributes).is_err()); + } + for size in [f32::NAN, f32::INFINITY, 0.0, 5.5, 130.5, 144.0, 144.5, 12.1] { + assert!(PreparedEdit::format(&source, sid, id, 0..4, &[A::FontSize(size)]).is_err()); + } + assert!(PreparedEdit::format(&source, sid, ExGuid::default(), 0..4, &[A::Bold(true)]).is_err()); +} +#[test] +fn formatting_publication_faults_preserve_complete_old_or_new_styles() { + let source = onestore::create_section("atomic.one", "Before 🦀 after", "Author").unwrap(); + let (sid, id) = target(&source); + let edit = PreparedEdit::format( + &source, + sid, + id, + 2..10, + &[A::Bold(true), A::Color(Some([8, 64, 128]))], + ) + .unwrap(); + let before = current::current(&source); + let after = current::current(edit.as_bytes()); + for write_limit in [17, 4096] { + let disk = |fail_at| disk::Disk { + visible: source.clone(), + durable: source.clone(), + operation: 0, + fail_at, + write_limit, + random: 946, + }; + let mut success = disk(None); + edit.commit(&mut success).unwrap(); + assert_eq!(success.durable, edit.as_bytes()); + for at in 1..=success.operation { + let mut interrupted = disk(Some(at)); + let failure = edit.commit(&mut interrupted).unwrap_err(); + let actual = current::current(&interrupted.durable); + assert!(actual == before || actual == after, "operation {at}"); + if failure.state == onestore::CommitState::NotCommitted { + assert_eq!(actual, before); + } + } + } +} + +#[test] +#[ignore = "exports public-API formatting candidates for cold native validation"] +fn export_native_formatting_candidates() { + use std::{fs, path::PathBuf}; + let output = PathBuf::from(std::env::var_os("ONESTORE_FORMAT_OUTPUT").unwrap()); + assert!(output.is_absolute()); + fs::create_dir(&output).unwrap(); + let source = onestore::create_section( + "format.one", + "Before café 東京 🦀 after", + "Formatting author", + ) + .unwrap(); + let (sid, id) = target(&source); + let mut manifest = Vec::new(); + let mut save = + |name: &str, source: &[u8], sid, id, range: std::ops::Range, attributes: &[A]| { + let prepared = + PreparedEdit::format(source, sid, id, range.clone(), attributes).unwrap(); + fs::write(output.join(format!("{name}.one")), prepared.as_bytes()).unwrap(); + manifest.push( + json!({"name":name,"space":sid,"object":id,"range":range,"attributes":attributes}), + ); + prepared.as_bytes().to_vec() + }; + save( + "partial-boolean", + &source, + sid, + id, + 2..18, + &[ + A::Bold(true), + A::Italic(true), + A::Underline(true), + A::Strike(true), + ], + ); + let colored = save( + "partial-font-color", + &source, + sid, + id, + 3..18, + &[ + A::Font("Arial".into()), + A::FontSize(13.5), + A::Color(Some([24, 96, 160])), + A::Highlight(Some([255, 255, 0])), + ], + ); + save( + "clear-color", + &colored, + sid, + id, + 6..14, + &[A::Color(None), A::Highlight(None)], + ); + let scripted = save("subscript", &source, sid, id, 0..23, &[A::Subscript(true)]); + save( + "superscript", + &scripted, + sid, + id, + 6..18, + &[A::Superscript(true)], + ); + let bold = save( + "bold", + &source, + sid, + id, + 0..23, + &[ + A::Bold(true), + A::Italic(true), + A::Underline(true), + A::Strike(true), + ], + ); + save( + "clear-boolean", + &bold, + sid, + id, + 7..14, + &[ + A::Bold(false), + A::Italic(false), + A::Underline(false), + A::Strike(false), + ], + ); + let empty = onestore::create_section("empty.one", "", "Author").unwrap(); + let (empty_sid, empty_id) = target(&empty); + let formatted = PreparedEdit::format( + &empty, + empty_sid, + empty_id, + 0..0, + &[A::FontSize(18.0), A::Italic(true)], + ) + .unwrap(); + let typed = PreparedEdit::text( + formatted.as_bytes(), + empty_sid, + empty_id, + 0..0, + "Typed café 🦀", + ) + .unwrap(); + fs::write(output.join("empty-then-type.one"), typed.as_bytes()).unwrap(); + let native = include_bytes!("../../../corpus/native-ink/20260905-ui/notebook/synthetic.one"); + let store = Store::parse(native).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let (native_sid, native_id) = document + .spaces + .iter() + .find_map(|(sid, s)| { + let r = &s.revisions[&s.contexts[&ExGuid::default()]]; + r.nodes.iter().find_map(|(id, n)| { + matches!(&n.kind,Kind::RichText{text,..} if text.starts_with("Fictitious:" )) + .then_some((*sid, *id)) + }) + }) + .unwrap(); + save( + "native-cross-runs", + native, + native_sid, + native_id, + 2..26, + &[A::Bold(false), A::Italic(true), A::Underline(true)], + ); + save( + "native-partial", + native, + native_sid, + native_id, + 3..8, + &[A::FontSize(14.0), A::Color(Some([16, 112, 48]))], + ); + save( + "native-font", + native, + native_sid, + native_id, + 0..27, + &[A::Font("Arial".into())], + ); + save("small-font", &source, sid, id, 0..23, &[A::FontSize(6.0)]); + save("large-font", &source, sid, id, 0..23, &[A::FontSize(130.0)]); + for points in [129.5, 130.0] { + save( + &format!("font-boundary-{points}"), + &source, + sid, + id, + 0..23, + &[A::FontSize(points)], + ); + } + fs::write(output.join("basic-baseline.one"), &source).unwrap(); + fs::write(output.join("native-baseline.one"), native).unwrap(); + fs::write( + output.join("manifest.json"), + serde_json::to_vec_pretty(&manifest).unwrap(), + ) + .unwrap(); +} diff --git a/crates/onestore/tests/insertion.rs b/crates/onestore/tests/insertion.rs new file mode 100644 index 0000000000000000000000000000000000000000..379b11a674a504f638b945b472cd855f1137e54b --- /dev/null +++ b/crates/onestore/tests/insertion.rs @@ -0,0 +1,377 @@ +#[path = "support/checkpoint.rs"] +mod checkpoint; +#[path = "support/current.rs"] +mod current; +#[path = "support/disk.rs"] +mod disk; + +use onestore::{ + ExGuid, Insertion, PreparedEdit, RevisionIndex, Store, + document::{Document, Kind}, +}; + +fn targets(source: &[u8]) -> (ExGuid, ExGuid, ExGuid, ExGuid, ExGuid) { + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let (sid, page) = document.pages().unwrap()[0]; + let space = &document.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let outline = view.nodes[&page] + .children + .iter() + .copied() + .find(|id| matches!(view.nodes[id].kind, Kind::Outline { .. })) + .unwrap(); + let paragraph = view.nodes[&outline].children[0]; + let text = view.nodes[¶graph].content[0]; + (sid, page, outline, paragraph, text) +} + +#[test] +fn paragraph_and_outline_insertions_publish_metadata_and_references_together() { + let source = onestore::create_section("insertion.one", "Original", "Original author").unwrap(); + let (sid, page, outline, paragraph, original_text) = targets(&source); + for (intent, expected_parent, expected_x, expected_y) in [ + ( + Insertion::paragraph(outline, Some(paragraph), "First 🦀\rSecond", "New author") + .unwrap(), + outline, + None, + None, + ), + ( + Insertion::outline(page, 144.0, 18.0, "First 🦀\rSecond", "New author").unwrap(), + page, + Some(144.0), + Some(18.0), + ), + ] { + let prepared = PreparedEdit::insert(&source, sid, &intent).unwrap(); + let store = Store::parse(prepared.as_bytes()).unwrap(); + assert_eq!( + store.header.transaction_count, + Store::parse(&source).unwrap().header.transaction_count + 1 + ); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + let document = Document::parse(&index).unwrap(); + let space = &document.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + assert!( + view.nodes[&expected_parent] + .children + .contains(&intent.object()) + ); + assert_eq!(view.nodes[&intent.object()].layout.x, expected_x); + assert_eq!(view.nodes[&intent.object()].layout.y, expected_y); + assert!( + matches!(&view.nodes[&original_text].kind, Kind::RichText { text, .. } if text == "Original") + ); + assert!( + matches!(&view.nodes[&intent.text_object()].kind, Kind::RichText { text, .. } if text == "First 🦀\rSecond") + ); + assert!( + matches!(&view.nodes[&page].kind, Kind::Page { alternate_title, .. } if alternate_title.as_deref() == Some("First 🦀")) + ); + assert!( + matches!(&view.nodes[&view.roots[&2]].kind, Kind::Metadata { title, .. } if title.as_deref() == Some("First 🦀")) + ); + assert!( + view.nodes + .values() + .any(|node| matches!(&node.kind, Kind::Author { name } if name.as_deref() == Some("New author"))) + ); + let runs = view.text_runs(intent.text_object()).unwrap(); + assert_eq!(runs.len(), 1); + assert_eq!(runs[0].format.font.as_deref(), Some("Calibri")); + assert_eq!(runs[0].format.font_size, Some(11.0)); + let previous_store = Store::parse(&source).unwrap(); + let previous = RevisionIndex::parse(&previous_store).unwrap(); + for (old_sid, old_space) in &previous.spaces { + for rid in old_space.revisions.keys() { + assert_eq!( + format!("{:?}", previous.resolve(*old_sid, *rid).unwrap()), + format!("{:?}", index.resolve(*old_sid, *rid).unwrap()) + ); + } + } + } +} + +#[test] +fn repeated_insertions_and_formatting_share_immutable_objects() { + let mut source = onestore::create_section("shared.one", "Original", "Same author").unwrap(); + let (sid, _, outline, _, _) = targets(&source); + let mut texts = Vec::new(); + for _ in 0..12 { + let insertion = + Insertion::paragraph(outline, None, "Repeated paragraph", "Same author").unwrap(); + source = PreparedEdit::insert(&source, sid, &insertion) + .unwrap() + .as_bytes() + .to_vec(); + source = PreparedEdit::format( + &source, + sid, + insertion.text_object(), + 1..8, + &[ + onestore::TextAttribute::Bold(true), + onestore::TextAttribute::FontSize(20.0), + ], + ) + .unwrap() + .as_bytes() + .to_vec(); + texts.push(insertion.text_object()); + } + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + let raw = index + .resolve(sid, index.spaces[&sid].labels[&(ExGuid::default(), 1)]) + .unwrap(); + let mut unique = std::collections::BTreeSet::new(); + let mut counts = Vec::new(); + for id in raw.reachable().unwrap() { + let object = &raw.objects[&id]; + if object.jcid & 0x100000 == 0 { + continue; + } + let onestore::ObjectData::Properties(bytes) = object.data else { + panic!() + }; + assert!( + unique.insert((object.jcid, bytes.to_vec())), + "Duplicate immutable object" + ); + counts.push((object.jcid, object.reference_count)); + } + counts.sort(); + assert_eq!(counts, [(0x120001, 26), (0x12004d, 12), (0x12004d, 25)]); + let document = Document::parse(&index).unwrap(); + let space = &document.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + for text in texts { + let runs = view.text_runs(text).unwrap(); + assert_eq!(runs.len(), 3); + assert_eq!(runs[1].format.bold, Some(true)); + assert_eq!(runs[1].format.font_size, Some(20.0)); + } +} + +#[test] +fn serialized_insertions_rebase_with_the_same_objects_and_preserve_remote_edits() { + let source = onestore::create_section("rebase.one", "Original", "Author").unwrap(); + let (sid, _, outline, paragraph, text) = targets(&source); + let intent = Insertion::paragraph(outline, Some(paragraph), "Inserted", "Author").unwrap(); + let encoded = serde_json::to_vec(&intent).unwrap(); + let restored: Insertion = serde_json::from_slice(&encoded).unwrap(); + assert_eq!(intent.object(), restored.object()); + assert_eq!(intent.text_object(), restored.text_object()); + let original_preparation = PreparedEdit::insert(&source, sid, &intent).unwrap(); + let remote = PreparedEdit::text(&source, sid, text, 0..0, "Remote ").unwrap(); + let updated = PreparedEdit::insert(remote.as_bytes(), sid, &restored).unwrap(); + let store = Store::parse(updated.as_bytes()).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let space = &document.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + assert_eq!( + view.nodes[&outline].children, + [restored.object(), paragraph] + ); + assert!( + matches!(&view.nodes[&text].kind, Kind::RichText { text, .. } if text == "Remote Original") + ); + assert!( + matches!(&view.nodes[&restored.text_object()].kind, Kind::RichText { text, .. } if text == "Inserted") + ); + assert!(PreparedEdit::insert(original_preparation.as_bytes(), sid, &restored).is_err()); + assert!(PreparedEdit::insert(updated.as_bytes(), sid, &restored).is_err()); +} + +#[test] +fn repositioning_preserves_intent_identity_and_kind() { + let source = onestore::create_section("placement.one", "Original", "Author").unwrap(); + let (sid, page, outline, paragraph, _) = targets(&source); + let p = Insertion::paragraph(outline, Some(paragraph), "Inserted", "Author").unwrap(); + let o = Insertion::outline(page, 144.0, 144.0, "Inserted", "Author").unwrap(); + assert!(p.reposition_outline(page, 72.0, 72.0).is_err()); + assert!(o.reposition_paragraph(outline, None).is_err()); + assert!(p.reposition_paragraph(ExGuid::default(), None).is_err()); + assert!(o.reposition_outline(page, f32::NAN, 72.0).is_err()); + for (original, moved) in [ + (&p, p.reposition_paragraph(outline, None).unwrap()), + (&o, o.reposition_outline(page, 288.0, 360.0).unwrap()), + ] { + let mut before = serde_json::to_value(original).unwrap(); + let mut after = serde_json::to_value(&moved).unwrap(); + for name in ["parent", "placement"] { + before.as_object_mut().unwrap().remove(name); + after.as_object_mut().unwrap().remove(name); + } + assert_eq!(before, after); + assert_eq!(original.object(), moved.object()); + assert_eq!(original.text_object(), moved.text_object()); + let restored: Insertion = + serde_json::from_value(serde_json::to_value(&moved).unwrap()).unwrap(); + let prepared = PreparedEdit::insert(&source, sid, &restored).unwrap(); + let store = Store::parse(prepared.as_bytes()).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + let doc = Document::parse(&index).unwrap(); + let space = &doc.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + if original == &p { + assert_eq!(view.nodes[&outline].children, [paragraph, moved.object()]); + } else { + assert_eq!( + ( + view.nodes[&moved.object()].layout.x, + view.nodes[&moved.object()].layout.y + ), + (Some(288.0), Some(360.0)) + ); + } + } +} + +#[test] +#[cfg(any(unix, windows))] +fn filesystem_insertion_uses_the_prepared_identity_and_rejects_stale_replay() { + use std::{fs, io::Write}; + let source = onestore::create_section("file.one", "Original", "Author").unwrap(); + let (sid, _, outline, _, _) = targets(&source); + let intent = Insertion::paragraph(outline, None, "File insertion", "Author").unwrap(); + let path = std::env::temp_dir().join(format!("onestore-insertion-{}.one", intent.object())); + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + .unwrap(); + file.write_all(&source).unwrap(); + file.sync_all().unwrap(); + drop(file); + let edit = PreparedEdit::insert(&source, sid, &intent).unwrap(); + let result = edit.commit_file(&path); + let written = onestore::read_file(&path).unwrap(); + let repeated = edit.commit_file(&path).unwrap_err(); + fs::remove_file(path).unwrap(); + result.unwrap(); + assert_eq!(written, edit.as_bytes()); + assert_eq!(repeated.state, onestore::CommitState::NotCommitted); + assert_eq!(repeated.error.kind(), std::io::ErrorKind::ResourceBusy); +} + +#[test] +fn anchors_targets_serialized_identities_and_text_are_validated_before_publication() { + let source = onestore::create_section("invalid.one", "Original", "Author").unwrap(); + let (sid, page, outline, paragraph, text) = targets(&source); + assert!(Insertion::outline(page, f32::NAN, 0.0, "Text", "Author").is_err()); + assert!(Insertion::outline(page, 0.0, f32::INFINITY, "Text", "Author").is_err()); + for content in ["a\0b", "a\nb", "a\u{fffc}b", "a\u{fddf}b"] { + assert!(Insertion::paragraph(outline, None, content, "Author").is_err()); + } + assert!(Insertion::paragraph(outline, None, "Text", "a\0b").is_err()); + for intent in [ + Insertion::paragraph(outline, Some(text), "Text", "Author").unwrap(), + Insertion::paragraph(text, None, "Text", "Author").unwrap(), + Insertion::paragraph(page, None, "Text", "Author").unwrap(), + Insertion::outline(paragraph, 0.0, 0.0, "Text", "Author").unwrap(), + ] { + assert!(PreparedEdit::insert(&source, sid, &intent).is_err()); + } + let intent = Insertion::paragraph(outline, None, "Text", "Author").unwrap(); + let created = PreparedEdit::insert(&source, sid, &intent).unwrap(); + let before_missing = + Insertion::paragraph(outline, Some(intent.object()), "Anchored", "Author").unwrap(); + assert!(PreparedEdit::insert(&source, sid, &before_missing).is_err()); + assert!(PreparedEdit::insert(created.as_bytes(), sid, &before_missing).is_ok()); + let mut encoded = serde_json::to_value(&intent).unwrap(); + encoded["parent"] = serde_json::to_value(intent.object()).unwrap(); + let collision: Insertion = serde_json::from_value(encoded).unwrap(); + assert!(PreparedEdit::insert(created.as_bytes(), sid, &collision).is_err()); +} + +#[test] +fn insertions_into_nested_paragraphs_and_native_table_cells_preserve_structure() { + let source = onestore::create_section("nested.one", "Original", "Author").unwrap(); + let (sid, _, outline, paragraph, _) = targets(&source); + let child = Insertion::paragraph(paragraph, None, "Nested", "Author").unwrap(); + let changed = PreparedEdit::insert(&source, sid, &child).unwrap(); + let store = Store::parse(changed.as_bytes()).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let space = &document.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + assert_eq!(view.nodes[&outline].children, [paragraph]); + assert_eq!(view.nodes[¶graph].children, [child.object()]); + let source = include_bytes!( + "../../../corpus/native/20260905-05/snapshots/07-table/notebook/synthetic.one" + ); + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let (sid, cell) = document + .spaces + .iter() + .find_map(|(sid, space)| { + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + view.nodes.iter().find_map(|(id, node)| { + matches!(node.kind, Kind::Cell { .. }).then_some((*sid, *id)) + }) + }) + .unwrap(); + let insertion = Insertion::paragraph(cell, None, "Added to cell", "Author").unwrap(); + let changed = PreparedEdit::insert(source, sid, &insertion).unwrap(); + current::current(changed.as_bytes()); +} + +#[test] +fn insertion_publication_faults_expose_only_complete_graphs_and_title_caches() { + let original = onestore::create_section("atomic.one", "Original", "Author").unwrap(); + let (sid, _, _, _, text) = targets(&original); + let checkpoint = checkpoint::pending(&original, sid, text, 0x14001d7a); + for source in [ + original.as_slice(), + include_bytes!("../../../corpus/append/round-01/tx-255/notebook/synthetic.one"), + &checkpoint, + ] { + let (sid, page, outline, paragraph, _) = targets(source); + for intent in [ + Insertion::paragraph(outline, Some(paragraph), "First 🦀", "New author").unwrap(), + Insertion::outline(page, 0.0, 0.0, "First 🦀", "New author").unwrap(), + ] { + let edit = PreparedEdit::insert(source, sid, &intent).unwrap(); + let before = current::current(source); + let after = current::current(edit.as_bytes()); + for write_limit in [17, 4096] { + let disk = |fail_at| disk::Disk { + visible: source.to_vec(), + durable: source.to_vec(), + operation: 0, + fail_at, + write_limit, + random: 945, + }; + let mut successful = disk(None); + edit.commit(&mut successful).unwrap(); + assert_eq!(successful.durable, edit.as_bytes()); + for at in 1..=successful.operation { + let mut interrupted = disk(Some(at)); + let failure = edit.commit(&mut interrupted).unwrap_err(); + let state = current::current(&interrupted.durable); + assert!(state == before || state == after, "interruption {at}"); + if failure.state == onestore::CommitState::NotCommitted { + assert_eq!(state, before); + } + if failure.state == onestore::CommitState::Committed { + assert_eq!(state, after); + } + } + } + } + } +} diff --git a/crates/onestore/tests/revisions.rs b/crates/onestore/tests/revisions.rs index 9b12b439b522d7aaf0ea25da326dbf82b673a2e6..8d49d368c9f9146c1c4008b16e5bc4c22c4b6cf4 100644 --- a/crates/onestore/tests/revisions.rs +++ b/crates/onestore/tests/revisions.rs @@ -3,6 +3,68 @@ use std::fs; const TABLE: &str = "../../corpus/native/20260905-05/snapshots/07-table/notebook/synthetic.one"; +#[test] +fn persisted_identities_preserve_native_byte_order_and_canonical_form() { + let text = "{00112233-4455-6677-8899-AABBCCDDEEFF},42"; + let id: ExGuid = text.parse().unwrap(); + assert_eq!( + id.guid, + [ + 0x33, 0x22, 0x11, 0, 0x55, 0x44, 0x77, 0x66, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, + 0xff + ] + ); + assert_eq!(id.n, 42); + assert_eq!(text.to_lowercase().parse::().unwrap(), id); + assert_eq!(id.to_string(), text); + let mut random = 7_u64; + for n in 0..1024 { + let guid = std::array::from_fn(|_| { + random = random.wrapping_mul(6364136223846793005).wrapping_add(1); + random.to_le_bytes()[0] + }); + let id = ExGuid { guid, n }; + assert_eq!(id.to_string().parse::().unwrap(), id); + assert_eq!( + serde_json::from_str::(&serde_json::to_string(&id).unwrap()).unwrap(), + id + ); + } + for id in [ + ExGuid::default(), + ExGuid { + guid: [255; 16], + n: u32::MAX, + }, + ] { + assert_eq!(id.to_string().parse::().unwrap(), id); + } + for at in 0..text.len() { + let mut changed = text.as_bytes().to_vec(); + changed[at] = b'?'; + assert!( + String::from_utf8(changed) + .unwrap() + .parse::() + .is_err() + ); + } + for changed in [ + text.replace(",42", ",+42"), + text.replace(",42", ",042"), + text.replace(",42", ",4294967296"), + text.replace(",42", ",-1"), + text.replace("00", "+0"), + text.replace("00", "é"), + "{00000000-0000-0000-0000-000000000000},1".to_owned(), + format!(" {text}"), + format!("{text} "), + ] { + assert!(changed.parse::().is_err(), "{changed}"); + assert!(serde_json::from_str::(&serde_json::to_string(&changed).unwrap()).is_err()); + } +} + #[test] fn native_encryption_remains_opaque() { let bytes = diff --git a/crates/onestore/tests/shared_snapshot.rs b/crates/onestore/tests/shared_snapshot.rs new file mode 100644 index 0000000000000000000000000000000000000000..22e81b09eb4e6bbc7b44dfa120cd80ddf3bd07c1 --- /dev/null +++ b/crates/onestore/tests/shared_snapshot.rs @@ -0,0 +1,420 @@ +#[path = "support/current.rs"] +mod current; +use current::current; + +#[path = "support/checkpoint.rs"] +mod checkpoint; +#[path = "support/trace.rs"] +mod trace; + +use onestore::read_snapshot; +use onestore::{ + ExGuid, RevisionIndex, Store, + document::{Document, Kind}, +}; +use std::{fs, io}; +use trace::{Event, Trace}; + +fn target(bytes: &[u8]) -> (ExGuid, ExGuid, u32) { + let store = Store::parse(bytes).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + doc.spaces + .iter() + .find_map(|(sid, space)| { + space.revisions[&space.contexts[&ExGuid::default()]] + .nodes + .iter() + .find_map(|(oid, node)| { + if let Kind::RichText { text, .. } = &node.kind + && (text.starts_with("Fictitious") || text.starts_with("Transaction")) + { + return Some((*sid, *oid, text.encode_utf16().count() as u32)); + } + None + }) + }) + .unwrap() +} + +#[test] +fn version_cached_readers_cannot_miss_a_completed_publication() { + for path in [ + "native/20260905-05/snapshots/03-format-unicode/notebook/synthetic.one", + "append/round-01/tx-255/notebook/synthetic.one", + ] { + let source = fs::read(format!("../../corpus/{path}")).unwrap(); + let (sid, oid, end) = target(&source); + let mut trace = Trace { + bytes: source.clone(), + events: Vec::new(), + }; + onestore::commit_text(&mut trace, &source, sid, oid, end..end, " [cached reader]").unwrap(); + let final_content = current(&trace.bytes); + let mut visible = source; + for event in &trace.events { + let Event::Write(offset, bytes) = event else { + continue; + }; + visible.resize(visible.len().max(offset + bytes.len()), 0); + for (index, byte) in bytes.iter().enumerate() { + visible[offset + index] = *byte; + if (212..252).contains(&(offset + index)) { + let cached_content = current(&visible); + let refreshed = if visible[212..252] == trace.bytes[212..252] { + cached_content + } else { + current(&trace.bytes) + }; + assert_eq!( + refreshed, + final_content, + "{path}: cached header at {}", + offset + index + ); + } + } + } + } +} + +#[test] +fn published_snapshots_survive_interleaved_commit_io() { + let cases = [ + ( + "unicode", + "native/20260905-05/snapshots/03-format-unicode/notebook/synthetic.one", + ), + ( + "attachment", + "native/20260905-05/snapshots/06-attachment/notebook/synthetic.one", + ), + ( + "rollover-256", + "append/round-01/tx-255/notebook/synthetic.one", + ), + ( + "rollover-65536", + "append/round-01/tx-65535/notebook/synthetic.one", + ), + ( + "checkpoint", + "native/20260905-05/snapshots/03-format-unicode/notebook/synthetic.one", + ), + ]; + for (name, path) in cases { + let mut source = fs::read(format!("../../corpus/{path}")).unwrap(); + let (sid, oid, _) = target(&source); + if name == "checkpoint" { + source = checkpoint::pending(&source, sid, oid, 0x14001d7a); + } + let (_, _, end) = target(&source); + let before = current(&source); + let mut trace = Trace { + bytes: source.clone(), + events: Vec::new(), + }; + onestore::commit_text(&mut trace, &source, sid, oid, end..end, " [reader café 🦀]") + .unwrap(); + let after = current(&trace.bytes); + assert_ne!(before, after); + let mut writes = Vec::new(); + for event in &trace.events { + if let Event::Write(offset, bytes) = event { + let piece = if *offset < 1024 { 17 } else { 4096 }; + writes.extend( + bytes + .chunks(piece) + .enumerate() + .map(|(i, bytes)| (offset + i * piece, bytes)), + ); + } + } + let mut accepted = [0; 2]; + let mut retried = 0; + let mut interleaved = 0; + for run in 0..writes.len() + 1 + 512 { + let paused = run <= writes.len(); + let mut step = if paused { run } else { 0 }; + let mut visible = source.clone(); + for &(offset, bytes) in &writes[..step] { + visible.resize(visible.len().max(offset + bytes.len()), 0); + visible[offset..offset + bytes.len()].copy_from_slice(bytes); + } + let mut seed = run as u64 + 1; + let mut read_calls = 0; + let mut overlapped = false; + let read_limit = [17, 193, 4096, 65536][run % 4]; + let result = read_snapshot( + |offset, output| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + if !paused { + let count = if seed.is_multiple_of(11) { + writes.len() + } else { + (seed % 4) as usize + }; + let end = writes.len().min(step + count); + for &(at, bytes) in &writes[step..end] { + visible.resize(visible.len().max(at + bytes.len()), 0); + visible[at..at + bytes.len()].copy_from_slice(bytes); + } + overlapped |= read_calls > 0 && end > step; + step = end; + } + read_calls += 1; + let offset = offset as usize; + let count = output + .len() + .min(read_limit) + .min(visible.len().saturating_sub(offset)); + if count != 0 { + output[..count].copy_from_slice(&visible[offset..offset + count]); + } + Ok(count) + }, + trace.bytes.len(), + ); + interleaved += usize::from(overlapped); + if paused && (run == 0 || run == writes.len()) { + assert!( + matches!(&result, Ok(Some(_))), + "{name}: quiescent run {run}" + ); + } + match result { + Ok(Some(bytes)) => { + let checked = std::panic::catch_unwind(|| { + let observed = current(&bytes); + assert!( + observed == before || observed == after, + "{name}: run {run}, write step {step}" + ); + if run == writes.len() { + assert_eq!(observed, after); + } + observed == after + }); + match checked { + Ok(new) => accepted[usize::from(new)] += 1, + Err(failure) => { + let time = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = std::path::PathBuf::from(format!( + "../../evidence/m9/read-interleaving-failure-{name}-{run}-{time}" + )); + fs::create_dir_all(&path).unwrap(); + fs::write(path.join("source.one"), &source).unwrap(); + fs::write(path.join("observed.one"), &bytes).unwrap(); + fs::write( + path.join("replay.json"), + serde_json::to_vec(&serde_json::json!({ + "run": run, "read_limit": read_limit, "writes": writes, + })) + .unwrap(), + ) + .unwrap(); + eprintln!("Replay: {}", path.display()); + std::panic::resume_unwind(failure); + } + } + } + Ok(None) => retried += 1, + Err(error) + if matches!( + error.kind(), + io::ErrorKind::UnexpectedEof | io::ErrorKind::InvalidData + ) => + { + retried += 1 + } + Err(error) => panic!("{name}: run {run}: {error}"), + } + } + assert!( + accepted[0] > 0 && accepted[1] > 0 && interleaved > 0 && retried > 0, + "{name}" + ); + println!( + "{name}: old={}, new={}, retry={retried}, overlap={interleaved}", + accepted[0], accepted[1] + ); + } +} + +#[test] +fn snapshot_rejects_short_io_and_unbounded_allocation() { + let source = onestore::create_section("test.one", "read", "test").unwrap(); + let mut calls = 0; + let result = read_snapshot( + |offset, output| { + calls += 1; + if calls == 1 { + return Err(io::ErrorKind::Interrupted.into()); + } + let offset = offset as usize; + let count = output.len().min(7).min(source.len().saturating_sub(offset)); + if count != 0 { + output[..count].copy_from_slice(&source[offset..offset + count]); + } + Ok(count) + }, + source.len(), + ) + .unwrap() + .unwrap(); + assert_eq!(current(&source), current(&result)); + assert!(calls > source.len() / 7); + let error = read_snapshot(|_, output| Ok(output.len() + 1), source.len()).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + let error = read_snapshot(|_, _| Ok(0), source.len()).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::UnexpectedEof); + let error = read_snapshot( + |_, output| { + output.copy_from_slice(&source[..1024]); + Ok(1024) + }, + 1023, + ) + .unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); +} + +#[test] +fn native_tail_truncation_before_header_update_is_retried() { + let compact = onestore::create_section("test.one", "Transaction retained", "test").unwrap(); + let mut expanded = compact.clone(); + expanded.resize(compact.len() + 216, 0); + let length = expanded.len() as u64; + expanded[196..204].copy_from_slice(&length.to_le_bytes()); + assert_eq!(current(&expanded), current(&compact)); + for block in [17, 193, 65536] { + for trigger in [0, 1024usize.div_ceil(block)] { + let mut visible = expanded.clone(); + let mut calls = 0; + let result = read_snapshot( + |offset, output| { + if calls == trigger { + visible.truncate(compact.len()); + } + calls += 1; + let offset = offset as usize; + let count = output + .len() + .min(block) + .min(visible.len().saturating_sub(offset)); + if count != 0 { + output[..count].copy_from_slice(&visible[offset..offset + count]); + } + Ok(count) + }, + expanded.len(), + ) + .unwrap(); + assert!( + result.is_none(), + "accepted a snapshot with a stale expected length" + ); + } + } + let result = read_snapshot( + |offset, output| { + let offset = offset as usize; + let count = output.len().min(compact.len().saturating_sub(offset)); + if count != 0 { + output[..count].copy_from_slice(&compact[offset..offset + count]); + } + Ok(count) + }, + expanded.len(), + ) + .unwrap() + .unwrap(); + assert_eq!(result, compact); +} + +#[test] +fn unpublished_append_remains_available_for_retry() { + let source = onestore::create_section("test.one", "Transaction before", "test").unwrap(); + let (sid, oid, end) = target(&source); + let mut trace = Trace { + bytes: source.clone(), + events: Vec::new(), + }; + onestore::commit_text(&mut trace, &source, sid, oid, end..end, " abandoned").unwrap(); + let Event::Write(offset, append) = &trace.events[0] else { + panic!() + }; + assert_eq!(*offset, source.len()); + for length in [1, 17, append.len()] { + let mut persisted = source.clone(); + persisted.extend_from_slice(&append[..length]); + let snapshot = read_snapshot( + |offset, output| { + let offset = offset as usize; + let count = output.len().min(persisted.len().saturating_sub(offset)); + output[..count].copy_from_slice(&persisted[offset..offset + count]); + Ok(count) + }, + persisted.len(), + ) + .unwrap() + .unwrap(); + assert_eq!(snapshot, persisted); + assert_eq!(current(&snapshot), current(&source)); + let mut retry = Trace { + bytes: persisted, + events: Vec::new(), + }; + onestore::commit_text(&mut retry, &snapshot, sid, oid, end..end, " retry").unwrap(); + assert_ne!(current(&retry.bytes), current(&source)); + } +} + +#[test] +fn header_comparison_cannot_replace_maintenance_exclusion() { + let mut source = onestore::create_section("test.one", "AAAA BBBB", "test").unwrap(); + source[212..228].fill(9); + source[236..252].fill(7); + let encoded: Vec<_> = "AAAA BBBB" + .encode_utf16() + .flat_map(u16::to_le_bytes) + .collect(); + let offset = source + .windows(encoded.len()) + .position(|bytes| bytes == encoded) + .unwrap(); + let mut changed = source.clone(); + let replacement: Vec<_> = "ZZZZ YYYY" + .encode_utf16() + .flat_map(u16::to_le_bytes) + .collect(); + changed[offset..offset + replacement.len()].copy_from_slice(&replacement); + let before = current(&source); + let after = current(&changed); + assert_ne!(before, after); + let mut visible = &source; + let result = read_snapshot( + |at, output| { + let at = at as usize; + if at >= offset + 8 { + visible = &changed; + } + let count = output.len().min(2).min(visible.len().saturating_sub(at)); + if count != 0 { + output[..count].copy_from_slice(&visible[at..at + count]); + } + Ok(count) + }, + source.len(), + ) + .unwrap() + .unwrap(); + let hybrid = current(&result); + assert_ne!(hybrid, before); + assert_ne!(hybrid, after); +} diff --git a/crates/onestore/tests/support/current.rs b/crates/onestore/tests/support/current.rs new file mode 100644 index 0000000000000000000000000000000000000000..7220c60fd58f090e3b25597377740ca812c95841 --- /dev/null +++ b/crates/onestore/tests/support/current.rs @@ -0,0 +1,26 @@ +use onestore::{ExGuid, RevisionIndex, Store, document::Document}; +use std::collections::BTreeMap; + +pub fn current(bytes: &[u8]) -> BTreeMap { + let store = Store::parse(bytes).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + Document::parse(&index).unwrap(); + index + .spaces + .iter() + .map(|(sid, space)| { + let rid = space.labels[&(ExGuid::default(), 1)]; + let revision = index.resolve(*sid, rid).unwrap(); + for object in revision.objects.values() { + if let Some(onestore::FileDataReference::Internal(guid)) = + object.file_reference().unwrap() + { + store.file_data(guid).unwrap(); + } + } + (*sid, format!("{revision:?}")) + }) + .collect() +} diff --git a/crates/onestore/tests/support/trace.rs b/crates/onestore/tests/support/trace.rs new file mode 100644 index 0000000000000000000000000000000000000000..5d8a79cf24266d8d2c80c821a363dcb9597d1159 --- /dev/null +++ b/crates/onestore/tests/support/trace.rs @@ -0,0 +1,36 @@ +use onestore::CommitIo; +use std::io; + +pub enum Event { + Write(usize, Vec), + Flush, +} + +pub struct Trace { + pub bytes: Vec, + pub events: Vec, +} + +impl CommitIo for Trace { + fn read_at(&mut self, offset: u64, output: &mut [u8]) -> io::Result { + let offset = offset as usize; + let count = output.len().min(self.bytes.len().saturating_sub(offset)); + if count != 0 { + output[..count].copy_from_slice(&self.bytes[offset..offset + count]); + } + Ok(count) + } + fn write_at(&mut self, offset: u64, bytes: &[u8]) -> io::Result { + let offset = offset as usize; + let count = bytes.len().min(4096); + self.bytes.resize(self.bytes.len().max(offset + count), 0); + self.bytes[offset..offset + count].copy_from_slice(&bytes[..count]); + self.events + .push(Event::Write(offset, bytes[..count].to_vec())); + Ok(count) + } + fn flush(&mut self) -> io::Result<()> { + self.events.push(Event::Flush); + Ok(()) + } +} diff --git a/readme.md b/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..61a6fb7e90326c57524e23e776edc4eb76f9e181 --- /dev/null +++ b/readme.md @@ -0,0 +1 @@ +# Snowbound — Freeform note taking diff --git a/tools/codex_usage_report.example.json b/tools/codex_usage_report.example.json new file mode 100644 index 0000000000000000000000000000000000000000..ce4d61158f7a0178d789b7024e83fde0b696b363 --- /dev/null +++ b/tools/codex_usage_report.example.json @@ -0,0 +1,18 @@ +{ + "threadIds": ["01a078fb-d5d2-7ad0-97f0-d14db4e4c94e"], + "sources": [ + "/Users/clo/.codex/sessions/2026/09/06/rollout-2026-09-06T16-09-25-01a078fb-d5d2-7ad0-97f0-d14db4e4c94e.jsonl" + ], + "boundaries": [ + { + "name": "Planning discussion", + "kind": "discussion", + "end": "2026-09-07T01:28:56Z" + }, + { + "name": "Canvas goal", + "kind": "goal", + "start": "2026-09-07T01:28:56Z" + } + ] +} diff --git a/tools/codex_usage_report.mjs b/tools/codex_usage_report.mjs new file mode 100644 index 0000000000000000000000000000000000000000..45cd27ff308511f6c86df1d755459e533085ea16 --- /dev/null +++ b/tools/codex_usage_report.mjs @@ -0,0 +1,129 @@ +#!/usr/bin/env node + +import { mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import { homedir } from "node:os"; +import { dirname, join, resolve } from "node:path"; + +const rates = { input: 10, cached: 1, output: 50 }; + +function usageCost(usage) { + return ((usage.input_tokens - usage.cached_input_tokens) * rates.input + + usage.cached_input_tokens * rates.cached + + usage.output_tokens * rates.output) / 1_000_000; +} + +function duration(ms) { + const seconds = Math.round(ms / 1000); + const hours = Math.floor(seconds / 3600); + const minutes = Math.floor((seconds % 3600) / 60); + const remainder = seconds % 60; + return hours ? `${hours}h ${minutes}m` : minutes ? `${minutes}m ${remainder}s` : `${remainder}s`; +} + +function unionDuration(intervals) { + const merged = []; + for (const interval of intervals.sort((a, b) => a[0] - b[0])) { + const previous = merged.at(-1); + if (previous && interval[0] <= previous[1]) previous[1] = Math.max(previous[1], interval[1]); + else merged.push([...interval]); + } + return merged.reduce((total, [start, end]) => total + end - start, 0); +} + +async function filesBelow(root) { + const entries = await readdir(root, { withFileTypes: true }); + const nested = await Promise.all(entries.map((entry) => { + const path = join(root, entry.name); + return entry.isDirectory() ? filesBelow(path) : entry.name.endsWith(".jsonl") ? [path] : []; + })); + return nested.flat(); +} + +async function loadEvents(paths) { + const events = []; + for (const path of paths) { + for (const line of (await readFile(path, "utf8")).trim().split("\n")) { + if (!line) continue; + try { events.push(JSON.parse(line)); } catch { /* Ignore a partially-written final line. */ } + } + } + return events; +} + +function snapshotBefore(records, time) { + return records.filter((record) => Date.parse(record.timestamp) <= time).at(-1)?.payload.thread_token_usage; +} + +function taskRows(events, start, end) { + const tasks = new Map(); + for (const event of events) { + if (event.type !== "event_msg" || !["task_started", "task_complete"].includes(event.payload.type)) continue; + const prior = tasks.get(event.payload.turn_id) ?? {}; + tasks.set(event.payload.turn_id, { ...prior, ...event.payload }); + } + return [...tasks.values()].filter((task) => task.started_at >= start && task.started_at < end && task.duration_ms); +} + +function toolIntervals(events, start, end) { + return events.flatMap((event) => { + const item = event.type === "event_msg" && event.payload.type === "item_completed" ? event.payload.item : undefined; + if (!item || !["CommandExecution", "McpToolCall"].includes(item.type)) return []; + const milliseconds = (item.duration?.secs ?? 0) * 1000 + (item.duration?.nanos ?? 0) / 1_000_000; + const finished = Date.parse(event.timestamp); + return milliseconds && finished >= start && finished < end ? [[finished - milliseconds, finished]] : []; + }); +} + +function segment(events, records, boundary, previous, finalTime) { + const start = boundary.start ? Date.parse(boundary.start) : -Infinity; + const end = boundary.end ? Date.parse(boundary.end) : finalTime; + const tasks = taskRows(events, start, end); + const tools = toolIntervals(events, start, end); + const finish = snapshotBefore(records, end); + const begin = previous ?? snapshotBefore(records, start); + return { + name: boundary.name, + kind: boundary.kind ?? "segment", + cost: finish ? usageCost(finish) - (begin ? usageCost(begin) : 0) : null, + durationMs: tasks.reduce((total, task) => total + task.duration_ms, 0), + apiTtftMs: tasks.reduce((total, task) => total + (task.time_to_first_token_ms ?? 0), 0), + toolWallMs: unionDuration(tools), + toolCoreMs: tools.reduce((total, [startTime, endTime]) => total + endTime - startTime, 0), + }; +} + +function chart(segments) { + const maximum = Math.max(...segments.map((segment) => segment.cost ?? 0), 1); + return segments.map((segment) => `
${escapeHtml(segment.name)}
${segment.cost === null ? "—" : `$${segment.cost.toFixed(2)}`}
`).join("\n"); +} + +function escapeHtml(value) { + return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">"); +} + +function html(report) { + const rows = report.segments.map((segment) => `${escapeHtml(segment.name)}${segment.kind}${segment.cost === null ? "—" : `$${segment.cost.toFixed(2)}`}${duration(segment.durationMs)}${duration(segment.toolWallMs)}${duration(segment.toolCoreMs)}≥${duration(segment.apiTtftMs)}`).join("\n"); + return `Codex usage report

Codex usage report

API-equivalent rate: Astra standard ($10/M uncached input, $1/M cached input, $50/M output). Tool core time counts parallel calls separately; API TTFT is a lower bound.

Cost

${chart(report.segments)}

Segments

${rows}
SegmentTypeCostDurationTool wallTool coreRecorded API
`; +} + +function usage() { + console.error("Usage: node tools/codex_usage_report.mjs report.json [output.html]"); + process.exit(1); +} + +const [configPath, outputPath] = process.argv.slice(2); +if (!configPath) usage(); +const config = JSON.parse(await readFile(resolve(configPath), "utf8")); +const codexHome = config.codexHome ?? join(homedir(), ".codex"); +const paths = config.sources ?? (await Promise.all(["sessions", "archived_sessions"].map((name) => filesBelow(join(codexHome, name))))).flat(); +const events = await loadEvents(paths); +const selected = events.filter((event) => event.type !== "token_usage_record" || config.threadIds.includes(event.payload.thread_id)); +const records = selected.filter((event) => event.type === "token_usage_record").sort((a, b) => Date.parse(a.timestamp) - Date.parse(b.timestamp)); +if (!records.length) throw new Error("No token records matched config.threadIds."); +const finalTime = Date.parse(records.at(-1).timestamp); +const segments = config.boundaries.map((boundary, index) => segment(selected, records, boundary, index ? snapshotBefore(records, Date.parse(config.boundaries[index - 1].end ?? new Date(finalTime).toISOString())) : undefined, finalTime)); +const report = { threadIds: config.threadIds, total: usageCost(records.at(-1).payload.thread_token_usage), segments }; +if (outputPath) { + await mkdir(dirname(resolve(outputPath)), { recursive: true }); + await writeFile(resolve(outputPath), html(report)); +} else console.log(JSON.stringify(report, null, 2)); diff --git a/tools/concurrent_rust.py b/tools/concurrent_rust.py index f303c8190fd938faeddd2a47de6be530a3b63f27..50eb6a182f4bdb530d425efb80e02a553568431f 100644 --- a/tools/concurrent_rust.py +++ b/tools/concurrent_rust.py @@ -70,21 +70,31 @@ def verify(logs, initial_transaction, writers, operations, edit=False): @contextmanager -def running_clients(output, source, writers, readers, operations, seed, timeout=600, edit=False): +def running_clients(output, source, writers, readers, operations, seed, timeout=600, edit=False, executable=CLIENT, environment=None, reader_executable=None): + if not 0 < timeout < 2**64 / 1000: + raise ValueError('Choose a finite positive client timeout.') + timeout_ms = int(timeout * 1000) + if not 0 < timeout_ms < 2**64: + raise ValueError('Client timeout does not fit the subprocess clock.') + environment = {**(os.environ if environment is None else environment), 'ONESTORE_CLIENT_TIMEOUT_MS': str(timeout_ms)} start, stop = output / 'start', output / 'stop' - (output / 'clients.json').write_text(json.dumps({'writers': writers, 'readers': readers, 'operations': operations, - 'seed': seed, 'edit': edit, 'client_sha256': hashlib.sha256(CLIENT.read_bytes()).hexdigest()}, indent=2)) + manifest = {'writers': writers, 'readers': readers, 'operations': operations, + 'seed': seed, 'edit': edit, 'timeout_ms': timeout_ms, 'client_sha256': hashlib.sha256(executable.read_bytes()).hexdigest(), 'executable': str(executable)} + if reader_executable is not None: + manifest.update(reader_executable=str(reader_executable), reader_sha256=hashlib.sha256(reader_executable.read_bytes()).hexdigest()) + (output / 'clients.json').write_text(json.dumps(manifest, indent=2)) processes = {} streams = [] try: for mode, count in [('write', writers), ('read', readers)]: + client = reader_executable if mode == 'read' and reader_executable is not None else executable for i in range(count): actor = mode[0] + str(i) out = (output / f'{actor}.jsonl').open('w') err = (output / f'{actor}.stderr').open('w') streams.extend([out, err]) - processes[actor] = subprocess.Popen([CLIENT, 'edit' if edit and mode == 'write' else mode, source, actor, str(operations), start, stop, - str(seed + i + (10000 if mode == 'read' else 0))], stdout=out, stderr=err) + processes[actor] = subprocess.Popen([client, 'edit' if edit and mode == 'write' else mode, source, actor, str(operations), start, stop, + str(seed + i + (10000 if mode == 'read' else 0))], stdout=out, stderr=err, env=environment) deadline = time.monotonic() + timeout while not all((output / f'{actor}.jsonl').stat().st_size for actor in processes): if any(p.poll() is not None for p in processes.values()) or time.monotonic() > deadline: diff --git a/tools/crash_recovery.py b/tools/crash_recovery.py index 3c371a3bacbc9f35731f87fd49dcdb45fe6af16a..e73874d7d2a2a5821dd08a2f42ae30aa4baa53a0 100644 --- a/tools/crash_recovery.py +++ b/tools/crash_recovery.py @@ -1,12 +1,22 @@ """Independent append-intent accounting across an abrupt storage interruption.""" from collections import Counter import re +from document_model import ordered_pages, walk + + +def active_text(model): + (_, _, revision, page), = ordered_pages(model) + text, = [node['kind']['text'] for _, node in walk(revision, page) + if node['kind']['type'] == 'RichText' and node['kind']['text'].startswith('Concurrent edits:')] + return text def verify_text(baseline, current, logs): events = [event for rows in logs.values() for event in rows] intents = {event['token'] for event in events if event['event'] == 'intent'} possible = set() + versions = {baseline} + predecessors = set() for rows in logs.values(): outcomes = {event['attempt']: event for event in rows if event['event'] in ('commit', 'retry', 'commit_error')} for event in rows: @@ -14,12 +24,17 @@ def verify_text(baseline, current, logs): outcome = outcomes.get(event['attempt']) if outcome is None or outcome['event'] == 'commit' or (outcome['event'] == 'commit_error' and outcome['state'] != 'NotCommitted'): possible.add(event['token']) + assert event['replacement'] == event['token'] and event['range'] == [len(event['before'].encode('utf-16-le')) // 2] * 2 + predecessors.add(event['before']) + versions.add(event['before'] + event['token']) + assert predecessors <= versions, 'A publication does not follow recorded history' acknowledged = [event for event in events if event['event'] == 'commit' or (event['event'] == 'commit_error' and event['state'] == 'Committed')] assert len({event['token'] for event in acknowledged}) == len(acknowledged), 'An edit was acknowledged twice' def tokens(text): assert text.startswith(baseline), 'Previously retained content changed' + assert text in versions, 'Text is not a recorded publication result' suffix = text[len(baseline):] found = re.findall(r' \[w\d+:\d+\]', suffix) assert ''.join(found) == suffix, 'Unexpected or partially persisted text' diff --git a/tools/diagnostic/editor.css b/tools/diagnostic/editor.css new file mode 100644 index 0000000000000000000000000000000000000000..50e395b1e1502ed094ce04fe85e2937bd929e093 --- /dev/null +++ b/tools/diagnostic/editor.css @@ -0,0 +1,24 @@ +.diagnostic{position:sticky;top:0;z-index:10;display:flex;align-items:center;gap:12px;flex-wrap:wrap;margin:-30px -40px 24px;padding:12px 20px;background:#f2f6fa;border-bottom:1px solid #aab9c8;font:13px/1.5 system-ui} +.diagnostic button,.edit-actions button{font:inherit;padding:6px 12px;cursor:pointer} +.diagnostic [role=status]{flex-basis:100%} +.diagnostic [role=status]:empty{display:none} +.editing [data-text-object]{cursor:pointer;outline:1px dashed #5182b1;outline-offset:2px} +.editing [data-text-object]:hover,.editing [data-text-object]:focus{outline:2px solid #175bb2} +.editing [data-text-object]:empty::before{content:'Empty text';color:#666;font:12px system-ui} +dialog{width:min(720px,90vw);max-height:90vh;overflow:auto;border:1px solid #8a9aab;border-radius:6px;padding:24px;color:#000;background:#fff;font:15px/1.5 system-ui} +dialog::backdrop{background:#0005} +dialog h2{margin:0 0 12px}dialog label{display:block;margin-top:16px;font-weight:600} +dialog textarea{display:block;width:100%;font:16px/1.6 system-ui;resize:vertical;min-height:140px} +dialog [hidden]{display:none} +dialog fieldset{margin:16px 0;padding:12px;border:1px solid #c1ccd6} +dialog fieldset label{margin-top:8px} +dialog input,dialog select,.diagnostic select{font:inherit;padding:4px;max-width:100%} +dialog select{display:block} +.format-grid{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:8px} +#coordinates:not([hidden]){display:flex;gap:20px} +#coordinates input{width:120px} +#format-range{margin:0} +dialog input[type=color]{width:64px;height:32px;vertical-align:middle} +.edit-actions{display:flex;justify-content:flex-end;gap:12px;margin-top:20px} +button:disabled{cursor:default} +@media(max-width:750px){.diagnostic{margin:-20px -20px 20px}} diff --git a/tools/diagnostic/editor.js b/tools/diagnostic/editor.js new file mode 100644 index 0000000000000000000000000000000000000000..0e3add9db1890afeffb643f732a1fae8f9f98ef9 --- /dev/null +++ b/tools/diagnostic/editor.js @@ -0,0 +1,246 @@ +const route = location.pathname.match(/^\/g\/(\d+)\/report\/(page-\d+\.html)$/); +const bar = document.createElement('aside'); +bar.className = 'diagnostic'; +bar.innerHTML = `Notebook copy + + + + + Latest notebook + `; +document.querySelector('main').prepend(bar); +const status = document.querySelector('#editor-status'); +const mode = document.querySelector('#edit-mode'); +for (const control of bar.querySelectorAll('button, select')) control.disabled = !route; +const dialog = document.createElement('dialog'); +dialog.setAttribute('aria-labelledby', 'edit-heading'); +dialog.innerHTML = `

Edit text

+

+
Paragraph placement + + +
+
Outline position + + +
+ + +
Character formatting +

+
+ + + + +
+

+ +
+
`; +document.body.append(dialog); +for (const name of ['Bold', 'Italic', 'Underline', 'Strike', 'Superscript', 'Subscript']) { + const label = document.createElement('label'); + label.textContent = name === 'Strike' ? 'Strikethrough' : name; + const select = document.createElement('select'); + select.dataset.attribute = name; + for (const [value, text] of [['', 'Keep'], ['true', 'On'], ['false', 'Off']]) select.add(new Option(text, value)); + label.append(select); + dialog.querySelector('.format-grid').append(label); +} +const draft = document.querySelector('#replacement'); +const save = document.querySelector('#save-edit'); +const cancel = document.querySelector('#cancel-edit'); +const message = document.querySelector('#save-status'); +const inspect = document.querySelector('#inspect-latest'); +const parent = document.querySelector('#parent'); +const before = document.querySelector('#before'); +let selected = null; +let opening = false; +let submitting = false; +let initial = ''; + +mode.addEventListener('click', () => { + const editing = document.body.classList.toggle('editing'); + mode.setAttribute('aria-pressed', String(editing)); + mode.textContent = editing ? 'Stop editing' : 'Edit text'; + for (const element of document.querySelectorAll('[data-text-object]')) { + if (editing) { + element.tabIndex = 0; + element.setAttribute('role', 'button'); + element.setAttribute('aria-label', 'Edit text: ' + element.textContent); + } else { + element.removeAttribute('tabindex'); + element.removeAttribute('role'); + element.removeAttribute('aria-label'); + } + } + status.textContent = editing ? 'Select a text run.' : ''; +}); + +function showEdit(selection, text) { + dialog.querySelector('form').reset(); + selected = selection; + const action = selection.action; + const titles = {text: 'Replace text', format: 'Format text', paragraph: 'Add paragraph', outline: 'Add outline'}; + document.querySelector('#edit-heading').textContent = titles[action]; + document.querySelector('#edit-hint').textContent = { + text: 'Replacement text keeps this run’s formatting.', + format: 'Select part of the text or leave the whole run selected. Unchanged attributes keep their current values.', + paragraph: 'Append to a container or insert before one of its children. Line breaks stay inside the new paragraph.', + outline: 'Add a text container to this page. Line breaks stay inside its first paragraph.' + }[action]; + for (const [id, visible] of [['placement', action === 'paragraph'], ['coordinates', action === 'outline'], ['formatting', action === 'format']]) { + const field = document.getElementById(id); + field.hidden = !visible; + field.disabled = !visible; + } + document.querySelector('#author-label').hidden = !['paragraph', 'outline'].includes(action); + draft.value = text; + draft.readOnly = action === 'format'; + save.disabled = false; + cancel.disabled = false; + inspect.hidden = true; + message.textContent = ''; + status.textContent = ''; + dialog.showModal(); + draft.focus(); + draft.setSelectionRange(0, text.length); + document.querySelector('#format-range').textContent = 'Selected: ' + text.length + ' UTF-16 units'; + initial = JSON.stringify(requestBody()); +} + +async function openEdit(element) { + if (opening) return; + opening = true; + status.textContent = 'Checking…'; + const selection = {generation: Number(route[1]), page: route[2], + object: element.dataset.textObject, run: Number(element.dataset.run)}; + try { + const result = await (await fetch('/api/run?' + new URLSearchParams(selection))).json(); + if (!result.ok) { + status.textContent = 'Edit not supported. ' + result.error; + return; + } + showEdit({...selection, action: document.querySelector('#text-action').value}, result.text); + } catch { + status.textContent = 'Unable to check this text. Reconnect to the diagnostic server.'; + } finally { + opening = false; + } +} + +for (const action of ['paragraph', 'outline']) document.querySelector('#add-' + action).addEventListener('click', async () => { + if (opening) return; + opening = true; + status.textContent = 'Loading…'; + const selection = {generation: Number(route[1]), page: route[2], action}; + try { + const result = await (await fetch('/api/page?' + new URLSearchParams(selection))).json(); + if (!result.ok) throw new Error(result.error); + if (action === 'paragraph' && !result.targets.length) throw new Error('Add an outline before adding a paragraph.'); + parent.replaceChildren(...result.targets.map(target => new Option(target.label, target.object))); + parent.onchange = () => { + const target = result.targets.find(target => target.object === parent.value); + before.replaceChildren(new Option('Append at end', ''), ...(target?.children || []).map(child => new Option(child.label, child.object))); + }; + parent.onchange(); + showEdit({...selection, object: result.object}, ''); + } catch (error) { + status.textContent = 'Unable to add content. ' + error.message; + } finally { + opening = false; + } +}); + +function requestBody() { + const body = {...selected}; + if (body.action === 'text') body.replacement = draft.value; + if (body.action === 'format') { + body.start = draft.selectionStart; + body.end = draft.selectionEnd; + body.attributes = [...dialog.querySelectorAll('[data-attribute]')].filter(select => select.value !== '') + .map(select => ({[select.dataset.attribute]: select.value === 'true'})); + const font = document.querySelector('#font').value; + const size = document.querySelector('#font-size').value; + if (font) body.attributes.push({Font: font}); + if (size) body.attributes.push({FontSize: Number(size)}); + for (const [field, attribute] of [['color', 'Color'], ['highlight', 'Highlight']]) { + const mode = document.getElementById(field + '-mode').value; + if (mode !== 'keep') body.attributes.push({[attribute]: mode === 'clear' ? null : document.getElementById(field).value.slice(1).match(/../g).map(hex => parseInt(hex, 16))}); + } + } + if (['paragraph', 'outline'].includes(body.action)) { + body.text = draft.value.replace(/\n/g, '\r'); + body.author = document.querySelector('#author').value; + if (body.action === 'paragraph') { + body.object = parent.value; + body.before = before.value || null; + } else { + body.x = document.querySelector('#outline-x').valueAsNumber; + body.y = document.querySelector('#outline-y').valueAsNumber; + } + } + return body; +} + +draft.addEventListener('select', () => { + document.querySelector('#format-range').textContent = 'Selected: ' + (draft.selectionEnd - draft.selectionStart) + ' UTF-16 units'; +}); +document.addEventListener('click', event => { + const element = event.target.closest('[data-text-object]'); + if (element && document.body.classList.contains('editing')) { + event.preventDefault(); + openEdit(element); + } +}); +document.addEventListener('keydown', event => { + if (event.target.matches('[data-text-object]') && document.body.classList.contains('editing') && ['Enter', ' '].includes(event.key)) { + event.preventDefault(); + openEdit(event.target); + } +}); +cancel.addEventListener('click', () => dialog.close()); +dialog.addEventListener('cancel', event => { if (submitting) event.preventDefault(); }); +dialog.addEventListener('close', () => { selected = null; }); +window.addEventListener('beforeunload', event => { + if (selected && JSON.stringify(requestBody()) !== initial) event.preventDefault(); +}); +dialog.querySelector('form').addEventListener('submit', async event => { + event.preventDefault(); + if (save.disabled || !selected) return; + const body = requestBody(); + save.disabled = true; + cancel.disabled = true; + submitting = true; + for (const control of dialog.querySelectorAll('input, select, textarea')) control.disabled = true; + message.textContent = 'Saving…'; + let result; + try { + result = await (await fetch('/api/save', {method: 'POST', headers: { + 'Content-Type': 'application/json', 'X-OneNote-Diagnostic': '1' + }, body: JSON.stringify(body)})).json(); + } catch { + result = {ok: false, state: 'Unknown'}; + } + submitting = false; + cancel.disabled = false; + for (const control of dialog.querySelectorAll('input, select, textarea')) control.disabled = false; + if (result.ok) { + selected = null; + location.assign(result.location); + return; + } + inspect.href = '/latest?' + new URLSearchParams({generation: selected.generation, page: selected.page}); + inspect.hidden = false; + if (result.state === 'Unknown') { + message.textContent = 'Save outcome unknown. Keep this draft and inspect the latest page before trying again.'; + } else if (result.state === 'Committed') { + message.textContent = 'Saved, but cleanup or refresh did not finish. Inspect the latest page; do not save this draft again.'; + } else if (result.kind === 'ResourceBusy') { + message.textContent = 'This section changed. Open the latest page before editing again. Your draft is still here.'; + } else { + message.textContent = 'Unable to save. ' + (result.error || result.report_error || 'Check this edit and try again.'); + save.disabled = false; + } +}); diff --git a/tools/native/network.ps1 b/tools/native/network.ps1 index 2e59b9258bc4a12fe52279e70706a8efdda3cd83..95fa83d5243ee6a2a653cc075128b063e2f09cf2 100644 --- a/tools/native/network.ps1 +++ b/tools/native/network.ps1 @@ -3,6 +3,10 @@ Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' $adapter = @(Get-WmiObject Win32_NetworkAdapterConfiguration | Where-Object { $_.MACAddress -eq $LabMac }) if ($adapter.Count -ne 1) { throw 'The clone lab adapter is ambiguous.' } +$enabled = $adapter[0].EnableDHCP() +if ($enabled.ReturnValue -notin @(0, 1)) { + throw ('Unable to enable DHCP on the clone lab adapter; result ' + $enabled.ReturnValue) +} $release = $adapter[0].ReleaseDHCPLease() $result = $adapter[0].RenewDHCPLease() $deadline = [DateTime]::UtcNow.AddSeconds(60) @@ -12,5 +16,5 @@ do { if ($addresses.Count -eq 1) { break } Start-Sleep -Milliseconds 250 } while ([DateTime]::UtcNow -lt $deadline) -if ($addresses.Count -ne 1) { throw ('The clone did not receive a lab IPv4 address; renewal result ' + $result.ReturnValue) } +if ($addresses.Count -ne 1) { throw ('The clone did not receive a lab IPv4 address; renewal result ' + $result.ReturnValue + '; adapter ' + ($adapter[0] | Select-Object MACAddress,DHCPEnabled,IPEnabled,IPAddress | ConvertTo-Json -Compress)) } @{mac=$LabMac;address=$addresses[0];releaseResult=$release.ReturnValue;renewalResult=$result.ReturnValue} | ConvertTo-Json -Compress diff --git a/tools/native/probe.ps1 b/tools/native/probe.ps1 index 09bdeec644fcb554e6f0a5de812c1403f626cbac..40c0226fb72ecc3b813ed55f81a8c974ea5f5f45 100644 --- a/tools/native/probe.ps1 +++ b/tools/native/probe.ps1 @@ -3,33 +3,57 @@ Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' $results = New-Object Collections.Generic.List[object] $index = 0 +function Record-Phase([string]$Phase) { + @{ name=$file.BaseName; phase=$Phase; utc=[DateTime]::UtcNow.ToString('o') } | + ConvertTo-Json -Compress | Add-Content "$Root\progress.jsonl" -Encoding UTF8 +} foreach ($file in @(Get-ChildItem "$Root\inputs" -Filter '*.one' | Sort-Object Name)) { - $case = "C:\one-tests\runs\probe-$index" $output = "$Root\results\$($file.BaseName)" - New-Item -ItemType Directory "$case\notebook", $output -Force | Out-Null - Copy-Item $file.FullName "$case\notebook\synthetic.one" - Get-Process ONENOTE -ErrorAction SilentlyContinue | ForEach-Object { - Stop-Process -InputObject $_ -Force - if (-not $_.WaitForExit(10000)) { throw 'OneNote did not exit before the cache reset.' } + New-Item -ItemType Directory $output -Force | Out-Null + Record-Phase 'cold-reset' + $cold = $false + for ($attempt = 0; $attempt -lt 5; $attempt++) { + $case = "C:\one-tests\runs\probe-$index-$attempt" + New-Item -ItemType Directory "$case\notebook" -Force | Out-Null + Copy-Item $file.FullName "$case\notebook\synthetic.one" + Get-Process ONENOTE -ErrorAction SilentlyContinue | ForEach-Object { + Stop-Process -InputObject $_ -Force -ErrorAction SilentlyContinue + if (-not $_.WaitForExit(10000)) { throw 'OneNote did not exit before the cache reset.' } + } + Start-Sleep -Milliseconds 250 + try { & "$PSScriptRoot\cold-current.ps1" -Root $case -CloneHost $CloneHost } + catch { + if ($_.Exception.Message -ne 'Close OneNote before resetting its test cache.') { throw } + continue + } + if (-not (Get-Process ONENOTE -ErrorAction SilentlyContinue)) { $cold = $true; break } } - & "$PSScriptRoot\cold-current.ps1" -Root $case -CloneHost $CloneHost + if (-not $cold) { throw 'OneNote kept reopening during the cache reset.' } + Record-Phase 'create-application' $app = New-Object -ComObject OneNote.Application $notebook = ''; $section = ''; $hierarchy = ''; $failure = $null; $pages = @() $started = [DateTime]::UtcNow try { + Record-Phase 'open-notebook' $app.OpenHierarchy("$case\notebook", '', [ref]$notebook, 0) + Record-Phase 'open-section' $app.OpenHierarchy("$case\notebook\synthetic.one", '', [ref]$section, 0) $deadline = [DateTime]::UtcNow.AddSeconds(30) do { + Record-Phase 'get-hierarchy' $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) [xml]$tree = $hierarchy $pages = @($tree.SelectNodes('//*[local-name()="Page"]')) - if ($pages.Count) { break } + if ($pages.Count -and $tree.DocumentElement.GetAttribute('areAllPagesAvailable') -ne 'false') { break } Start-Sleep -Milliseconds 250 } while ([DateTime]::UtcNow -lt $deadline) + if (-not $pages.Count -or $tree.DocumentElement.GetAttribute('areAllPagesAvailable') -eq 'false') { + throw 'The section did not finish loading its pages.' + } $n = 0 foreach ($page in $pages) { $content = '' + Record-Phase "get-page-$n" $app.GetPageContent($page.GetAttribute('ID'), [ref]$content, 1, 1) [IO.File]::WriteAllText("$output\page-$n.xml", $content, [Text.Encoding]::UTF8) $n++ @@ -41,11 +65,13 @@ foreach ($file in @(Get-ChildItem "$Root\inputs" -Filter '*.one' | Sort-Object N seconds=([DateTime]::UtcNow - $started).TotalSeconds; source_sha256=(Get-FileHash $file.FullName).Hash.ToLowerInvariant() }) $results | ConvertTo-Json -Depth 5 | Set-Content "$Root\results.json" -Encoding UTF8 + Record-Phase 'close-notebook' try { if ($notebook) { $app.CloseNotebook($notebook, $false) } } catch {} [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) $app = $null [GC]::Collect() [GC]::WaitForPendingFinalizers() + Record-Phase 'complete' } $index++ } diff --git a/tools/native/stress.ps1 b/tools/native/stress.ps1 index e4f7b69778914c34aa28a7a972bd2f3d4f24a5ce..a0ab5bc69b4dabe970373e481873dfd951a1a69a 100644 --- a/tools/native/stress.ps1 +++ b/tools/native/stress.ps1 @@ -1,6 +1,7 @@ function Invoke-Stress($app, $section, $command, $output, $shared) { $hierarchy = '' $app.GetHierarchy($section, 4, [ref]$hierarchy, 1) + [IO.File]::WriteAllText("$output\hierarchy.xml", $hierarchy, [Text.Encoding]::UTF8) [xml]$tree = $hierarchy $pages = @($tree.SelectNodes('//*[local-name()="Page"]')) if ($pages.Count -ne 1) { throw 'Expected one stress-test page.' } @@ -23,6 +24,13 @@ function Invoke-Stress($app, $section, $command, $output, $shared) { Start-Sleep -Milliseconds 20 } for ($i = 0; $i -lt $command.operations; $i++) { + if (($command.PSObject.Properties.Name -contains 'maintenance') -and $command.maintenance -and $i -eq [Math]::Floor($command.operations / 2)) { + [IO.File]::WriteAllText("$shared\maintenance-paused-n$($command.actor)", 'paused') + while (-not (Test-Path "$shared\maintenance-resume")) { + if ([DateTime]::UtcNow -gt $deadline) { throw 'Maintenance pause timed out.' } + Start-Sleep -Milliseconds 100 + } + } Start-Sleep -Milliseconds $random.Next(10, 100) $started = [DateTime]::UtcNow.Ticks $content = '' diff --git a/tools/native_collaboration.py b/tools/native_collaboration.py index 747d8e3b26ed15f13a77d0cf8ee6c6d786eb1548..e51f7e04d0ba24f45006018565460e6619f7dc17 100644 --- a/tools/native_collaboration.py +++ b/tools/native_collaboration.py @@ -47,20 +47,27 @@ def verify_final_state(model): return {'main_space': sid, 'conflict_space': conflict_sid, 'competing_edits': sorted(edits)} -def replay(output, server, stress_clients=0, stress_operations=30, sync_every=1, rust_writers=4, rust_readers=3, edit=False, seed=710, conflict_clients=0, abrupt=False): +def replay(output, server, stress_clients=0, stress_operations=30, sync_every=1, rust_writers=4, rust_readers=3, edit=False, seed=710, conflict_clients=0, abrupt=False, embedded_smb=False, maintenance=False, fixture=None, disconnect=False, client_timeout=600, offline=False, offline_outage=False, offline_lost_reply=False, client_profile="debug", document_operations=False, record_writes=False, offline_client_reply=False): + if fixture is not None and not stress_clients: + raise ValueError('Use a fixture with stress mode.') if linux_vm.instance_path(server).exists(): raise ValueError('Choose a new Linux VM name; existing machines are not owned by this run.') output = output.resolve() output.mkdir(parents=True, exist_ok=False) mount = output / 'mount' mount.mkdir() - mounted = False scripts = output / 'scripts' scripts.mkdir() for name in ('cold.ps1', 'collaborate.ps1', 'network.ps1', 'stress.ps1', 'text.ps1'): shutil.copyfile(ROOT / 'tools/native' / name, scripts / name) - (output / 'run.json').write_text(json.dumps({'server': server, 'stress_clients': stress_clients, 'conflict_clients': conflict_clients, 'stress_operations': stress_operations, 'sync_every': sync_every, 'rust_writers': rust_writers, 'rust_readers': rust_readers, 'edit': edit, 'seed': seed, 'abrupt': abrupt, - 'harness_sha256': {name: hashlib.sha256((ROOT / 'tools' / name).read_bytes()).hexdigest() for name in ('native_collaboration.py', 'native_stress.py', 'concurrent_rust.py', 'native_runner.py')}, 'scripts': {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in scripts.iterdir()}}, indent=2)) + harness = ('native_collaboration.py', 'native_maintenance.py', 'native_disconnect.py', 'native_stress.py', 'offline_history.py', 'offline_document_history.py', 'offline_outage.py', 'verify_offline.py', 'concurrent_rust.py', 'native_runner.py', + 'crash_recovery.py', 'smb-proxy.py', 'verify_smb_overlap.py', 'w7/crash.py', 'w7/vm.py', 'w7/linux_vm.py') + for name in harness: + saved = output / 'harness' / name + saved.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(ROOT / 'tools' / name, saved) + (output / 'run.json').write_text(json.dumps({'server': server, 'stress_clients': stress_clients, 'conflict_clients': conflict_clients, 'stress_operations': stress_operations, 'sync_every': sync_every, 'rust_writers': rust_writers, 'rust_readers': rust_readers, 'edit': edit, 'seed': seed, 'abrupt': abrupt, 'embedded_smb': embedded_smb, 'maintenance': maintenance, 'fixture': str(fixture) if fixture is not None else None, + 'disconnect': disconnect, 'client_timeout': client_timeout, 'client_profile': client_profile, 'offline': offline, 'document_operations': document_operations, 'record_writes': record_writes, 'offline_outage': offline_outage, 'offline_lost_reply': offline_lost_reply, 'offline_client_reply': offline_client_reply, 'harness_sha256': {name: hashlib.sha256((output / 'harness' / name).read_bytes()).hexdigest() for name in harness}, 'scripts': {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in scripts.iterdir()}}, indent=2)) def ssh(text): result = linux_vm.run_ssh(server, text, timeout=90) @@ -76,18 +83,39 @@ def replay(output, server, stress_clients=0, stress_operations=30, sync_every=1, linux_vm.wait_instance(server, 600) config = linux_vm.load_instance(server) (output / 'linux.json').write_text(json.dumps(config, indent=2)) - def reconnect_mount(): - nonlocal mounted + if embedded_smb: + subprocess.run(linux_vm.ssh_argv(server, 'cat > /tmp/smb-proxy.py'), + input=(output / 'harness/smb-proxy.py').read_bytes(), check=True) + ssh("sudo sed -i '/^\\[global\\]/a smb ports = 1445' /etc/samba/smb.conf && sudo systemctl restart smbd") + subprocess.run(linux_vm.ssh_argv(server, 'cat > /tmp/smb-control.json'), + input=json.dumps({'record_writes': record_writes}).encode(), check=True) + ssh("sudo sh -c 'nohup python3 /tmp/smb-proxy.py /tmp/smb-control.json --port 445 --bind 0.0.0.0 --server 127.0.0.1 --server-port 1445 > /tmp/smb-trace.jsonl 2>&1 < /dev/null &'") + ssh("sleep 1; sudo ss -ltn | grep ':445 '") + os.environ['ONESTORE_SMB_LAB'] = f'127.0.0.1:{config["samba_port"]}' + os.environ['ONESTORE_SMB_SHARE'] = 'agent' + def unmount(force=False): + for options in ([['-f']] if force else [[], ['-f']]): + if not os.path.ismount(mount): return + result = subprocess.run(['/sbin/umount', *options, str(mount)], capture_output=True, text=True, timeout=60) + with (output / 'unmounts.jsonl').open('a') as log: + log.write(json.dumps({'force': bool(options), 'exit': result.returncode, 'stderr': result.stderr}) + '\n') if os.path.ismount(mount): - subprocess.run(['/sbin/umount', str(mount)], check=True) - mounted = False + raise RuntimeError('The owned SMB mount remains attached; preserve its server until it is unmounted.') + def reconnect_mount(): + unmount() subprocess.run(['/sbin/mount_smbfs', '-N', f'//guest@127.0.0.1:{config["samba_port"]}/agent', mount], check=True, stdin=subprocess.DEVNULL) - mounted = True ssh('mkdir /srv/agent/m6-collaboration') source = ROOT / 'corpus/native-ink/cold-ui-ink/notebook' if stress_clients or conflict_clients or abrupt: source = output / 'input' - subprocess.run([ROOT / 'target/debug/examples/create_notebook', source, 'Concurrent edits:', 'Concurrency test'], check=True) + if fixture is not None: + source.mkdir() + for name in ('synthetic.one', 'Open Notebook.onetoc2'): + shutil.copyfile(Path(fixture) / name, source / name) + elif maintenance: + subprocess.run([ROOT / 'target/debug/examples/maintenance_fixture', source, 'Concurrent edits:'], check=True) + else: + subprocess.run([ROOT / 'target/debug/examples/create_notebook', source, 'Concurrent edits:', 'Concurrency test'], check=True) with tarfile.open(output / 'input.tar', 'w', dereference=True) as archive: for name in ('synthetic.one', 'Open Notebook.onetoc2'): archive.add(source / name, arcname=name) @@ -127,8 +155,7 @@ def replay(output, server, stress_clients=0, stress_operations=30, sync_every=1, # Joining workers before stack exit retains ownership even if another boot fails. with ThreadPoolExecutor(max_workers=len(labels)) as pool: names = dict(zip(labels, pool.map(start_clone, labels))) - clients = [] - for label in labels: + def prepare_client(label): folder = output / label name = names[label] for local in scripts.iterdir(): @@ -139,10 +166,13 @@ def replay(output, server, stress_clients=0, stress_operations=30, sync_every=1, command(name, 'powershell -NoProfile -ExecutionPolicy Bypass -File C:\\one-tests\\network.ps1 -LabMac ' + vm.lab_mac(name), folder) command(name, 'ipconfig', folder) command(name, 'dir \\\\192.168.77.1\\agent\\m6-collaboration', folder) - clients.append({'name': name, 'folder': folder, 'sequence': 0}) - start_controller(clients[-1]) + client = {'name': name, 'folder': folder, 'sequence': 0} + start_controller(client) command(name, 'ipconfig', folder) print('Collaboration ready:', label, name, flush=True) + return client + with ThreadPoolExecutor(max_workers=len(labels)) as pool: + clients = list(pool.map(prepare_client, labels)) def action(client, action, wait=True, **parameters): client['sequence'] += 1 @@ -166,10 +196,19 @@ def replay(output, server, stress_clients=0, stress_operations=30, sync_every=1, time.sleep(.5) else: raise TimeoutError(f'Native command {sequence} did not complete') if action == 'snapshot': - destination = client['folder'] / f'snapshot-{sequence:04}.xml' - result = windows.do_get(remote + '\\page-0.xml', destination, client['name']) + hierarchy = client['folder'] / f'hierarchy-{sequence:04}.xml' + result = windows.do_get(remote + '\\hierarchy.xml', hierarchy, client['name']) if result.get('error'): raise RuntimeError(result['error']) - return texts(ET.parse(destination).getroot()) + xml = hierarchy.read_text(encoding='utf-8-sig').strip() + if xml == '': return [] + pages = [node for node in ET.fromstring(xml).iter() if node.tag.endswith('}Page')] + observed = [] + for i in range(len(pages)): + destination = client['folder'] / f'snapshot-{sequence:04}-{i}.xml' + result = windows.do_get(remote + f'\\page-{i}.xml', destination, client['name']) + if result.get('error'): raise RuntimeError(result['error']) + observed.extend(texts(ET.parse(destination).getroot())) + return observed def wait_text(client, expected): deadline = time.monotonic() + 120 @@ -201,6 +240,7 @@ def replay(output, server, stress_clients=0, stress_operations=30, sync_every=1, return stream.read() def checkpoint(label): + if embedded_smb: reconnect_mount() deadline, previous, incomplete = time.monotonic() + 120, None, 0 while time.monotonic() < deadline: snapshot = {name: snapshot_file(name) for name in ('synthetic.one', 'Open Notebook.onetoc2')} @@ -228,7 +268,7 @@ def replay(output, server, stress_clients=0, stress_operations=30, sync_every=1, if abrupt and not conflict_clients: from concurrent_rust import running_clients - from crash_recovery import verify_text + from crash_recovery import active_text, verify_text import crash action(clients[0], 'prepare-stress', clients=len(clients)) @@ -244,6 +284,8 @@ def replay(output, server, stress_clients=0, stress_operations=30, sync_every=1, changed = native_text[i] + ' Before server stop.' action(client, 'edit', expected=native_text[i], text=changed) native_text[i] = changed + for client in clients: action(client, 'sync') + for client in clients: wait_text(client, native_text) deadline = time.monotonic() + 60 while True: commits = sum(line.count('"event":"commit"') for path in folder.glob('w*.jsonl') for line in path.read_text().splitlines()) @@ -254,6 +296,8 @@ def replay(output, server, stress_clients=0, stress_operations=30, sync_every=1, assert all(p.poll() is None for p in processes.values()), 'A client stopped before the planned interruption' (output / 'server-crash.json').write_text(json.dumps(crash.stop('linux', server), indent=2)) for client in clients: vm.qmp(client['name'], 'set_link', {'name': 'lab', 'up': False}) + for process in processes.values(): process.terminate() + unmount(force=True) raise InterruptedError('Recorded server interruption') except InterruptedError: pass @@ -267,11 +311,12 @@ def replay(output, server, stress_clients=0, stress_operations=30, sync_every=1, subprocess.run(linux_vm.ssh_argv(server, 'cat /srv/agent/m6-collaboration/synthetic.one'), stdout=stream, check=True, timeout=60) subprocess.run([ROOT / 'target/debug/examples/document', recovered / 'synthetic.one', recovered / 'model'], check=True) model = json.loads((recovered / 'model/document.json').read_text()) - text, = [text for values in reachable_page_text(model).values() for text in values if text.startswith('Concurrent edits:')] + text = active_text(model) retained = verify_text('Concurrent edits:', text, logs) (output / 'server-retention.json').write_text(json.dumps(retained, indent=2)) reconnect_mount() for client in clients: vm.qmp(client['name'], 'set_link', {'name': 'lab', 'up': True}) + for client in clients: action(client, 'sync') for client in clients: wait_text(client, [text, *native_text]) checkpoint('server-recovered') @@ -302,6 +347,14 @@ def replay(output, server, stress_clients=0, stress_operations=30, sync_every=1, (output / 'native-cache-result.json').write_text(json.dumps({'cache_acknowledged': pending_text, 'retained_after_abrupt_stop': survived, 'server_durability_acknowledged': False}, indent=2)) for client in clients: wait_text(client, [text, *native_text]) checkpoint('client-recovered') + recovered_model = json.loads((output / 'client-recovered/model/document.json').read_text()) + recovered_pages = reachable_page_text(recovered_model) + main_space, *conflict_spaces = recovered_pages + known = {'Concurrent edits:', *[f'Native {i}:' for i in range(len(clients))], *native_text, pending_text} + conflicts = {sid: recovered_pages[sid] for sid in conflict_spaces} + assert all(value in known or (value.endswith(']') and text.startswith(value)) + for values in conflicts.values() for value in values), 'A recovered conflict contains unrecorded content' + (output / 'cache-conflicts.json').write_text(json.dumps(conflicts, indent=2)) continued = output / 'rust-continued' continued.mkdir() @@ -310,7 +363,7 @@ def replay(output, server, stress_clients=0, stress_operations=30, sync_every=1, continuation = {actor: [json.loads(line) for line in (continued / f'{actor}.jsonl').read_text().splitlines()] for actor in processes} checkpoint('continued') model = json.loads((output / 'continued/model/document.json').read_text()) - final_text, = [value for values in reachable_page_text(model).values() for value in values if value.startswith('Concurrent edits:')] + final_text = active_text(model) result = verify_text(text, final_text, continuation) for client in clients: wait_text(client, [final_text, *native_text]) (output / 'result.json').write_text(json.dumps({'server': retained, 'continuation': result, 'native_cache_retained': survived, 'expected_text': sorted([final_text, *native_text])}, indent=2)) @@ -370,7 +423,7 @@ def replay(output, server, stress_clients=0, stress_operations=30, sync_every=1, for client in clients: action(client, 'sync') elif stress_clients: from native_stress import exercise - exercise(output, shared, clients, action, wait_action, wait_text, checkpoint, stress_operations, sync_every, rust_writers, rust_readers, edit, seed) + exercise(output, shared, clients, action, wait_action, wait_text, checkpoint, stress_operations, sync_every, rust_writers, rust_readers, edit, seed, embedded_smb) else: a, b = clients original = 'Fictitious: café, 東京, مرحبا' @@ -450,8 +503,9 @@ def replay(output, server, stress_clients=0, stress_operations=30, sync_every=1, if abrupt and not conflict_clients: checkpoint('crash-closed') model = json.loads((output / 'crash-closed/model/document.json').read_text()) - actual = sorted(value for values in reachable_page_text(model).values() for value in values) - assert actual == sorted([final_text, *native_text]), 'Application closure changed recovered edits' + actual = reachable_page_text(model) + assert actual.pop(main_space) == sorted([final_text, *native_text]), 'Application closure changed recovered edits' + assert actual == conflicts, 'Application closure changed recovered conflict pages' elif stress_clients: checkpoint('stress-closed') elif conflict_clients: @@ -476,9 +530,13 @@ def replay(output, server, stress_clients=0, stress_operations=30, sync_every=1, (failure / 'capture-error.txt').write_text(str(error)) raise finally: - if mounted: - result = subprocess.run(['/sbin/umount', str(mount)], capture_output=True, text=True) - (output / 'unmount.json').write_text(json.dumps({'exit': result.returncode, 'stderr': result.stderr})) + if embedded_smb and linux_vm.instance_path(server).exists() and linux_vm.running(server): + try: + with (output / 'smb-trace.jsonl').open('wb') as trace: + subprocess.run(linux_vm.ssh_argv(server, 'cat /tmp/smb-trace.jsonl'), stdout=trace, check=True, timeout=30) + except Exception as error: + (output / 'trace-error.txt').write_text(str(error)) + if os.path.ismount(mount): unmount() if linux_vm.instance_path(server).exists(): try: if linux_vm.running(server): linux_vm.shutdown(server, 60) @@ -488,6 +546,20 @@ def replay(output, server, stress_clients=0, stress_operations=30, sync_every=1, while linux_vm.running(server) and time.monotonic() < deadline: time.sleep(.1) linux_vm.delete_instance(server) (output / 'teardown.json').write_text(json.dumps({'linux_absent': not linux_vm.instance_path(server).exists()}, indent=2)) + if embedded_smb: + from verify_smb_overlap import verify + with (output / 'smb-trace.jsonl').open() as trace: + overlap = verify(json.loads(line) for line in trace) + (output / 'overlap.json').write_text(json.dumps(overlap, indent=2)) + if offline_lost_reply: + from offline_outage import verify_lost_reply + (output / 'offline-lost-reply-verification.json').write_text(json.dumps(verify_lost_reply(output), indent=2)) + if offline_outage: + from offline_outage import verify_outage + (output / 'offline-outage-verification.json').write_text(json.dumps(verify_outage(output), indent=2)) + if disconnect: + from native_disconnect import verify_disconnect + (output / 'disconnect-verification.json').write_text(json.dumps(verify_disconnect(output), indent=2)) if __name__ == '__main__': @@ -503,7 +575,29 @@ if __name__ == '__main__': parser.add_argument('--edit', action='store_true', help='Use random text replacements in every writer.') parser.add_argument('--seed', type=int, default=710) parser.add_argument('--abrupt', action='store_true', help='Abrupt server and client stops with preserved-disk recovery and intent accounting.') + parser.add_argument('--embedded-smb', action='store_true', help='Run Rust stress clients through the embedded SMB adapter.') + parser.add_argument('--maintenance', action='store_true', help='Pause the workload for the owned maintenance controller.') + parser.add_argument('--offline', action='store_true', help='Use durable local queues and traced offline workers for Rust writers.') + parser.add_argument('--document-operations', action='store_true', help='Queue paragraph/outline creation and formatting alongside offline text edits.') + parser.add_argument('--record-writes', action='store_true', help='Retain owned lab write payloads for revision replay.') + parser.add_argument('--offline-lost-reply', action='store_true', help='Drop a publication reply and require confirmation of its original revision.') + parser.add_argument('--offline-client-reply', action='store_true', help='Disconnect only the formatting writer and require peer publication before it reconciles.') + parser.add_argument('--offline-outage', action='store_true', help='Queue local edits during an owned SMB outage, then require recovery.') + parser.add_argument('--disconnect', action='store_true', help='Interrupt and reconnect the embedded append workload twice.') + parser.add_argument('--client-profile', choices=('debug', 'release'), default='debug', help='Cargo build profile for Rust stress clients') + parser.add_argument('--client-timeout', type=float, default=600, help='Maximum seconds for the Rust workload, including its start barrier.') + parser.add_argument('--fixture', type=Path, help='Copy this fixture directory into the disposable stress notebook.') args = parser.parse_args() + if not 0 < args.client_timeout < 2**64 / 1000: parser.error('Choose a finite positive client timeout.') + if args.maintenance and not (args.embedded_smb and args.stress_clients): parser.error('--maintenance requires embedded SMB stress mode.') + if args.document_operations and not args.offline: parser.error('--document-operations requires --offline.') + if args.record_writes and not args.embedded_smb: parser.error('--record-writes requires --embedded-smb.') + if args.offline_client_reply and not (args.offline_lost_reply and args.document_operations): parser.error('--offline-client-reply requires --offline-lost-reply and --document-operations.') + if args.offline_lost_reply and (not args.offline or args.offline_outage or args.sync_every or args.stress_operations < 8): parser.error('--offline-lost-reply requires --offline, --sync-every 0, at least eight operations and no --offline-outage.') + if args.offline_outage and (not args.offline or args.sync_every or args.stress_operations < 8): parser.error('--offline-outage requires --offline, --sync-every 0 and at least eight operations.') + if args.offline and (not args.embedded_smb or not args.stress_clients or args.edit or args.disconnect or args.maintenance): parser.error('--offline requires embedded append stress without disconnect or maintenance.') + if args.embedded_smb and not args.stress_clients: parser.error('--embedded-smb requires --stress-clients.') + if args.disconnect and (not args.embedded_smb or args.edit or args.maintenance or args.sync_every): parser.error('--disconnect requires embedded append stress with --sync-every 0 and no maintenance.') if args.rust_writers < 2 or args.rust_readers < 1: parser.error('Use at least two Rust writers and one reader.') if args.sync_every < 0 or args.stress_operations <= 0: parser.error('Use a nonnegative sync interval and positive operation count.') if args.stress_clients and args.stress_clients < 3: parser.error('Stress mode requires at least three native clients.') @@ -511,4 +605,4 @@ if __name__ == '__main__': if args.abrupt and (args.stress_clients or args.edit): parser.error('Abrupt recovery uses append intents or the offline-conflict workload.') def interrupted(_signal, _frame): raise KeyboardInterrupt signal.signal(signal.SIGTERM, interrupted) - replay(args.output, args.linux, args.stress_clients, args.stress_operations, args.sync_every, args.rust_writers, args.rust_readers, args.edit, args.seed, args.conflict_clients, args.abrupt) + replay(args.output, args.linux, args.stress_clients, args.stress_operations, args.sync_every, args.rust_writers, args.rust_readers, args.edit, args.seed, args.conflict_clients, args.abrupt, args.embedded_smb, args.maintenance, args.fixture, args.disconnect, args.client_timeout, args.offline, args.offline_outage, args.offline_lost_reply, args.client_profile, args.document_operations, args.record_writes, args.offline_client_reply) diff --git a/tools/native_disconnect.py b/tools/native_disconnect.py new file mode 100644 index 0000000000000000000000000000000000000000..606a0d308254658e8b36ce874ad6e867511ef4d7 --- /dev/null +++ b/tools/native_disconnect.py @@ -0,0 +1,137 @@ +"""Interrupt an owned mixed append workload and require progress after reconnection.""" +import json +import shlex +import subprocess +import time + +from native_runner import windows +import linux_vm +from verify_smb_overlap import verify + + +def interrupt(output, clients, sequences, processes): + config = json.loads((output / 'run.json').read_text()) + server = config['server'] + samples = [] + subprocess.run(linux_vm.ssh_argv(server, 'cat > /tmp/verify_smb_overlap.py'), + input=(output / 'harness/verify_smb_overlap.py').read_bytes(), check=True) + + def counts(transport=False): + result = {} + for actor in processes: + data = (output / 'rust' / (actor + '.jsonl')).read_text() + events = [json.loads(line) for line in data[:data.rfind('\n') + 1].splitlines()] + names = ('transport_read_error', 'transport_commit_error') if transport else ('commit' if actor.startswith('w') else 'read',) + result[actor] = sum(event['event'] in names for event in events) + return result + + def wait_for(predicate, message): + deadline = time.monotonic() + 90 + while not predicate(): + assert all(process.poll() is None for process in processes.values()), 'A Rust client exited during the interruption campaign' + if time.monotonic() > deadline: raise TimeoutError(message) + time.sleep(.1) + + def ssh(command): + result = linux_vm.run_ssh(server, command, timeout=15) + with (output / 'disconnect-server.jsonl').open('a') as stream: + stream.write(json.dumps({'command': command, 'exit': result.returncode, 'stdout': result.stdout, 'stderr': result.stderr}) + '\n') + result.check_returncode() + return result.stdout + + def phase(control): + text = json.dumps(control) + ssh("printf '%s' '" + text + "' > /tmp/smb-control.tmp && mv /tmp/smb-control.tmp /tmp/smb-control.json") + wait_for(lambda: text in ssh("grep -F '\"control\":' /tmp/smb-trace.jsonl | tail -n 1"), 'Proxy did not acknowledge the disconnect phase') + + previous = {actor: 0 for actor in processes} + for cycle in range(2): + wait_for(lambda: all(value >= previous[actor] + 3 for actor, value in counts().items()), 'Clients made no progress before the interruption') + before = counts() + native = [] + for actor, (client, sequence) in enumerate(zip(clients, sequences)): + capture = output / f'disconnect-{cycle}-n{actor}.jsonl' + result = windows.do_get(f'C:\\one-tests\\runs\\capture\\outbox\\{sequence}\\events.jsonl', capture, client['name']) + assert not result.get('error'), result + data = capture.read_text(encoding='utf-8-sig') + rows = [json.loads(line) for line in data[:data.rfind('\n') + 1].splitlines()] + assert 0 < len(rows) < config['stress_operations'], 'A native writer was inactive before the interruption' + native.append(len(rows)) + before_errors = counts(transport=True) + assert all(process.poll() is None for process in processes.values()), 'A Rust client finished before the interruption' + try: + phase({'phase': f'disconnect-{cycle}', 'cut': 9, 'peer': '10.0.2.2', 'offset': 96, + 'direction': 'request' if cycle == 0 else 'response'}) + wait_for(lambda: f'"phase": "disconnect-{cycle}"' in ssh("grep -F '\"control\":' /tmp/smb-trace.jsonl | tail -n 1") + and int(ssh("grep -c '\"cut\": {' /tmp/smb-trace.jsonl || true").strip()) == cycle + 1, + 'The planned write interruption did not occur') + time.sleep(3) + finally: + phase({'phase': f'reconnected-{cycle}'}) + wait_for(lambda: all(value >= before[actor] + 3 for actor, value in counts().items()), 'A client failed to progress after reconnecting') + script = '\n'.join([ + 'import json', 'from verify_smb_overlap import verify, PendingOverlap', + "data = open('/tmp/smb-trace.jsonl').read()", + "events = [json.loads(line) for line in data[:data.rfind('\\n') + 1].splitlines()]", + 'try:', f" result = verify(events, phase='reconnected-{cycle}')", + "except PendingOverlap: result = None", 'print(json.dumps(result))']) + wait_for(lambda: json.loads(ssh('cd /tmp && python3 -c ' + shlex.quote(script))) is not None, + 'Native and Rust guarded I/O did not overlap after reconnection') + previous = counts() + samples.append({'cycle': cycle, 'before': before, 'after': previous, 'native_before': native, + 'before_errors': before_errors, 'after_errors': counts(transport=True)}) + (output / 'disconnect-progress.json').write_text(json.dumps(samples, indent=2)) + + +def verify_disconnect(output): + config = json.loads((output / 'run.json').read_text()) + assert config['stress_clients'] + config['rust_writers'] + config['rust_readers'] >= 12 + samples = json.loads((output / 'disconnect-progress.json').read_text()) + assert [sample['cycle'] for sample in samples] == [0, 1] + actors = {f'w{i}' for i in range(config['rust_writers'])} | {f'r{i}' for i in range(config['rust_readers'])} + events = [json.loads(line) for line in (output / 'smb-trace.jsonl').read_text().splitlines()] + assert sum('cut' in event for event in events) == 2, 'Unexpected number of connection interruptions' + errors = {} + progress = {} + progress_limit = 120 + started = (output / 'rust/start').stat().st_mtime_ns // 1000 + stopped = (output / 'rust/stop').stat().st_mtime_ns // 1000 + + def max_gap(actor, times, units): + assert len(times) > 1, 'A client made no progress' + gaps = [(end - begin) / units for begin, end in zip(times, times[1:])] + assert all(0 <= gap <= progress_limit for gap in gaps), f'{actor}: client progress stalled or went backwards' + return max(gaps) + + for i in range(config['stress_clients']): + rows = [json.loads(line) for line in (output / f'n{i}/stress-events.jsonl').read_text(encoding='utf-8-sig').splitlines()] + assert len(rows) == config['stress_operations'], 'A native writer did not finish' + progress[f'n{i}'] = max_gap(f'n{i}', [rows[0]['update_started_ticks'], *[row['updated_ticks'] for row in rows]], 10**7) + uncertain = set() + for actor in sorted(actors): + rows = [json.loads(line) for line in (output / 'rust' / (actor + '.jsonl')).read_text().splitlines()] + errors[actor] = sum(row['event'] in ('transport_read_error', 'transport_commit_error') for row in rows) + assert errors[actor] and any(row['event'] == 'transport_connected' for row in rows), 'A Rust client did not exercise reconnection' + times = [started, *[row['finished_us'] for row in rows if row['event'] == ('commit' if actor.startswith('w') else 'read')]] + if actor.startswith('r') and len(times) > 1: times.append(max(times[-1], stopped)) + progress[actor] = max_gap(actor, times, 10**6) + pending = None + for row in rows: + if row['event'] == 'transport_commit_error': pending = row + if row['event'] != 'transport_reconciled': continue + assert pending is not None and pending['token'] == row['token'] + if row['published']: assert row.get('flush_confirmed'), 'Visible recovery lacks a durable acknowledgement' + if pending['state'] == 'Unknown': uncertain.add('after' if row['published'] else 'before') + pending = None + assert uncertain == {'before', 'after'}, 'The mixed workload did not resolve both uncertain outcomes' + overlap = [] + for sample in samples: + assert all(set(sample[field]) == actors for field in ('before', 'after', 'before_errors', 'after_errors')) + assert all(sample['after'][actor] >= value + 3 for actor, value in sample['before'].items()) + assert all(sample['after_errors'][actor] > value for actor, value in sample['before_errors'].items()), 'A client did not encounter this interruption' + assert len(sample['native_before']) == config['stress_clients'] + assert all(0 < value < config['stress_operations'] for value in sample['native_before']) + end = next((i for i, event in enumerate(events) if event.get('control', {}).get('phase') == f'disconnect-{sample["cycle"] + 1}'), len(events)) + overlap.append(verify(events[:end], phase=f'reconnected-{sample["cycle"]}')) + return {'interruptions': 2, 'transport_errors': errors, 'uncertain_outcomes': sorted(uncertain), 'resumed_overlap': overlap, + 'max_progress_gap_seconds': progress, 'progress_limit_seconds': progress_limit} diff --git a/tools/native_maintenance.py b/tools/native_maintenance.py new file mode 100644 index 0000000000000000000000000000000000000000..c7b4d61b7e31f94eaafb8877a8ef6104a5ee238f --- /dev/null +++ b/tools/native_maintenance.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""Compact a shared section between two halves of a twelve-client editing run.""" +import argparse +import base64 +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import time +import xml.etree.ElementTree as ET + +from native_runner import ROOT, windows +import linux_vm +from verify_smb_overlap import verify + + +def maintenance_locks(events): + pending, files, peers = {}, {}, {} + phase, attempts = None, [] + for event in events: + assert not event.get('trace_error') and not event.get('encrypted') + phase = event.get('control', {}).get('phase', phase) + connection = event.get('connection') + if event.get('opened'): peers[connection] = event['peer'][0] + if event.get('closed'): + files = {key: value for key, value in files.items() if key[0] != connection} + if 'command' not in event: continue + key = connection, event['message'] + if event['direction'] == 'request': + pending[key] = event, phase + continue + if event['status'] == '0x103': continue + pair = pending.pop(key, None) + if pair is None: continue + request, issued = pair + command = request['command'] + if command == 5 and event['status'] == '0x0': + files[connection, event['file_id']] = request['path'].replace('\\', '/').lower() + elif command == 6: + files.pop((connection, request['file_id']), None) + elif command == 10 and peers[connection].startswith('192.168.77.'): + path = files.get((connection, request['file_id']), '') + if path != 'm6-collaboration/synthetic.one': continue + for offset, length, flags in request['locks']: + if (offset, length) == (0xffffeffc, 4096) and flags & 3 == 2: + attempts.append({'phase': issued, 'status': event['status'], 'connection': connection, + 'message': event['message'], 'flags': flags}) + assert any(a['phase'] == 'maintenance-held' and a['status'] in ('0xc0000054', '0xc0000055') for a in attempts), 'No section maintenance conflict observed while the reader held its guard' + assert any(a['phase'] == 'maintenance-released' and a['status'] == '0x0' for a in attempts), 'No section maintenance guard acquired after reader release' + return attempts + + +def run(output, server, fixture, operations, seed): + output = output.resolve() + assert not output.exists(), 'Choose a new output directory' + output.parent.mkdir(parents=True, exist_ok=True) + guardian = None + with output.with_suffix('.log').open('x') as log: + process = subprocess.Popen([sys.executable, ROOT / 'tools/native_collaboration.py', output, + '--linux', server, '--stress-clients', '4', '--rust-writers', '4', '--rust-readers', '4', + '--stress-operations', str(operations), '--seed', str(seed), '--edit', '--embedded-smb', + '--maintenance', '--fixture', fixture], stdout=log, stderr=subprocess.STDOUT) + + def wait_for(predicate, timeout, message): + deadline = time.monotonic() + timeout + while not predicate(): + if process.poll() is not None: raise RuntimeError(f'Workload exited with {process.returncode}: {message}') + if guardian is not None and guardian.poll() is not None and guardian.returncode != 0: + raise RuntimeError('The reader guardian failed') + if time.monotonic() > deadline: raise TimeoutError(message) + time.sleep(.2) + + def ssh(command): + result = linux_vm.run_ssh(server, command, timeout=30) + with (output / 'maintenance-server.jsonl').open('a') as stream: + stream.write(json.dumps({'command': command, 'exit': result.returncode, 'stdout': result.stdout, 'stderr': result.stderr}) + '\n') + result.check_returncode() + return result.stdout + + def phase(name): + ssh('printf \'{"phase":"' + name + '"}\' > /tmp/smb-control.json') + wait_for(lambda: name in ssh(f'grep -F \'"control": {{"phase": "{name}"}}\' /tmp/smb-trace.jsonl || true'), + 10, 'Proxy did not acknowledge the maintenance phase') + + def stat(): + inode, size = ssh("stat -c '%i %s' /srv/agent/m6-collaboration/synthetic.one").split() + return {'inode': int(inode), 'size': int(size)} + + def ui(name, script): + (output / f'{name}.ahk').write_text(script) + result = windows.do_exec(script, target=target, timeout_ms=60000, shot_delay_ms=500) + screenshot = result.pop('png_b64', None) + if screenshot: (output / f'{name}.png').write_bytes(base64.b64decode(screenshot)) + (output / f'{name}.json').write_text(json.dumps(result, indent=2)) + if result.get('error') or result.get('exit') != 0: raise RuntimeError(str(result)) + + try: + wait_for(lambda: (output / 'maintenance-ready').exists(), 900, 'Native clients did not reach the initial checkpoint') + (output / 'maintenance-start').touch() + shared, rust = output / 'mount/m6-collaboration', output / 'rust' + wait_for(lambda: all((shared / f'maintenance-paused-n{i}').exists() and (rust / f'paused-w{i}').exists() for i in range(4)), + 300, 'Writers did not reach the maintenance barrier') + (rust / 'pause').touch() + wait_for(lambda: all((rust / f'paused-r{i}').exists() for i in range(4)), 60, 'Readers did not pause') + target = json.loads((output / 'n0/machine.json').read_text())['name'] + page = ET.parse(sorted((output / 'n0').glob('snapshot-*.xml'))[-1]).getroot().attrib['ID'] + ui('maintenance-options', f'''if A_ScreenWidth != 800 || A_ScreenHeight != 600 + throw Error("The maintenance controller requires an 800 by 600 desktop.") +ComObject("OneNote.Application").NavigateTo("{page}", "", false) +WinWait("ahk_exe ONENOTE.EXE", , 10) +WinActivate("ahk_exe ONENOTE.EXE") +WinWaitActive("ahk_exe ONENOTE.EXE", , 10) +Send("!ft") +WinWait("OneNote Options", , 10) +WinActivate("OneNote Options") +WinWaitActive("OneNote Options", , 10) +Sleep(1000) +CoordMode("Mouse", "Screen") +Click(74, 134) +Sleep(1000) +''') + hold = output / 'guardian' + hold.mkdir() + config = json.loads((output / 'linux.json').read_text()) + with (hold / 'run.log').open('w') as guardian_log: + guardian = subprocess.Popen(['cargo', 'test', '-p', 'onestore-smb', 'live_reader_hold', '--', '--ignored', '--nocapture'], + cwd=ROOT, stdout=guardian_log, stderr=subprocess.STDOUT, + env={**os.environ, 'ONESTORE_SMB_LAB': f'127.0.0.1:{config["samba_port"]}', + 'ONESTORE_SMB_PATH': 'm6-collaboration/synthetic.one', 'ONESTORE_SMB_HOLD': str(hold)}) + wait_for(lambda: (hold / 'ready').exists(), 60, 'Reader guard was not acquired') + before = stat() + phase('maintenance-held') + ui('maintenance-denied', '''CoordMode("Mouse", "Screen") +WinActivate("OneNote Options") +WinWaitActive("OneNote Options", , 10) +Click(254, 440) +if !WinWait("Microsoft OneNote ahk_class #32770", , 30) + throw Error("The maintenance conflict dialog did not appear.") +FileAppend(WinGetText("Microsoft OneNote ahk_class #32770"), "*") +''') + held = stat() + assert held == before, 'Section changed while the reader held its maintenance exclusion guard' + (hold / 'release').touch() + wait_for(lambda: guardian.poll() is not None, 30, 'Reader guardian did not release') + assert guardian.returncode == 0 + assert json.loads((hold / 'released.json').read_text())['accepted'] > 0 + phase('maintenance-released') + ui('maintenance-dismiss', '''WinActivate("Microsoft OneNote ahk_class #32770") +ControlClick("Button1", "Microsoft OneNote ahk_class #32770") +if !WinWaitClose("Microsoft OneNote ahk_class #32770", , 10) + throw Error("The maintenance conflict dialog did not close.") +''') + replacements = [] + for attempt in range(5): + ui(f'maintenance-optimize-{attempt}', '''CoordMode("Mouse", "Screen") +WinActivate("OneNote Options") +WinWaitActive("OneNote Options", , 10) +Click(254, 440) +Sleep(3000) +if WinExist("Microsoft OneNote ahk_class #32770") { + FileAppend(WinGetText("Microsoft OneNote ahk_class #32770"), "*") + ControlClick("Button1", "Microsoft OneNote ahk_class #32770") + if !WinWaitClose("Microsoft OneNote ahk_class #32770", , 10) + throw Error("The maintenance conflict dialog did not close.") +} +''') + replacements.append(stat()) + if replacements[-1]['inode'] != before['inode'] and replacements[-1]['size'] < before['size']: break + time.sleep(2) + (output / 'maintenance-stat.json').write_text(json.dumps({'before': before, 'held': held, 'attempts': replacements}, indent=2)) + assert replacements[-1]['inode'] != before['inode'] and replacements[-1]['size'] < before['size'], 'Native optimization did not replace and shrink the section' + ui('maintenance-options-close', '''WinActivate("OneNote Options") +CoordMode("Mouse", "Screen") +Click(663, 533) +WinWaitClose("OneNote Options", , 10) +''') + phase('maintenance-resumed') + (rust / 'resume').touch() + (shared / 'maintenance-resume').touch() + wait_for(lambda: process.poll() is not None, 600, 'Post-maintenance workload did not complete') + assert process.returncode == 0, 'Mixed workload failed after maintenance' + events = [json.loads(line) for line in (output / 'smb-trace.jsonl').read_text().splitlines()] + result = {'maintenance_locks': maintenance_locks(events), 'resumed_overlap': verify(events, phase='maintenance-resumed')} + (output / 'maintenance-verification.json').write_text(json.dumps(result, indent=2)) + finally: + if guardian is not None and guardian.poll() is None: + (output / 'guardian/release').touch() + try: guardian.wait(timeout=30) + except subprocess.TimeoutExpired: + guardian.terminate() + guardian.wait(timeout=30) + if process.poll() is None: + process.terminate() + process.wait(timeout=180) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('output', type=Path) + parser.add_argument('--linux', required=True) + parser.add_argument('--fixture', type=Path, required=True) + parser.add_argument('--operations', type=int, default=40) + parser.add_argument('--seed', type=int, default=908) + args = parser.parse_args() + if args.operations < 4 or args.operations % 2: parser.error('Use an even operation count of at least four.') + def interrupted(_signal, _frame): raise KeyboardInterrupt + signal.signal(signal.SIGTERM, interrupted) + run(args.output, args.linux, args.fixture.resolve(), args.operations, args.seed) diff --git a/tools/native_probe.py b/tools/native_probe.py index c1d8b354b6641070ac16d02db6f65179e691e581..c7515236ce5acd14af7278d569f5ca368a7f5f85 100644 --- a/tools/native_probe.py +++ b/tools/native_probe.py @@ -9,11 +9,14 @@ import zipfile from native_runner import ROOT, clone, command, windows, collect_artifacts -def run(inputs, output): +def run(inputs, output, scripts=None): output.mkdir(parents=True, exist_ok=False) files = sorted(inputs.glob('*.one')) assert files, 'No section candidates' - scripts = [ROOT / 'tools/native/cold.ps1', ROOT / 'tools/native/probe.ps1'] + if scripts is None: + (output / 'scripts').mkdir() + scripts = [output / 'scripts' / name for name in ('cold.ps1', 'probe.ps1')] + for path in scripts: path.write_bytes((ROOT / 'tools/native' / path.name).read_bytes()) (output / 'run.json').write_text(json.dumps({'inputs': {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in files}, 'scripts': {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in scripts}}, indent=2)) archive = output / 'inputs.zip' @@ -28,7 +31,7 @@ def run(inputs, output): if result.get('error'): raise RuntimeError(result['error']) command(name, r'powershell -NoProfile -Command "Expand-Archive C:\one-tests\inputs.zip C:\one-tests\runs\capture\inputs"', output) command(name, r'powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\one-tests\probe.ps1 -Root C:\one-tests\runs\capture -CloneHost ONE-' + name.upper(), output, (len(files) * 60 + 120) * 1000) - collect_artifacts(name, output, r'results, C:\one-tests\runs\capture\results.json') + collect_artifacts(name, output, r'results, C:\one-tests\runs\capture\results.json, C:\one-tests\runs\capture\progress.jsonl') finally: archive.unlink(missing_ok=True) diff --git a/tools/native_stress.py b/tools/native_stress.py index d0e217d02347275063fdfcc1204a143805b35eab..c82f2a642db999599a2830250e748e37aadd6ef1 100644 --- a/tools/native_stress.py +++ b/tools/native_stress.py @@ -1,5 +1,6 @@ """Overlapping native and Rust editing histories on one shared section.""" import json +import os import time from native_runner import windows @@ -7,28 +8,36 @@ from concurrent_rust import running_clients from document_model import ordered_pages, walk -def edit_history(logs, operations, edit=False): - links = {} - for actor, events in logs.items(): - assert events[0]['event'] == 'ready' and events[-1]['event'] == 'done', 'Incomplete client log' - if not actor.startswith('w'): continue - intents = {event['attempt']: event for event in events if event['event'] == 'intent'} - commits = [event for event in events if event['event'] == 'commit'] - assert sorted(event['operation'] for event in commits) == list(range(operations)), 'Missing or duplicate Rust acknowledgements' - for event in commits: - intent = intents[event['attempt']] - token = f' [{actor}:{event["operation"]}]' - offset = len(intent['before'].encode('utf-16-le')) // 2 - assert event['token'] == intent['token'] == token, 'Acknowledgement differs from intended edit' - assert intent['replacement'] == (' café 🦀' if edit else '') + token, 'Unexpected replacement text' - assert intent['operation'] == event['operation'] and intent['source_transaction'] == event['source_transaction'], 'Acknowledgement used another intent' - start, end = intent['range'] - assert len('Concurrent edits:') <= start <= end <= offset, 'Invalid edit range' - if not edit: assert start == end == offset, 'Expected an append intent' - units = intent['before'].encode('utf-16-le') - after = units[:start * 2].decode('utf-16-le') + intent['replacement'] + units[end * 2:].decode('utf-16-le') - assert intent['before'] not in links, 'Acknowledged Rust edits branched from the same content' - links[intent['before']] = event, after +def edit_history(logs, operations, edit=False, *, partial=False, offline=False): + if offline: + assert not edit, 'Offline acceptance currently uses append intents' + from offline_history import publication_links + links = publication_links(logs, operations, partial) + else: + links = {} + for actor, events in logs.items(): + assert events and events[0]['event'] == 'ready', 'Missing client start' + assert partial or events[-1]['event'] == 'done', 'Incomplete client log' + if not actor.startswith('w'): continue + intents = {event['attempt']: event for event in events if event['event'] == 'intent'} + commits = [event for event in events if event['event'] == 'commit'] + assert len(commits) <= operations, 'Unexpected Rust acknowledgement' + expected = len(commits) if partial else operations + assert sorted(event['operation'] for event in commits) == list(range(expected)), 'Missing or duplicate Rust acknowledgements' + for event in commits: + intent = intents[event['attempt']] + token = f' [{actor}:{event["operation"]}]' + offset = len(intent['before'].encode('utf-16-le')) // 2 + assert event['token'] == intent['token'] == token, 'Acknowledgement differs from intended edit' + assert intent['replacement'] == (' café 🦀' if edit else '') + token, 'Unexpected replacement text' + assert intent['operation'] == event['operation'] and intent['source_transaction'] == event['source_transaction'], 'Acknowledgement used another intent' + start, end = intent['range'] + assert len('Concurrent edits:') <= start <= end <= offset, 'Invalid edit range' + if not edit: assert start == end == offset, 'Expected an append intent' + units = intent['before'].encode('utf-16-le') + after = units[:start * 2].decode('utf-16-le') + intent['replacement'] + units[end * 2:].decode('utf-16-le') + assert intent['before'] not in links, 'Acknowledged Rust edits branched from the same content' + links[intent['before']] = event, after text = 'Concurrent edits:' versions = {text: 0} ordered = [] @@ -79,10 +88,15 @@ def verify_capture(output, capture): operations, edit = config['stress_operations'], config['edit'] actors = [f'w{i}' for i in range(config['rust_writers'])] + [f'r{i}' for i in range(config['rust_readers'])] logs = {actor: [json.loads(line) for line in (output / 'rust' / f'{actor}.jsonl').read_text().splitlines()] for actor in actors} - commits, rust_text = edit_history(logs, operations, edit) + commits, rust_text = edit_history(logs, operations, edit, offline=config.get('offline', False)) native = [[json.loads(line) for line in (output / f'n{i}' / 'stress-events.jsonl').read_text(encoding='utf-8-sig').splitlines()] for i in range(config['stress_clients'])] expected = [rust_text, *(native_history(events, i, operations, edit) for i, events in enumerate(native))] + documents = {} + if config.get('document_operations'): + from offline_document_history import document_history + documents = document_history(logs, operations) + expected.extend(document['text'] for document in documents.values()) page_file, = capture.glob('page-*.xml') page = ET.parse(page_file).getroot() paragraphs = native_characters(page, page.findall('one:Outline', ns)) @@ -95,16 +109,31 @@ def verify_capture(output, capture): for field in ('bold', 'italic'): assert bool(style.get(field)) == events[-1][field], 'Fresh native formatting differs from its last recorded editing intent' checks += 1 - return {'rust_intents': len(commits), 'native_intents': sum(map(len, native)), + if documents: + from offline_document_history import verify_native + checks += verify_native(paragraphs, documents) + return {'rust_intents': len(commits), 'document_intents': len(documents)*2, 'native_intents': sum(map(len, native)), 'exact_paragraphs': len(expected), 'native_intended_format_checks': checks} -def exercise(output, shared, clients, action, wait_action, wait_text, checkpoint, operations, sync_every, rust_writers, rust_readers, edit=False, seed=710): +def exercise(output, shared, clients, action, wait_action, wait_text, checkpoint, operations, sync_every, rust_writers, rust_readers, edit=False, seed=710, embedded_smb=False): + wait_text(clients[0], ['Concurrent edits:']) action(clients[0], 'prepare-stress', clients=len(clients)) prefixes = [f'Native {i}:' for i in range(len(clients))] for client in clients: wait_text(client, ['Concurrent edits:', *prefixes]) checkpoint('stress-initial') + config = json.loads((output / 'run.json').read_text()) + maintenance = config.get('maintenance', False) + disconnect = config.get('disconnect', False) + offline_outage = config.get('offline_outage', False) + offline_lost_reply = config.get('offline_lost_reply', False) + if maintenance: + (output / 'maintenance-ready').touch() + deadline = time.monotonic() + 300 + while not (output / 'maintenance-start').exists(): + if time.monotonic() > deadline: raise TimeoutError('Maintenance controller did not start the workload') + time.sleep(.1) clocks = [] for client in clients: before = time.time_ns() // 1000 @@ -113,7 +142,7 @@ def exercise(output, shared, clients, action, wait_action, wait_text, checkpoint native = result['utc_us'] clocks.append({'native_minus_host_us': [native - after - 15625, native - before + 15625]}) (output / 'clocks.json').write_text(json.dumps(clocks, indent=2)) - sequences = [action(client, 'stress', wait=False, actor=i, prefix=prefixes[i], operations=operations, seed=seed + 10000 + i, sync_every=sync_every, edit=edit) + sequences = [action(client, 'stress', wait=False, actor=i, prefix=prefixes[i], operations=operations, seed=seed + 10000 + i, sync_every=sync_every, edit=edit, maintenance=maintenance) for i, client in enumerate(clients)] for client, sequence in zip(clients, sequences): deadline = time.monotonic() + 60 @@ -125,18 +154,63 @@ def exercise(output, shared, clients, action, wait_action, wait_text, checkpoint folder = output / 'rust' folder.mkdir() start = folder / 'start' - with running_clients(folder, shared / 'synthetic.one', rust_writers, rust_readers, operations, seed, edit=edit) as processes: - (shared / 'stress-start').write_text('start') + from concurrent_rust import ROOT + binaries = ROOT / 'target' / config.get('client_profile', 'debug') / 'examples' + source = 'm6-collaboration\\synthetic.one' if embedded_smb else shared / 'synthetic.one' + executable = binaries / ('smb_concurrent_client' if embedded_smb else 'concurrent_client') + if disconnect: executable = binaries / 'smb_reconnect_client' + if config.get('offline'): executable = binaries / 'smb_offline_client' + environment = {**os.environ, 'ONESTORE_MAINTENANCE_DIR': str(folder)} if maintenance else None + reader_executable = None + if offline_outage: + environment = {**os.environ, 'ONESTORE_OFFLINE_OUTAGE_DIR': str(folder)} + reader_executable = binaries / 'smb_reconnect_client' + if offline_lost_reply: + captures = folder / 'confirmations' + captures.mkdir() + environment = {**os.environ, 'ONESTORE_OFFLINE_CONFIRM_DIR': str(captures)} + reader_executable = binaries / 'smb_reconnect_client' + if config.get('document_operations'): + environment = {**(environment or os.environ), 'ONESTORE_OFFLINE_DOCUMENTS': '1'} + if offline_lost_reply: + environment['ONESTORE_OFFLINE_FORMAT_REPLY_DIR'] = str(folder) + try: + with running_clients(folder, source, rust_writers, rust_readers, operations, seed, timeout=config.get('client_timeout', 600), edit=edit, executable=executable, environment=environment, reader_executable=reader_executable) as processes: + (shared / 'stress-start').write_text('start') + for client, sequence in zip(clients, sequences): + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + result = windows.do_cmd(f'if exist C:\\one-tests\\runs\\capture\\outbox\\{sequence}\\editing echo editing', target=client['name']) + if 'editing' in result.get('stdout', ''): break + time.sleep(.1) + else: raise TimeoutError('Native stress client did not acknowledge its first edit') + start.touch() + if disconnect: + from native_disconnect import interrupt + interrupt(output, clients, sequences, processes) + if offline_outage or offline_lost_reply: + from offline_outage import interrupt + interrupt(output, clients, sequences, processes) + if embedded_smb: + import linux_vm + server = json.loads((output / 'run.json').read_text())['server'] + with (output / 'server-locks.jsonl').open('w') as trace: + for _ in range(100): + captured = linux_vm.run_ssh(server, 'sudo smbstatus --byterange --json', timeout=5) + captured.check_returncode() + trace.write(json.dumps(json.loads(captured.stdout)) + '\n') + trace.flush() + time.sleep(.1) + for client, sequence in zip(clients, sequences): + wait_action(client, sequence, 'stress') + finally: for client, sequence in zip(clients, sequences): - deadline = time.monotonic() + 60 - while time.monotonic() < deadline: - result = windows.do_cmd(f'if exist C:\\one-tests\\runs\\capture\\outbox\\{sequence}\\editing echo editing', target=client['name']) - if 'editing' in result.get('stdout', ''): break - time.sleep(.1) - else: raise TimeoutError('Native stress client did not acknowledge its first edit') - start.touch() - for client, sequence in zip(clients, sequences): - wait_action(client, sequence, 'stress') + local = client['folder'] / 'stress-events.jsonl' + try: + result = windows.do_get(f'C:\\one-tests\\runs\\capture\\outbox\\{sequence}\\events.jsonl', local, client['name']) + except Exception as error: + result = {'error': str(error)} + (client['folder'] / 'stress-capture.json').write_text(json.dumps(result, indent=2)) for client, clock in zip(clients, clocks): before = time.time_ns() // 1000 native = windows.do_health(client['name'])['utc_us'] @@ -144,23 +218,31 @@ def exercise(output, shared, clients, action, wait_action, wait_text, checkpoint clock['after_native_minus_host_us'] = [native - after - 15625, native - before + 15625] (output / 'clocks.json').write_text(json.dumps(clocks, indent=2)) rust = {actor: [json.loads(line) for line in (folder / f'{actor}.jsonl').read_text().splitlines()] for actor in processes} - commits, expected_rust = edit_history(rust, operations, edit) + commits, expected_rust = edit_history(rust, operations, edit, offline=config.get('offline', False)) assert len(commits) == rust_writers * operations, 'Missing Rust acknowledgements' native_events = [] - for i, (client, sequence) in enumerate(zip(clients, sequences)): + for client in clients: local = client['folder'] / 'stress-events.jsonl' - result = windows.do_get(f'C:\\one-tests\\runs\\capture\\outbox\\{sequence}\\events.jsonl', local, client['name']) - if result.get('error'): raise RuntimeError(str(result)) events = [json.loads(line) for line in local.read_text(encoding='utf-8-sig').splitlines()] - prefixes[i] = native_history(events, i, operations, edit) native_events.append(events) + prefixes = [native_history(events, i, operations, edit) for i, events in enumerate(native_events)] expected = [expected_rust, *prefixes] + documents = {} + if config.get('document_operations'): + from offline_document_history import document_history + documents = document_history(rust, operations) + expected.extend(document['text'] for document in documents.values()) for client in clients: wait_text(client, expected) checkpoint('stress-final') model = json.loads((output / 'stress-final/model/document.json').read_text()) + for _, _, revision, _ in ordered_pages(model): + assert not revision['nodes'][revision['roots']['1']]['spaces'], 'Disjoint edits created conflict pages' paragraphs = [n['kind']['text'] for _, _, revision, page in ordered_pages(model) for _, n in walk(revision, page) if n['kind']['type'] == 'RichText'] assert sorted(paragraphs) == sorted(expected), 'Final shared state lost, duplicated or added unrecorded content' + if documents: + from offline_document_history import verify_model + verify_model(model, documents) if edit: resolved = json.loads((output / 'stress-final/model/text.json').read_text()) for i, events in enumerate(native_events): @@ -180,7 +262,7 @@ def exercise(output, shared, clients, action, wait_action, wait_text, checkpoint earliest_end = (native['updated_ticks'] - 621355968000000000) // 10 - high overlap += sum(max(latest_start, rust['started_us']) < min(earliest_end, rust['finished_us']) for rust in commits) result = {'native_writers': len(clients), 'rust_writers': rust_writers, 'rust_readers': rust_readers, - 'sync_every': sync_every, 'edit': edit, 'seed': seed, 'native_edits': operations * len(clients), 'rust_commits': len(commits), 'rust_reads': len(reads), + 'sync_every': sync_every, 'edit': edit, 'seed': seed, 'offline': config.get('offline', False), 'native_edits': operations * len(clients), 'rust_commits': len(commits), 'document_commits': len(documents)*2, 'rust_reads': len(reads), 'clock_bounded_native_rust_call_overlaps': overlap, 'converged_paragraphs': len(expected)} (output / 'result.json').write_text(json.dumps(result, indent=2)) assert overlap, 'No native/Rust call overlap established within clock uncertainty' diff --git a/tools/notebook_editor.py b/tools/notebook_editor.py new file mode 100644 index 0000000000000000000000000000000000000000..fa45ab62b3a8b6723d43f2825580674583bfaab3 --- /dev/null +++ b/tools/notebook_editor.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +"""Serve the HTML diagnostic editor on a new notebook copy.""" +import argparse +import hashlib +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import mimetypes +import os +from pathlib import Path +import re +import shutil +import subprocess +from threading import Lock +import time +from urllib.parse import parse_qs, urlsplit +import uuid + +from document_model import walk +from notebook_report import generate + +ROOT = Path(__file__).resolve().parent.parent +BRIDGE = ROOT / 'target/debug/onestore-diagnostic' + + +def bridge(mode, source, destination, edit=None): + try: + result = subprocess.run([BRIDGE, mode, source, destination], + input=json.dumps(edit) if edit is not None else None, + capture_output=True, text=True, timeout=120) + if result.returncode: + raise RuntimeError(result.stderr or f'Diagnostic process exited {result.returncode}') + return json.loads(result.stdout) + except (OSError, ValueError, RuntimeError, subprocess.TimeoutExpired) as error: + return {'ok': False, 'state': 'Unknown' if mode == 'commit' else 'NotCommitted', + 'kind': 'Process', 'error': str(error)} + + +class Session: + def __init__(self, source, output): + source = source.resolve(strict=True) + self.output = output.resolve() + if self.output.is_relative_to(source): + raise ValueError('Choose a session directory outside the source notebook.') + self.output.mkdir(parents=True, exist_ok=False) + self.lock = Lock() + hashes = {p.relative_to(source).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest() + for p in source.rglob('*') if p.is_file()} + shutil.copytree(source, self.output / 'notebook') + for name, digest in hashes.items(): + copied = self.output / 'notebook' / name + if hashlib.sha256(copied.read_bytes()).hexdigest() != digest or hashlib.sha256((source / name).read_bytes()).hexdigest() != digest: + raise ValueError('The source changed during copying; start a fresh session.') + copied.chmod(copied.stat().st_mode | 0o600) + (self.output / 'source.json').write_text(json.dumps({'root': str(source), 'sha256': hashes}, indent=2)) + (self.output / 'g').mkdir() + (self.output / 'objects').mkdir() + self.snapshot() + + def snapshot(self): + number = max((int(p.name) for p in (self.output / 'g').iterdir() if p.name.isdecimal()), default=-1) + 1 + pending = self.output / 'g' / (str(number) + '-' + uuid.uuid4().hex + '.building') + source = pending / 'snapshot' + source.mkdir(parents=True) + for path in sorted((self.output / 'notebook').rglob('*')): + if path.suffix.lower() not in ('.one', '.onetoc2'): continue + saved = source / path.relative_to(self.output / 'notebook') + saved.parent.mkdir(parents=True, exist_ok=True) + result = bridge('snapshot', path, saved) + if not result['ok']: raise RuntimeError(result['error']) + digest = hashlib.sha256(saved.read_bytes()).hexdigest() + blob = self.output / 'objects' / digest + if blob.exists(): saved.unlink() + else: + saved.rename(blob) + blob.chmod(0o444) + os.link(blob, saved) + previous = self.output / 'g' / str(number - 1) / 'report' if number else None + generate(source, pending / 'report', editable=True, previous=previous) + pending.rename(self.output / 'g' / str(number)) + return number + + def page(self, generation, page): + if type(generation) is not int or generation < 0: + raise ValueError('Choose a page from this report.') + folder = self.output / 'g' / str(generation) + pages = json.loads((folder / 'report/pages.json').read_text()) + row = next(p for p in pages if p['report'] == page) + if row['category'] != 'Page': + raise ValueError('Choose an active page. Conflicts, templates, deleted pages and history are read-only here.') + sources = json.loads((folder / 'report/source.json').read_text()) + index = next(i for i, source in enumerate(sources) if source['path'] == row['section']) + model = folder / 'report/model' / str(index) + document = json.loads((model / 'document.json').read_text()) + revision = document['spaces'][row['space']]['revisions'][row['revision']] + return row, folder / 'snapshot' / row['section'], revision, model + + def selection(self, generation, page, oid, run): + if type(run) is not int or run < 0: + raise ValueError('Choose a text run from this report.') + row, source, revision, model = self.page(generation, page) + node = next(node for key, node in walk(revision, row['object']) if key == oid) + if node['kind']['type'] != 'RichText': raise ValueError('Choose a text run.') + selected = node['kind']['runs'][run] + text = json.loads((model / 'text.json').read_text())[row['space']][row['revision']][oid][run]['text'] + request = {'space': row['space'], 'object': oid, + 'action': {'type': 'Text', 'start': selected['start'], 'end': selected['end'], 'replacement': text}} + return row, source, request + + def location(self, generation, row=None): + page = 'index.html' + if row: + pages = json.loads((self.output / 'g' / str(generation) / 'report/pages.json').read_text()) + page = next((p['report'] for p in pages if (p['section'], p['space'], p['object'], p['context']) == + (row['section'], row['space'], row['object'], row['context'])), page) + return f'/g/{generation}/report/{page}' + + def save(self, data): + fields = {'text': {'run', 'replacement'}, 'format': {'run', 'start', 'end', 'attributes'}, + 'paragraph': {'before', 'text', 'author'}, 'outline': {'x', 'y', 'text', 'author'}} + action = data.get('action') + if action not in fields or set(data) != {'generation', 'page', 'object', 'action'} | fields[action]: + raise ValueError('Choose text, formatting, a paragraph or an outline to save.') + if action in ('text', 'format'): + row, source, edit = self.selection(data['generation'], data['page'], data['object'], data['run']) + selected = edit['action'] + if action == 'text': + if not isinstance(data['replacement'], str): raise ValueError('Enter replacement text.') + selected['replacement'] = data['replacement'] + else: + if (type(data['start']) is not int or type(data['end']) is not int + or not 0 <= data['start'] <= data['end'] <= selected['end'] - selected['start']): + raise ValueError('Select text within this run.') + edit['action'] = {'type': 'Format', 'start': selected['start'] + data['start'], + 'end': selected['start'] + data['end'], 'attributes': data['attributes']} + else: + row, source, revision, _ = self.page(data['generation'], data['page']) + if data['object'] not in {oid for oid, _ in walk(revision, row['object'])}: + raise ValueError('Choose a container on this page.') + edit = {'space': row['space'], 'object': data['object'], + 'action': {'type': action.title(), **{key: data[key] for key in fields[action]}}} + operation = uuid.uuid4().hex + result = {'ok': False, 'state': 'NotCommitted'} + try: + with (self.output / 'operations.jsonl').open('a') as log: + intent = {'event': 'intent', 'operation': operation, 'started_ms': time.time_ns() // 1000000, + 'selection': {key: data[key] for key in ('generation', 'page', 'object', 'action')}, + 'edit': edit, 'section': row['section']} + log.write(json.dumps(intent, ensure_ascii=True) + '\n') + log.flush(); os.fsync(log.fileno()) + result = {'ok': False, 'state': 'Unknown'} + result = bridge('commit', self.output / 'notebook' / row['section'], source, edit) + log.write(json.dumps({'event': 'outcome', 'operation': operation, 'finished_ms': time.time_ns() // 1000000, **result}) + '\n') + log.flush(); os.fsync(log.fileno()) + except Exception as error: + result = {**result, 'ok': False, 'error': str(error)} + try: + generation = self.snapshot() + result['location'] = self.location(generation, row) + except Exception as error: + result = {**result, 'ok': False, 'report_error': str(error)} + return result + + +class Handler(BaseHTTPRequestHandler): + def redirect(self, location): + self.send_response(302) + self.send_header('Location', location) + self.send_header('Cache-Control', 'no-store') + self.send_header('Content-Length', '0') + self.end_headers() + + def reply(self, status, value): + content = json.dumps(value, ensure_ascii=True).encode() + self.send_response(status) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(content))) + self.send_header('Cache-Control', 'no-store') + self.end_headers() + self.wfile.write(content) + + def do_GET(self): + session = self.server.session + url = urlsplit(self.path) + try: + if url.path == '/': + latest = max(int(p.name) for p in (session.output / 'g').iterdir() if p.name.isdecimal()) + self.redirect(session.location(latest)) + return + if url.path in ('/api/run', '/api/page', '/latest'): + query = parse_qs(url.query, strict_parsing=True) + if url.path == '/latest': + row = session.page(int(query['generation'][0]), query['page'][0])[0] if query else None + with session.lock: + self.redirect(session.location(session.snapshot(), row)) + return + if url.path == '/api/page': + row, _, revision, _ = session.page(int(query['generation'][0]), query['page'][0]) + def label(oid): + node = revision['nodes'][oid] + text = ' '.join(n['kind']['text'] for _, n in walk(revision, oid) if n['kind']['type'] == 'RichText') + return node['kind']['type'] + (' · ' + text[:80] if text else '') + targets = [] + pending = [row['object']] + while pending: + oid = pending.pop() + node = revision['nodes'][oid] + if node['kind']['type'] == 'Title': continue + if node['kind']['type'] in ('Outline', 'Paragraph', 'OutlineGroup', 'Cell'): + targets.append({'object': oid, 'label': label(oid), + 'children': [{'object': child, 'label': label(child)} for child in node['children']]}) + pending.extend(reversed(node['structure'] + node['content'] + node['children'])) + self.reply(200, {'ok': True, 'object': row['object'], 'targets': targets}) + return + row, source, edit = session.selection(int(query['generation'][0]), query['page'][0], query['object'][0], int(query['run'][0])) + result = bridge('check', source, '-', edit) + self.reply(200 if result['ok'] else 422, {**result, 'text': edit['action']['replacement']}) + return + if url.path in ('/editor.js', '/editor.css'): + path = ROOT / 'tools/diagnostic' / url.path[1:] + else: + match = re.fullmatch(r'/g/(\d+)/report/(.+)', url.path) + if not match: raise FileNotFoundError() + root = session.output / 'g' / match[1] / 'report' + path = (root / match[2]).resolve(strict=True) + if not path.is_relative_to(root): raise FileNotFoundError() + with path.open('rb') as file: + self.send_response(200) + self.send_header('Content-Type', mimetypes.guess_type(path)[0] or 'application/octet-stream') + self.send_header('Content-Length', str(os.fstat(file.fileno()).st_size)) + self.send_header('Cache-Control', 'no-store') + self.end_headers() + shutil.copyfileobj(file, self.wfile) + except (ValueError, KeyError, IndexError, StopIteration) as error: + self.reply(422, {'ok': False, 'error': str(error) or 'The selected text is unavailable.'}) + except OSError: + self.reply(404, {'ok': False, 'error': 'Report unavailable. Open the notebook index.'}) + except Exception as error: + self.reply(503, {'ok': False, 'error': str(error)}) + + def do_POST(self): + session = self.server.session + host = self.headers.get('Host') + allowed = {f'127.0.0.1:{self.server.server_port}', f'localhost:{self.server.server_port}'} + if host not in allowed or self.headers.get('Origin', 'http://' + host) != 'http://' + host or self.headers.get('X-OneNote-Diagnostic') != '1': + self.reply(403, {'ok': False, 'error': 'Open this editor on its local address.'}) + return + if self.path != '/api/save': + self.reply(404, {'ok': False, 'error': 'Unknown diagnostic action.'}) + return + try: + length = int(self.headers.get('Content-Length', '0')) + if not 0 < length <= 65536 or self.headers.get('Content-Type') != 'application/json': + raise ValueError('Send a JSON edit smaller than 64 KiB.') + data = json.loads(self.rfile.read(length)) + with session.lock: + result = session.save(data) + status = 200 if result['ok'] else 409 if result.get('kind') == 'ResourceBusy' else 422 if result.get('kind') in ('InvalidData', 'Input') else 503 + self.reply(status, result) + except (ValueError, KeyError, IndexError, StopIteration, TypeError) as error: + self.reply(422, {'ok': False, 'state': 'NotCommitted', 'error': str(error) or 'The selected text is unavailable.'}) + except Exception as error: + self.reply(503, {'ok': False, 'state': 'Unknown' if self.path == '/api/save' else 'NotCommitted', 'error': str(error)}) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('source', type=Path) + parser.add_argument('session', type=Path) + parser.add_argument('--port', type=int, default=8782) + args = parser.parse_args() + server = ThreadingHTTPServer(('127.0.0.1', args.port), Handler) + server.session = Session(args.source, args.session) + print(f'Diagnostic editor: http://127.0.0.1:{server.server_port}/', flush=True) + print(f'Editable notebook copy: {server.session.output / "notebook"}', flush=True) + try: + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + server.server_close() diff --git a/tools/notebook_report.py b/tools/notebook_report.py index e76607c5614013c5a413dd041edbd4441c27de02..b61522470926a4288c2aaed6d1e75ef8e9b47eea 100644 --- a/tools/notebook_report.py +++ b/tools/notebook_report.py @@ -6,6 +6,7 @@ from datetime import datetime, timedelta, timezone import hashlib from html import escape as esc import json +import os from io import BytesIO from pathlib import Path, PureWindowsPath from PIL import Image @@ -64,13 +65,16 @@ def css(fmt): return ';'.join(rules) -def html_page(title, nav, body): +def html_page(title, nav, body, editable=False): + policy = "default-src 'none'; img-src 'self' data:; style-src 'self' 'unsafe-inline'; media-src 'self'; base-uri 'none'" + if editable: + policy += "; script-src 'self'; connect-src 'self'" return ''' - + ''' + esc(title) + '''
' + body + '
' +''' + ('' if editable else '') + '
' + body + '
' class Page: @@ -117,7 +121,7 @@ class Page: body = '' structured_math = any(c in kind['text'] for c in '\ufdd0\ufdee\ufdef') equation = False - for run in self.text[oid]: + for run_index, run in enumerate(self.text[oid]): if run['format']['hidden']: continue if structured_math and run['format'].get('math'): @@ -129,7 +133,7 @@ class Page: style = {**run['format'], **tag_format} if (style['superscript'] or style['subscript']) and style['font_size'] is not None: style['font_size'] *= 2 / 3 - fragment = '' + esc(run['text']) + '' + fragment = '' + esc(run['text']) + '' if run['format']['superscript']: fragment = '' + fragment + '' if run['format']['subscript']: @@ -262,19 +266,28 @@ class Page: return '
' + body + '
' if typ not in ('RichText', 'Row', 'Cell') else body -def generate(source, destination, native=None, versions=(), zone=timezone.utc): +def generate(source, destination, native=None, versions=(), zone=timezone.utc, editable=False, previous=None): source = source.resolve(strict=True) if destination.resolve().is_relative_to(source): raise ValueError('Choose an export directory outside the source notebook.') destination.mkdir(parents=True, exist_ok=False) (destination / 'model').mkdir(); (destination / 'assets').mkdir() + cached = {row['path']: (row['sha256'], previous / 'model' / str(index)) + for index, row in enumerate(json.loads((previous / 'source.json').read_text()))} if previous else {} sections = []; tocs = {}; manifest = [] for index, path in enumerate(sorted(p for p in source.rglob('*') if p.suffix.lower() in ('.one', '.onetoc2'))): relative = path.relative_to(source) before = path.read_bytes() manifest.append({'path': relative.as_posix(), 'sha256': hashlib.sha256(before).hexdigest(), 'bytes': len(before)}) exported = destination / 'model' / str(index) - subprocess.run([EXPORTER, path, exported], check=True) + reusable = cached.get(relative.as_posix()) + reused = reusable is not None and reusable[0] == manifest[-1]['sha256'] + if reused: + exported.mkdir() + for name in ('document.json', 'text.json', 'assets.json'): + os.link(reusable[1] / name, exported / name) + else: + subprocess.run([EXPORTER, path, exported], check=True) document = json.loads((exported / 'document.json').read_text()) if path.read_bytes() != before: raise ValueError('A source file changed during export.') @@ -291,6 +304,18 @@ def generate(source, destination, native=None, versions=(), zone=timezone.utc): } rows = json.loads((exported / 'assets.json').read_text()) for asset in rows: + reference = json.dumps(asset['reference'], sort_keys=True) + if reused: + name = Path(asset['path']).name + assets[reference] = 'assets/' + name + if not (destination / 'assets' / name).exists(): + os.link(previous / 'assets' / name, destination / 'assets' / name) + preview = name + '.png' + if name.endswith('.tiff') and reference in image_references: + if not (destination / 'assets' / preview).exists(): + os.link(previous / 'assets' / preview, destination / 'assets' / preview) + previews['assets/' + name] = 'assets/' + preview + continue original = exported / asset['path']; data = original.read_bytes() extension = '.png' if data.startswith(b'\x89PNG') else '.jpg' if data.startswith(b'\xff\xd8') else '.gif' if data.startswith(b'GIF8') else '.bmp' if data.startswith(b'BM') else '.tiff' if data.startswith((b'II*\0', b'MM\0*')) else '.bin' name = hashlib.sha256(data).hexdigest() + extension @@ -300,7 +325,6 @@ def generate(source, destination, native=None, versions=(), zone=timezone.utc): else: original.rename(target) asset['path'] = '../../assets/' + name - reference = json.dumps(asset['reference'], sort_keys=True) assets[reference] = 'assets/' + name if extension == '.tiff' and reference in image_references: preview = name + '.png' @@ -309,8 +333,9 @@ def generate(source, destination, native=None, versions=(), zone=timezone.utc): raise ValueError('Multipage TIFF requires frame interpretation before report generation.') image.convert('RGBA').save(destination / 'assets' / preview) previews['assets/' + name] = 'assets/' + preview - (exported / 'assets').rmdir() - (exported / 'assets.json').write_text(json.dumps(rows, indent=2)) + if not reused: + (exported / 'assets').rmdir() + (exported / 'assets.json').write_text(json.dumps(rows, indent=2)) if path.suffix.lower() == '.one': sections.append({'path': relative, 'export': exported.relative_to(destination), 'document': document, 'text': json.loads((exported / 'text.json').read_text()), 'assets': assets, 'previews': previews}) @@ -443,14 +468,14 @@ def generate(source, destination, native=None, versions=(), zone=timezone.utc): body += page.render(oid) body += '
Document structure and source identities
' + esc(json.dumps(page.space, indent=2, ensure_ascii=False)) + '
' body += '

Document JSONAsset references

' - (destination / filename).write_text(html_page(title, nav, body)) + (destination / filename).write_text(html_page(title, nav, body, editable and category == 'Page')) accounting.append({'section': str(section['path']), 'ordinal': ordinal, 'space': sid, 'revision': rid, 'object': oid, 'title': title, 'report': filename, 'category': category, 'context': context, 'version_modified': version['modified'] if version else None, 'source_report': source_page, 'native_reference': reference.as_posix() if reference else None, 'rendered': dict(page.counts)}) (destination / 'source.json').write_text(json.dumps(manifest, indent=2)) (destination / 'pages.json').write_text(json.dumps(accounting, indent=2, ensure_ascii=False)) intro = '

Notebook review

' + str(len(sections)) + ' sections · ' + str(len(pages)) + ' stored pages

Readable content follows the stored object order. Outline positions are shown in points. The document structure retains properties and identities that the readable view does not interpret.

Source hashes · Page inventory

' if locked_sections: intro += '

Page counts are unavailable for locked sections: ' + esc(', '.join(locked_sections)) + '.

' - (destination / 'index.html').write_text(html_page('Notebook review', nav, intro)) + (destination / 'index.html').write_text(html_page('Notebook review', nav, intro, editable)) if __name__ == '__main__': diff --git a/tools/offline_cache_crash.py b/tools/offline_cache_crash.py new file mode 100644 index 0000000000000000000000000000000000000000..8c6bb7ff83e06a02aa2a6a7564337df1b7d87519 --- /dev/null +++ b/tools/offline_cache_crash.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Interrupt owned offline-cache writers and verify every retained intent and image.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import select +import signal +import subprocess +import time + + +def run(binary, output): + output.mkdir(parents=True, exist_ok=False) + cache = output / "cache.sqlite" + source = os.environ.get("ONESTORE_CACHE_PROBE_SOURCE") + source_hash = hashlib.sha256(Path(source).read_bytes()).hexdigest() if source else None + payload_bytes = int(os.environ.get("ONESTORE_CACHE_PROBE_BYTES", 2 * 1024 * 1024)) + assert 0 < payload_bytes <= 2 * 1024 * 1024 + (output / "run.json").write_text(json.dumps({ + "binary": str(binary), + "binary_sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), + "controller_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "payload_bytes": payload_bytes, + "operation": os.environ.get("ONESTORE_CACHE_PROBE_OPERATION", "text"), + "source": source, + "source_sha256": source_hash, + }, indent=2)) + subprocess.run([binary, "init", cache], check=True, timeout=30) + retained = [] + retained_ids = [] + results = [] + delays = [None, "ack", "unack", "journal", "database-write", 0, .001, .01, .02, .04, .08, .16, .32, .64, 1.28, 2.56, None, "ack"] + for operation, delay in enumerate(delays, 1): + with (output / f"owner-{operation}.stderr").open("w") as stderr: + owner = subprocess.Popen([binary, "edit", cache], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=stderr, text=True, bufsize=1) + try: + def expect(prefix): + if not select.select([owner.stdout], [], [], 60)[0]: + raise TimeoutError(prefix) + actual = owner.stdout.readline().strip() + assert actual == prefix or actual.startswith(prefix + " "), (prefix, actual, owner.poll()) + return actual + + expect("ready") + contender = subprocess.run([binary, "read", cache], capture_output=True, text=True, timeout=15) + (output / f"contender-{operation}.stderr").write_text(contender.stderr) + assert contender.returncode != 0 and "DatabaseBusy" in contender.stderr, contender + before_write = cache.stat().st_mtime_ns + owner.stdin.write(f"{operation} {'unack' if delay == 'unack' else 'ack'}\n") + owner.stdin.flush() + expect(f"editing {operation}") + acknowledgement = None + journal_observation = None + if delay == "ack": + acknowledgement = expect(f"ack {operation}") + elif delay == "unack": + expect(f"durable {operation}") + elif delay in ("journal", "database-write"): + deadline = time.monotonic() + 10 + journal = cache.with_name(cache.name + "-journal") + while time.monotonic() < deadline: + try: + with journal.open("rb") as active: + header = active.read(28) + size = journal.stat().st_size + database_changed = cache.stat().st_mtime_ns != before_write + if (header[:8] == bytes.fromhex("d9d505f920a163d7") and + (delay == "journal" or database_changed)): + journal_observation = {"header": header.hex(), "bytes": size, + "database_changed": database_changed} + break + except FileNotFoundError: + pass + time.sleep(.0001) + assert journal_observation, f"No active {delay} phase observed" + elif delay is not None: + time.sleep(delay) + owner.kill() + assert owner.wait(timeout=10) == -signal.SIGKILL, "Writer did not terminate at the requested process cut" + extra = owner.stdout.read() + if f"ack {operation} " in extra: + acknowledgement = extra.strip() + read = subprocess.run([binary, "read", cache], check=True, capture_output=True, + text=True, timeout=60) + actual = json.loads(read.stdout) + operations = actual["operations"] + ids = actual["ids"] + assert operations in (retained, retained + [operation]), (retained, operation, actual) + assert ids[:len(retained_ids)] == retained_ids, (retained_ids, actual) + if acknowledgement: + assert operations[-1] == operation + assert ids[-1] == int(acknowledgement.split()[2]) + if delay == "unack": + assert operations[-1] == operation and acknowledgement is None + assert actual["complete_payloads"] + results.append({"operation": operation, "delay": delay, + "process_exit": owner.returncode, + "acknowledged": acknowledgement is not None, + "retained": operation in operations, + "retained_operations": operations, "ids": ids, + "section_bytes": actual["section_bytes"], + "complete_payloads": True, "exclusive_owner": True, + "journal_observation": journal_observation}) + retained, retained_ids = operations, ids + (output / "results.json").write_text(json.dumps(results, indent=2)) + print(json.dumps(results[-1]), flush=True) + finally: + if owner.poll() is None: + owner.kill() + owner.wait(timeout=10) + owner.stdin.close() + owner.stdout.close() + assert any(row["acknowledged"] for row in results) + assert any(row["retained"] and not row["acknowledged"] for row in results) + assert any(not row["retained"] for row in results) + subprocess.run([binary, "read", cache, output / "recovered.one"], check=True, timeout=60) + if source: + assert hashlib.sha256(Path(source).read_bytes()).hexdigest() == source_hash + print(f"Passed {len(results)} offline-cache process interruptions", flush=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("output", type=Path) + parser.add_argument("--binary", type=Path, default=Path(__file__).resolve().parents[1] / "target/release/examples/cache_probe") + args = parser.parse_args() + run(args.binary.resolve(), args.output.resolve()) diff --git a/tools/offline_document_history.py b/tools/offline_document_history.py new file mode 100644 index 0000000000000000000000000000000000000000..c1fd88a48abcb84d5cd91f3f33e1233d7ebea36e --- /dev/null +++ b/tools/offline_document_history.py @@ -0,0 +1,163 @@ +"""Account for offline document intents, receipts, reader states and native formatting.""" +import uuid + +from document_model import ordered_pages, walk + + +def identity(insertion, extension): + return '{' + str(uuid.UUID(bytes_le=bytes(insertion['guid']))).upper() + '},' + str(extension) + + +def characters(observed): + actual = [(char, run['bold'], run['size'], run['color']) + for run in observed['runs'] for char in run['text']] + assert ''.join(row[0] for row in actual) == observed['text'], 'Document runs omit or duplicate text' + return actual + + +def document_history(logs, operations): + documents = {} + for actor, events in logs.items(): + if not actor.startswith('w'): continue + assert events[0].get('document_operations') is True, 'Writer omitted document operations' + edits = [row for row in events if row['event'] == 'local_document_commit'] + assert [(row['operation'], row['kind']) for row in edits] == [ + (i, kind) for i in range(operations) for kind in ('insert', 'format')], 'Missing or duplicate document intent' + ids = [row['id'] for row in edits] + assert ids == sorted(set(ids)), 'Document intent IDs are duplicated or unordered' + assert not set(ids) & {row['id'] for row in events if row['event'] == 'local_commit'}, 'Text and document intents share an ID' + receipts = [row for row in events if row['event'] == 'document_receipt'] + reopened = [row for row in events if row['event'] == 'reopened_document_receipt'] + assert [row['id'] for row in receipts] == ids, 'Document receipt inventory differs' + assert [(r['id'], r['revision']) for r in reopened] == [(r['id'], r['revision']) for r in receipts], 'Document receipts changed across reopen' + linked = {} + for intent, receipt in zip(edits, receipts, strict=True): + attempts = [row for row in events if row['event'] == 'remote_attempt' and row['revision'] == receipt['revision'] + and set(row.get('document_changes') or {}) == {intent['object']}] + if not attempts and intent['kind'] == 'format': + attempts = [row for row in events if row['event'] == 'remote_attempt' and row['state'] == 'Unknown' + and set(row.get('document_changes') or {}) == {intent['object']}] + assert len(attempts) == 1, 'Document receipt lacks one publication attempt' + attempt, = attempts + assert attempt['state'] in ('Committed', 'Unknown'), 'Receipt identifies an unpublished document operation' + assert intent['object'] in attempt['documents'] and attempt['document_changes'] == {intent['object']: attempt['documents'][intent['object']]}, 'Document publication changed another target' + successful = [row for row in events if row['event'] == 'remote_attempt' + and row['state'] in ('Committed', 'Unknown') + and row.get('document_changes') == attempt['document_changes']] + assert successful == [attempt], 'Document intent was published or attempted uncertainly more than once' + assert intent['started_us'] <= attempt['started_us'] <= attempt['finished_us'] <= receipt['at_us'] and intent['started_us'] <= intent['finished_us'] <= receipt['at_us'], 'Document acknowledgement order is invalid' + if attempt['state'] == 'Unknown': + confirmed, observed = False, None + for row in events: + if row['event'] == 'read': observed = row + if (row['event'] != 'remote_confirm' or row['state'] != 'Committed' + or receipt['revision'] not in row.get('revisions', {}).get(intent['space'], []) + or not attempt['finished_us'] <= row['started_us'] <= row['finished_us'] <= receipt['at_us']): + continue + if receipt['revision'] != attempt['revision']: + assert intent['kind'] == 'format' and attempt['revision'] not in row['revisions'][intent['space']], 'Replacement receipt did not retire the original attempt' + assert row.get('current_revisions', {}).get(intent['space']) == receipt['revision'], 'Effect receipt does not identify the confirmed current revision' + assert observed and observed['finished_us'] <= row['started_us'] and observed['text'] == row['text'] + assert observed.get('documents', {}).get(intent['object']) == attempt['document_changes'][intent['object']], 'Effect confirmation differs from the uncertain formatting intent' + confirmed = True + assert confirmed, 'Uncertain document publication lacks confirmation' + else: + assert receipt['revision'] == attempt['revision'] + linked[intent['id']] = {**attempt, 'acknowledged_us': receipt['at_us'], 'receipt_revision': receipt['revision']} + for inserted, formatted in zip(edits[::2], edits[1::2], strict=True): + number = inserted['operation'] + text = f'Document {actor}:{number} 🦀' + insertion = inserted['insertion'] + target = identity(insertion, 2) + assert target == inserted['object'] == formatted['object'], 'Dependent formatting addresses another object' + assert inserted['space'] == formatted['space'] and inserted['text'] == formatted['text'] == insertion['text'] == text + assert insertion['author'] == 'Offline document writer' + if number % 2 == 0: + assert insertion['placement'] == {'Outline': {'x': 144 + int(actor[1:]) * 240, 'y': 144 + number * 72}}, 'Outline placement differs from intent' + else: + assert insertion['placement'] == {'Paragraph': {'before': None}} + assert insertion['parent'] == identity(edits[(number-1)*2]['insertion'], 1), 'Paragraph lost its outline parent' + assert formatted['range'] == [1, len(text.encode('utf-16-le')) // 2 - 2] + assert formatted['attributes'] == [{'Bold': True}, {'FontSize': 18 + number % 9}, {'Color': [18, 52, 86]}] + old = [(char, False, 11, 0xff000000) for char in text] + new = [(char, True, 18 + number % 9, 0x563412) if 0 < i < len(text)-1 else old[i] for i, char in enumerate(text)] + assert target not in documents, 'Two insertion intents share an object identity' + created, changed = linked[inserted['id']], linked[formatted['id']] + assert created['finished_us'] <= changed['started_us'], 'Formatting preceded its insertion' + assert characters(created['documents'][target]) == old, 'Insertion publication differs from its local intent' + assert characters(changed['documents'][target]) == new, 'Formatting publication differs from its local intent' + documents[target] = {'text': text, 'insertion': insertion, 'space': inserted['space'], + 'old': old, 'new': new, 'insert': created, 'format': changed} + assert documents, 'No document operations were recorded' + for actor, events in logs.items(): + previous = {} + reads = [row for row in events if row['event'] in ('read', 'document_read') and row.get('documents') is not None] + assert reads, f'{actor} did not observe document snapshots' + assert any(row['documents'] for row in reads), f'{actor} never observed a created document object' + for read in reads: + observed = read['documents'] + assert set(previous) <= set(observed) <= set(documents), 'Reader lost an object or observed an unrecorded insertion' + for target, document in documents.items(): + if read['started_us'] > document['insert']['acknowledged_us']: + assert target in observed, 'Reader missed an acknowledged insertion' + if target not in observed: continue + assert read['finished_us'] >= document['insert']['started_us'], 'Reader observed a future insertion' + actual = characters(observed[target]) + assert actual in (document['old'], document['new']), 'Reader observed partial or invented formatting' + formatted = actual == document['new'] + assert not previous.get(target, False) or formatted, 'Reader reverted acknowledged formatting' + if read['started_us'] > document['format']['acknowledged_us']: + assert formatted, 'Reader missed acknowledged formatting' + if formatted: + assert read['finished_us'] >= document['format']['started_us'], 'Reader observed future formatting' + previous[target] = formatted + return documents + + +def verify_model(model, documents): + children = {} + for document in documents.values(): + insertion = document['insertion'] + if 'Outline' in insertion['placement']: + children[identity(insertion, 1)] = [identity(insertion, 3)] + for document in documents.values(): + insertion = document['insertion'] + if 'Paragraph' in insertion['placement']: + children[insertion['parent']].append(identity(insertion, 1)) + found = set() + for sid, _, revision, page in ordered_pages(model): + nodes = revision['nodes'] + for target, node in walk(revision, page): + if target not in documents: continue + assert target not in found, 'Inserted text is reachable twice' + found.add(target) + expected = documents[target] + insertion = expected['insertion'] + assert sid == expected['space'] and node['kind']['text'] == expected['text'] + object_id = identity(insertion, 1) + assert object_id in nodes[insertion['parent']]['children'], 'Insertion lost its parent' + paragraph = identity(insertion, 3) if 'Outline' in insertion['placement'] else object_id + assert nodes[paragraph]['content'] == [target], 'Inserted paragraph content changed' + if 'Outline' in insertion['placement']: + position = insertion['placement']['Outline'] + assert all(nodes[object_id]['layout'][key] == position[key] for key in ('x', 'y')), 'Outline coordinates changed' + assert nodes[object_id]['children'] == children[object_id], 'Inserted paragraph order changed' + assert found == set(documents), 'Final model omitted an inserted object' + + +def verify_native(paragraphs, documents): + from PIL import ImageColor + by_text = {''.join(char for char, _ in paragraph): paragraph for paragraph in paragraphs} + checks = 0 + for document in documents.values(): + actual = by_text[document['text']] + for (char, style), (wanted, bold, size, color) in zip(actual, document['new'], strict=True): + assert char == wanted and bool(style.get('bold')) == bold, 'Native text or bold differs from intent' + assert style.get('font_size', 11) == size, 'Native font size differs from intent' + native_color = style.get('color', 'automatic') + if color == 0xff000000: + assert native_color in ('automatic', None), 'Native automatic color changed' + else: + assert ImageColor.getrgb(native_color) == (18, 52, 86), 'Native color differs from intent' + checks += 3 + return checks diff --git a/tools/offline_history.py b/tools/offline_history.py new file mode 100644 index 0000000000000000000000000000000000000000..62556cc93dff529752d849a8559254da18c4eb09 --- /dev/null +++ b/tools/offline_history.py @@ -0,0 +1,73 @@ +"""Verify local intent acknowledgements separately from the remote publication chain.""" +import re + + +def tokens(text): + assert text.startswith('Concurrent edits:'), 'Unexpected append prefix' + tail = text[len('Concurrent edits:'):] + found = re.findall(r' \[w[0-9]+:[0-9]+\]', tail) + assert ''.join(found) == tail and len(set(found)) == len(found), 'Malformed or duplicated append history' + return found + + +def publication_links(logs, operations, partial=False): + local = {} + for actor, events in logs.items(): + assert events and events[0]['event'] == 'ready', 'Missing client start' + assert partial or events[-1]['event'] == 'done', 'Incomplete client log' + if not actor.startswith('w'): continue + assert events[0].get('offline') is True, 'Expected an offline writer' + edits = [event for event in events if event['event'] == 'local_commit'] + assert [event['operation'] for event in edits] == list(range(len(edits))), 'Missing or duplicate local acknowledgement' + assert len(edits) <= operations and (partial or len(edits) == operations), 'Local operation count differs' + assert len({event['id'] for event in edits}) == len(edits), 'Duplicate local intent ID' + assert [event['id'] for event in edits] == sorted(event['id'] for event in edits), 'Local IDs went backwards' + for event in edits: + token = f' [{actor}:{event["operation"]}]' + assert event['token'] == token and token not in local, 'Local token differs from its operation' + assert event['started_us'] <= event['finished_us'], 'Invalid local acknowledgement interval' + prior = tokens(event['before']) + own = [item for item in prior if item.startswith(f' [{actor}:')] + assert own == [f' [{actor}:{i}]' for i in range(event['operation'])], 'Local view lost or invented its own edit' + local[token] = event + for event in local.values(): + for token in tokens(event['before']): + assert token in local and local[token]['started_us'] <= event['finished_us'], 'Local view invented a future token' + links = {} + seen_revisions = set() + for actor, events in logs.items(): + if not actor.startswith('w'): continue + edits = {event['id']: event for event in events if event['event'] == 'local_commit'} + receipts = [event for event in events if event['event'] == 'remote_receipt'] + assert len({event['id'] for event in receipts}) == len(receipts), 'Duplicate remote receipt' + assert set(event['id'] for event in receipts) <= set(edits), 'Receipt lacks a local intent' + assert partial or len(receipts) == len(edits), 'Local success lacks remote acknowledgement' + reopened = [event for event in events if event['event'] == 'reopened_receipt'] + if not partial: + assert [(event['id'], event['revision']) for event in reopened] == [(event['id'], event['revision']) for event in receipts], 'Receipt changed across reopen' + for receipt in receipts: + intent = edits[receipt['id']] + attempts = [event for event in events if event['event'] == 'remote_attempt' and event['revision'] == receipt['revision']] + assert len(attempts) == 1, 'Receipt does not identify one publication attempt' + attempt, = attempts + assert attempt['state'] in ('Committed', 'Unknown'), 'Receipt identifies a proven-unpublished attempt' + if attempt['state'] == 'Unknown': + assert all(field in attempt and field in intent for field in ('space', 'object')), 'Uncertain target identity is missing' + assert attempt['space'] == intent['space'] and attempt['object'] == intent['object'], 'Confirmation identifies another target' + confirmed = [event for event in events if event['event'] == 'remote_confirm' + and event['state'] == 'Committed' + and receipt['revision'] in event.get('revisions', {}).get(intent['space'], []) + and event.get('text', '').startswith(attempt['after']) + and attempt['finished_us'] <= event['started_us'] <= event['finished_us'] <= receipt['at_us']] + assert confirmed, 'Uncertain publication lacks a successful retained-revision confirmation' + assert not any(event['event'] == 'remote_attempt' and event['started_us'] >= attempt['finished_us'] + and event['after'] == event['before'] + intent['token'] for event in events), 'An uncertain intent was replayed' + assert receipt['revision'] not in seen_revisions, 'A revision was acknowledged twice' + seen_revisions.add(receipt['revision']) + assert intent['started_us'] <= attempt['started_us'] <= attempt['finished_us'] <= receipt['at_us'], 'Receipt precedes its publication' + assert attempt['after'] == attempt['before'] + intent['token'], 'Remote publication differs from local intent' + tokens(attempt['after']) + assert attempt['before'] not in links, 'Remote publications branched from the same content' + event = {**attempt, 'event': 'commit', 'operation': intent['operation'], 'token': intent['token'], 'finished_us': receipt['at_us']} + links[attempt['before']] = event, attempt['after'] + return links diff --git a/tools/offline_outage.py b/tools/offline_outage.py new file mode 100644 index 0000000000000000000000000000000000000000..1e066857d23636b27b274dbac8aa49015c7f7dec --- /dev/null +++ b/tools/offline_outage.py @@ -0,0 +1,313 @@ +"""Require durable local progress during a confirmed outage of the owned SMB proxy.""" +import hashlib +import json +from pathlib import Path +import shlex +import time + +from native_runner import windows +import linux_vm +from verify_smb_overlap import verify + + +def interrupt(output, clients, sequences, processes): + config = json.loads((output / 'run.json').read_text()) + folder = output / 'rust' + samples = {} + + def logs(): + result = {} + for actor in processes: + text = (folder / f'{actor}.jsonl').read_text() + result[actor] = [json.loads(line) for line in text[:text.rfind('\n') + 1].splitlines()] + return result + + def wait_for(predicate, message, timeout=90): + deadline = time.monotonic() + timeout + while not predicate(): + assert all(p.poll() in (None, 0) for p in processes.values()), 'A client failed during the offline outage' + if time.monotonic() > deadline: raise TimeoutError(message) + time.sleep(.05) + + def ssh(command): + result = linux_vm.run_ssh(config['server'], command, timeout=15) + with (output / 'offline-outage-server.jsonl').open('a') as stream: + stream.write(json.dumps({'command': command, 'exit': result.returncode, 'stdout': result.stdout, 'stderr': result.stderr}) + '\n') + result.check_returncode() + return result.stdout + + def phase(control): + text = json.dumps(control) + ssh("printf '%s' " + shlex.quote(text) + ' > /tmp/smb-control.tmp && mv /tmp/smb-control.tmp /tmp/smb-control.json') + wait_for(lambda: text in ssh("grep -F '\"control\":' /tmp/smb-trace.jsonl | tail -n 1"), 'Proxy did not acknowledge the outage phase') + + def native_counts(label): + counts = [] + for actor, (client, sequence) in enumerate(zip(clients, sequences)): + capture = output / f'offline-{label}-n{actor}.jsonl' + result = windows.do_get(f'C:\\one-tests\\runs\\capture\\outbox\\{sequence}\\events.jsonl', capture, client['name']) + assert not result.get('error'), result + text = capture.read_text(encoding='utf-8-sig') + rows = [json.loads(line) for line in text[:text.rfind('\n') + 1].splitlines()] + assert 0 < len(rows) < config['stress_operations'], 'Native client was inactive during the outage campaign' + counts.append(len(rows)) + return counts + + writers = [actor for actor in processes if actor.startswith('w')] + readers = [actor for actor in processes if actor.startswith('r')] + if config.get('offline_lost_reply'): + isolated = config.get('offline_client_reply', False) + def counts(): + return {actor: sum(row['event'] == ('remote_receipt' if actor in writers else 'read') for row in rows) + for actor, rows in logs().items()} + if config.get('document_operations'): + wait_for(lambda: all((folder / f'offline-paused-{actor}').exists() for actor in writers) + and all(counts()[actor] > 0 for actor in readers), 'Clients did not reach the formatting publication barrier') + else: + wait_for(lambda: all(value >= 3 for value in counts().values()), 'Clients made no progress before the reply cut') + samples['before'] = counts() + samples['native_before'] = native_counts('reply-before') + try: + control = {'phase': 'offline-reply-cut', 'cut': 9, 'peer': '10.0.2.2', 'offset': 96, 'direction': 'response'} + if isolated: + control['scope'] = 'connection' + (folder / 'offline-paused-w0.isolate').touch() + phase(control) + if config.get('document_operations'): + samples['format_released_us'] = time.time_ns() // 1000 + (folder / 'offline-paused-w0.resume').touch() + wait_for(lambda: int(ssh("grep -c '\"cut\": {' /tmp/smb-trace.jsonl || true").strip()) == 1, + 'The publication reply was not interrupted') + samples['down_started_us'] = time.time_ns() // 1000 + if isolated: + wait_for(lambda: any(row['event'] == 'confirmation_paused' for row in logs()['w0']), + 'The disconnected writer did not retain its uncertain publication') + for actor in writers[1:]: (folder / f'offline-paused-{actor}.resume').touch() + wait_for(lambda: all(value >= samples['before'][actor] + 3 for actor, value in counts().items() if actor != 'w0'), + 'Peers did not advance while the writer was disconnected') + wait_for(lambda: (folder / 'offline-retired.one').exists(), + 'Native maintenance did not retire the isolated revision') + time.sleep(3) + if isolated: samples['during'] = counts() + samples['native_during'] = native_counts('reply-during') + finally: + samples['up_started_us'] = time.time_ns() // 1000 + phase({'phase': 'offline-reply-reconnected'}) + if config.get('document_operations'): + for actor in writers: (folder / f'offline-paused-{actor}.resume').touch() + if isolated: (folder / 'offline-paused-w0.confirmation-resume').touch() + (output / 'offline-lost-reply-progress.json').write_text(json.dumps(samples, indent=2)) + wait_for(lambda: all(value >= samples['before'][actor] + 3 for actor, value in counts().items()), + 'A client failed to progress after the lost publication reply', timeout=120) + samples['after'] = counts() + (output / 'offline-lost-reply-progress.json').write_text(json.dumps(samples, indent=2)) + return + wait_for(lambda: all((folder / f'offline-paused-{actor}').exists() for actor in writers) + and (not config.get('document_operations') or all( + sum(row['event'] == 'local_document_commit' for row in logs()[actor]) == 2 for actor in writers)) + and all(any(row['event'] == 'read' for row in logs()[actor]) for actor in readers), + 'Clients did not reach the pre-publication outage barrier') + samples['native_before'] = native_counts('before') + samples['reader_errors_before'] = {actor: sum(row['event'] == 'transport_read_error' for row in logs()[actor]) for actor in readers} + try: + phase({'phase': 'offline-down', 'mode': 'down'}) + samples['down_started_us'] = time.time_ns() // 1000 + (folder / 'offline-outage-down').touch() + wait_for(lambda: all(sum(row['event'] == 'local_commit' for row in events) == 8 for actor, events in logs().items() if actor in writers) + and (not config.get('document_operations') or all( + sum(row['event'] == 'local_document_commit' for row in logs()[actor]) == 16 for actor in writers)) + and all(sum(row['event'] == 'transport_read_error' for row in logs()[actor]) > samples['reader_errors_before'][actor] for actor in readers), + 'Local queues or disconnected readers failed to progress during the outage') + time.sleep(3) + samples['native_during'] = native_counts('during') + samples['up_started_us'] = time.time_ns() // 1000 + finally: + phase({'phase': 'offline-reconnected'}) + (folder / 'offline-outage-resumed').touch() + (output / 'offline-outage-progress.json').write_text(json.dumps(samples, indent=2)) + wait_for(lambda: all(sum(row['event'] == 'remote_receipt' for row in logs()[actor]) >= 3 for actor in writers) + and all(sum(row['event'] == 'read' and row['started_us'] > samples['up_started_us'] for row in logs()[actor]) >= 3 for actor in readers), + 'A client failed to progress after the offline outage') + + +def native_progress(output, config, sample): + down, up = sample['down_started_us'], sample['up_started_us'] + assert len(sample['native_before']) == len(sample['native_during']) == config['stress_clients'] + native = list(zip(sample['native_before'], sample['native_during'])) + assert all(0 < before < during < config['stress_operations'] for before, during in native), 'Native local edits did not advance during the outage' + clocks = json.loads((output / 'clocks.json').read_text()) + native_inside = [] + assert len(clocks) == config['stress_clients'] + for index, clock in enumerate(clocks): + low = min(clock['native_minus_host_us'][0], clock['after_native_minus_host_us'][0]) + high = max(clock['native_minus_host_us'][1], clock['after_native_minus_host_us'][1]) + rows = [json.loads(line) for line in (output / f'n{index}/stress-events.jsonl').read_text(encoding='utf-8-sig').splitlines()] + inside = sum((row['update_started_ticks'] - 621355968000000000) // 10 - high > down + and (row['updated_ticks'] - 621355968000000000) // 10 - low < up for row in rows) + assert inside, 'Native timestamps do not establish local edits inside the confirmed outage' + native_inside.append(inside) + return native_inside + + +def verify_outage(output): + config = json.loads((output / 'run.json').read_text()) + assert config['offline'] and config['offline_outage'] and config['embedded_smb'] + assert config['stress_clients'] + config['rust_writers'] + config['rust_readers'] >= 12 + sample = json.loads((output / 'offline-outage-progress.json').read_text()) + down, up = sample['down_started_us'], sample['up_started_us'] + assert up - down >= 3_000_000, 'Confirmed outage lasted less than three seconds' + native_inside = native_progress(output, config, sample) + queues, reconnects = {}, {} + for mode, count in [('w', config['rust_writers']), ('r', config['rust_readers'])]: + for index in range(count): + actor = f'{mode}{index}' + events = [json.loads(line) for line in (output / 'rust' / f'{actor}.jsonl').read_text().splitlines()] + assert events[0]['event'] == 'ready' and events[-1]['event'] == 'done' + connected = [row for row in events if row['event'] == 'transport_connected'] + assert any(row['at_us'] > up for row in connected), 'No fresh transport after outage' + reconnects[actor] = len(connected) + if mode == 'r': + assert sum(row['event'] == 'transport_read_error' for row in events) > sample['reader_errors_before'][actor] + assert sum(row['event'] == 'read' and row['started_us'] > up for row in events) >= 3 + continue + local = [row for row in events if row['event'] == 'local_commit'] + assert len(local) == config['stress_operations'] + assert local[0]['finished_us'] < down + queued = [row for row in local if down < row['started_us'] <= row['finished_us'] < up] + assert [row['operation'] for row in queued] == list(range(1, 8)), 'Seven local edits were not accepted while SMB was down' + queues[actor] = len(queued) + paused = [row for row in events if row['event'] == 'publication_paused'] + assert len(paused) == 1 and paused[0]['at_us'] < down + attempts = [row for row in events if row['event'] == 'remote_attempt'] + assert attempts and all(row['started_us'] > up for row in attempts), 'Publication escaped the outage barrier' + assert attempts[0]['state'] == 'NotCommitted' and attempts[0]['revision'] == paused[0]['revision'], 'Disconnected pre-I/O attempt was not safely rejected' + assert all(row['state'] != 'Unknown' for row in attempts), 'Unexpected uncertain publication requires separate recovery evidence' + receipts = [row for row in events if row['event'] == 'remote_receipt'] + assert len(receipts) == config['stress_operations'] and all(row['at_us'] > up for row in receipts) + if config.get('document_operations'): + edits = [row for row in events if row['event'] == 'local_document_commit'] + assert [(row['operation'], row['kind']) for row in edits] == [ + (operation, kind) for operation in range(config['stress_operations']) for kind in ('insert', 'format')] + assert all(row['finished_us'] < down for row in edits[:2]), 'Initial document edits missed the outage barrier' + queued = [row for row in edits if down < row['started_us'] <= row['finished_us'] < up] + assert [(row['operation'], row['kind']) for row in queued] == [ + (operation, kind) for operation in range(1, 8) for kind in ('insert', 'format')], 'Document edits did not persist during the outage' + queues[actor] += len(queued) + receipts = [row for row in events if row['event'] == 'document_receipt'] + assert len(receipts) == config['stress_operations'] * 2 and all(row['at_us'] > up for row in receipts) + trace = [json.loads(line) for line in (output / 'smb-trace.jsonl').read_text().splitlines()] + controls = [row['control'] for row in trace if row.get('control', {}).get('phase', '').startswith('offline-')] + assert controls == [{'phase': 'offline-down', 'mode': 'down'}, {'phase': 'offline-reconnected'}], 'Unexpected outage control sequence' + return {'outages': 1, 'confirmed_down_seconds': (up - down) / 1_000_000, + 'local_edits_while_down': queues, 'transport_connection_events': reconnects, + 'native_local_edits_inside_confirmed_outage': native_inside, + 'resumed_guarded_io_overlap': verify(trace, phase='offline-reconnected')} + + +def verify_lost_reply(output): + from offline_history import publication_links, tokens + config = json.loads((output / 'run.json').read_text()) + assert config['offline'] and config['offline_lost_reply'] and config['embedded_smb'] + assert config['stress_clients'] + config['rust_writers'] + config['rust_readers'] >= 12 + sample = json.loads((output / 'offline-lost-reply-progress.json').read_text()) + isolated = config.get('offline_client_reply', False) + assert sample['up_started_us'] - sample['down_started_us'] >= 3_000_000 + actors = [*(f'w{i}' for i in range(config['rust_writers'])), *(f'r{i}' for i in range(config['rust_readers']))] + assert set(sample['before']) == set(sample['after']) == set(actors) + assert all(sample['after'][actor] >= sample['before'][actor] + 3 for actor in actors) + if not config.get('document_operations'): + assert all(sample['before'][actor] >= 3 for actor in actors) + logs = {actor: [json.loads(line) for line in (output / 'rust' / f'{actor}.jsonl').read_text().splitlines()] for actor in actors} + publication_links(logs, config['stress_operations']) + documents = {} + if config.get('document_operations'): + from offline_document_history import document_history + documents = document_history(logs, config['stress_operations']) + for actor, rows in logs.items(): + if isolated and actor != 'w0': continue + assert any(row['event'] == 'transport_connected' and row['at_us'] > sample['up_started_us'] for row in rows), f'{actor} did not reconnect' + unknown = [(actor, row) for actor, rows in logs.items() for row in rows if row['event'] == 'remote_attempt' and row['state'] == 'Unknown'] + assert len(unknown) == 1, 'The reply cut did not establish exactly one uncertain publication' + actor, attempt = unknown[0] + assert attempt['started_us'] < sample['down_started_us'], 'Uncertain attempt started after the reply cut' + receipts = [row for row in logs[actor] if row['event'] in ('remote_receipt', 'document_receipt') and row['revision'] == attempt['revision']] + if not receipts and documents: + target, = attempt['document_changes'] + confirmed_revision = documents[target]['format']['receipt_revision'] + receipts = [row for row in logs[actor] if row['event'] == 'document_receipt' and row['revision'] == confirmed_revision] + assert len(receipts) == 1 and receipts[0]['at_us'] > sample['up_started_us'] + peer_progress = {} + if isolated: + assert sample['during']['w0'] == sample['before']['w0'] + retired, = [row for row in logs['r0'] if row['event'] == 'revision_retired'] + assert retired['space'] == attempt['space'] and retired['revision'] == attempt['revision'] + assert sample['down_started_us'] < retired['started_us'] <= retired['finished_us'] < sample['up_started_us'] + assert receipts[0]['revision'] != attempt['revision'], 'The client-disconnect gate did not exercise confirmation after revision retirement' + paused, = [row for row in logs['w0'] if row['event'] == 'confirmation_paused'] + assert paused['revision'] == attempt['revision'] and attempt['finished_us'] <= paused['at_us'] < sample['up_started_us'] + for peer, rows in logs.items(): + if peer == 'w0': continue + assert sample['during'][peer] >= sample['before'][peer] + 3 + if peer.startswith('w'): + progress = [row for row in rows if row['event'] == 'remote_attempt' and row['state'] == 'Committed' + and sample['down_started_us'] < row['started_us'] <= row['finished_us'] < sample['up_started_us']] + else: + progress = [row for row in rows if row['event'] == 'read' + and sample['down_started_us'] < row['started_us'] <= row['finished_us'] < sample['up_started_us']] + assert len(progress) >= 3, f'{peer} has insufficient completed I/O while the writer was disconnected' + peer_progress[peer] = len(progress) + if config.get('document_operations'): + assert actor == 'w0' and sample['format_released_us'] <= attempt['started_us'] + intent, = [row for row in logs[actor] if row['event'] == 'local_document_commit' and row['id'] == receipts[0]['id']] + assert intent['kind'] == 'format', 'The interrupted publication was not formatting' + for writer in (f'w{i}' for i in range(config['rust_writers'])): + paused = [row for row in logs[writer] if row['event'] == 'publication_paused'] + assert len(paused) == 1 and paused[0]['kind'] == 'format' and paused[0]['at_us'] < sample['format_released_us'] + paused, = [row for row in logs[actor] if row['event'] == 'publication_paused'] + if paused['revision'] != attempt['revision']: + prior, = [row for row in logs[actor] if row['event'] == 'remote_attempt' and row['revision'] == paused['revision']] + assert prior['state'] == 'NotCommitted' and sample['format_released_us'] <= prior['started_us'] <= prior['finished_us'] <= attempt['started_us'], 'Paused revision was replaced without proving it unpublished' + captures = {} + for rows in logs.values(): + for row in rows: + if row['event'] != 'remote_confirm': continue + name = row['capture'] + assert Path(name).name == name and name not in captures + tokens(row['text']) + assert row['started_us'] > sample['up_started_us'] + captures[name] = {'sha256': hashlib.sha256((output / 'rust/confirmations' / name).read_bytes()).hexdigest(), 'state': row['state']} + assert captures and set(captures) == {path.name for path in (output / 'rust/confirmations').glob('*.one')} + trace = [json.loads(line) for line in (output / 'smb-trace.jsonl').read_text().splitlines()] + controls = [row['control'] for row in trace if row.get('control', {}).get('phase', '').startswith('offline-')] + expected = {'phase': 'offline-reply-cut', 'cut': 9, 'peer': '10.0.2.2', 'offset': 96, 'direction': 'response'} + if isolated: expected['scope'] = 'connection' + assert controls == [expected, {'phase': 'offline-reply-reconnected'}] + cuts = [row['cut'] for row in trace if 'cut' in row] + assert len(cuts) == 1 and cuts[0]['direction'] == 'response' and cuts[0]['command'] == 9 and cuts[0]['status'] == '0x0' + request, = [row for row in trace if row.get('direction') == 'request' and row['connection'] == cuts[0]['connection'] and row['message'] == cuts[0]['message']] + assert request['offset'] == 96 and request['command'] == 9 + native_writes = 0 + if isolated: + peers = {row['connection']: row['peer'][0] for row in trace if row.get('opened')} + pending, files = {}, {} + for row in trace: + if row.get('command') not in (5, 6, 9): continue + key = row['connection'], row['message'] + if row['direction'] == 'request': + pending[key] = row + if row['command'] == 6: files.pop((key[0], row['file_id']), None) + elif row['status'] == '0x0' and key in pending: + request = pending.pop(key) + if row['command'] == 5: + files[key[0], row['file_id']] = request['path'].lower() + elif (row['command'] == 9 and peers[key[0]].startswith('192.168.77.') and row.get('written', 0) > 0 + and files.get((key[0], request['file_id']), '').endswith('synthetic.one') + and sample['down_started_us'] < request['time'] * 1_000_000 <= row['time'] * 1_000_000 < sample['up_started_us']): + native_writes += 1 + assert native_writes, 'No successful native writes while the isolated writer awaited reconciliation' + return {'reply_cuts': 1, 'uncertain_actor': actor, 'attempted_revision': attempt['revision'], 'confirmed_revision': receipts[0]['revision'], + 'peer_operations_during_client_disconnect': peer_progress, + 'native_writes_during_client_disconnect': native_writes, + 'retired_snapshot_sha256': hashlib.sha256((output / 'rust/offline-retired.one').read_bytes()).hexdigest() if isolated else None, + 'confirmation_snapshots': captures, 'native_local_edits_inside_confirmed_outage': native_progress(output, config, sample), + 'resumed_guarded_io_overlap': verify(trace, phase='offline-reply-reconnected')} diff --git a/tools/offline_publication_crash.py b/tools/offline_publication_crash.py new file mode 100644 index 0000000000000000000000000000000000000000..c428255856dd15ed2abdc3167d95997aa17fe419 --- /dev/null +++ b/tools/offline_publication_crash.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +"""Kill owned processes across local-cache/remote-file publication boundaries.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import queue +import shutil +import signal +import sqlite3 +import subprocess +import threading +import time + +ROOT = Path(__file__).resolve().parent.parent +BINARY = ROOT / 'target/debug/examples/recovery_probe' +TOKEN = ' [offline-recovery]' + + +def run_command(binary, args, output): + result = subprocess.run([str(binary), *map(str, args)], capture_output=True, text=True, timeout=60, + env={**os.environ, 'ONESTORE_RECOVERY_PAUSE': ''}) + output.with_suffix('.jsonl').write_text(result.stdout) + output.with_suffix('.stderr').write_text(result.stderr) + result.check_returncode() + return [json.loads(line) for line in result.stdout.splitlines()] + + +def kill_at(binary, args, phase, output, receipt_window=None): + events = queue.Queue() + database = Path(args[1]) / 'cache.sqlite' + journal = database.with_name(database.name + '-journal') + def header(): + try: + with journal.open('rb') as stream: return stream.read(8) + except FileNotFoundError: return b'' + with output.with_suffix('.jsonl').open('w') as log, output.with_suffix('.stderr').open('w') as error: + process = subprocess.Popen([str(binary), *map(str, args)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=error, text=True, + env={**os.environ, 'ONESTORE_RECOVERY_PAUSE': phase}) + def collect(): + try: + for line in process.stdout: + log.write(line) + log.flush() + events.put(json.loads(line)) + finally: + events.put(None) + reader = threading.Thread(target=collect) + reader.start() + try: + deadline = time.monotonic() + 60 + while True: + event = events.get(timeout=max(0, deadline-time.monotonic())) + assert event is not None, f'Process exited before {phase}' + if event.get('event') == 'phase' and event['name'] == phase: + if receipt_window is not None: + assert args[0] == 'sync' and phase == 'publish-after' + assert receipt_window in ('journal', 'database') + before_mtime = database.stat().st_mtime_ns + process.stdin.write('\n') + process.stdin.flush() + deadline = time.monotonic() + 5 + while True: + if header() == bytes.fromhex('d9d505f920a163d7') and (receipt_window == 'journal' or database.stat().st_mtime_ns != before_mtime): break + assert process.poll() is None, 'Receipt transaction finished before the requested cut' + assert time.monotonic() < deadline, 'Receipt transaction window was not observed' + time.sleep(.0001) + process.kill() + break + assert process.wait(timeout=10) == -signal.SIGKILL, 'Expected an actual process kill' + proof = {'pid': process.pid, 'exit': process.returncode, 'phase': phase} + if receipt_window is not None: + proof.update(journal_header_after_kill=header().hex(), database_changed=database.stat().st_mtime_ns != before_mtime, receipt_window=receipt_window) + assert proof['journal_header_after_kill'] == 'd9d505f920a163d7', 'Receipt transaction committed before the process died' + assert receipt_window != 'database' or proof['database_changed'] + output.with_suffix('.cut.json').write_text(json.dumps(proof, indent=2)) + finally: + if process.poll() is None: process.kill() + process.wait(timeout=10) + process.stdin.close() + reader.join(timeout=10) + assert not reader.is_alive(), 'Trace reader did not finish' + process.stdout.close() + + +def state(rows, original): + found = [row for row in rows if row['event'] == 'state'] + assert len(found) == 1, 'Missing independent post-reopen state' + result, = found + assert result['local_text'] == original + TOKEN, 'Locally acknowledged text was lost or duplicated' + assert result['remote_text'] in [original, original+TOKEN], 'Remote current text is partial, duplicated or invented' + assert result['status'] in ('pending', 'uncertain', 'published'), 'Intent disappeared or became an unexplained conflict' + if result['status'] == 'pending': + assert result['revision'] is None, 'Unattempted intent acquired a publication identity' + else: + assert isinstance(result['revision'], str) and result['revision'], 'Attempted identity was lost' + assert (result['revision'] == result['remote_revision']) == (result['remote_text'] == original+TOKEN), 'Publication identity disagrees with the visible effect' + if result['status'] == 'published': + assert result['pending'] == [] and result['remote_text'] == original+TOKEN + assert result['revision'] == result['remote_revision'], 'Receipt identifies another remote revision' + else: + assert len(result['pending']) == 1, 'Unacknowledged intent was lost or duplicated' + pending, = result['pending'] + assert pending['id'] == 1 and pending['before'] == original and pending['replacement'] == TOKEN + at = len(original.encode('utf-16-le')) // 2 + assert pending['range'] == [at, at], 'Durable intent range changed' + return result + + +def save_image(output, data): + digest = hashlib.sha256(data).hexdigest() + path = output / 'images' / f'{digest}.one' + if not path.exists(): path.write_bytes(data) + return digest + + +def confirmation_only(events, before, after): + assert not any(row['event'] == 'phase' and row['name'] == 'publish-before' for row in events) + assert all(row['offset'] == 212 and row['bytes'] == 40 for row in events if row['event'] == 'write'), 'Recovery republished the remote edit' + assert before[:212] == after[:212] and before[252:] == after[252:], 'Confirmation changed content or the transaction count' + + +def prepare_run(source, output): + output.mkdir(parents=True, exist_ok=False) + (output / 'images').mkdir() + binary = output / 'recovery_probe' + shutil.copyfile(BINARY, binary) + binary.chmod(0o755) + shutil.copyfile(__file__, output / Path(__file__).name) + source_hash = hashlib.sha256(source.read_bytes()).hexdigest() + (output / 'run.json').write_text(json.dumps({'source': str(source), 'source_sha256': source_hash, + 'binary_sha256': hashlib.sha256(binary.read_bytes()).hexdigest(), 'controller_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest()}, indent=2)) + return binary, source_hash + + +def run(source, output): + binary, source_hash = prepare_run(source, output) + baseline = output / 'baseline' + initialized = run_command(binary, ['init', baseline, source], output / 'baseline-init') + original = next(row['remote_text'] for row in initialized if row['event'] == 'state') + state(initialized, original) + published = run_command(binary, ['sync', baseline], output / 'baseline-sync') + assert state(published, original)['status'] == 'published' + phases = [row['name'] for row in published if row['event'] == 'phase'] + assert len(phases) == len(set(phases)), 'Baseline has ambiguous phase names' + assert 'publish-after' in phases and any(name.startswith('write-') for name in phases) + confirmation = output / 'confirmation-baseline' + run_command(binary, ['init', confirmation, source], output / 'confirmation-init') + kill_at(binary, ['sync', confirmation], 'publish-after', output / 'confirmation-setup') + confirmed = run_command(binary, ['sync', confirmation], output / 'confirmation-sync') + confirmation_phases = [row['name'] for row in confirmed if row['event'] == 'phase'] + assert len(confirmation_phases) == len(set(confirmation_phases)) and 'confirm-after' in confirmation_phases + assert state(confirmed, original)['status'] == 'published' + cases = [('local-after', False), *((phase, False) for phase in phases), *((phase, True) for phase in confirmation_phases)] + results = [] + for index, (phase, confirmation) in enumerate(cases): + folder = output / f'case-{index:02}' + trace = output / f'case-{index:02}-kill' + if phase == 'local-after': + kill_at(binary, ['init', folder, source], phase, trace) + else: + run_command(binary, ['init', folder, source], output / f'case-{index:02}-init') + if confirmation: + kill_at(binary, ['sync', folder], 'publish-after', output / f'case-{index:02}-setup') + kill_at(binary, ['sync', folder], phase, trace) + before = state(run_command(binary, ['inspect', folder], output / f'case-{index:02}-inspect'), original) + before_bytes = (folder / 'remote.one').read_bytes() + first = run_command(binary, ['sync', folder], output / f'case-{index:02}-recover') + after = state(first, original) + if before['status'] == 'uncertain' and before['remote_text'] == original: + assert after['status'] == 'uncertain' and after['revision'] == before['revision'], 'Absent uncertain attempt was replayed' + assert not any(row['event'] == 'write' for row in first) + else: + assert after['status'] == 'published' + if before['status'] != 'pending': + assert after['revision'] == before['revision'], 'Recovery published another revision' + confirmation_only(first, before_bytes, (folder / 'remote.one').read_bytes()) + remote_hash = hashlib.sha256((folder / 'remote.one').read_bytes()).hexdigest() + repeated = run_command(binary, ['sync', folder], output / f'case-{index:02}-repeat') + assert state(repeated, original) == after, 'Repeated recovery changed durable intent state' + assert not any(row['event'] == 'write' for row in repeated), 'Repeated recovery published again' + assert hashlib.sha256((folder / 'remote.one').read_bytes()).hexdigest() == remote_hash + remote_image = save_image(output, (folder / 'remote.one').read_bytes()) + connection = sqlite3.connect(folder / 'cache.sqlite') + try: + assert connection.execute('PRAGMA quick_check').fetchall() == [('ok',)] + assert connection.execute('PRAGMA foreign_key_check').fetchall() == [] + local_image = save_image(output, connection.execute('SELECT working FROM replica WHERE id=1').fetchone()[0]) + finally: + connection.close() + result = {'case': index, 'phase': phase, 'during_confirmation': confirmation, 'before_status': before['status'], 'after_status': after['status'], + 'visible_before_recovery': before['remote_text'] != original, 'remote_image': remote_image, 'local_image': local_image, + 'remote_text': after['remote_text'], 'local_text': after['local_text']} + results.append(result) + (output / 'results.json').write_text(json.dumps(results, indent=2)) + print(json.dumps({key: value for key, value in result.items() if not key.endswith('_text')}), flush=True) + assert hashlib.sha256(source.read_bytes()).hexdigest() == source_hash, 'The frozen source changed' + summary = {'cases': len(results), 'process_kills': 1 + len(results) + sum(confirmation for _, confirmation in cases), 'uncertain_absent_preserved': sum(row['after_status']=='uncertain' for row in results), + 'durable_receipts': sum(row['after_status']=='published' for row in results), 'unique_images': len(list((output/'images').glob('*.one')))} + (output/'summary.json').write_text(json.dumps(summary, indent=2)) + print(json.dumps(summary), flush=True) + + +def receipt_windows(source, output): + binary, source_hash = prepare_run(source, output) + results = [] + for window in ('journal', 'database'): + folder = output / window + initialized = run_command(binary, ['init', folder, source], output / (window+'-init')) + original = next(row['remote_text'] for row in initialized if row['event'] == 'state') + state(initialized, original) + kill_at(binary, ['sync', folder], 'publish-after', output / (window+'-kill'), receipt_window=window) + before = state(run_command(binary, ['inspect', folder], output / (window+'-inspect')), original) + assert before['status'] == 'uncertain' and before['remote_text'] == original+TOKEN, 'Hot-journal recovery lost the pending confirmation' + before_bytes = (folder / 'remote.one').read_bytes() + recovered = run_command(binary, ['sync', folder], output / (window+'-recover')) + after = state(recovered, original) + assert after['status'] == 'published' and before['revision'] == after['revision'] + confirmation_only(recovered, before_bytes, (folder / 'remote.one').read_bytes()) + repeated = run_command(binary, ['sync', folder], output / (window+'-repeat')) + assert state(repeated, original) == after and not any(row['event']=='write' for row in repeated) + digest = save_image(output, (folder/'remote.one').read_bytes()) + results.append({'receipt_window':window, 'remote_image':digest, 'remote_text':after['remote_text'], 'revision':after['revision']}) + print(json.dumps({'receipt_window':window, 'status':after['status'], 'remote_image':digest}), flush=True) + assert hashlib.sha256(source.read_bytes()).hexdigest() == source_hash + (output/'results.json').write_text(json.dumps(results, indent=2)) + (output/'summary.json').write_text(json.dumps({'process_kills':2, 'recovered_receipts':2, 'republished_edits':0}, indent=2)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('source', type=Path) + parser.add_argument('output', type=Path) + parser.add_argument('--receipt-windows', action='store_true') + args = parser.parse_args() + (receipt_windows if args.receipt_windows else run)(args.source.resolve(), args.output.resolve()) diff --git a/tools/smb-proxy.py b/tools/smb-proxy.py index f2cf7bc630bdeea90fe141683f7af929afb910f4..1e9024d778b065f85b293946b3d4c9972ffe476d 100644 --- a/tools/smb-proxy.py +++ b/tools/smb-proxy.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Trace a dedicated test SMB session and cut selected responses before delivery.""" +"""Trace a dedicated test SMB session and cut selected requests or responses.""" import argparse import asyncio import json @@ -8,28 +8,44 @@ import struct import time +def header_fields(offset, data): + result = {} + for name, start, length in [('transactions', 96, 4), ('version', 212, 16), + ('generation', 228, 8), ('deny_read', 236, 16)]: + if offset <= start and start + length <= offset + len(data): + value = data[start - offset:start - offset + length] + result[name] = value.hex() if length == 16 else int.from_bytes(value, 'little') + return result + + async def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('control', type=Path) parser.add_argument('--port', type=int, default=11445) parser.add_argument('--server', default='10.0.0.1') + parser.add_argument('--server-port', type=int, default=445) + parser.add_argument('--bind', default='127.0.0.1') args = parser.parse_args() writers = set() state = {'mode': 'up'} previous = None blocked = False matched = 0 + connections = 0 + record_writes = False def record(**fields): print(json.dumps({'time': time.time(), **fields}), flush=True) async def controls(): - nonlocal state, previous, blocked, matched + nonlocal state, previous, blocked, matched, record_writes while True: try: raw = args.control.read_bytes() if raw != previous: state = json.loads(raw) + if 'record_writes' in state: + record_writes = bool(state['record_writes']) previous, matched = raw, 0 blocked = state.get('mode') == 'down' record(control=state) @@ -41,13 +57,16 @@ async def main(): await asyncio.sleep(0.05) async def connection(client, client_writer): - nonlocal blocked, matched + nonlocal blocked, matched, connections + connections += 1 + connection_id = connections + record(connection=connection_id, peer=client_writer.get_extra_info("peername"), opened=True) if blocked: client_writer.close() return server_writer = None try: - server, server_writer = await asyncio.open_connection(args.server, 445) + server, server_writer = await asyncio.open_connection(args.server, args.server_port) writers.update((client_writer, server_writer)) async def forward(reader, writer, direction): @@ -58,25 +77,68 @@ async def main(): at = 0 while frame[at:at+4] == b'\xfeSMB': command = struct.unpack_from('= 100: + if command == 9: data_offset = struct.unpack_from('{changed}') with self.assertRaises(AssertionError): verify_capture(root, root) + + +class FailureArtifacts(unittest.TestCase): + def test_failed_clients_preserve_native_logs_without_masking_the_failure(self): + @contextmanager + def failed_clients(*args, **kwargs): + yield {} + raise RuntimeError('Rust client exited') + + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) + (output / 'run.json').write_text(json.dumps({'maintenance': False})) + shared = output / 'shared' + shared.mkdir() + clients = [{'name': f'n{i}', 'folder': output / f'n{i}'} for i in range(2)] + for client in clients: client['folder'].mkdir() + + def capture(remote, local, name): + self.assertTrue(remote.endswith('\\outbox\\7\\events.jsonl')) + if name == 'n0': raise OSError('Native client unavailable') + local.write_text('{"operation":0}\n') + return {'error': None} + + with patch.object(native_stress, 'running_clients', failed_clients), \ + patch.object(native_stress.windows, 'do_health', return_value={'utc_us': 0}), \ + patch.object(native_stress.windows, 'do_cmd', return_value={'stdout': 'ready editing'}), \ + patch.object(native_stress.windows, 'do_get', side_effect=capture): + with self.assertRaisesRegex(RuntimeError, 'Rust client exited'): + native_stress.exercise(output, shared, clients, lambda *a, **kw: 7, + lambda *a: None, lambda *a: None, lambda *a: None, 40, 1, 4, 4) + self.assertEqual(json.loads((output / 'n0/stress-capture.json').read_text())['error'], 'Native client unavailable') + self.assertEqual((output / 'n1/stress-events.jsonl').read_text(), '{"operation":0}\n') diff --git a/tools/test_notebook_editor.py b/tools/test_notebook_editor.py new file mode 100644 index 0000000000000000000000000000000000000000..a92e5019b12c5d13ba73d3413f62ab42e415bc1f --- /dev/null +++ b/tools/test_notebook_editor.py @@ -0,0 +1,237 @@ +import json +from pathlib import Path +import tempfile +from threading import Thread +import unittest +from unittest.mock import patch +from urllib.error import HTTPError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +from document_model import view, walk +import notebook_editor as editor +from random_edit_campaign import export, verify + + +class EditorTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + source = editor.ROOT / 'corpus/native/20260905-05/snapshots/06-attachment/notebook' + self.session = editor.Session(source, Path(self.temporary.name) / 'session') + self.server = editor.ThreadingHTTPServer(('127.0.0.1', 0), editor.Handler) + self.server.session = self.session + self.thread = Thread(target=self.server.serve_forever) + self.thread.start() + self.addCleanup(self.stop) + self.url = f'http://127.0.0.1:{self.server.server_port}' + self.file = self.session.output / 'notebook/synthetic.one' + self.before = export(self.file) + self.row = json.loads((self.session.output / 'g/0/report/pages.json').read_text())[0] + _, revision = view(self.before[0], self.row['space']) + self.oid, node = next((oid, node) for oid, node in walk(revision, self.row['object']) + if node['kind']['type'] == 'RichText' and node['kind']['text'].startswith('Fictitious')) + self.selection = {'generation': 0, 'page': self.row['report'], 'object': self.oid, 'run': 0, 'action': 'text'} + self.text = self.before[1][self.row['space']][self.row['revision']][self.oid][0]['text'] + + def stop(self): + self.server.shutdown() + self.thread.join() + self.server.server_close() + + def request(self, path, data=None): + request = Request(self.url + path, data=json.dumps(data).encode() if data is not None else None, + headers={'Content-Type': 'application/json', 'X-OneNote-Diagnostic': '1'}) + try: + response = urlopen(request) + except HTTPError as error: + response = error + with response: + return response.status, json.loads(response.read()) + + def test_unicode_stale_snapshot_and_unrelated_content(self): + status, checked = self.request('/api/run?' + urlencode(self.selection)) + self.assertEqual((status, checked['ok'], checked['text']), (200, True, self.text)) + replacement = self.text + ' café 🦀 e\u0301 ' + status, saved = self.request('/api/save', {**self.selection, 'replacement': replacement}) + self.assertEqual((status, saved['state'], saved['ok']), (200, 'Committed', True)) + intent, outcome = [json.loads(s) for s in (self.session.output / 'operations.jsonl').read_text().splitlines()] + edit = intent['edit'] + verify(self.before, export(self.file), {'page': self.row['object'], 'space': edit['space'], 'object': self.oid, + 'range': [edit['action']['start'], edit['action']['end']], 'replacement': replacement, 'run_start': edit['action']['start'], + 'run_before': self.text, 'started_ms': intent['started_ms'], 'finished_ms': outcome['finished_ms']}) + after = self.file.read_bytes() + fresh = self.session.output / 'fresh-report' + editor.generate(self.session.output / 'g/1/snapshot', fresh, editable=True) + cached = self.session.output / 'g/1/report' + self.assertEqual({p.relative_to(fresh): p.read_bytes() for p in fresh.rglob('*') if p.is_file()}, + {p.relative_to(cached): p.read_bytes() for p in cached.rglob('*') if p.is_file()}) + status, stale = self.request('/api/save', {**self.selection, 'replacement': 'another draft'}) + self.assertEqual((status, stale['state'], stale['kind']), (409, 'NotCommitted', 'ResourceBusy')) + self.assertEqual(self.file.read_bytes(), after) + self.assertEqual((self.session.output / 'g/0/snapshot/synthetic.one').read_bytes(), + (editor.ROOT / 'corpus/native/20260905-05/snapshots/06-attachment/notebook/synthetic.one').read_bytes()) + self.assertEqual((self.session.output / 'g/1/snapshot/synthetic.one').stat().st_ino, + (self.session.output / 'g/2/snapshot/synthetic.one').stat().st_ino) + with urlopen(self.url + saved['location']) as response: + html = response.read().decode() + self.assertIn('<diagnostic>', html) + self.assertNotIn('', html) + + def test_invalid_edits_and_selection_never_write(self): + before = self.file.read_bytes() + for changes in [{'replacement': 'first\nsecond'}, {'replacement': '\ud800'}, + {'run': -1}, {'generation': True}, {'object': 'missing'}, {'extra': 'field'}]: + status, result = self.request('/api/save', {**self.selection, 'replacement': 'changed', **changes}) + self.assertEqual((status, result['state']), (422, 'NotCommitted')) + self.assertEqual(self.file.read_bytes(), before) + + def test_unknown_response_does_not_replay_a_real_commit(self): + original = editor.bridge + calls = [] + def uncertain(mode, *args): + result = original(mode, *args) + if mode == 'commit': + calls.append(result) + self.assertTrue(result['ok']) + return {'ok': False, 'state': 'Unknown', 'error': 'Simulated lost outcome'} + return result + with patch.object(editor, 'bridge', side_effect=uncertain): + status, result = self.request('/api/save', {**self.selection, 'replacement': self.text + ' once'}) + self.assertEqual((status, result['state'], len(calls)), (503, 'Unknown', 1)) + after = export(self.file) + rid, _ = view(after[0], self.row['space']) + self.assertEqual(after[1][self.row['space']][rid][self.oid][0]['text'], self.text + ' once') + status, stale = self.request('/api/save', {**self.selection, 'replacement': self.text + ' twice'}) + self.assertEqual((status, stale['kind']), (409, 'ResourceBusy')) + + def test_refresh_failure_preserves_committed_outcome(self): + with patch.object(self.session, 'snapshot', side_effect=OSError('Report storage unavailable')): + status, result = self.request('/api/save', {**self.selection, 'replacement': self.text + ' saved'}) + self.assertEqual((status, result['state'], result['ok']), (503, 'Committed', False)) + self.assertIn('Report storage', result['report_error']) + after = export(self.file) + rid, _ = view(after[0], self.row['space']) + self.assertEqual(after[1][self.row['space']][rid][self.oid][0]['text'], self.text + ' saved') + with urlopen(self.url + '/latest?' + urlencode(self.selection)) as response: + self.assertIn(' saved', response.read().decode()) + + def test_document_actions_preserve_placement_unicode_and_unselected_formatting(self): + status, page = self.request('/api/page?' + urlencode(self.selection)) + self.assertEqual(status, 200) + outline = next(target for target in page['targets'] if target['label'].startswith('Outline')) + base = {'generation': 0, 'page': self.row['report'], 'object': outline['object'], 'action': 'paragraph', + 'before': outline['children'][0]['object'], 'text': 'A🦀 café\rsecond line', 'author': 'Diagnostic test'} + status, saved = self.request('/api/save', base) + self.assertEqual((status, saved['state']), (200, 'Committed')) + after = export(self.file) + _, revision = view(after[0], self.row['space']) + children = revision['nodes'][outline['object']]['children'] + self.assertEqual(children[1], base['before']) + oid, node = next((key, node) for key, node in walk(revision, children[0]) if node['kind']['type'] == 'RichText') + self.assertEqual(node['kind']['text'], base['text']) + self.assertEqual(self.before[2], after[2]) + attrs = [{'Bold': True}, {'Italic': True}, {'Underline': True}, {'Strike': True}, + {'Superscript': True}, {'Subscript': False}, {'Font': 'Arial'}, {'FontSize': 20.5}, + {'Color': [18, 52, 86]}, {'Highlight': [255, 255, 0]}] + selected = {'generation': 1, 'page': saved['location'].split('/')[-1], 'object': oid, + 'run': 0, 'action': 'format', 'start': 1, 'end': 3, 'attributes': attrs} + status, formatted = self.request('/api/save', selected) + self.assertEqual((status, formatted['state']), (200, 'Committed')) + after_format = export(self.file) + rid, _ = view(after_format[0], self.row['space']) + runs = after_format[1][self.row['space']][rid][oid] + self.assertEqual([run['text'] for run in runs], ['A', '🦀', ' café\rsecond line']) + original_rid, _ = view(after[0], self.row['space']) + original_format = after[1][self.row['space']][original_rid][oid][0]['format'] + self.assertEqual(runs[0]['format'], original_format) + self.assertEqual(runs[2]['format'], original_format) + for key, value in {'bold': True, 'italic': True, 'underline': True, 'strike': True, + 'superscript': True, 'subscript': False, 'font': 'Arial', 'font_size': 20.5, + 'color': 0x563412, 'highlight': 0xffff}.items(): + self.assertEqual(runs[1]['format'][key], value, key) + status, cleared = self.request('/api/save', {**selected, 'generation': 2, 'run': 1, 'start': 0, 'end': 2, + 'attributes': [{'Subscript': True}, {'Color': None}, {'Highlight': None}]}) + self.assertEqual(status, 200) + cleared_model = export(self.file) + rid, _ = view(cleared_model[0], self.row['space']) + fmt = cleared_model[1][self.row['space']][rid][oid][1]['format'] + self.assertEqual((fmt['superscript'], fmt['subscript'], fmt['color'], fmt['highlight']), (False, True, 0xff000000, 0xff000000)) + status, added = self.request('/api/save', {'generation': 3, 'page': selected['page'], 'action': 'outline', + 'object': self.row['object'], 'x': 216.5, 'y': 360, + 'text': 'New outline 🦀', 'author': 'Diagnostic test'}) + self.assertEqual(status, 200) + _, revision = view(export(self.file)[0], self.row['space']) + new_outline = revision['nodes'][revision['nodes'][self.row['object']]['children'][-1]] + self.assertEqual(new_outline['kind']['type'], 'Outline') + self.assertEqual((new_outline['layout']['x'], new_outline['layout']['y']), (216.5, 360)) + committed = self.file.read_bytes() + for request in (base, selected): + status, stale = self.request('/api/save', request) + self.assertEqual((status, stale['state'], stale['kind']), (409, 'NotCommitted', 'ResourceBusy')) + self.assertEqual(self.file.read_bytes(), committed) + + def test_document_rejections_do_not_publish(self): + before = self.file.read_bytes() + common = {'generation': 0, 'page': self.row['report'], 'object': self.oid} + requests = [ + {**common, 'action': 'format', 'run': 0, 'start': 0, 'end': 1, 'attributes': []}, + {**common, 'action': 'format', 'run': 0, 'start': True, 'end': 1, 'attributes': [{'Bold': True}]}, + {**common, 'action': 'format', 'run': 0, 'start': 0, 'end': 99999, 'attributes': [{'Bold': True}]}, + {**common, 'action': 'format', 'run': 0, 'start': 0, 'end': 1, 'attributes': [{'FontSize': 144}]}, + {**common, 'action': 'format', 'run': 0, 'start': 0, 'end': 1, 'attributes': [{'Bold': True}, {'Bold': False}]}, + {**common, 'action': 'outline', 'x': 1, 'y': 1, 'text': 'No page parent', 'author': 'test'}, + {**common, 'action': 'paragraph', 'before': None, 'text': 'No paragraph parent', 'author': 'test'}, + ] + for request in requests: + status, result = self.request('/api/save', request) + self.assertEqual((status, result['state']), (422, 'NotCommitted')) + self.assertEqual(self.file.read_bytes(), before) + + def test_uncertain_insertion_has_one_publication_and_a_stale_retry(self): + request = {'generation': 0, 'page': self.row['report'], 'object': self.row['object'], + 'action': 'outline', 'x': 144, 'y': 288, 'text': 'Only once 🦀', 'author': 'test'} + original = editor.bridge + def uncertain(mode, *args): + result = original(mode, *args) + return {'ok': False, 'state': 'Unknown'} if mode == 'commit' and result['ok'] else result + with patch.object(editor, 'bridge', side_effect=uncertain): + status, result = self.request('/api/save', request) + self.assertEqual((status, result['state']), (503, 'Unknown')) + status, result = self.request('/api/save', request) + self.assertEqual((status, result['kind']), (409, 'ResourceBusy')) + _, revision = view(export(self.file)[0], self.row['space']) + self.assertEqual(sum(n['kind'].get('text') == 'Only once 🦀' for _, n in walk(revision, self.row['object'])), 1) + + def test_generated_fields_are_read_only(self): + source = editor.ROOT / 'corpus/m6/native-structure-01/notebook' + session = editor.Session(source, Path(self.temporary.name) / 'fields') + self.server.session = session + pages = json.loads((session.output / 'g/0/report/pages.json').read_text()) + document = export(source / 'synthetic.one')[0] + for row in pages: + _, revision = view(document, row['space']) + fields = [oid for oid, node in walk(revision, row['object']) + if node['kind']['type'] == 'RichText' and node['kind']['boilerplate']] + if fields: break + self.assertTrue(fields) + status, result = self.request('/api/run?' + urlencode({'generation': 0, 'page': row['report'], 'object': fields[0], 'run': 0})) + self.assertEqual((status, result['ok']), (422, False)) + + def test_templates_keep_reader_only_controls(self): + session = editor.Session(editor.ROOT / 'corpus/m6/native-template-controls-01/notebook', + Path(self.temporary.name) / 'template') + self.server.session = session + pages = json.loads((session.output / 'g/0/report/pages.json').read_text()) + protected = [row for row in pages if row['category'] == 'Default page template'] + self.assertTrue(protected) + for row in protected: + status, result = self.request('/api/run?' + urlencode({'generation': 0, 'page': row['report'], 'object': row['object'], 'run': 0})) + self.assertEqual(status, 422) + self.assertIn('active page', result['error']) + html = (session.output / 'g/0/report' / row['report']).read_text() + self.assertNotIn('src="/editor.js"', html) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/test_notebook_report.py b/tools/test_notebook_report.py index 9841f16e30a06d40c73b8e97f230f46e05693c8a..f08c11cb927597ef52d81c7574cbf6b42646c912 100644 --- a/tools/test_notebook_report.py +++ b/tools/test_notebook_report.py @@ -4,7 +4,9 @@ import json from pathlib import Path import re from tempfile import TemporaryDirectory +import shutil import unittest +from unittest.mock import patch import xml.etree.ElementTree as ET from PIL import Image @@ -32,6 +34,37 @@ class NotebookReportTest(unittest.TestCase): references = [a['path'] for p in (output / 'model').glob('*/assets.json') for a in json.loads(p.read_text())] self.assertIn('../../assets/' + original.name, references) + def test_reused_models_and_assets_match_a_fresh_report_after_source_order_changes(self): + fixture = Path(__file__).resolve().parent.parent / 'corpus/m6/native-features-01/notebook' + with TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / 'notebook' + shutil.copytree(fixture, source) + generate(source, root / 'first') + before = {p.relative_to(root / 'first'): p.read_bytes() for p in (root / 'first').rglob('*') if p.is_file()} + with patch('notebook_report.subprocess.run', side_effect=AssertionError('Unchanged model exported again')): + generate(source, root / 'same', previous=root / 'first') + self.assertEqual(before, {p.relative_to(root / 'same'): p.read_bytes() for p in (root / 'same').rglob('*') if p.is_file()}) + shutil.copyfile(source / 'synthetic.one', source / 'a.one') + import notebook_report + original = notebook_report.subprocess.run + with patch('notebook_report.subprocess.run', wraps=original) as run: + generate(source, root / 'changed', previous=root / 'first') + self.assertEqual([Path(call.args[0][1]).name for call in run.call_args_list], ['a.one']) + generate(source, root / 'fresh') + self.assertEqual({p.relative_to(root / 'fresh'): p.read_bytes() for p in (root / 'fresh').rglob('*') if p.is_file()}, + {p.relative_to(root / 'changed'): p.read_bytes() for p in (root / 'changed').rglob('*') if p.is_file()}) + for name, contents in before.items(): + self.assertEqual((root / 'first' / name).read_bytes(), contents) + old_index = next(i for i, row in enumerate(json.loads((root / 'first/source.json').read_text())) if row['path'] == 'synthetic.one') + new_index = next(i for i, row in enumerate(json.loads((root / 'changed/source.json').read_text())) if row['path'] == 'synthetic.one') + self.assertEqual((root / f'first/model/{old_index}/document.json').stat().st_ino, + (root / f'changed/model/{new_index}/document.json').stat().st_ino) + (source / 'a.one').unlink() + with patch('notebook_report.subprocess.run', side_effect=AssertionError('Unchanged model exported again')): + generate(source, root / 'deleted', previous=root / 'changed') + self.assertEqual(before, {p.relative_to(root / 'deleted'): p.read_bytes() for p in (root / 'deleted').rglob('*') if p.is_file()}) + if __name__ == '__main__': unittest.main() diff --git a/tools/test_offline_confirmation.py b/tools/test_offline_confirmation.py new file mode 100644 index 0000000000000000000000000000000000000000..47d33448819c4af15e0713a7124068e4c30b247c --- /dev/null +++ b/tools/test_offline_confirmation.py @@ -0,0 +1,88 @@ +import hashlib +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from verify_offline_confirmation import verify + + +class ConfirmationOracle(unittest.TestCase): + def test_native_images_must_match_the_confirmation_and_preserved_native_prefix(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + cold = root / 'cold' + (root / 'rust/confirmations').mkdir(parents=True) + (root / 'n0').mkdir() + (cold / 'results/123-456').mkdir(parents=True) + (root / 'run.json').write_text(json.dumps(dict(rust_writers=1, rust_readers=0, stress_clients=1, stress_operations=1))) + rows = [dict(event='ready', pid=123), dict(event='remote_attempt', state='Unknown', revision='revision', space='space'), + dict(event='remote_confirm', capture='123-456.one', revisions={'space': ['revision']}, text='Concurrent edits: [w0:0]')] + (root / 'rust/w0.jsonl').write_text('\n'.join(map(json.dumps, rows))) + (root / 'n0/stress-events.jsonl').write_text(json.dumps(dict(operation=0, token=' [n0:0]', before='Native 0:'))) + snapshot = root / 'rust/confirmations/123-456.one' + snapshot.write_bytes(b'captured snapshot') + sha = hashlib.sha256(snapshot.read_bytes()).hexdigest() + (cold / 'run.json').write_text(json.dumps(dict(inputs={'123-456.one': sha}))) + result = dict(name='123-456', error=None, pages=1, source_sha256=sha, seconds=1) + (cold / 'results.json').write_text(json.dumps(result)) + (cold / 'teardown.json').write_text(json.dumps(dict(absent=True))) + page = cold / 'results/123-456/page-0.xml' + def xml(text, native): + return f'{text}{native}' + page.write_text(xml('Concurrent edits: [w0:0]', 'Native 0:')) + with patch('verify_offline_confirmation.publication_links') as ledger: + self.assertEqual(verify(root, cold)['validated_paragraphs'], 2) + self.assertTrue(ledger.called) + config = json.loads((root / 'run.json').read_text()) + config['stress_operations'] = 2 + (root / 'run.json').write_text(json.dumps(config)) + with self.assertRaisesRegex(AssertionError, 'Missing native acknowledgements'): + verify(root, cold) + self.assertFalse(verify(root, cold, partial=True)['complete_workload']) + config['stress_operations'] = 1 + (root / 'run.json').write_text(json.dumps(config)) + for text, native in [('Concurrent edits:', 'Native 0:'), ('Concurrent edits: [w0:0]', 'Native 0: [n0:1]')]: + page.write_text(xml(text, native)) + with self.assertRaises(AssertionError): verify(root, cold) + page.write_text(xml('Concurrent edits: [w0:0]', 'Native 0: [n0:0]')) + self.assertEqual(verify(root, cold)['native_images'], 1) + snapshot.write_bytes(b'changed') + with self.assertRaises(AssertionError): verify(root, cold) + snapshot.write_bytes(b'captured snapshot') + config['document_operations'] = True + (root / 'run.json').write_text(json.dumps(config)) + document = dict(text='x', runs=[dict(text='x', bold=True, size=18, color=0x563412)]) + rows[1]['document_changes'] = {'target': document} + rows[2]['started_us'] = 200 + observed = dict(event='read', finished_us=100, text=rows[2]['text'], documents={'target': document}) + rows.insert(2, observed) + (root / 'rust/w0.jsonl').write_text('\n'.join(map(json.dumps, rows))) + markup = 'x]]>' + content = xml('Concurrent edits: [w0:0]', 'Native 0:').replace('', markup + '') + page.write_text(content) + with patch('offline_document_history.document_history', return_value={'target': {'text': 'x', 'format': {'receipt_revision': 'revision'}}}) as documents: + result = verify(root, cold) + self.assertEqual(result['validated_paragraphs'], 3) + self.assertEqual(result['native_intended_format_checks'], 3) + documents.assert_called_once_with({'w0': rows}, 1) + rows[3]['revisions'] = {'space': ['current']} + rows[3]['current_revisions'] = {'space': 'current'} + documents.return_value['target']['format']['receipt_revision'] = 'current' + (root / 'rust/w0.jsonl').write_text('\n'.join(map(json.dumps, rows))) + self.assertEqual(verify(root, cold)['confirmed_revision'], 'current') + rows[3]['current_revisions'] = {'space': 'unrelated'} + (root / 'rust/w0.jsonl').write_text('\n'.join(map(json.dumps, rows))) + with self.assertRaisesRegex(AssertionError, 'current effect-confirmation'): verify(root, cold) + rows[3]['current_revisions'] = {'space': 'current'} + (root / 'rust/w0.jsonl').write_text('\n'.join(map(json.dumps, rows))) + page.write_text(content.replace('18pt', '19pt')) + with self.assertRaisesRegex(AssertionError, 'font size'): verify(root, cold) + page.write_text(content) + observed['documents'] = {} + (root / 'rust/w0.jsonl').write_text('\n'.join(map(json.dumps, rows))) + with self.assertRaisesRegex(AssertionError, 'omitted'): verify(root, cold) + + +if __name__ == '__main__': unittest.main() diff --git a/tools/test_offline_document_history.py b/tools/test_offline_document_history.py new file mode 100644 index 0000000000000000000000000000000000000000..f25d7c486041cf40e7fbb93cda00eaff1ebe553e --- /dev/null +++ b/tools/test_offline_document_history.py @@ -0,0 +1,151 @@ +import copy +import unittest +import uuid +from unittest.mock import patch + +from offline_document_history import document_history, identity, verify_model, verify_native + + +class DocumentHistoryTests(unittest.TestCase): + def setUp(self): + self.logs = {'w0': [{'event': 'ready', 'document_operations': True}]} + events = self.logs['w0'] + observed = {} + previous = None + for operation in range(2): + insertion = {'guid': list(uuid.UUID(int=operation+1).bytes_le), 'text': f'Document w0:{operation} 🦀', + 'parent': 'page' if operation == 0 else identity(previous, 1), 'author': 'Offline document writer', + 'placement': {'Outline': {'x': 144, 'y': 144}} if operation == 0 else {'Paragraph': {'before': None}}} + previous = insertion + target = identity(insertion, 2) + for step, kind in enumerate(('insert', 'format')): + timestamp = 10 + operation*30 + step*10 + local_id = operation*3 + step + 2 + event = {'event': 'local_document_commit', 'id': local_id, 'operation': operation, 'kind': kind, + 'space': 'space', 'object': target, 'text': insertion['text'], 'insertion': insertion if step == 0 else None, + 'range': [1, len(insertion['text'].encode('utf-16-le'))//2-2], + 'attributes': [{'Bold': True}, {'FontSize': 18+operation}, {'Color': [18, 52, 86]}], + 'started_us': timestamp-2, 'finished_us': timestamp-1} + events.append(event) + runs = [] + for index, char in enumerate(insertion['text']): + selected = kind == 'format' and 0 < index < len(insertion['text'])-1 + runs.append({'text': char, 'bold': selected, 'size': 18+operation if selected else 11, + 'color': 0x563412 if selected else 0xff000000}) + observed[target] = {'text': insertion['text'], 'runs': runs} + revision = f'revision-{local_id}' + events.append({'event': 'remote_attempt', 'revision': revision, 'state': 'Committed', + 'document_changes': {target: copy.deepcopy(observed[target])}, + 'documents': copy.deepcopy(observed), 'started_us': timestamp, 'finished_us': timestamp+1}) + events.append({'event': 'document_receipt', 'id': local_id, 'revision': revision, 'at_us': timestamp+2}) + events.extend({'event': 'reopened_document_receipt', 'id': row['id'], 'revision': row['revision']} + for row in list(events) if row['event'] == 'document_receipt') + read = {'event': 'read', 'started_us': 100, 'finished_us': 101, 'documents': observed} + events.extend([read, {'event': 'done'}]) + self.logs['r0'] = [{'event': 'ready'}, copy.deepcopy(read), {'event': 'done'}] + + def test_document_receipts_and_reader_states_match_the_intents(self): + documents = document_history(self.logs, 2) + self.assertEqual(len(documents), 2) + paragraphs = [[(char, {'bold': bold, 'font_size': size, 'color': 'automatic' if color == 0xff000000 else '#123456'}) + for char, bold, size, color in row['new']] for row in documents.values()] + self.assertEqual(verify_native(paragraphs, documents), sum(len(row['new'])*3 for row in documents.values())) + paragraphs[0][1][1]['font_size'] = 19 + with self.assertRaisesRegex(AssertionError, 'font size'): verify_native(paragraphs, documents) + + def test_missing_intents_receipts_or_reopen_records_are_rejected(self): + for name in ('local_document_commit', 'document_receipt', 'reopened_document_receipt', 'remote_attempt'): + logs = copy.deepcopy(self.logs) + events = logs['w0'] + events.remove(next(row for row in events if row['event'] == name)) + with self.subTest(event=name), self.assertRaises(AssertionError): document_history(logs, 2) + + def test_retired_format_receipt_requires_current_revision_and_exact_observed_effect(self): + events = self.logs['w0'] + attempt = next(row for row in events if row['event'] == 'remote_attempt' and row['revision'] == 'revision-3') + attempt['state'] = 'Unknown' + receipt = next(row for row in events if row['event'] == 'document_receipt' and row['id'] == 3) + receipt.update(revision='current-revision', at_us=25) + next(row for row in events if row['event'] == 'reopened_document_receipt' and row['id'] == 3)['revision'] = receipt['revision'] + read = dict(event='read', started_us=21, finished_us=22, text='Concurrent edits:', documents=copy.deepcopy(attempt['documents'])) + confirmation = dict(event='remote_confirm', started_us=23, finished_us=24, state='Committed', text=read['text'], + revisions={'space': ['current-revision']}, current_revisions={'space': 'current-revision'}) + index = events.index(receipt) + events[index:index] = [read, confirmation] + result = document_history(self.logs, 2) + target, = attempt['document_changes'] + self.assertEqual(result[target]['format']['receipt_revision'], 'current-revision') + for field, value in [('current_revisions', {'space': 'unrelated'}), + ('revisions', {'space': ['revision-3', 'current-revision']}), + ('state', 'NotCommitted')]: + original = confirmation[field] + confirmation[field] = value + with self.subTest(field=field), self.assertRaises(AssertionError): document_history(self.logs, 2) + confirmation[field] = original + read['documents'][target]['runs'][1]['size'] = 12 + with self.assertRaisesRegex(AssertionError, 'differs from the uncertain formatting intent'): document_history(self.logs, 2) + + def test_model_rejects_reordered_or_reparented_insertions(self): + documents = document_history(self.logs, 2) + first, second = [row['insertion'] for row in documents.values()] + outline, paragraph, appended = identity(first, 1), identity(first, 3), identity(second, 1) + nodes = {key: {'structure': [], 'content': [], 'children': [], 'kind': {}, 'layout': {}} + for key in ['page', outline, paragraph, appended, *documents]} + nodes['page']['children'] = [outline] + nodes[outline].update(children=[paragraph, appended], layout={'x': 144, 'y': 144}) + for parent, (target, document) in zip([paragraph, appended], documents.items(), strict=True): + nodes[parent]['content'] = [target] + nodes[target]['kind'] = {'text': document['text']} + revision = {'nodes': nodes} + with patch('offline_document_history.ordered_pages', return_value=[('space', 'revision', revision, 'page')]): + verify_model({}, documents) + nodes[outline]['children'].reverse() + with self.assertRaisesRegex(AssertionError, 'order'): verify_model({}, documents) + nodes[outline]['children'] = [paragraph] + nodes['page']['children'].append(appended) + with self.assertRaises(AssertionError): verify_model({}, documents) + nodes[outline]['children'] = [paragraph, appended] + nodes['page']['children'] = [outline] + nodes[paragraph]['content'].append(identity(second, 2)) + with self.assertRaisesRegex(AssertionError, 'content'): verify_model({}, documents) + + def test_wrong_attributes_targets_and_unconfirmed_receipts_are_rejected(self): + for event, field, value in [('local_document_commit', 'object', 'wrong'), + ('local_document_commit', 'text', 'changed'), + ('document_receipt', 'revision', 'wrong'), + ('remote_attempt', 'state', 'Unknown'), + ('remote_attempt', 'state', 'NotCommitted')]: + logs = copy.deepcopy(self.logs) + next(row for row in logs['w0'] if row['event'] == event)[field] = value + with self.subTest(event=event, field=field), self.assertRaises(AssertionError): document_history(logs, 2) + for field, value in [('attributes', [{'Bold': False}]), ('range', [0, 1])]: + logs = copy.deepcopy(self.logs) + next(row for row in logs['w0'] if row.get('kind') == 'format')[field] = value + with self.subTest(field=field), self.assertRaises(AssertionError): document_history(logs, 2) + + def test_readers_cannot_lose_revert_or_invent_document_content(self): + for mutation in ('missing', 'partial', 'future', 'reverted'): + logs = copy.deepcopy(self.logs) + read = logs['r0'][1] + target = next(iter(read['documents'])) + if mutation == 'missing': del read['documents'][target] + elif mutation == 'partial': read['documents'][target]['runs'][1]['bold'] = False + elif mutation == 'future': read.update(started_us=0, finished_us=1) + else: + old = copy.deepcopy(read) + old.update(started_us=102, finished_us=103) + for run in old['documents'][target]['runs']: + run.update(bold=False, size=11, color=0xff000000) + logs['r0'].insert(2, old) + with self.subTest(mutation=mutation), self.assertRaises(AssertionError): document_history(logs, 2) + + def test_an_uncertain_document_attempt_cannot_be_replayed_under_another_revision(self): + logs = copy.deepcopy(self.logs) + first = copy.deepcopy(next(row for row in logs['w0'] if row['event'] == 'remote_attempt')) + first.update(state='Unknown', revision='earlier-uncertain', started_us=8, finished_us=9) + logs['w0'].insert(2, first) + with self.assertRaisesRegex(AssertionError, 'more than once'): document_history(logs, 2) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/test_offline_history.py b/tools/test_offline_history.py new file mode 100644 index 0000000000000000000000000000000000000000..eb27594af0378f16ec5e2ad3f07a190ad7ce417a --- /dev/null +++ b/tools/test_offline_history.py @@ -0,0 +1,156 @@ +import copy +import unittest +from native_stress import edit_history + + +class OfflineHistoryTests(unittest.TestCase): + def setUp(self): + base = 'Concurrent edits:' + self.logs = {} + for actor, started, before in [('w1', 5, base), ('w0', 10, base + ' [w1:0]')]: + token = f' [{actor}:0]' + self.logs[actor] = [ + {'event': 'ready', 'offline': True}, + {'event': 'local_commit', 'id': 1, 'operation': 0, 'before': base, 'token': token, 'started_us': 1, 'finished_us': 2}, + {'event': 'read', 'text': before, 'started_us': started - 1, 'finished_us': started}, + {'event': 'remote_attempt', 'revision': actor, 'before': before, 'after': before + token, 'state': 'Committed', 'started_us': started, 'finished_us': started + 1}, + {'event': 'remote_receipt', 'id': 1, 'revision': actor, 'at_us': started + 2}, + {'event': 'reopened_receipt', 'id': 1, 'revision': actor}, + {'event': 'done'}, + ] + self.logs['r0'] = [{'event': 'ready'}, {'event': 'read', 'text': base + ' [w1:0] [w0:0]', 'started_us': 14, 'finished_us': 15}, {'event': 'done'}] + + def test_private_local_branches_require_one_separate_remote_history(self): + commits, text = edit_history(self.logs, 1, offline=True) + self.assertEqual([event['token'] for event in commits], [' [w1:0]', ' [w0:0]']) + self.assertEqual(text, self.logs['r0'][1]['text']) + + def test_false_receipts_lost_local_intents_and_changed_reopen_state_fail(self): + for index, field, value in [(1, 'id', 2), (1, 'token', ' [w0:9]'), (3, 'state', 'Unknown'), + (3, 'after', 'Concurrent edits: [w1:0]'), (4, 'at_us', 0), + (5, 'revision', 'changed'), (3, 'revision', 'missing')]: + with self.subTest(index=index, field=field): + logs = copy.deepcopy(self.logs) + logs['w0'][index][field] = value + with self.assertRaises(AssertionError): edit_history(logs, 1, offline=True) + for event in ('local_commit', 'remote_receipt', 'remote_attempt', 'reopened_receipt'): + logs = copy.deepcopy(self.logs) + logs['w0'] = [item for item in logs['w0'] if item['event'] != event] + with self.assertRaises(AssertionError): edit_history(logs, 1, offline=True) + + def test_stale_reads_future_local_tokens_and_duplicate_acknowledgements_fail(self): + logs = copy.deepcopy(self.logs) + logs['r0'][1]['text'] = 'Concurrent edits:' + with self.assertRaises(AssertionError): edit_history(logs, 1, offline=True) + logs = copy.deepcopy(self.logs) + logs['w0'][1]['before'] += ' [w2:0]' + with self.assertRaises(AssertionError): edit_history(logs, 1, offline=True) + for event in ('local_commit', 'remote_receipt', 'remote_attempt', 'reopened_receipt'): + logs = copy.deepcopy(self.logs) + copied = next(item for item in logs['w0'] if item['event'] == event) + logs['w0'].insert(-1, copy.deepcopy(copied)) + with self.assertRaises(AssertionError): edit_history(logs, 1, offline=True) + + def test_uncertain_receipt_requires_the_original_target_revision_and_successful_confirmation(self): + rows = self.logs['w0'] + intent = next(row for row in rows if row['event'] == 'local_commit') + attempt = next(row for row in rows if row['event'] == 'remote_attempt') + receipt = next(row for row in rows if row['event'] == 'remote_receipt') + intent.update(space='page-space', object='paragraph') + attempt.update(space='page-space', object='paragraph', state='Unknown') + receipt['at_us'] = 14 + confirmation = dict(event='remote_confirm', state='Committed', started_us=12, finished_us=13, + revisions={'page-space': [attempt['revision']]}, text=attempt['after']) + rows.insert(4, confirmation) + self.assertEqual(len(edit_history(self.logs, 1, offline=True)[0]), 2) + original = copy.deepcopy(self.logs) + for field, value in [('state', 'NotCommitted'), ('state', 'Unknown'), ('finished_us', 15), + ('revisions', {'other-space': ['w0']}), ('revisions', {'page-space': ['wrong']}), ('text', 'Concurrent edits:')]: + self.logs = copy.deepcopy(original) + next(row for row in self.logs['w0'] if row['event'] == 'remote_confirm')[field] = value + with self.subTest(field=field, value=value), self.assertRaises(AssertionError): edit_history(self.logs, 1, offline=True) + self.logs = copy.deepcopy(original) + replay = dict(attempt, revision='another-attempt', state='NotCommitted', started_us=11, finished_us=12) + self.logs['w0'].insert(4, replay) + with self.assertRaisesRegex(AssertionError, 'replayed'): edit_history(self.logs, 1, offline=True) + + def test_partial_capture_does_not_promote_pending_local_success(self): + logs = copy.deepcopy(self.logs) + logs['w0'] = logs['w0'][:2] + logs['r0'] = [{'event': 'ready'}] + commits, text = edit_history(logs, 1, offline=True, partial=True) + self.assertEqual(len(commits), 1) + self.assertEqual(text, 'Concurrent edits: [w1:0]') + with self.assertRaises(AssertionError): edit_history(logs, 1, offline=True) + + +class OfflineLedgerTests(unittest.TestCase): + def test_independent_ledger_queue_depth_and_progress_checks(self): + import json + import os + from pathlib import Path + import sqlite3 + import tempfile + from verify_offline import verify + with tempfile.TemporaryDirectory() as folder: + output = Path(folder) + (output / 'rust').mkdir() + (output / 'run.json').write_text(json.dumps({'offline': True, 'embedded_smb': True, 'edit': False, 'stress_clients': 4, 'rust_writers': 4, 'rust_readers': 4, 'stress_operations': 2})) + (output / 'rust/stop').touch() + os.utime(output / 'rust/stop', ns=(0, 0)) + (output / 'rust/start').touch() + os.utime(output / 'rust/start', ns=(0, 0)) + logs = {f'w{i}': [{'event': 'ready', 'offline': True}] for i in range(4)} + for actor, events in logs.items(): + before = 'Concurrent edits:' + for op in range(2): + token = f' [{actor}:{op}]' + events.append({'event': 'local_commit', 'id': op+1, 'operation': op, 'token': token, 'before': before, 'started_us': 1+op*2, 'finished_us': 2+op*2}) + before += token + before = 'Concurrent edits:' + timestamp = 100 + for op in range(2): + for actor, events in logs.items(): + token, revision = f' [{actor}:{op}]', f'{actor}-{op}' + events.append({'event': 'remote_attempt', 'before': before, 'after': before+token, 'revision': revision, 'state': 'Committed', 'started_us': timestamp, 'finished_us': timestamp+1}) + events.append({'event': 'remote_receipt', 'id': op+1, 'revision': revision, 'at_us': timestamp+2}) + before += token + timestamp += 10 + for actor, events in logs.items(): + events.extend({'event': 'reopened_receipt', 'id': op+1, 'revision': f'{actor}-{op}'} for op in range(2)) + events.append({'event': 'done'}) + connection = sqlite3.connect(output / 'rust' / f'{actor}.sqlite') + connection.executescript('CREATE TABLE receipts(edit_id INTEGER, revision TEXT); CREATE TABLE edits(id INTEGER); CREATE TABLE attempt(id INTEGER); CREATE TABLE conflicts(id INTEGER); CREATE TABLE replica(id INTEGER, base BLOB, working BLOB);') + connection.executemany('INSERT INTO receipts VALUES (?,?)', [(op+1, f'{actor}-{op}') for op in range(2)]) + connection.execute('INSERT INTO replica VALUES (1, ?, ?)', (b'opaque image', b'opaque image')) + connection.commit() + connection.close() + for i in range(4): + logs[f'r{i}'] = [{'event': 'ready'}, {'event': 'read', 'text': before, 'started_us': timestamp, 'finished_us': timestamp+1}, {'event': 'done'}] + (output / f'n{i}').mkdir() + prefix = f'Native {i}:' + native = [] + for op in range(2): + token = f' [n{i}:{op}]' + native.append({'operation': op, 'token': token, 'before': prefix, 'updated_ticks': op*10_000_000}) + prefix += token + (output / f'n{i}/stress-events.jsonl').write_text('\n'.join(json.dumps(event) for event in native)) + for actor, events in logs.items(): + (output / 'rust' / f'{actor}.jsonl').write_text('\n'.join(json.dumps(event) for event in events)) + result = verify(output, max_gap=2) + self.assertEqual(result['remote_publications'], 8) + self.assertEqual(result['queued_before_publication_lower_bound'], {f'w{i}': 2 for i in range(4)}) + with self.assertRaisesRegex(AssertionError, 'progress exceeded'): verify(output, max_gap=.5) + os.utime(output / 'rust/stop', ns=(3_000_000_000, 3_000_000_000)) + with self.assertRaisesRegex(AssertionError, 'progress exceeded'): verify(output, max_gap=2) + os.utime(output / 'rust/stop', ns=(0, 0)) + connection = sqlite3.connect(output / 'rust/w0.sqlite') + for sql, undo in [("UPDATE receipts SET revision='wrong' WHERE edit_id=1", "UPDATE receipts SET revision='w0-0' WHERE edit_id=1"), + ('INSERT INTO attempt VALUES (1)', 'DELETE FROM attempt'), + ("UPDATE replica SET working=X'00'", "UPDATE replica SET working=base")]: + connection.execute(sql) + connection.commit() + with self.assertRaises(AssertionError): verify(output) + connection.execute(undo) + connection.commit() + connection.close() diff --git a/tools/test_offline_outage.py b/tools/test_offline_outage.py new file mode 100644 index 0000000000000000000000000000000000000000..15ad50c69a0d3f9e62c2f2b86a7c0e8123d1fbf7 --- /dev/null +++ b/tools/test_offline_outage.py @@ -0,0 +1,216 @@ +import copy +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from offline_outage import verify_outage, verify_lost_reply + + +class OutageOracle(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + (self.root / 'rust').mkdir() + self.config = dict(offline=True, offline_outage=True, embedded_smb=True, stress_clients=4, rust_writers=4, rust_readers=4, stress_operations=8) + self.sample = dict(down_started_us=1_000_000, up_started_us=5_000_000, native_before=[1]*4, native_during=[4]*4, reader_errors_before={f'r{i}': 0 for i in range(4)}) + (self.root / 'clocks.json').write_text(json.dumps([dict(native_minus_host_us=[-10, 10], after_native_minus_host_us=[-10, 10])]*4)) + for i in range(4): + (self.root / f'n{i}').mkdir() + (self.root / f'n{i}/stress-events.jsonl').write_text(json.dumps(dict(update_started_ticks=621355968020000000, updated_ticks=621355968030000000))) + self.logs = {} + for i in range(4): + self.logs[f'w{i}'] = [dict(event='ready'), dict(event='transport_connected', at_us=0), + dict(event='publication_paused', revision=f'w{i}', at_us=100), + *[dict(event='local_commit', operation=j, started_us=10 if j == 0 else 2_000_000+j, finished_us=20 if j == 0 else 2_000_010+j) for j in range(8)], + dict(event='remote_attempt', started_us=6_000_000, state='NotCommitted', revision=f'w{i}'), + dict(event='transport_connected', at_us=6_000_001), + *[dict(event='remote_receipt', at_us=7_000_000+j) for j in range(8)], dict(event='done')] + self.logs[f'r{i}'] = [dict(event='ready'), dict(event='transport_read_error'), dict(event='transport_connected', at_us=6_000_000), + *[dict(event='read', started_us=7_000_000+j, finished_us=7_000_001+j) for j in range(3)], dict(event='done')] + self.trace = [dict(control=dict(phase='offline-down', mode='down')), dict(control=dict(phase='offline-reconnected'))] + + def verify(self): + (self.root / 'run.json').write_text(json.dumps(self.config)) + (self.root / 'offline-outage-progress.json').write_text(json.dumps(self.sample)) + (self.root / 'smb-trace.jsonl').write_text('\n'.join(map(json.dumps, self.trace))) + for actor, events in self.logs.items(): + (self.root / 'rust' / f'{actor}.jsonl').write_text('\n'.join(map(json.dumps, events))) + with patch('offline_outage.verify', return_value={'guarded_pairs': 1}) as overlap: + result = verify_outage(self.root) + overlap.assert_called_once_with(self.trace, phase='offline-reconnected') + return result + + def test_confirmed_outage_requires_local_and_native_progress_and_fresh_sessions(self): + self.assertEqual(self.verify()['local_edits_while_down'], {f'w{i}': 7 for i in range(4)}) + original = copy.deepcopy(self.logs) + for event, field, value in [('local_commit', 'started_us', 0), ('publication_paused', 'at_us', 3_000_000), + ('remote_attempt', 'state', 'Unknown'), ('remote_attempt', 'started_us', 3_000_000), + ('remote_attempt', 'revision', 'other'), ('remote_receipt', 'at_us', 0)]: + self.logs = copy.deepcopy(original) + row = next(row for row in self.logs['w0'] if row['event'] == event and (event != 'local_commit' or row['operation'] == 1)) + row[field] = value + with self.subTest(event=event, field=field), self.assertRaises(AssertionError): self.verify() + for actor, event in [('r0', 'transport_connected'), ('r0', 'transport_read_error'), ('r0', 'read'), ('w0', 'publication_paused')]: + self.logs = copy.deepcopy(original) + self.logs[actor] = [row for row in self.logs[actor] if row['event'] != event] + with self.subTest(actor=actor, event=event), self.assertRaises(AssertionError): self.verify() + + def test_lost_reply_requires_one_original_revision_receipt_and_captured_confirmation(self): + self.config.update(offline_outage=False, offline_lost_reply=True) + self.sample.update(before={actor: 3 for actor in self.logs}, after={actor: 6 for actor in self.logs}) + attempt = next(row for row in self.logs['w0'] if row['event'] == 'remote_attempt') + attempt.update(state='Unknown', started_us=500_000, finished_us=1_100_000) + receipt = next(row for row in self.logs['w0'] if row['event'] == 'remote_receipt') + receipt['revision'] = attempt['revision'] + for row in self.logs['w0']: + if row['event'] == 'remote_receipt' and row is not receipt: row['revision'] = 'other' + folder = self.root / 'rust/confirmations' + folder.mkdir() + (folder / 'confirmation.one').write_bytes(b'owned captured image') + confirmation = dict(event='remote_confirm', state='Committed', started_us=6_000_000, finished_us=6_000_001, + text='Concurrent edits: [w0:0]', capture='confirmation.one') + self.logs['w0'].insert(-1, confirmation) + self.trace = [dict(control=dict(phase='offline-reply-cut', cut=9, peer='10.0.2.2', offset=96, direction='response')), + dict(direction='request', command=9, offset=96, connection=1, message=2), + dict(cut=dict(direction='response', command=9, status='0x0', connection=1, message=2)), + dict(control=dict(phase='offline-reply-reconnected'))] + def check(): + (self.root / 'run.json').write_text(json.dumps(self.config)) + (self.root / 'offline-lost-reply-progress.json').write_text(json.dumps(self.sample)) + (self.root / 'smb-trace.jsonl').write_text('\n'.join(map(json.dumps, self.trace))) + for actor, rows in self.logs.items(): + (self.root / 'rust' / f'{actor}.jsonl').write_text('\n'.join(map(json.dumps, rows))) + with patch('offline_history.publication_links') as ledger, patch('offline_document_history.document_history') as documents, patch('offline_outage.verify', return_value={'guarded_pairs': 1}) as overlap: + if self.config.get('offline_client_reply'): + documents.return_value = {'target': {'format': {'receipt_revision': 'effect-revision'}}} + result = verify_lost_reply(self.root) + ledger.assert_called_once_with(self.logs, self.config['stress_operations']) + if self.config.get('document_operations'): + documents.assert_called_once_with(self.logs, self.config['stress_operations']) + else: + documents.assert_not_called() + overlap.assert_called_once_with(self.trace, phase='offline-reply-reconnected') + return result + self.assertEqual(check()['confirmed_revision'], 'w0') + self.config['document_operations'] = True + self.sample['format_released_us'] = 200_000 + receipt['event'] = 'document_receipt' + receipt['id'] = 17 + intent = dict(event='local_document_commit', id=17, kind='format') + self.logs['w0'].insert(-1, intent) + for actor, rows in self.logs.items(): + if actor.startswith('w'): + next(row for row in rows if row['event'] == 'publication_paused')['kind'] = 'format' + self.assertEqual(check()['confirmed_revision'], 'w0') + intent['kind'] = 'insert' + with self.assertRaisesRegex(AssertionError, 'not formatting'): check() + intent['kind'] = 'format' + self.sample['format_released_us'] = 900_000 + with self.assertRaises(AssertionError): check() + self.sample['format_released_us'] = 200_000 + paused = next(row for row in self.logs['w0'] if row['event'] == 'publication_paused') + paused['revision'] = 'stale-prepared-revision' + prior = dict(event='remote_attempt', revision=paused['revision'], state='NotCommitted', + started_us=200_001, finished_us=200_002) + self.logs['w0'].insert(-1, prior) + self.assertEqual(check()['confirmed_revision'], 'w0') + prior['finished_us'] = 600_000 + with self.assertRaisesRegex(AssertionError, 'proving it unpublished'): check() + self.logs['w0'].remove(prior) + paused['revision'] = attempt['revision'] + self.config['offline_client_reply'] = True + receipt['revision'] = 'effect-revision' + attempt.update(space='space', document_changes={'target': {}}) + retirement = dict(event='revision_retired', space='space', revision=attempt['revision'], started_us=2_200_000, finished_us=2_300_000) + self.logs['r0'].append(retirement) + (self.root / 'rust/offline-retired.one').write_bytes(b'retired revision snapshot') + self.sample['during'] = {actor: 3 if actor == 'w0' else 6 for actor in self.logs} + self.trace[0]['control']['scope'] = 'connection' + barrier = dict(event='confirmation_paused', revision=attempt['revision'], at_us=1_200_000) + self.logs['w0'].append(barrier) + peer_rows = [] + for actor, rows in self.logs.items(): + if actor == 'w0': continue + for i in range(3): + row = dict(event='remote_attempt' if actor.startswith('w') else 'read', state='Committed', + started_us=2_000_000+i, finished_us=2_000_010+i) + rows.append(row) + peer_rows.append((rows, row)) + wire = [dict(connection=7, opened=True, peer=['192.168.77.12', 445]), + dict(connection=7, message=2, command=5, direction='request', path='owned/synthetic.one'), + dict(connection=7, message=2, command=5, direction='response', status='0x0', file_id='native-file'), + dict(connection=7, message=3, command=9, direction='request', time=2.0, file_id='native-file'), + dict(connection=7, message=3, command=9, direction='response', time=2.1, status='0x0', written=4)] + self.trace.extend(wire) + result = check() + self.assertEqual(result['native_writes_during_client_disconnect'], 1) + self.assertEqual(result['peer_operations_during_client_disconnect'], {actor: 3 for actor in self.logs if actor != 'w0'}) + peer_rows[0][1]['finished_us'] = 6_000_000 + with self.assertRaisesRegex(AssertionError, 'insufficient completed I/O'): check() + peer_rows[0][1]['finished_us'] = 2_000_010 + wire[-1]['time'] = 6.0 + with self.assertRaisesRegex(AssertionError, 'No successful native writes'): check() + wire[-1]['time'] = 2.1 + barrier['revision'] = 'other' + with self.assertRaises(AssertionError): check() + self.logs['w0'].remove(barrier) + self.logs['r0'].remove(retirement) + receipt['revision'] = attempt['revision'] + for rows, row in peer_rows: rows.remove(row) + del self.trace[-len(wire):] + del self.trace[0]['control']['scope'] + self.config['offline_client_reply'] = False + self.config['document_operations'] = False + receipt['event'] = 'remote_receipt' + self.logs['w0'].remove(intent) + attempt['state'] = 'Committed' + with self.assertRaises(AssertionError): check() + attempt['state'] = 'Unknown' + self.trace[1]['offset'] = 100 + with self.assertRaises(AssertionError): check() + self.trace[1]['offset'] = 96 + (folder / 'unrecorded.one').write_bytes(b'extra') + with self.assertRaises(AssertionError): check() + + def test_document_outage_requires_both_local_operations_and_delayed_receipts(self): + self.config['document_operations'] = True + for actor, events in self.logs.items(): + if not actor.startswith('w'): continue + events[-1:-1] = [dict(event='local_document_commit', operation=operation, kind=kind, + started_us=10 if operation == 0 else 2_100_000+operation, + finished_us=20 if operation == 0 else 2_100_010+operation) + for operation in range(8) for kind in ('insert', 'format')] + events[-1:-1] = [dict(event='document_receipt', at_us=7_100_000+i) for i in range(16)] + self.assertEqual(self.verify()['local_edits_while_down'], {f'w{i}': 21 for i in range(4)}) + original = copy.deepcopy(self.logs) + for event, field, value in [('local_document_commit', 'finished_us', 6_000_000), + ('local_document_commit', 'kind', 'insert'), + ('document_receipt', 'at_us', 0)]: + self.logs = copy.deepcopy(original) + row = next(row for row in self.logs['w0'] if row['event'] == event + and (event != 'local_document_commit' or row['operation'] == 1 and row['kind'] == 'format')) + row[field] = value + with self.subTest(event=event, field=field), self.assertRaises(AssertionError): self.verify() + + def test_native_edits_outside_confirmed_window_fail(self): + (self.root / 'n0/stress-events.jsonl').write_text(json.dumps(dict(update_started_ticks=621355968000000000, updated_ticks=621355968000000100))) + with self.assertRaisesRegex(AssertionError, 'timestamps'): self.verify() + + def test_native_stall_short_outage_and_wrong_control_fail(self): + self.sample['native_before'].append(1) + with self.assertRaises(AssertionError): self.verify() + self.sample['native_before'].pop() + self.sample['native_during'][0] = 1 + with self.assertRaises(AssertionError): self.verify() + self.sample['native_during'][0] = 4 + self.sample['up_started_us'] = 1_000_001 + with self.assertRaises(AssertionError): self.verify() + self.sample['up_started_us'] = 5_000_000 + self.trace[0]['control'].pop('mode') + with self.assertRaises(AssertionError): self.verify() + + +if __name__ == '__main__': unittest.main() diff --git a/tools/test_offline_publication_crash.py b/tools/test_offline_publication_crash.py new file mode 100644 index 0000000000000000000000000000000000000000..0080ac05cb3158a7bc9b8c17f4ebcde393c9e045 --- /dev/null +++ b/tools/test_offline_publication_crash.py @@ -0,0 +1,81 @@ +import copy +import hashlib +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +from offline_publication_crash import state, TOKEN, confirmation_only +from verify_offline_recovery import verify + + +class RecoveryOracleTests(unittest.TestCase): + def test_confirmation_cannot_hide_revision_or_content_writes(self): + source = bytes(1024) + changed = bytearray(source) + changed[212:252] = bytes(range(40)) + events = [{'event':'write', 'offset':212, 'bytes':40}] + confirmation_only(events, source, changed) + for offset in (96, 211, 252, 1023): + bad = changed.copy() + bad[offset] ^= 1 + with self.assertRaises(AssertionError): confirmation_only(events, source, bad) + for event in ({'event':'write','offset':96,'bytes':1}, {'event':'phase','name':'publish-before'}): + with self.assertRaises(AssertionError): confirmation_only([*events,event], source, changed) + + def test_text_acknowledgements_and_revision_identity_must_agree(self): + original = 'Concurrent edits: 🦀' + at = len(original.encode('utf-16-le')) // 2 + row = {'event': 'state', 'status': 'uncertain', 'revision': 'new', 'remote_revision': 'old', + 'local_text': original+TOKEN, 'remote_text': original, + 'pending': [{'id': 1, 'before': original, 'replacement': TOKEN, 'range': [at, at]}]} + self.assertEqual(state([row], original), row) + for field, value in [('revision', None), ('revision', 'old'), ('local_text', original), + ('remote_text', original+TOKEN+TOKEN), ('status', 'missing'), ('pending', [])]: + changed = copy.deepcopy(row) + changed[field] = value + with self.subTest(field=field): + with self.assertRaises(AssertionError): state([changed], original) + for field, value in [('id', 2), ('before', 'other'), ('replacement', 'other'), ('range', [at-1, at-1])]: + changed = copy.deepcopy(row) + changed['pending'][0][field] = value + with self.assertRaises(AssertionError): state([changed], original) + row.update(status='published', remote_revision='new', remote_text=original+TOKEN, pending=[]) + self.assertEqual(state([row], original), row) + for field, value in [('revision', 'old'), ('remote_text', original), ('pending', [{'id': 1}])]: + changed = copy.deepcopy(row) + changed[field] = value + with self.assertRaises(AssertionError): state([changed], original) + + def test_native_inventory_hashes_errors_and_content_are_independently_checked(self): + with tempfile.TemporaryDirectory() as folder: + run, cold, source = [Path(folder)/name for name in ('run', 'cold', 'source.xml')] + (run/'images').mkdir(parents=True) + cold.mkdir() + data = b'owned fixture bytes' + digest = hashlib.sha256(data).hexdigest() + (run/'images'/f'{digest}.one').write_bytes(data) + expected = ['Concurrent edits:'+TOKEN, 'Other paragraph'] + (run/'results.json').write_text(json.dumps([{'remote_image': digest, 'remote_text': expected[0]}])) + (cold/'run.json').write_text(json.dumps({'inputs': {digest+'.one': digest}})) + record = {'name': digest, 'source_sha256': digest, 'error': None, 'pages': 1, 'seconds': 1} + (cold/'results.json').write_text(json.dumps(record)) + (cold/'teardown.json').write_text(json.dumps({'absent': True})) + (cold/'results'/digest).mkdir(parents=True) + (cold/'results'/digest/'page-0.xml').touch() + def content(path): + return ['Concurrent edits:', 'Other paragraph'] if path == source else expected + with patch('verify_offline_recovery.paragraphs', side_effect=content): + self.assertEqual(verify(run, cold, source)['exact_images'], 1) + for key, value in [('source_sha256', 'wrong'), ('error', 'failed'), ('pages', 0), ('name', 'wrong')]: + (cold/'results.json').write_text(json.dumps({**record, key:value})) + with self.subTest(key=key): + with self.assertRaises(AssertionError): verify(run, cold, source) + (cold/'results.json').write_text(json.dumps([record, record])) + with self.assertRaises(AssertionError): verify(run, cold, source) + (cold/'results.json').write_text(json.dumps(record)) + expected[0] = 'Concurrent edits:' + with self.assertRaises(AssertionError): verify(run, cold, source) + expected[0] += TOKEN + (run/'images'/f'{digest}.one').write_bytes(b'changed') + with self.assertRaises(AssertionError): verify(run, cold, source) diff --git a/tools/test_smb_overlap.py b/tools/test_smb_overlap.py new file mode 100644 index 0000000000000000000000000000000000000000..51e1ba965bb7df46c54c3e9f0de700521107c003 --- /dev/null +++ b/tools/test_smb_overlap.py @@ -0,0 +1,144 @@ +import unittest + +from verify_smb_overlap import verify + + +class Overlap(unittest.TestCase): + def history(self, serialized=False, failed_write=False, writer_reads=False): + events = [{'connection': 1, 'opened': True, 'peer': ['10.0.2.2', 1]}, + {'connection': 2, 'opened': True, 'peer': ['192.168.77.2', 2]}] + def exchange(connection, command, status='0x0', **fields): + message = len(events) + request = {'connection': connection, 'command': command, 'message': message, + 'direction': 'request', **fields} + response = {'connection': connection, 'command': command, 'message': message, + 'direction': 'response', 'status': status, 'file_id': str(connection)} + events.extend((request, response)) + for connection in (1, 2): + exchange(connection, 5, path='synthetic.one', access=0xc0000000 if connection == 2 or writer_reads else 0x80000000) + exchange(connection, 10, file_id=str(connection), locks=[(0xfffffffb, 1, 0x11)]) + if serialized: + exchange(1, 8, file_id='1', length=32) + exchange(1, 6, file_id='1') + exchange(2, 10, file_id='2', locks=[(0xfffffffd, 1, 0x12)]) + if not serialized: + exchange(1, 8, file_id='1', length=32) + exchange(2, 9, status='0xc0000054' if failed_write else '0x0', file_id='2', length=32) + exchange(2, 6, file_id='2') + events.append({'connection': 3, 'opened': True, 'peer': ['10.0.2.2', 3]}) + exchange(3, 5, path='synthetic.one', access=0xc0000000) + exchange(3, 10, file_id='3', locks=[(0xfffffffb, 1, 0x11), (0xfffffffd, 1, 0x12)]) + if not serialized: + exchange(1, 8, file_id='1', length=32) + exchange(3, 9, file_id='3', length=32) + return events + + def test_active_read_and_native_write(self): + result = verify(self.history()) + self.assertEqual(result['active_native_writer_pairs'], 1) + self.assertEqual(result['active_rust_writer_pairs'], 1) + self.assertEqual(result['overlapping_reads'], 2) + self.assertEqual(result['overlapping_writes'], 2) + + def test_progress_before_resume_does_not_satisfy_after_gate(self): + events = self.history() + events.append({'control': {'phase': 'resumed'}}) + with self.assertRaises(AssertionError): + verify(events, phase='resumed') + with self.assertRaises(AssertionError): + verify(self.history(), phase='resumed') + self.assertEqual(verify([{'control': {'phase': 'resumed'}}, *self.history()], phase='resumed')['active_rust_writer_pairs'], 1) + + def test_native_only_traffic_does_not_count(self): + events = self.history() + for event in events: + if event.get('opened'): event['peer'][0] = '192.168.77.' + str(event['connection'] + 1) + with self.assertRaises(AssertionError): + verify(events) + + def test_delayed_io_responses_do_not_count_as_resumed_requests(self): + events = self.history() + end = next(i for i, event in enumerate(events) if event.get('direction') == 'request' and event['command'] == 6) + responses = [event for event in events[:end] if event.get('direction') == 'response' and event['command'] in (8, 9)] + events = [event for event in events if event not in responses] + events[end - len(responses):end - len(responses)] = [{'control': {'phase': 'resumed'}}, *responses] + self.assertEqual(verify(events)['active_native_writer_pairs'], 1) + with self.assertRaises(AssertionError): + verify(events, phase='resumed') + + def test_serialized_calls_do_not_count(self): + with self.assertRaises(AssertionError): + verify(self.history(serialized=True)) + + def test_failed_write_does_not_count(self): + with self.assertRaises(AssertionError): + verify(self.history(failed_write=True)) + + def test_writers_validation_reads_do_not_count(self): + with self.assertRaises(AssertionError): + verify(self.history(writer_reads=True)) + + def test_lock_failure_does_not_count(self): + events = self.history() + for event in events: + if event.get('direction') == 'response' and event['command'] == 10: + event['status'] = '0xc0000055' + with self.assertRaises(AssertionError): + verify(events) + + def test_separate_lock_lifetimes_do_not_combine(self): + events = self.history() + inserted = [] + for message, flags in [(1000, 4), (1001, 0x12)]: + inserted.extend([ + {'connection': 3, 'direction': 'request', 'command': 10, 'message': message, + 'file_id': '3', 'locks': [(0xfffffffd, 1, flags)]}, + {'connection': 3, 'direction': 'response', 'command': 10, 'message': message, + 'status': '0x0'}, + ]) + events[-2:-2] = inserted + with self.assertRaises(AssertionError): + verify(events) + + def test_renamed_path_does_not_combine_file_versions(self): + events = self.history() + events[-2:-2] = [ + {'connection': 3, 'direction': 'request', 'command': 17, 'message': 1000, + 'info_type': 1, 'info_class': 10}, + {'connection': 3, 'direction': 'response', 'command': 17, 'message': 1000, 'status': '0x0'}, + ] + with self.assertRaises(AssertionError): + verify(events) + + def test_open_response_crossing_rename_is_not_attributed(self): + events = self.history() + at = next(i for i, event in enumerate(events) if event.get('connection') == 3 + and event.get('command') == 5 and event['direction'] == 'response') + inserted = [ + {'connection': 4, 'opened': True, 'peer': ['192.168.77.4', 4]}, + {'connection': 4, 'direction': 'request', 'command': 17, 'message': 1000, + 'info_type': 1, 'info_class': 10}, + {'connection': 4, 'direction': 'response', 'command': 17, 'message': 1000, 'status': '0x0'}, + ] + for message, command, fields in [ + (1001, 5, {'path': 'synthetic.one', 'access': 0x80000000}), + (1002, 10, {'file_id': '1', 'locks': [(0xfffffffb, 1, 0x11)]}), + ]: + inserted.extend([ + {'connection': 1, 'direction': 'request', 'command': command, 'message': message, **fields}, + {'connection': 1, 'direction': 'response', 'command': command, 'message': message, + 'status': '0x0', 'file_id': '1'}, + ]) + events[at:at] = inserted + with self.assertRaises(AssertionError): + verify(events) + + def test_connection_close_ends_guard(self): + events = self.history() + events.insert(-2, {'connection': 1, 'closed': True}) + with self.assertRaises(AssertionError): + verify(events) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/test_smb_proxy.py b/tools/test_smb_proxy.py new file mode 100644 index 0000000000000000000000000000000000000000..ad2ecabf412a3bfecf7ad1e818b7eeed713e9a55 --- /dev/null +++ b/tools/test_smb_proxy.py @@ -0,0 +1,196 @@ +import importlib.util +import asyncio +import json +from pathlib import Path +import socket +import struct +import sys +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location('smb_proxy', Path(__file__).with_name('smb-proxy.py')) +proxy = importlib.util.module_from_spec(spec) +spec.loader.exec_module(proxy) + + +class HeaderTrace(unittest.TestCase): + def test_fields_follow_complete_payload_ranges(self): + header = bytearray(1024) + header[96:100] = (257).to_bytes(4, 'little') + header[212:228] = bytes(range(16)) + header[228:236] = (999).to_bytes(8, 'little') + header[236:252] = bytes(range(16, 32)) + expected = {'transactions': 257, 'version': bytes(range(16)).hex(), + 'generation': 999, 'deny_read': bytes(range(16, 32)).hex()} + self.assertEqual(proxy.header_fields(0, header), expected) + self.assertEqual(proxy.header_fields(212, header[212:252]), {k:v for k,v in expected.items() if k != 'transactions'}) + self.assertEqual(proxy.header_fields(96, header[96:99]), {}) + self.assertEqual(proxy.header_fields(100, header[100:212]), {}) + self.assertEqual(proxy.header_fields(228, header[228:251]), {'generation':999}) + self.assertEqual(proxy.header_fields(252, bytes(1024)), {}) + + +class WriteTrace(unittest.IsolatedAsyncioTestCase): + async def test_opt_in_payload_and_partial_write_response_are_traced(self): + frames = [] + + async def respond(reader, writer): + try: + for _ in range(3): + prefix = await reader.readexactly(4) + frame = await reader.readexactly(int.from_bytes(prefix[1:], 'big')) + frames.append(frame) + reply = bytearray(80) + reply[:64] = frame[:64] + struct.pack_into('= 12 + actors = [*(f'w{i}' for i in range(config['rust_writers'])), *(f'r{i}' for i in range(config['rust_readers']))] + logs = {actor: [json.loads(line) for line in (output / 'rust' / f'{actor}.jsonl').read_text().splitlines()] for actor in actors} + commits, text = edit_history(logs, config['stress_operations'], offline=True) + documents = {} + if config.get('document_operations'): + from offline_document_history import document_history + documents = document_history(logs, config['stress_operations']) + started = (output / 'rust/start').stat().st_mtime_ns // 1000 + stopped = (output / 'rust/stop').stat().st_mtime_ns // 1000 + progress, queues, caches = {}, {}, {} + for actor, events in logs.items(): + times = [event['at_us'] for event in events if event['event'] in ('remote_receipt', 'document_receipt')] if actor.startswith('w') else [event['finished_us'] for event in events if event['event'] == 'read'] + assert times and times == sorted(times), 'Client progress is absent or went backwards' + if actor.startswith('r'): times.append(max(times[-1], stopped)) + progress[actor] = max(b-a for a, b in zip([started, *times], times)) / 1_000_000 + if not actor.startswith('w'): continue + edits = {event['id']: event for event in events if event['event'] in ('local_commit', 'local_document_commit')} + receipts = {event['id']: event for event in events if event['event'] in ('remote_receipt', 'document_receipt')} + attempts = {event['revision']: event for event in events if event['event'] == 'remote_attempt'} + publication_starts = {id: documents[edit['object']][edit['kind']]['started_us'] if edit['event'] == 'local_document_commit' + else attempts[receipts[id]['revision']]['started_us'] for id, edit in edits.items()} + queues[actor] = max(sum(edit['finished_us'] <= at < publication_starts[id] for id, edit in edits.items()) for at in [edit['finished_us'] for edit in edits.values()]) + assert queues[actor] >= 2, 'Writer did not establish a durable local queue before publication' + path = output / 'rust' / f'{actor}.sqlite' + connection = sqlite3.connect(path.resolve().as_uri() + '?mode=ro', uri=True) + try: + assert connection.execute('PRAGMA quick_check').fetchall() == [('ok',)], 'Cache integrity failed' + assert connection.execute('PRAGMA foreign_key_check').fetchall() == [], 'Cache foreign keys failed' + for table in ('edits', 'attempt', 'conflicts'): + assert connection.execute(f'SELECT count(*) FROM {table}').fetchone() == (0,), 'Completed cache retains unresolved state' + persisted = dict(connection.execute('SELECT edit_id, revision FROM receipts')) + assert persisted == {id: event['revision'] for id, event in receipts.items()}, 'SQLite receipts differ from observed acknowledgements' + base, working = connection.execute('SELECT base, working FROM replica WHERE id=1').fetchone() + assert base == working, 'A drained cache retained a divergent local branch' + caches[actor] = {'receipts': len(persisted), 'image_bytes': len(base), 'image_sha256': hashlib.sha256(base).hexdigest(), 'database_sha256': hashlib.sha256(path.read_bytes()).hexdigest()} + finally: + connection.close() + for i in range(config['stress_clients']): + events = [json.loads(line) for line in (output / f'n{i}/stress-events.jsonl').read_text(encoding='utf-8-sig').splitlines()] + native_history(events, i, config['stress_operations'], False) + times = [event['updated_ticks'] for event in events] + assert times == sorted(times) + progress[f'n{i}'] = max((b-a for a, b in zip(times, times[1:])), default=0) / 10_000_000 + assert all(gap <= max_gap for gap in progress.values()), f'Client progress exceeded {max_gap}s: {progress}' + return {'remote_publications': len(commits), 'document_publications': len(documents)*2, 'remote_text_sha256': hashlib.sha256(text.encode()).hexdigest(), 'maximum_progress_gap_seconds': progress, + 'queued_before_publication_lower_bound': queues, 'reviewed_placements': sum(event['event'] == 'reviewed_append' for events in logs.values() for event in events), 'caches': caches} + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('output', type=Path) + parser.add_argument('--max-gap', type=float, default=120) + args = parser.parse_args() + result = verify(args.output, args.max_gap) + (args.output / 'offline-verification.json').write_text(json.dumps(result, indent=2)) + print(json.dumps(result, indent=2)) diff --git a/tools/verify_offline_confirmation.py b/tools/verify_offline_confirmation.py new file mode 100644 index 0000000000000000000000000000000000000000..2ae74fb22e13fd771d2656f69eff2f3db983777d --- /dev/null +++ b/tools/verify_offline_confirmation.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Compare cold native reads of confirmation snapshots with the recorded editing history.""" +import argparse +import hashlib +import json +from pathlib import Path +import xml.etree.ElementTree as ET + +from native_format import native_characters +from native_stress import native_history +from native_xml import ns +from offline_history import publication_links + + +def verify(output, cold, *, partial=False): + config = json.loads((output / 'run.json').read_text()) + actors = [*(f'w{i}' for i in range(config['rust_writers'])), *(f'r{i}' for i in range(config['rust_readers']))] + logs = {actor: [json.loads(line) for line in (output / 'rust' / f'{actor}.jsonl').read_text().splitlines()] for actor in actors} + publication_links(logs, config['stress_operations'], partial=partial) + documents = {} + if config.get('document_operations'): + from offline_document_history import document_history + assert not partial, 'Document confirmation audit requires a complete workload' + documents = document_history(logs, config['stress_operations']) + unknown = [row for rows in logs.values() for row in rows if row['event'] == 'remote_attempt' and row['state'] == 'Unknown'] + assert len(unknown) == 1 + attempt, = unknown + confirmations = {} + for actor, rows in logs.items(): + observed = None + for row in rows: + if row['event'] == 'read': observed = row + if row['event'] != 'remote_confirm': continue + name = row['capture'] + assert Path(name).name == name and name.startswith(str(rows[0]['pid']) + '-') + assert name not in confirmations + if attempt['revision'] not in row['revisions'].get(attempt['space'], []): + assert documents and row.get('current_revisions', {}).get(attempt['space']) in row['revisions'].get(attempt['space'], []), 'Retired attempt lacks a current effect-confirmation revision' + if documents: + assert observed and observed['text'] == row['text'] and observed['finished_us'] <= row['started_us'] + assert isinstance(observed.get('documents'), dict) + assert all(observed['documents'].get(target) == value for target, value in attempt['document_changes'].items()), 'Confirmation omitted the uncertain document change' + confirmations[name] = row, observed['documents'] if documents else {} + assert confirmations + manifest = json.loads((cold / 'run.json').read_text())['inputs'] + assert set(manifest) == set(confirmations) + assert set(manifest) == {path.name for path in (output / 'rust/confirmations').glob('*.one')} + results = json.loads((cold / 'results.json').read_text(encoding='utf-8-sig')) + if isinstance(results, dict): results = [results] + assert len(results) == len(confirmations) and len({row['name'] for row in results}) == len(results) + native = [] + for i in range(config['stress_clients']): + rows = [json.loads(line) for line in (output / f'n{i}/stress-events.jsonl').read_text(encoding='utf-8-sig').splitlines()] + assert len(rows) <= config['stress_operations'] + native_history(rows, i, len(rows) if partial else config['stress_operations'], False) + native.append({f'Native {i}:', *(row['before'] + row['token'] for row in rows)}) + checks = format_checks = 0 + for result in results: + name = result['name'] + '.one' + assert result['error'] is None and result['pages'] == 1 + assert result['source_sha256'] == manifest[name] == hashlib.sha256((output / 'rust/confirmations' / name).read_bytes()).hexdigest() + page = ET.parse(cold / 'results' / result['name'] / 'page-0.xml').getroot() + formatted = native_characters(page, page.findall('one:Outline', ns)) + paragraphs = [''.join(c for c, _ in paragraph) for paragraph in formatted] + confirmation, observed = confirmations[name] + expected = confirmation['text'] + assert len(paragraphs) == len(native) + 1 + len(observed) and paragraphs.count(expected) == 1 + paragraphs.remove(expected) + if observed: + from offline_document_history import characters, verify_native + expected_documents = {target: {'text': documents[target]['text'], 'new': characters(value)} for target, value in observed.items()} + format_checks += verify_native(formatted, expected_documents) + for document in expected_documents.values(): + assert paragraphs.count(document['text']) == 1, 'Confirmation duplicated a document paragraph' + paragraphs.remove(document['text']) + for index, versions in enumerate(native): + selected = [text for text in paragraphs if text.startswith(f'Native {index}:')] + assert len(selected) == 1 and selected[0] in versions, 'Native confirmation image contains an unrecorded edit or loses a prefix' + checks += len(native) + 1 + len(observed) + assert json.loads((cold / 'teardown.json').read_text()) == {'absent': True} + confirmed_revision = documents[next(iter(attempt['document_changes']))]['format']['receipt_revision'] if documents else attempt['revision'] + return {'complete_workload': not partial, 'attempted_revision': attempt['revision'], 'confirmed_revision': confirmed_revision, 'native_images': len(results), 'exact_rust_paragraphs': len(results), + 'validated_paragraphs': checks, 'native_intended_format_checks': format_checks, + 'maximum_native_export_seconds': max(row['seconds'] for row in results)} + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('run', type=Path) + parser.add_argument('cold', type=Path) + parser.add_argument('--partial', action='store_true', help='Validate preserved confirmations from an interrupted workload') + args = parser.parse_args() + result = verify(args.run, args.cold, partial=args.partial) + (args.run / 'confirmation-cold-verification.json').write_text(json.dumps(result, indent=2)) + print(json.dumps(result, indent=2)) diff --git a/tools/verify_offline_recovery.py b/tools/verify_offline_recovery.py new file mode 100644 index 0000000000000000000000000000000000000000..9e4a939bcc46151b8713bc34a509b55ebff56994 --- /dev/null +++ b/tools/verify_offline_recovery.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Compare every interrupted/recovered image with an independent cold native export.""" +import argparse +import hashlib +import json +from pathlib import Path +import xml.etree.ElementTree as ET +from native_format import native_characters +from native_xml import ns + + +def paragraphs(path): + root = ET.parse(path).getroot() + return [''.join(char for char, _ in text) for text in native_characters(root, root.findall('one:Outline', ns))] + + +def verify(run, cold, source_capture): + original = paragraphs(source_capture) + target, = [text for text in original if text.startswith('Concurrent edits:')] + other = [text for text in original if text != target] + expected = {} + for case in json.loads((run/'results.json').read_text()): + for side in ('remote', 'local'): + if side+'_image' not in case: continue + digest, text = case[side+'_image'], case[side+'_text'] + assert text in (target, target+' [offline-recovery]'), 'Recovery oracle is unrelated to the native source' + wanted = sorted([text, *other]) + if digest in expected: assert expected[digest] == wanted, 'One image has contradictory expected states' + expected[digest] = wanted + inputs = json.loads((cold/'run.json').read_text())['inputs'] + assert inputs == {digest+'.one': digest for digest in expected}, 'Cold input inventory differs from recovery images' + records = json.loads((cold/'results.json').read_text(encoding='utf-8-sig')) + if isinstance(records, dict): records = [records] + assert len(records) == len(expected) and {record['name'] for record in records} == set(expected), 'Cold results omit or duplicate an image' + for record in records: + digest = record['name'] + assert record['source_sha256'] == digest, 'Native input hash differs' + assert hashlib.sha256((run/'images'/f'{digest}.one').read_bytes()).hexdigest() == digest, 'Retained image changed' + assert record['error'] is None and record['pages'] == 1, f'Native open failed: {record}' + pages = list((cold/'results'/digest).glob('page-*.xml')) + assert len(pages) == 1 + assert sorted(paragraphs(pages[0])) == expected[digest], f'Native content differs for {digest}' + assert json.loads((cold/'teardown.json').read_text())['absent'], 'Cold VM remains' + return {'exact_images':len(records), 'exact_paragraphs':sum(map(len, expected.values())), 'native_errors':0, + 'maximum_native_export_seconds':max(record['seconds'] for record in records)} + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('run', type=Path) + parser.add_argument('cold', type=Path) + parser.add_argument('source_capture', type=Path) + args = parser.parse_args() + result = verify(args.run, args.cold, args.source_capture) + (args.run/'cold-verification.json').write_text(json.dumps(result, indent=2)) + print(json.dumps(result, indent=2)) diff --git a/tools/verify_smb_faults.py b/tools/verify_smb_faults.py new file mode 100644 index 0000000000000000000000000000000000000000..582bc79e15fbd5c29bd323f34300118114c6dd59 --- /dev/null +++ b/tools/verify_smb_faults.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Cold-open every SMB fault artifact and compare its complete native text.""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +from pathlib import Path +import signal +import xml.etree.ElementTree as ET + +from native_format import native_characters +from native_xml import ns +from native_probe import ROOT, run as probe + + +def verify(root, captures): + cases = root / 'cases' + records = json.loads((cases / 'results.json').read_text()) + assert records and len({record['case'] for record in records}) == len(records), 'Missing or duplicate fault cases' + packets = {f'{role}-{path.stem}': path for role in ('source', 'interrupted', 'recovered') + for path in (cases / role).glob('*.one')} + expected_names = {f'{role}-{record["case"]}' for record in records for role in ('interrupted', 'recovered')} + fixtures = {record['case'].rsplit('-', 1)[0] for record in records} + expected_names.update(f'source-{fixture}' for fixture in fixtures) + assert set(packets) == expected_names, 'Artifact inventory differs from completed cases' + observed = {} + for worker in captures.glob('worker-*'): + result = json.loads((worker / 'results.json').read_text(encoding='utf-8-sig')) + for record in result if isinstance(result, list) else [result]: + name = record['name'] + assert name in packets and name not in observed, 'Unexpected or duplicate native result' + assert record['error'] is None and record['pages'] == 1, f'{name}: native open failed' + assert record['source_sha256'] == hashlib.sha256(packets[name].read_bytes()).hexdigest(), f'{name}: native input differs' + page, = (worker / 'results' / name).glob('page-*.xml') + page = ET.parse(page).getroot() + containers = page.findall('one:Title', ns) + page.findall('one:Outline', ns) + observed[name] = [''.join(char for char, _ in paragraph) for paragraph in native_characters(page, containers)] + assert set(observed) == expected_names, 'Missing native captures' + for record in records: + fixture = record['case'].rsplit('-', 1)[0] + intent = json.loads((cases / f'{fixture}-intent.json').read_text()) + baseline = observed[f'source-{fixture}'] + assert baseline.count(intent['before']) == 1, 'The native source lacks a unique editing target' + for role in ('interrupted', 'recovered'): + replacement = intent[record['visible']] if role == 'interrupted' else intent['after'] + intent['suffix'] + expected = [replacement if text == intent['before'] else text for text in baseline] + assert observed[f'{role}-{record["case"]}'] == expected, f'{role}-{record["case"]}: native content differs from the recorded outcome' + return {'cases': len(records), 'cold_native_opens': len(observed), 'exact_native_text': True} + + +def run(root, output): + assert json.loads((root / 'verification.json').read_text())['server_hashes_match'] + output.mkdir(parents=True, exist_ok=False) + (output / 'scripts').mkdir() + scripts = [output / 'scripts' / name for name in ('cold.ps1', 'probe.ps1')] + for path in scripts: path.write_bytes((ROOT / 'tools/native' / path.name).read_bytes()) + packets = [(role, path) for role in ('source', 'interrupted', 'recovered') + for path in sorted((root / 'cases' / role).glob('*.one'))] + workers = min(8, len(packets)) + inputs = [output / f'input-{i}' for i in range(workers)] + for path in inputs: path.mkdir() + for i, (role, path) in enumerate(packets): + (inputs[i % workers] / f'{role}-{path.name}').symlink_to(path.resolve()) + def capture(i): probe(inputs[i], output / f'worker-{i}', scripts) + with ThreadPoolExecutor(max_workers=workers) as pool: + list(pool.map(capture, range(workers))) + result = verify(root, output) + (output / 'verification.json').write_text(json.dumps(result, indent=2)) + print(json.dumps(result), flush=True) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('run', type=Path) + parser.add_argument('captures', type=Path) + parser.add_argument('--verify-only', action='store_true') + args = parser.parse_args() + def interrupted(_signal, _frame): raise KeyboardInterrupt + signal.signal(signal.SIGTERM, interrupted) + if args.verify_only: print(json.dumps(verify(args.run.resolve(), args.captures.resolve()), indent=2)) + else: run(args.run.resolve(), args.captures.resolve()) diff --git a/tools/verify_smb_overlap.py b/tools/verify_smb_overlap.py new file mode 100644 index 0000000000000000000000000000000000000000..47e17dce760ddb3e8a624203067ff2cf1294683c --- /dev/null +++ b/tools/verify_smb_overlap.py @@ -0,0 +1,123 @@ +"""Verify successful I/O inside overlapping SMB reader and writer guards.""" +import json +from collections import Counter + + +class PendingOverlap(AssertionError): + pass + + +def renames(request): + return request['command'] == 17 and request.get('info_type') == 1 and request.get('info_class') == 10 + + +def verify(events, phase=None): + pending, files, peers, pairs = {}, {}, {}, {} + progress = Counter() + path_epoch = 0 + active_since = -1 if phase is None else None + for index, event in enumerate(events): + assert not event.get('trace_error'), 'Trace collection failed' + assert not event.get('encrypted'), 'Encrypted traffic cannot establish I/O overlap' + if phase is not None and event.get('control', {}).get('phase') == phase: + active_since = index + connection = event.get('connection') + if event.get('opened'): + peers[connection] = event['peer'][0] + if event.get('closed'): + files = {key: value for key, value in files.items() if key[0] != connection} + if 'command' not in event: + continue + message = connection, event['message'] + if event['direction'] == 'request': + if renames(event): + files.clear() + path_epoch += 1 + pending[message] = index, event, path_epoch + if event['command'] == 6: + files.pop((connection, event['file_id']), None) + elif event['command'] == 10: + file = files.get((connection, event['file_id'])) + if file: + for offset, length, flags in event['locks']: + if flags & 4: + file['locks'] = {at: lock for at, lock in file['locks'].items() + if not offset <= at < offset + length} + continue + if event['status'] == '0x103': + continue + if message not in pending and event['command'] in (0, 18): + continue + request_index, request, request_epoch = pending.pop(message) + if renames(request): + files.clear() + path_epoch += 1 + if event['status'] != '0x0': + continue + command = event['command'] + if command == 5: + if request_epoch != path_epoch or any(renames(request) for _, request, _ in pending.values()): + continue + files[connection, event['file_id']] = { + 'path': request['path'].lower(), 'access': request['access'], + 'locks': {}, + } + continue + if command not in (8, 9, 10): + continue + key = connection, request['file_id'] + if key not in files: + continue + file = files[key] + if command == 10: + for offset, length, flags in request['locks']: + if length != 1 or offset not in (0xfffffffb, 0xfffffffd): + continue + if flags & 4: + file['locks'].pop(offset, None) + else: + file['locks'][offset] = index, flags & 3 + continue + if active_since is None or request_index <= active_since or not file['path'].endswith('synthetic.one') or request['length'] == 0: + continue + native = peers[connection].startswith('192.168.77.') + progress[('native' if native else 'host', connection, 'read' if command == 8 else 'write')] += 1 + for other_key, other in files.items(): + if key[0] == other_key[0] or file['path'] != other['path']: + continue + reader, writer = (file, other) if command == 8 else (other, file) + reader_key, writer_key = (key, other_key) if command == 8 else (other_key, key) + reader_lock = reader['locks'].get(0xfffffffb) + writer_lock = writer['locks'].get(0xfffffffd) + # Only read-only handles distinguish a reader from a writer's own validation reads. + if reader['access'] & 0x40000002 or not reader_lock or not writer_lock: + continue + if reader_lock[1] != 1 or writer_lock[1] != 2: + continue + if request_index <= max(reader_lock[0], writer_lock[0]): + continue + pair = reader_key + (reader_lock[0],) + writer_key + (writer_lock[0],) + observed = pairs.setdefault(pair, {'read': 0, 'write': 0, + 'reader_peer': peers[reader_key[0]], 'writer_peer': peers[writer_key[0]]}) + observed['read' if command == 8 else 'write'] += 1 + assert active_since is not None, 'Requested trace phase was not observed' + both = [pair for pair in pairs.values() if pair['read'] and pair['write']] + native_writer_pairs = [pair for pair in both if pair['writer_peer'].startswith('192.168.77.') + and not pair['reader_peer'].startswith('192.168.77.')] + rust_writer_pairs = [pair for pair in both if not pair['writer_peer'].startswith('192.168.77.')] + if not both: raise PendingOverlap('No reader/writer guard pair performed both successful reads and writes while overlapping') + if not native_writer_pairs: raise PendingOverlap('No active Rust reader overlapped successful native writes') + if not rust_writer_pairs: raise PendingOverlap('No active reader overlapped successful Rust writes') + return {'active_guard_pairs': len(both), 'active_native_writer_pairs': len(native_writer_pairs), + 'active_rust_writer_pairs': len(rust_writer_pairs), + 'overlapping_reads': sum(pair['read'] for pair in both), + 'overlapping_writes': sum(pair['write'] for pair in both), + 'connections': [{'kind': kind, 'connection': connection, 'operation': operation, 'count': count} + for (kind, connection, operation), count in sorted(progress.items())]} + + +if __name__ == '__main__': + import sys + with open(sys.argv[1]) as stream: + result = verify(json.loads(line) for line in stream) + print(json.dumps(result, indent=2)) -- 2.54.0