diff --git a/Cargo.lock b/Cargo.lock index 83fa0aca68d4f841b8bb4b95a13d153854fd6cd0..773eff13833a5ec5fa04a8116edd92ce7c27b988 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1561,6 +1561,22 @@ dependencies = [ "libc", ] +[[package]] +name = "notebook" +version = "0.1.0" +dependencies = [ + "onestore", + "rusqlite", + "serde", + "serde_json", + "sha2", + "smb2", + "tempfile", + "thiserror 2.0.20", + "tokio", + "zeroize", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1923,54 +1939,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "onestore-diagnostic" -version = "0.1.0" -dependencies = [ - "onestore", - "onestore-notebook", - "serde", - "serde_json", -] - -[[package]] -name = "onestore-notebook" -version = "0.1.0" -dependencies = [ - "onestore", - "onestore-smb", - "serde", - "serde_json", - "tempfile", - "thiserror 2.0.20", - "zeroize", -] - -[[package]] -name = "onestore-offline" -version = "0.1.0" -dependencies = [ - "onestore", - "onestore-notebook", - "onestore-smb", - "rusqlite", - "serde", - "serde_json", - "sha2", - "tempfile", - "thiserror 2.0.20", -] - -[[package]] -name = "onestore-smb" -version = "0.1.0" -dependencies = [ - "onestore", - "serde_json", - "smb2", - "tokio", -] - [[package]] name = "orbclient" version = "0.3.55" diff --git a/crates/notebook/Cargo.toml b/crates/notebook/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..6c7a47eadcdb9156907dd56c131fe16884961f10 --- /dev/null +++ b/crates/notebook/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "notebook" +version = "0.1.0" +edition = "2024" +publish = false + +[features] +smb = ["dep:smb2", "dep:tokio"] +protected = ["onestore/protected", "dep:zeroize"] + +[dependencies] +onestore = { path = "../onestore" } +rusqlite = { version = "=0.40.2", features = ["bundled", "backup"] } +thiserror = "2" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +tempfile = "3" +sha2 = "0.11" +smb2 = { version = "=0.21.0", optional = true } +tokio = { version = "1", features = ["rt-multi-thread", "time"], optional = true } +zeroize = { version = "1.9.0", optional = true } + +[dev-dependencies] +serde_json = "1" +tempfile = "3" + +[[example]] +name = "smb_offline_client" +required-features = ["smb"] + +[[example]] +name = "smb_reconnect_client" +required-features = ["smb"] + +[[example]] +name = "smb_concurrent_client" +required-features = ["smb"] diff --git a/crates/notebook/README.md b/crates/notebook/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7c6a6a3d4371e36f277e891cb5265351e6061968 --- /dev/null +++ b/crates/notebook/README.md @@ -0,0 +1,415 @@ +# notebook + +The application-facing crate: notebook discovery, a durable local replica with +reconnect reconciliation, external-asset caching, recovery export and optional +embedded SMB access. Durable local editing for a OneNote file: the current +foundation stores a complete working image and typed editing intents in a local +SQLite database. `sync_once` provides a reconciliation step and `start_sync` owns +automatic polling and reconnects. Local success does not +acknowledge publication to a shared notebook. + +```no_run +use onestore::ExGuid; +use notebook::Replica; +# fn example(path: &std::path::Path, source: &[u8], space: ExGuid, text: ExGuid) +# -> Result<(), Box> { +// `space` and `text` are identities from the supplied section's document model. +let cache = Replica::create(path, source)?; +let snapshot = cache.snapshot()?; +let local_id = cache.edit_text(&snapshot, space, text, 0..0, "Offline edit ")?; +drop(cache); + +let reopened = Replica::open(path)?; +let current = reopened.snapshot()?; +let pending = reopened.pending()?; +assert_eq!(pending.last().map(|edit| edit.id), local_id); +# Ok(()) +# } +``` + +`insert` accepts the core library's `Insertion` value and durably retains its +intent identities. `Insertion::with_formatting` queues text and +its styles as one intent and one publication. Keep that value across retries; its `text_object()` identifies +the new text for subsequent offline edits. Pending entries expose +`Operation::Text(TextEdit)`, `Operation::Insert(Insertion)`, +`Operation::Format(FormatEdit)`, `Operation::Split(SplitEdit)`, +`Operation::Join(JoinEdit)`, `Operation::Outline(OutlineEdit)`, +`Operation::Tree(TreeEdit)`, `Operation::CreatePage(PageCreation)` and +`Operation::Pages(PageEdits)` through their +`operation` field. Synchronization applies these in queue order, so an inserted +outline can precede its paragraphs and their later edits. Missing anchors or +existing insertion identities preserve a conflict and the complete local image. + +`create_page` accepts the core `PageCreation` intent and queues its page space +and section entry as one publication. Subsequent outlines and title edits use +the intent's stable identities immediately after local acknowledgement. +Independent page additions rebase against the current section order; duplicate +titles remain distinct. An unavailable or no-longer-leading insertion anchor +produces `StructureChanged`. `rebase_page_creation_conflict(id, local, remote, +before)` reviews a replacement anchor against both cache images while retaining +the new page and dependent object identities. Existing page identities require +reconciliation; a matching page alone does not establish a receipt. + +`pages(snapshot, edits)` queues a slice of core `PageEdit` intents as one atomic +publication. `PagePosition::Keep` preserves remote movement during indentation-only +edits. Every requested level remains explicit: a competing remote level produces +`StructureChanged` unless it already matches the requested level. Moves compare +the selected page's position relative to surviving observed pages; an unchanged +or already-satisfied position can proceed, and new remote pages remain present. +Indistinguishable competing moves retain a conflict and the complete local image. + +`rebase_pages_conflict(id, local, remote, edits)` reviews the batch against both +cache images. Use `PageEdit::reposition(position, level)` on the retained intents +to revise anchors or indentation; replacing page or allocated series identities +is rejected. Dependent edits remain queued. Schema 11 adds the durable batch; +schema 10 migration preserves existing multi-space publication evidence unchanged. +Attempt evidence includes every changed space plus the receipt space when that +space is unchanged. An existing section revision alone cannot confirm a page edit. +The [offline page corpus](../../corpus/page-lifecycle/offline-edits/README.md) +contains native comparisons, reproduction commands and stateful sanitizer seeds. + +`split` and `join` accept the core `ParagraphSplit` and `ParagraphJoin` intents. +Splits retain allocated identities when the original UTF-16 boundary rebases; +dependent edits can address `ParagraphSplit::text_object()` immediately after +local acknowledgement. Both operations retain observed parent paths, children, +indentation, paragraph/list state and tags. Changed structure produces +`StructureChanged`. Remote text changes must leave unambiguous boundary mappings; +current remote character styles are preserved. Reconciliation does not silently +discard a newly added right tag or move an unobserved child. + +`rebase_conflict` accepts a zero-length reviewed range for a split, preserving its +allocated identities. `rebase_join_conflict(id, local, remote)` reviews the original +join against current images. It rejects a change in which text identity survives, +because dependent edits still address that identity. As with other conflict +reviews, stale images, pending operations and uncertain attempts cannot be rebased. +An uncertain structural operation requires its original attempted revision for +confirmation; matching text or structure does not establish a receipt. + +`rebase_paragraph_conflict(id, local, remote, parent, before)` and +`rebase_outline_conflict(id, local, remote, page, x, y)` accept reviewed replacement +placements for the oldest insertion conflict. They preserve creation time, text, +author and object identities, keeping later edits attached to their original targets. +The images must still match the cache; the new placement must be valid in the remote +image. Paragraph intents cannot become outlines or vice versa. Pending edits and +uncertain publication attempts cannot be repositioned through conflict review. + +```no_run +use onestore::{ExGuid, Insertion, TextAttribute}; +use notebook::{EditStatus, Replica}; +# fn add_outline(cache: &Replica, space: ExGuid, page: ExGuid) +# -> Result<(), Box> { +let outline = Insertion::outline(page, 144.0, 216.0, "Offline outline", "Author")? + .with_formatting(0..7, &[TextAttribute::Bold(true)])?; +let id = cache.insert(&cache.snapshot()?, space, &outline)?; +if let Some(id) = id { + // A running worker may already have advanced this state. + match cache.status(id)? { + Some(EditStatus::Published { revision }) => println!("{revision}"), + state => println!("{state:?}"), + } +} +# Ok(()) +# } +``` + +`format` accepts the core `TextAttribute` slice and a UTF-16 range. Its durable intent +retains the observed text and selected attribute values. Remote text changes must +leave an unambiguous range; independent remote attributes merge, while competing +values preserve `FormattingChanged`. A remote value that already matches the +requested value is accepted. Enabling superscript or subscript also checks the +opposite attribute that the operation clears. If the remote image already satisfies +the whole operation, guarded confirmation still precedes a durable receipt. + +`outline` accepts a target object and the core `onestore::OutlineEdit`: outline +position/width or a paragraph's saved collapse default. It retains the target's +ancestry and the properties the operation changes. Independent content, formatting +and layout properties merge; competing selected values produce `LayoutChanged`, +and changed ancestry produces `StructureChanged`. Width changes also check the +reserved wrapping width that they clear. Missing targets preserve the local branch. +`rebase_layout_conflict(id, local, remote)` explicitly reviews the original change +against both current images, preserving dependent intents. An uncertain layout +attempt requires its original revision; converged values cannot establish its receipt. + +`tree` accepts the core `onestore::TreeEdit`. Moves preserve independent remote +text, formatting and descendants. A competing ancestor, indentation or sibling +crossing produces `StructureChanged`; unrelated sibling insertions/deletions can +merge. An anchor must remain a direct child of the requested destination. An +already satisfied move still requires guarded confirmation before acknowledgement. +Deletion compares the selected raw property graph, including referenced styles, +tags, unknown fields and internal attachments. Changed content produces +`ContentChanged`; property order, CompactID numbering and modification timestamps +do not affect that comparison. Immutable records compare by content, and empty +child lists compare equally to absent child lists. Mutable object identities +remain significant. Missing targets retain `TargetUnavailable`. +`rebase_tree_conflict(id, local, remote)` reviews the original move/deletion against +both current images. An emptied cell's replacement paragraph/text identities must +remain the same before replay or review; later queued edits keep their targets. +Uncertain tree attempts require their original revision for confirmation, even +when an independent move or deletion has the same visible effect. + +Recognized earlier caches migrate transactionally to version ten. Publication +attempts retain every changed space's revision; legacy attempts retain their +single-space evidence. Earlier recovery archives remain readable without migration. +Version-eight +deletion observations retain their original identity-sensitive preconditions; +explicit conflict review upgrades them to the immutable-content comparison. +The migration retains images, local IDs, publication attempts, conflicts, +receipts and the autoincrement sequence; it does not reuse acknowledged IDs when +the pending queue is empty. + +Share one `Replica` between application threads. Each edit compares its supplied +snapshot under the cache transaction; stale snapshots return `Io(ResourceBusy)`. +The intent and its resulting image commit together. No-op edits return `None`. +Keep the cache on a local filesystem: the connection holds exclusive ownership +between transactions, and a second open fails busy. No network wait occurs in a +local edit. After a database error, reopen and inspect the durable state before +retrying. + +`sync_once(&mut remote)` processes the oldest pending edit through a `Remote` +implementation, returning its ID and `EditStatus`. A durable `Published` receipt +survives reopening. Publication attempts are recorded before network I/O; a retained +attempt requires every recorded revision to remain present, followed by comparison, +flushing and refreshed header version +metadata before acknowledgement. If a formatting attempt's revision is missing, +the complete requested effect can instead be confirmed on a uniquely aligned +range; its receipt identifies that confirmed current revision. Otherwise the +missing attempt remains `AwaitingConfirmation`. Neither path replays an uncertain +publication. Overlapping or ambiguous edits retain `Conflict` status, their complete +local image and the last observed remote image returned by `remote_snapshot`. +Transport errors return `Error::Remote` or `Error::RemoteIo`; inspect `status(id)` +after the error to distinguish a retained attempt from a pending edit or receipt. +The error return does not roll back a locally acknowledged intent. + +Rebasing accepts only character mappings shared by every minimum insertion/deletion +alignment. UTF-16 ranges must preserve Unicode scalar boundaries. A bounded +alignment search also leaves a conflict when it cannot establish a unique mapping. +The current operation processes one queue head; while edits remain, `snapshot` +preserves the complete local working image. An empty queue can refresh from the +remote image. Synchronization holds a separate owner lock, so local edits can +continue during network waits; competing synchronization calls return `WouldBlock`. + +`rebase_conflict(id, local, remote, range)` lets a caller explicitly place the +oldest conflicting text or formatting intent at a reviewed UTF-16 range in the remote image. +The supplied images must still match `snapshot()` and `remote_snapshot()`. +It preserves the original replacement or requested attributes, intent ID, complete local working image +and every later intent; the selected range and remote paragraph become the +intent's new comparison base in one local transaction. Formatting also captures +the reviewed attribute values as its new precondition. This operation performs +no network I/O, clears the conflict to `Pending`, and wakes the worker. +Publication still reads the latest remote image and uses exact guarded comparison; +another overlapping remote edit can produce a new conflict. An uncertain attempt +cannot be rebased, and selecting text that already equals the replacement does +not create a publication acknowledgement. + +With the optional `smb` feature, `SmbRemote::new(client, path, limit)` binds an +`notebook::smb::Client` to one share-relative file and snapshot limit. Remote identity uses the logical root +object space, which survives the tested native compaction that replaces the file ID. + +An `Arc` can own one background worker. Supply a connection factory, poll +interval and observer; successful publications drain immediately, durable local +edits wake the worker, and `wake()` requests an immediate reachability retry. +Transport failures discard the old connection and retry through the factory; +Read contention and `NotCommitted` operations with `WouldBlock` or `ResourceBusy` +reuse the connection. Contended `NotCommitted` operations use randomized backoff, +capped at one second, to separate competing retry cycles. Cancellation interrupts this delay; local wake notifications +remain coalesced until its end. +`RemoteIo` distinguishes connection/read failures from +local `Io` errors. Cache/document errors stop the worker. Observers receive every +attempt's result on the worker thread, including unchanged conflict/uncertain +statuses; durable edit state remains available through `status`. + +```no_run +# #[cfg(feature = "smb")] +# fn example(cache: std::sync::Arc, username: String, password: String) +# -> Result<(), Box> { +use notebook::SmbRemote; +use notebook::smb::{Client, Credentials}; +use std::time::Duration; + +let worker = cache.start_sync( + Duration::from_secs(2), + move || { + Client::connect( + "server:445", "notes", + Credentials { username: &username, password: &password, domain: "" }, + Duration::from_secs(5), + ).map(|client| SmbRemote::new(client, "Personal/Video.one", 64 * 1024 * 1024)) + }, + |result| { + if let Err(error) = result { eprintln!("{error}"); } + }, +)?; +// Retain `worker` while synchronization should run; local edits wake it automatically. +worker.stop()?; +# Ok(()) +# } +``` + +`stop()` cancels future steps and joins the worker, returning a fatal cache error +or worker panic. Dropping it requests cancellation without waiting. An in-flight +step completes before releasing ownership; a replacement worker cannot start +while the old one still owns the replica. Remote operations and callbacks must +have bounded execution times if shutdown latency matters. A dropped worker can +briefly retain the cache; use `stop()` before requiring an immediate reopen. +Cancellation and application restart retain pending edits and publication attempts. +Credentials belong to the factory, not the cache database. + +Creation refuses existing paths. Opening recognizes the application identity and +schema version, validates database integrity and both notebook images, and rejects +unsupported journal modes without converting them. Failed initialization preserves +the file for inspection. SQLite uses DELETE journaling, EXTRA synchronization and +fullfsync; each required setting is queried back. + +The cache and its pending intents contain notebook content. The core `onestore` +crate stays independent of SQLite and network runtimes. Device and simulator +examples compile and link for iOS; recorded process-interruption tests do not +establish physical power-loss durability. Evidence is tracked in [Milestone 9](../../evidence/MILESTONE9.md). + +The SMB-enabled `smb_offline_client` example is an owned-lab workload for +`tools/native_collaboration.py --offline --embedded-smb`. It separates local +acknowledgements, remote publication attempts, persisted receipts and cache reopen +checks. Its append-specific conflict review policy lives in the test client; +the library continues to preserve conflicts requiring an explicit decision. + +## Recovery archives + +`export_recovery(new_path)` captures both complete notebook images, the typed +queue, uncertain attempts, conflicts, receipts, downloaded media and the edit-ID sequence in one +SQLite snapshot. It refuses existing destinations and leaves the live queue +unchanged. Export to a local directory from a background thread: copying holds +the cache mutex while capturing the database. Failure after the final rename can +leave a complete archive at the requested path; it never acknowledges a remote +edit. Archives contain notebook content and use a separate database identity, so +`Replica::open` rejects them as writable caches. + +```no_run +use notebook::{Recovery, Replica}; +# fn example(cache: &Replica) -> Result<(), Box> { +cache.export_recovery("review.sqlite")?; +let review = Recovery::open("review.sqlite")?; +let counts = review.summary()?; +let local = review.snapshot()?; +let remote = review.remote_snapshot()?; +let pending = review.pending()?; +let receipts = review.receipts()?; +# Ok(()) +# } +``` + +`Recovery` provides read-only inspection and no synchronization or restore method. +`status(id)` preserves the same state interpretation as the live replica. +`recovery_summary()` on the live replica and `summary()` on an archive return +counts and byte sizes without notebook text, paths, authors or credentials. +Opening an archive validates its schema and images without migration. A recovery +archive is evidence for a reviewed recovery decision, not a second active queue. + +## Downloaded media + +`fetch_asset(source, section, filename, limit)` resolves a declared external +payload through `notebook::discover::Source` and durably caches its exact bytes. +The section path is relative to the source root; the filename comes from a +`FileDataReference::External` in the retained working or remote image. Local and +SMB sources use the same API. Downloads release the cache mutex during network +I/O and recheck the reference before committing; edits can continue meanwhile. + +`cached_asset(filename, limit)` reads previously downloaded bytes without network +access. `None` means never downloaded; `Some(Vec::new())` is a downloaded empty +payload. Each read checks the stored SHA-256 and enforces the byte limit before +loading the payload. Cache contents describe the prior download, not current +server reachability or presence. A failed fetch returns its error without +silently substituting cached bytes. Different bytes for an already cached file +identity return `AssetChanged` and preserve the previous download. + +Downloads do not change pending edits, publication attempts or receipts. Recovery +archives include cached media and expose the same bounded `cached_asset` lookup. +Schema-4 archives remain readable without migration and contain no media cache. + +## Discovery + +`notebook::discover` provides read-only notebook discovery over a caller-supplied +root, keeping directory traversal out of the single-file storage parser. + +Discovery returns ordered sections and nested groups with file identities and +share-relative paths. Section display-name overrides remain distinct from file +names. TOC references whose identities are absent from the directory remain +inspectable; a cached filename never substitutes for an identity match. +Reserved `_onefiles` directories are excluded from section-group traversal. +Encrypted sections and valid storage with an unreadable document graph retain +their identity as `Locked` or `Unreadable`, without being presented as empty pages. +Malformed storage, failed reads and ambiguous identities reject the discovery. + +Each file read must be a consistent, bounded snapshot. The result is an +observation across multiple files, not an atomic notebook transaction or +authorization to publish an edit. Refresh rejects observed topology changes and +duplicate physical files with the same logical identity. Retain the last accepted +catalog if discovery fails; a connection failure does not mean files were deleted. + +`read_external_asset` resolves a validated UUID `.onebin` filename beneath the +selected section's sibling `_onefiles` folder and returns exact bounded bytes. +Missing files, permissions and size failures retain their I/O error kinds; an +empty payload is a successful empty buffer. Embedded payloads remain available +directly from the core document model. + +## SMB (feature `smb`) + +`notebook::smb` provides blocking SMB access for OneNote sections and +table-of-contents files. The core `onestore` crate remains independent of network runtimes. This is an +experimental Rust API with native interoperability evidence in the repository's +[Milestone 9](../../evidence/MILESTONE9.md). + +```no_run +use notebook::smb::{Client, Credentials}; +use std::time::Duration; + +let client = Client::connect( + "server:445", + "notes", + Credentials { username: "user", password: "password", domain: "" }, + Duration::from_secs(5), +)?; +let snapshot = client.read("Personal/Video.one", 64 * 1024 * 1024)?; +let store = onestore::Store::parse(&snapshot)?; +let revisions = onestore::RevisionIndex::parse(&store)?; +let document = onestore::document::Document::parse(&revisions)?; +# Ok::<(), Box>(()) +``` + +Paths are relative to the share. The read limit bounds the complete physical +snapshot. Call from a background thread outside a Tokio runtime. Use identities +from the document and the same snapshot with `Client::commit_text` or +`Client::commit_property_bytes`; their errors retain `onestore::CommitState`. +`PreparedEdit::{text,insert,format}` separate preparation from I/O: inspect the immutable image +and persist the intended revision identity before `Client::commit_prepared`. +`Client::confirm_snapshot` compares and flushes an observed image, then refreshes +its header version metadata without adding a revision. The caller must first +establish which intents that image contains and reread before another commit. + +Readers use shared native guards while writers publish under native write-open +and byte-lock exclusion. Maintenance is excluded during each operation; pathname +identity is checked after acquiring the guards. Connection loss retires the +client. Reconnect for subsequent operations, and reconcile an `Unknown` edit +before retrying it. The transport does not automatically replay requests. + +`Client::read_dir(path, entry_limit)` enumerates a directory, including the share +root with an empty path. It follows every response page and returns no partial +list on interruption, entry-limit overflow or close failure. Entries retain exact +Unicode names, observed sizes and MS-FSCC attributes, including directory/reparse +flags. Concurrent directory changes are not an atomic snapshot; repeated names +are rejected with `ResourceBusy`. Notebook identities come from the files, not +directory names or sizes. Missing paths, denied access and non-directory paths +have distinct I/O error kinds. + +`Client::read_asset(path, byte_limit)` reads an external payload under a read-only +share handle that excludes writes and deletion. Empty files succeed; limits, +interrupted reads and failed close never return partial bytes. This payload read +does not parse a OneStore header or acquire its reader-coordination bytes. + +`python3 tools/test_smb_directory.py VM OUTPUT` checks a caller-owned disposable +Linux lab VM against its filesystem listing and interrupts directory requests, +responses and close. It creates synthetic files in that VM; the caller retains +responsibility for VM teardown. The parser also has bounded-record/truncation +tests independent of the server. + +Device and simulator builds link for iOS. Native acceptance uses disposable +OneNote 2010 clients and Samba; it does not establish on-device execution or +physical power-loss durability. diff --git a/crates/notebook/examples/cache_probe.rs b/crates/notebook/examples/cache_probe.rs new file mode 100644 index 0000000000000000000000000000000000000000..aef2d6e75b0d81db0965cedf76778955b6d73208 --- /dev/null +++ b/crates/notebook/examples/cache_probe.rs @@ -0,0 +1,238 @@ +use notebook::Replica; +use onestore::{ + ExGuid, Insertion, RevisionIndex, Store, TextAttribute, + document::{Document, Kind}, +}; +use std::{ + io::{self, BufRead, Write}, + path::Path, +}; + +fn content(bytes: &[u8]) -> (ExGuid, ExGuid, String) { + let store = Store::parse(bytes).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + let document = Document::parse(&index).unwrap(); + document + .spaces + .iter() + .find_map(|(sid, space)| { + let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; + revision + .nodes + .iter() + .find_map(|(oid, node)| match &node.kind { + Kind::RichText { text, .. } => Some((*sid, *oid, text.clone())), + _ => None, + }) + }) + .unwrap() +} + +fn payload(operation: u64, size: usize) -> String { + format!("{operation}:🦀{}", "x".repeat(size)) +} + +fn main() -> Result<(), Box> { + let args: Vec<_> = std::env::args().collect(); + let mode = &args[1]; + let path = Path::new(&args[2]); + let operation_kind = + std::env::var("ONESTORE_CACHE_PROBE_OPERATION").unwrap_or_else(|_| "text".into()); + assert!(matches!( + operation_kind.as_str(), + "text" | "insert" | "format" + )); + let size = match std::env::var("ONESTORE_CACHE_PROBE_BYTES") { + Ok(value) => value.parse::()?, + Err(std::env::VarError::NotPresent) => 2 * 1024 * 1024, + Err(error) => return Err(error.into()), + }; + assert!(size > 0 && size <= 2 * 1024 * 1024); + let seed = std::env::var_os("ONESTORE_CACHE_PROBE_SOURCE") + .map(std::fs::read) + .transpose()?; + if mode == "init" { + let source = match seed { + Some(source) => source, + None => onestore::create_section( + "cache.one", + &if operation_kind == "format" { + payload(0, size) + } else { + "Base".into() + }, + "Fixture", + )?, + }; + Replica::create(path, &source)?; + return Ok(()); + } + let cache = Replica::open(path)?; + if mode == "read" { + let snapshot = cache.snapshot()?; + let base = cache.remote_snapshot()?; + let (sid, target, mut expected) = content(&base); + let store = Store::parse(&snapshot)?; + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store)?; + index.validate_current()?; + let document = Document::parse(&index)?; + let space = &document.spaces[&sid]; + let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; + let mut operations = Vec::new(); + let mut ids = Vec::new(); + let mut font_size = None; + for pending in cache.pending()? { + let operation = match pending.operation { + notebook::Operation::Text(edit) => { + assert_eq!(operation_kind, "text"); + assert_eq!(edit.object, target); + assert_eq!(edit.before, expected); + assert_eq!( + edit.range, + 0..u32::try_from(expected.encode_utf16().count())? + ); + let operation: u64 = edit.replacement.split_once(':').unwrap().0.parse()?; + expected = payload(operation, size); + assert_eq!(edit.replacement, expected); + operation + } + notebook::Operation::Insert(insertion) => { + assert_eq!(operation_kind, "insert"); + let Kind::RichText { text, .. } = + &revision.nodes[&insertion.text_object()].kind + else { + panic!("Missing inserted text") + }; + let operation: u64 = text.split_once(':').unwrap().0.parse()?; + assert_eq!(*text, payload(operation, size)); + assert_eq!(serde_json::to_value(&insertion)?["text"], *text); + let outline = &revision.nodes[&insertion.object()]; + assert!(matches!(outline.kind, Kind::Outline { .. })); + assert_eq!( + (outline.layout.x, outline.layout.y), + (Some(144.0), Some(operation as f32 * 72.0)) + ); + let (_, page) = document + .pages()? + .into_iter() + .find(|(space, _)| *space == sid) + .unwrap(); + assert!(revision.nodes[&page].children.contains(&insertion.object())); + operation + } + notebook::Operation::Format(edit) => { + assert_eq!(operation_kind, "format"); + assert_eq!(edit.object, target); + assert_eq!(edit.before, expected); + assert_eq!( + edit.range, + 0..u32::try_from(expected.encode_utf16().count())? + ); + let [TextAttribute::FontSize(value)] = edit.attributes.as_slice() else { + panic!("Unexpected formatting intent") + }; + assert!((7.0..=130.0).contains(value) && value.fract() == 0.0); + font_size = Some(*value); + *value as u64 - 6 + } + notebook::Operation::Split(_) + | notebook::Operation::CreatePage(_) + | notebook::Operation::Pages(_) + | notebook::Operation::Join(_) + | notebook::Operation::Outline(_) + | notebook::Operation::Tree(_) => { + panic!("Unexpected operation for this fixture") + } + }; + assert!(operations.last().is_none_or(|last| *last < operation)); + assert!(ids.last().is_none_or(|last| *last < pending.id)); + operations.push(operation); + ids.push(pending.id); + } + assert!(matches!(&revision.nodes[&target].kind,Kind::RichText{text,..} if *text==expected)); + if let Some(font_size) = font_size { + assert!( + revision + .text_runs(target)? + .iter() + .all(|run| run.format.font_size == Some(font_size)) + ); + } + if operations.is_empty() { + assert!( + snapshot == base, + "An empty local queue changed its working image" + ); + } + if let Some(output) = args.get(3) { + std::fs::write(output, &snapshot)?; + } + println!( + "{}", + serde_json::json!({"operations": operations, "ids": ids, "section_bytes": snapshot.len(), "complete_payloads": true}) + ); + return Ok(()); + } + assert_eq!(mode, "edit"); + assert!( + matches!(Replica::open(path), Err(notebook::Error::Database(error)) if error.sqlite_error_code() == Some(rusqlite::ErrorCode::DatabaseBusy)) + ); + println!("ready"); + io::stdout().flush()?; + let input = io::stdin(); + let mut lines = input.lock().lines(); + let instruction = lines.next().unwrap()?; + let (operation, acknowledgement) = instruction.split_once(' ').unwrap(); + let operation: u64 = operation.parse()?; + let source = cache.snapshot()?; + let (sid, oid, text) = content(&source); + let replacement = payload(operation, size); + println!("editing {operation}"); + io::stdout().flush()?; + let id = match operation_kind.as_str() { + "text" => cache.edit_text( + &source, + sid, + oid, + 0..text.encode_utf16().count().try_into()?, + &replacement, + )?, + "insert" => { + let store = Store::parse(&source)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + let (sid, page) = document.pages()?[0]; + let insertion = Insertion::outline( + page, + 144.0, + operation as f32 * 72.0, + &replacement, + "Fixture", + )?; + cache.insert(&source, sid, &insertion)? + } + "format" => { + assert!((1..=124).contains(&operation)); + cache.format( + &source, + sid, + oid, + 0..text.encode_utf16().count().try_into()?, + &[TextAttribute::FontSize(6.0 + operation as f32)], + )? + } + _ => unreachable!(), + } + .unwrap(); + if acknowledgement == "unack" { + println!("durable {operation} {id}"); + } else { + println!("ack {operation} {id}"); + } + io::stdout().flush()?; + lines.next().transpose()?; + Ok(()) +} diff --git a/crates/notebook/examples/discover.rs b/crates/notebook/examples/discover.rs new file mode 100644 index 0000000000000000000000000000000000000000..c243228318a5733038cf1eee4accf536ad8eae66 --- /dev/null +++ b/crates/notebook/examples/discover.rs @@ -0,0 +1,40 @@ +use notebook::discover::{Limits, Local, discover}; + +fn main() -> Result<(), Box> { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + let limits = Limits { + entries: 100_000, + bytes_per_file: 256 * 1024 * 1024, + depth: 64, + }; + let catalog = match args.as_slice() { + [root] => discover(&mut Local::open(root)?, limits)?, + #[cfg(feature = "smb")] + [flag, address, share, root] if flag == "--smb" => { + use notebook::smb::{Client, Credentials}; + let username = std::env::var("ONESTORE_SMB_USERNAME").unwrap_or_default(); + let password = std::env::var("ONESTORE_SMB_PASSWORD").unwrap_or_default(); + let domain = std::env::var("ONESTORE_SMB_DOMAIN").unwrap_or_default(); + let client = Client::connect( + address.to_str().ok_or("Use a UTF-8 server address")?, + share.to_str().ok_or("Use a UTF-8 share name")?, + Credentials { + username: &username, + password: &password, + domain: &domain, + }, + std::time::Duration::from_secs(5), + )?; + discover( + &mut notebook::discover::Smb::new( + &client, + root.to_str().ok_or("Use a UTF-8 notebook path")?, + )?, + limits, + )? + } + _ => return Err("Provide ROOT, or --smb ADDRESS SHARE ROOT with the smb feature".into()), + }; + serde_json::to_writer_pretty(std::io::stdout().lock(), &catalog)?; + Ok(()) +} diff --git a/crates/notebook/examples/document.rs b/crates/notebook/examples/document.rs new file mode 100644 index 0000000000000000000000000000000000000000..d1a68175db8f8b35932e59946be4ff683f581a7b --- /dev/null +++ b/crates/notebook/examples/document.rs @@ -0,0 +1,170 @@ +use onestore::{ + FileDataReference, RevisionIndex, Store, + document::{Document, Kind}, +}; +use std::{ + borrow::Cow, + collections::BTreeSet, + env, fs, + io::{self, BufWriter, Write}, + path::{Path, PathBuf}, +}; + +fn write_json( + path: impl AsRef, + value: &impl serde::Serialize, +) -> Result<(), Box> { + let mut output = BufWriter::new(fs::File::create(path)?); + serde_json::to_writer(&mut output, value)?; + output.flush()?; + Ok(()) +} + +fn main() -> Result<(), Box> { + let mut args = env::args_os().skip(1); + let path = args.next().ok_or("Provide a .one or .onetoc2 file.")?; + let next = args.next(); + let (destination, option) = if next.as_deref() == Some(std::ffi::OsStr::new("--password-file")) + { + (None, next) + } else { + (next.map(PathBuf::from), args.next()) + }; + let password_file = match (option, args.next(), args.next()) { + (None, None, None) => None, + (Some(flag), Some(path), None) if flag == "--password-file" => Some(PathBuf::from(path)), + _ => return Err("Provide a source file, an optional new export directory, and optionally --password-file PATH.".into()), + }; + let source_path = PathBuf::from(path); + let bytes = onestore::read_file(&source_path)?; + let store = Store::parse(&bytes)?; + let index = RevisionIndex::parse(&store)?; + #[cfg(feature = "protected")] + let unlocked = if let Some(path) = &password_file { + use std::io::Read; + let mut password = zeroize::Zeroizing::new(String::new()); + fs::File::open(path)? + .take(65537) + .read_to_string(&mut password)?; + if password.len() > 65536 { + return Err("The password file exceeds 64 KiB.".into()); + } + Some(onestore::protected::UnlockedSection::open( + &index, + &password, + Default::default(), + )?) + } else { + None + }; + #[cfg(not(feature = "protected"))] + if password_file.is_some() { + return Err( + "Build this exporter with the protected feature to open a password-protected section." + .into(), + ); + } + #[cfg(feature = "protected")] + let document = match &unlocked { + Some(section) => section.document()?, + None => Document::parse(&index)?, + }; + #[cfg(not(feature = "protected"))] + let document = Document::parse(&index)?; + if let Some(destination) = destination { + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + fs::DirBuilder::new() + .mode(if password_file.is_some() { + 0o700 + } else { + 0o777 + }) + .create(&destination)?; + } + #[cfg(not(unix))] + fs::create_dir(&destination)?; + fs::create_dir(destination.join("assets"))?; + let parent = source_path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let mut source = notebook::discover::Local::open(parent)?; + let section = source_path + .file_name() + .and_then(|name| name.to_str()) + .ok_or("Use a UTF-8 section filename")?; + let mut assets = Vec::new(); + let mut seen = BTreeSet::new(); + for node in document + .spaces + .values() + .flat_map(|s| s.revisions.values()) + .flat_map(|r| r.nodes.values()) + { + if let Kind::File { + reference, payload, .. + } = &node.kind + && seen.insert(serde_json::to_string(reference)?) + { + let data = match reference { + FileDataReference::Internal(_) => { + Cow::Borrowed(payload.ok_or("Missing embedded file data")?) + } + FileDataReference::External(filename) => { + if password_file.is_some() { + assets.push(serde_json::json!({"reference":reference,"path":null,"error":{"kind":"Unsupported","message":"Protected external payload export is not supported"}})); + continue; + } + match notebook::discover::read_external_asset( + &mut source, + section, + filename, + 256 * 1024 * 1024, + ) { + Ok(bytes) => Cow::Owned(bytes), + Err(error) => { + let kind = match &error { + notebook::discover::Error::Io { error, .. } => { + format!("{:?}", error.kind()) + } + _ => "InvalidData".into(), + }; + assets.push(serde_json::json!({"reference":reference,"path":null,"error":{"kind":kind,"message":error.to_string()}})); + continue; + } + } + } + FileDataReference::Invalid => { + assets.push(serde_json::json!({"reference":reference,"path":null,"error":{"kind":"InvalidData","message":"The document marks this payload as unavailable"}})); + continue; + } + }; + let path = format!("assets/{}.bin", assets.len()); + fs::write(destination.join(&path), data)?; + assets.push(serde_json::json!({"reference":reference,"path":path})); + } + } + write_json(destination.join("assets.json"), &assets)?; + let mut text = std::collections::BTreeMap::new(); + for (sid, space) in &document.spaces { + let mut revisions = std::collections::BTreeMap::new(); + for (rid, revision) in &space.revisions { + let mut objects = std::collections::BTreeMap::new(); + for (oid, node) in &revision.nodes { + if matches!(node.kind, Kind::RichText { .. }) { + objects.insert(*oid, revision.text_runs(*oid)?); + } + } + revisions.insert(*rid, objects); + } + text.insert(*sid, revisions); + } + write_json(destination.join("text.json"), &text)?; + write_json(destination.join("document.json"), &document)?; + } else { + serde_json::to_writer(io::stdout().lock(), &document)?; + } + Ok(()) +} diff --git a/crates/notebook/examples/externalize_fixture.rs b/crates/notebook/examples/externalize_fixture.rs new file mode 100644 index 0000000000000000000000000000000000000000..2b82cd3798d8c710e79c14cd3f60a308588661b0 --- /dev/null +++ b/crates/notebook/examples/externalize_fixture.rs @@ -0,0 +1,101 @@ +//! Builds a new external-payload test fixture from a native capture with reserved fragment space. +use onestore::{FileDataReference, RevisionIndex, Store}; +use std::{fs, path::PathBuf}; +fn main() -> Result<(), Box> { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + if args.len() != 2 { + return Err("Provide a native source section and a new fixture directory".into()); + } + let source = fs::read(&args[0])?; + let destination = PathBuf::from(&args[1]); + fs::create_dir(&destination)?; + fs::create_dir(destination.join("synthetic_onefiles"))?; + let store = Store::parse(&source)?; + assert!(store.checksum_mismatches.is_empty()); + RevisionIndex::parse(&store)?.validate_current()?; + let mut output = source.clone(); + let mut changed = 0; + for list in store.lists.values() { + for chunk in &list.fragments { + let start = chunk.offset as usize; + let end = start + chunk.length as usize - 20; + let mut body = Vec::new(); + let mut touched = false; + for node in list + .nodes + .iter() + .filter(|node| node.offset >= start + 16 && node.offset < end) + { + let raw = u32::from_le_bytes(source[node.offset..node.offset + 4].try_into()?); + let size = ((raw >> 10) & 0x1fff) as usize; + let mut encoded = source[node.offset..node.offset + size].to_vec(); + if matches!(node.id, 0x72 | 0x73) { + let offset = 4 + 4 + 4 + if node.id == 0x72 { 1 } else { 4 }; + let count = + u32::from_le_bytes(encoded[offset..offset + 4].try_into()?) as usize; + let text = String::from_utf16( + &encoded[offset + 4..offset + 4 + count * 2] + .chunks_exact(2) + .map(|v| u16::from_le_bytes(v.try_into().unwrap())) + .collect::>(), + )?; + if let FileDataReference::Internal(guid) = text.parse()? { + let filename = format!( + "{}.onebin", + text.strip_prefix("{") + .unwrap() + .strip_suffix('}') + .unwrap() + ); + let payload = store.file_data(guid)?; + let path = destination.join("synthetic_onefiles").join(&filename); + if path.exists() { + assert_eq!(fs::read(&path)?, payload); + } else { + fs::write(path, payload)?; + } + let reference = format!("{filename}"); + let data: Vec<_> = reference + .encode_utf16() + .flat_map(u16::to_le_bytes) + .collect(); + encoded.splice(offset + 4..offset + 4 + count * 2, data.iter().copied()); + encoded[offset..offset + 4] + .copy_from_slice(&(data.len() as u32 / 2).to_le_bytes()); + assert!(encoded.len() <= 0x1fff); + let header = (raw & !(0x1fff << 10)) | ((encoded.len() as u32) << 10); + encoded[..4].copy_from_slice(&header.to_le_bytes()); + changed += 1; + touched = true; + } + } + body.extend(encoded); + } + if touched { + assert!( + body.len() + 4 <= end - start - 16, + "Native fragment has insufficient reserved space" + ); + body.extend_from_slice(&(0xff_u32 | (4 << 10)).to_le_bytes()); + output[start + 16..end].fill(0); + output[start + 16..start + 16 + body.len()].copy_from_slice(&body); + } + } + } + assert!(changed > 0); + let converted = Store::parse(&output)?; + assert!(converted.checksum_mismatches.is_empty()); + RevisionIndex::parse(&converted)?.validate_current()?; + fs::write(destination.join("synthetic.one"), &output)?; + fs::write( + destination.join("Open Notebook.onetoc2"), + onestore::create_table_of_contents( + "Open Notebook.onetoc2", + &[("synthetic.one", converted.header.file_id)], + )?, + )?; + println!( + "Converted {changed} file-data declarations without changing payload identities or bytes" + ); + Ok(()) +} diff --git a/crates/notebook/examples/recovery_probe.rs b/crates/notebook/examples/recovery_probe.rs new file mode 100644 index 0000000000000000000000000000000000000000..a573a56a213c052f6e00ede4e86e5241fcd4b925 --- /dev/null +++ b/crates/notebook/examples/recovery_probe.rs @@ -0,0 +1,160 @@ +mod support { + pub mod view; +} +use support::view::view; + +use notebook::{EditStatus, Remote, Replica}; +use onestore::{CommitError, CommitIo, PreparedEdit}; +use serde_json::json; +use std::{ + env, + fs::{self, File, OpenOptions}, + io::{self, Write}, + os::unix::fs::FileExt, + path::Path, +}; + +fn phase(name: &str) { + println!("{}", json!({"event":"phase", "name":name})); + io::stdout().flush().unwrap(); + if env::var("ONESTORE_RECOVERY_PAUSE").ok().as_deref() == Some(name) { + let mut line = String::new(); + assert!( + io::stdin().read_line(&mut line).unwrap() > 0, + "Controller closed a paused operation" + ); + } +} + +struct Disk { + file: File, + writes: usize, + flushes: usize, +} + +impl CommitIo for Disk { + fn read_at(&mut self, offset: u64, output: &mut [u8]) -> io::Result { + self.file.read_at(output, offset) + } + fn write_at(&mut self, offset: u64, data: &[u8]) -> io::Result { + self.writes += 1; + println!( + "{}", + json!({"event":"write", "number":self.writes, "offset":offset, "bytes":data.len()}) + ); + phase(&format!("write-{}-before", self.writes)); + let result = self.file.write_at(data, offset); + phase(&format!("write-{}-after", self.writes)); + result + } + fn flush(&mut self) -> io::Result<()> { + self.flushes += 1; + phase(&format!("flush-{}-before", self.flushes)); + let result = self.file.sync_all(); + phase(&format!("flush-{}-after", self.flushes)); + result + } +} + +impl Remote for Disk { + fn read(&mut self) -> io::Result> { + phase("read-before"); + let result = onestore::read_snapshot( + |offset, output| self.file.read_at(output, offset), + 256 * 1024 * 1024, + ) + .and_then(|snapshot| snapshot.ok_or_else(|| io::ErrorKind::WouldBlock.into())); + phase("read-after"); + result + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + phase("publish-before"); + let result = edit.commit(self); + phase("publish-after"); + result + } + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + phase("confirm-before"); + let result = onestore::confirm_snapshot(self, snapshot); + phase("confirm-after"); + result + } +} + +fn report(cache: &Replica, root: &Path) -> Result<(), Box> { + let local = view(&cache.snapshot()?)?; + let remote = view(&fs::read(root.join("remote.one"))?)?; + let (status, revision) = match cache.status(1)? { + Some(EditStatus::Pending) => ("pending", None), + Some(EditStatus::AwaitingConfirmation { revision }) => { + ("uncertain", Some(revision.to_string())) + } + Some(EditStatus::Published { revision }) => ("published", Some(revision.to_string())), + Some(EditStatus::Conflict(_)) => ("conflict", None), + None => ("missing", None), + }; + println!( + "{}", + json!({"event":"state", "status":status, "revision":revision, + "local_text":local.text, "remote_text":remote.text, "remote_revision":remote.revision.to_string(), + "pending":cache.pending()?.iter().map(|pending|match &pending.operation { + notebook::Operation::Text(edit) => json!({"id":pending.id,"before":edit.before,"replacement":edit.replacement,"range":[edit.range.start,edit.range.end]}), + operation => json!({"id":pending.id,"operation":operation}), + }).collect::>() }) + ); + Ok(()) +} + +fn main() -> Result<(), Box> { + let args: Vec<_> = env::args().skip(1).collect(); + let mode = args + .first() + .ok_or("Expected init|inspect|sync DIRECTORY [SOURCE]")?; + let root = Path::new(args.get(1).ok_or("Missing owned directory")?); + if mode == "init" && args.len() == 3 { + let source = fs::read(&args[2])?; + let target = view(&source)?; + fs::create_dir(root)?; + let mut remote = OpenOptions::new() + .write(true) + .create_new(true) + .open(root.join("remote.one"))?; + remote.write_all(&source)?; + remote.sync_all()?; + drop(remote); + let cache = Replica::create(root.join("cache.sqlite"), &source)?; + let at = u32::try_from(target.text.encode_utf16().count())?; + assert_eq!( + cache.edit_text( + &source, + target.space, + target.object, + at..at, + " [offline-recovery]" + )?, + Some(1) + ); + phase("local-after"); + report(&cache, root)?; + } else if args.len() == 2 && ["inspect", "sync"].contains(&mode.as_str()) { + let cache = Replica::open(root.join("cache.sqlite"))?; + if mode == "sync" { + let mut disk = Disk { + file: OpenOptions::new() + .read(true) + .write(true) + .open(root.join("remote.one"))?, + writes: 0, + flushes: 0, + }; + phase("sync-before"); + let result = cache.sync_once(&mut disk); + phase("sync-after"); + result?; + } + report(&cache, root)?; + } else { + return Err("Expected init|inspect|sync DIRECTORY [SOURCE]".into()); + } + Ok(()) +} diff --git a/crates/notebook/examples/smb_concurrent_client.rs b/crates/notebook/examples/smb_concurrent_client.rs new file mode 100644 index 0000000000000000000000000000000000000000..9c55d513e961b1c553f0b0957032c8fb94cd6d2e --- /dev/null +++ b/crates/notebook/examples/smb_concurrent_client.rs @@ -0,0 +1,22 @@ +#[path = "../../onestore/examples/support/concurrent.rs"] +mod concurrent; + +use notebook::smb::{Client, Credentials}; +use std::{env, time::Duration}; + +fn main() -> Result<(), Box> { + let client = Client::connect( + &env::var("ONESTORE_SMB_LAB")?, + &env::var("ONESTORE_SMB_SHARE")?, + Credentials::default(), + Duration::from_secs(10), + )?; + let args: Vec<_> = env::args().skip(1).collect(); + concurrent::run( + &args, + |path| client.read(path, 256 * 1024 * 1024), + |path, source, space, object, range, replacement| { + client.commit_text(path, source, space, object, range, replacement) + }, + ) +} diff --git a/crates/notebook/examples/smb_offline_client.rs b/crates/notebook/examples/smb_offline_client.rs new file mode 100644 index 0000000000000000000000000000000000000000..aa68d9082ca051bc17989c20f05b555e9678e48b --- /dev/null +++ b/crates/notebook/examples/smb_offline_client.rs @@ -0,0 +1,863 @@ +#[path = "../../onestore/examples/support/concurrent.rs"] +mod concurrent; + +use notebook::smb::{Client, Credentials}; +use notebook::{EditStatus, Error, Remote, Replica, SmbRemote}; +use onestore::{ + CommitError, CommitState, ExGuid, Insertion, ParagraphJoin, ParagraphSplit, PreparedEdit, + RevisionIndex, Store, TextAttribute, TreeEdit, document::Document, +}; +use serde_json::json; +use std::{ + env, + io::{self, Write}, + path::{Path, PathBuf}, + sync::Arc, + thread, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +mod support { + pub mod view; +} +use concurrent::{DocumentView, document_view}; +use support::view::view; + +impl DocumentView { + fn changes(&self, before: &Self) -> Self { + let difference = + |old: &serde_json::Map, + new: &serde_json::Map| { + old.keys() + .chain(new.keys()) + .filter(|id| old.get(*id) != new.get(*id)) + .map(|id| (id.clone(), new.get(id).cloned().unwrap_or_default())) + .collect() + }; + Self { + texts: difference(&before.texts, &self.texts), + graph: difference(&before.graph, &self.graph), + } + } +} + +const DOCUMENT_OPERATIONS: [&str; 10] = [ + "insert", + "format", + "text", + "split", + "right_text", + "nest", + "unnest", + "join", + "tail_split", + "delete", +]; + +fn now() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_micros() +} + +#[derive(Clone)] +enum Pause { + Outage(PathBuf), + FormatReply(PathBuf), +} + +struct Traced { + remote: R, + before: Option<(String, Option)>, + pause: Option, + documents: bool, +} + +impl Remote for Traced { + fn read(&mut self) -> io::Result> { + let started = now(); + let bytes = self.remote.read()?; + let observed = view(&bytes).map_err(|error| io::Error::other(error.to_string()))?; + let documents = if self.documents { + Some(document_view(&bytes).map_err(io::Error::other)?) + } else { + None + }; + println!( + "{}", + json!({"event":"read", "started_us":started, "finished_us":now(), "text":observed.text, "documents":documents.as_ref().map(|view| &view.texts), "document_graph":documents.as_ref().map(|view| &view.graph)}) + ); + self.before = Some((observed.text, documents)); + Ok(bytes) + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + let after = view(edit.as_bytes()).map_err(|error| CommitError { + state: CommitState::NotCommitted, + error: io::Error::other(error.to_string()), + })?; + let documents = if self.documents { + Some(document_view(edit.as_bytes()).map_err(|error| CommitError { + state: CommitState::NotCommitted, + error: io::Error::other(error), + })?) + } else { + None + }; + let changes = if let Some(after) = &documents { + let before = self + .before + .as_ref() + .and_then(|(_, documents)| documents.as_ref()) + .ok_or_else(|| CommitError { + state: CommitState::NotCommitted, + error: io::Error::other("Document publication has no observed source"), + })?; + Some(after.changes(before)) + } else { + None + }; + let pause = match &self.pause { + Some(Pause::Outage(marker)) => Some(( + marker, + marker.parent().unwrap().join("offline-outage-resumed"), + "outage", + )), + Some(Pause::FormatReply(marker)) + if changes.as_ref().is_some_and(|changes| { + changes.texts.len() == 1 + && self + .before + .as_ref() + .and_then(|(_, before)| before.as_ref()) + .is_some_and(|before| { + changes.texts.keys().all(|id| before.texts.contains_key(id)) + }) + }) => + { + Some((marker, marker.with_extension("resume"), "format")) + } + _ => None, + }; + if let Some((marker, resumed, kind)) = pause + && !resumed.exists() + { + std::fs::write(marker, after.revision.to_string()).map_err(|error| CommitError { + state: CommitState::NotCommitted, + error, + })?; + println!( + "{}", + json!({"event":"publication_paused", "revision":after.revision.to_string(), "kind":kind, "at_us":now()}) + ); + let deadline = Instant::now() + Duration::from_secs(120); + while !resumed.exists() { + if Instant::now() >= deadline { + return Err(CommitError { + state: CommitState::NotCommitted, + error: io::Error::new( + io::ErrorKind::TimedOut, + "Offline publication barrier timed out", + ), + }); + } + thread::sleep(Duration::from_millis(10)); + } + } + let started = now(); + let result = self.remote.publish(edit); + let finished = now(); + println!( + "{}", + json!({"event":"remote_attempt", "started_us":started, "finished_us":finished, + "revision":after.revision.to_string(), "space":after.space.to_string(), "object":after.object.to_string(), "before":self.before.as_ref().map(|(text,_)|text), "after":after.text, + "state":format!("{:?}", result.as_ref().map_or_else(|error| error.state, |_| CommitState::Committed)), + "documents":documents.as_ref().map(|view| &view.texts), "document_graph":documents.as_ref().map(|view| &view.graph), + "document_changes":changes.as_ref().map(|view| &view.texts), "document_graph_changes":changes.as_ref().map(|view| &view.graph)}) + ); + if result + .as_ref() + .is_err_and(|error| error.state == CommitState::Unknown) + && let Some(Pause::FormatReply(marker)) = &self.pause + && marker.with_extension("isolate").exists() + { + std::fs::write(marker.with_extension("isolate"), serde_json::to_vec(&json!({"space":after.space.to_string(), "revision":after.revision.to_string(), "after_us":finished})).unwrap()) + .map_err(|error| CommitError { state:CommitState::Unknown, error })?; + println!( + "{}", + json!({"event":"confirmation_paused", "revision":after.revision.to_string(), "at_us":now()}) + ); + let deadline = Instant::now() + Duration::from_secs(120); + while !marker.with_extension("confirmation-resume").exists() { + if Instant::now() >= deadline { + return Err(CommitError { + state: CommitState::Unknown, + error: io::Error::new( + io::ErrorKind::TimedOut, + "Offline confirmation barrier timed out", + ), + }); + } + thread::sleep(Duration::from_millis(10)); + } + } + result + } + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + let captured = (|| -> Result<_, Box> { + let store = Store::parse(snapshot)?; + let index = RevisionIndex::parse(&store)?; + let revisions = index + .spaces + .iter() + .map(|(id, space)| { + ( + id.to_string(), + space + .revisions + .keys() + .map(ToString::to_string) + .collect::>(), + ) + }) + .collect::>(); + let current = index + .spaces + .iter() + .filter_map(|(id, space)| { + space + .labels + .get(&(ExGuid::default(), 1)) + .map(|revision| (id.to_string(), revision.to_string())) + }) + .collect::>(); + let path = env::var_os("ONESTORE_OFFLINE_CONFIRM_DIR").map(|directory| { + PathBuf::from(directory).join(format!("{}-{}.one", std::process::id(), now())) + }); + if let Some(path) = &path { + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path)? + .write_all(snapshot)?; + } + Ok((revisions, current, path)) + })(); + let (revisions, current, capture) = captured.map_err(|error| CommitError { + state: CommitState::NotCommitted, + error: io::Error::other(error.to_string()), + })?; + let started = now(); + let result = self.remote.confirm(snapshot); + println!( + "{}", + json!({"event":"remote_confirm", "started_us":started, "finished_us":now(), "revisions":revisions, "current_revisions":current, "capture":capture.as_ref().and_then(|path| path.file_name()).map(|name| name.to_string_lossy()), "text":self.before.as_ref().map(|(text,_)|text), + "state":format!("{:?}", result.as_ref().map_or_else(|error| error.state, |_| CommitState::Committed)), "error":result.as_ref().err().map(|error|error.error.to_string())}) + ); + result + } +} + +fn tokens(text: &str) -> Option> { + let mut remaining = text.strip_prefix("Concurrent edits:")?; + let mut tokens = Vec::new(); + while !remaining.is_empty() { + let body = remaining.strip_prefix(" [w")?; + let (token, tail) = body.split_once(']')?; + let (actor, operation) = token.split_once(':')?; + if actor.is_empty() + || operation.is_empty() + || !actor + .bytes() + .chain(operation.bytes()) + .all(|b| b.is_ascii_digit()) + || tokens.contains(&token) + { + return None; + } + tokens.push(token); + remaining = tail; + } + Some(tokens) +} + +// This owned workload explicitly resolves append conflicts after all retained tokens. +fn append_position(before: &str, current: &str, token: &str) -> Option { + let original = tokens(before)?; + let present = tokens(current)?; + let mut retained = present.iter(); + for token in original { + retained.find(|&&candidate| candidate == token)?; + } + if current.contains(token) { + return None; + } + u32::try_from(current.encode_utf16().count()).ok() +} + +fn queue_document( + cache: &Replica, + actor: &str, + operation: usize, + parent: Option, + deadline: Instant, +) -> Result<(ExGuid, [u64; DOCUMENT_OPERATIONS.len()]), Box> { + let source = cache.snapshot()?; + let store = Store::parse(&source)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + let (space, page) = document.pages()?[0]; + let text = format!("Document {actor}:{operation} 🦀"); + let insertion = if operation.is_multiple_of(2) { + let column: u32 = actor + .strip_prefix('w') + .ok_or("Missing writer number")? + .parse()?; + Insertion::outline( + page, + 144.0 + column as f32 * 240.0, + 144.0 + operation as f32 * 72.0, + &text, + "Offline document writer", + )? + } else { + Insertion::paragraph( + parent.ok_or("Missing prior outline")?, + None, + &text, + "Offline document writer", + )? + } + .with_formatting( + 0..u32::try_from(text.encode_utf16().count())?, + &[ + TextAttribute::FontSize(13.5), + TextAttribute::Color(Some([0x44, 0x55, 0x66])), + ], + )?; + let parent = if operation.is_multiple_of(2) { + insertion.object() + } else { + parent.unwrap() + }; + let end = u32::try_from(text.encode_utf16().count())?; + let split = ParagraphSplit::new(insertion.text_object(), end - 2, "Offline document writer")?; + let tail_split = + ParagraphSplit::new(insertion.text_object(), end + 1, "Offline document writer")?; + let join = ParagraphJoin::new( + insertion.text_object(), + split.text_object(), + "Offline document writer", + )?; + let attributes = [ + TextAttribute::Bold(true), + TextAttribute::FontSize(18.0 + (operation % 9) as f32), + TextAttribute::Color(Some([0x12, 0x34, 0x56])), + ]; + let mut ids = [0; DOCUMENT_OPERATIONS.len()]; + for (step, id) in ids.iter_mut().enumerate() { + let kind = DOCUMENT_OPERATIONS[step]; + let tree = match kind { + "nest" => { + let source = cache.snapshot()?; + let store = Store::parse(&source)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + let section = &document.spaces[&space]; + let view = §ion.revisions[§ion.contexts[&ExGuid::default()]]; + let paragraph = view + .nodes + .iter() + .find(|(_, node)| node.content == [insertion.text_object()]) + .map(|(id, _)| *id) + .ok_or("Missing inserted paragraph")?; + Some(TreeEdit::move_to( + split.object(), + paragraph, + None, + "Offline document writer", + )?) + } + "unnest" => Some(TreeEdit::move_to( + split.object(), + parent, + None, + "Offline document writer", + )?), + "delete" => Some(TreeEdit::delete( + tail_split.object(), + "Offline document writer", + )?), + _ => None, + }; + let range = match kind { + "text" => end - 3..end, + "split" => end - 2..end - 2, + "tail_split" => end + 1..end + 1, + "right_text" => 0..2, + _ => 1..end - 2, + }; + let target = if kind == "right_text" { + split.text_object() + } else if kind == "delete" { + tail_split.text_object() + } else { + insertion.text_object() + }; + let replacement = match kind { + "text" => Some(" e\u{301}🐈"), + "right_text" => Some("B🦋"), + _ => None, + }; + loop { + if Instant::now() >= deadline { + return Err("Document queue timed out; cache retained".into()); + } + let source = cache.snapshot()?; + let started = now(); + let result = match kind { + "insert" => cache.insert(&source, space, &insertion), + "format" => cache.format(&source, space, target, range.clone(), &attributes), + "text" | "right_text" => { + cache.edit_text(&source, space, target, range.clone(), replacement.unwrap()) + } + "split" => cache.split(&source, space, &split), + "tail_split" => cache.split(&source, space, &tail_split), + "join" => cache.join(&source, space, &join), + "nest" | "unnest" | "delete" => cache.tree(&source, space, tree.as_ref().unwrap()), + _ => unreachable!(), + }; + match result { + Ok(Some(acknowledged)) => { + *id = acknowledged; + println!( + "{}", + json!({"event":"local_document_commit","id":acknowledged,"operation":operation,"kind":kind, + "space":space.to_string(),"object":target.to_string(),"document":insertion.text_object().to_string(), + "text":text,"insertion":if kind=="insert" {Some(&insertion)} else {None}, + "split":match kind { "split" => Some(&split), "tail_split" => Some(&tail_split), _ => None }, + "tree":tree, + "joined":if kind=="join" {Some(join.texts().map(|id| id.to_string()))} else {None}, + "range":[range.start,range.end],"attributes":attributes,"replacement":replacement, + "started_us":started,"finished_us":now()}) + ); + break; + } + Err(Error::Io(error)) if error.kind() == io::ErrorKind::ResourceBusy => {} + other => return Err(format!("Unexpected document queue result: {other:?}").into()), + } + } + } + Ok((parent, ids)) +} + +fn main() -> Result<(), Box> { + let args: Vec<_> = env::args().skip(1).collect(); + if args.len() != 7 + || !["read", "write"].contains(&args[0].as_str()) + || !args[2].bytes().all(|b| b.is_ascii_alphanumeric()) + { + return Err("Expected read|write FILE ACTOR OPERATIONS START_FILE STOP_FILE SEED".into()); + } + let documents = env::var_os("ONESTORE_OFFLINE_DOCUMENTS").is_some(); + let address = env::var("ONESTORE_SMB_LAB")?; + let share = env::var("ONESTORE_SMB_SHARE")?; + let initial = Client::connect( + &address, + &share, + Credentials::default(), + Duration::from_secs(5), + )?; + if args[0] == "read" { + return concurrent::run( + &args, + |path| initial.read(path, 256 * 1024 * 1024), + |_, _, _, _, _, _| unreachable!(), + ); + } + let timeout: u64 = env::var("ONESTORE_CLIENT_TIMEOUT_MS") + .unwrap_or_else(|_| "600000".into()) + .parse()?; + let deadline = Instant::now() + .checked_add(Duration::from_millis(timeout)) + .ok_or("Invalid timeout")?; + let operations: usize = args[3].parse()?; + let mut seed: u64 = args[6].parse()?; + if operations == 0 { + return Err("Expected positive operations".into()); + } + let source = initial.read(&args[1], 256 * 1024 * 1024)?; + drop(initial); + let cache_path = Path::new(&args[4]) + .parent() + .ok_or("Missing workload directory")? + .join(format!("{}.sqlite", args[2])); + let cache = Arc::new(Replica::create(&cache_path, &source)?); + println!( + "{}", + json!({"event":"ready", "pid":std::process::id(), "actor":args[2], "offline":true, "document_operations":documents,"document_graph":documents,"document_kinds":if documents {DOCUMENT_OPERATIONS.as_slice()} else {&[]}}) + ); + while !Path::new(&args[4]).exists() { + if Instant::now() >= deadline { + return Err("Start barrier timed out".into()); + } + thread::sleep(Duration::from_millis(5)); + } + let path = args[1].clone(); + let outage = env::var_os("ONESTORE_OFFLINE_OUTAGE_DIR").map(PathBuf::from); + let pause = outage + .as_ref() + .map(|directory| Pause::Outage(directory.join(format!("offline-paused-{}", args[2])))) + .or_else(|| { + env::var_os("ONESTORE_OFFLINE_FORMAT_REPLY_DIR").map(|directory| { + Pause::FormatReply( + PathBuf::from(directory).join(format!("offline-paused-{}", args[2])), + ) + }) + }); + let (fatal_tx, fatal_rx) = std::sync::mpsc::channel(); + let worker = cache.start_sync(Duration::from_millis(50), move || { + let client = Client::connect(&address, &share, Credentials::default(), Duration::from_secs(5))?; + println!("{}", json!({"event":"transport_connected", "at_us":now()})); + Ok(Traced { remote: SmbRemote::new(client, &path, 256 * 1024 * 1024), before:None, pause:pause.clone(), documents }) + }, move |result| { + if let Err(error) = result { + println!("{}", json!({"event":"sync_error", "error":error.to_string(), "at_us":now()})); + if !matches!(error, Error::RemoteIo(_) | Error::Remote(_)) && !matches!(error, Error::Io(error) if error.kind() == io::ErrorKind::WouldBlock) { + let _ = fatal_tx.send(error.to_string()); + } + } + })?; + let mut ids = Vec::new(); + let mut document_ids = std::collections::BTreeSet::new(); + let mut document_parent = None; + let result = (|| -> Result<(), Box> { + let mut generated = 0; + let mut received = 0; + loop { + match fatal_rx.try_recv() { + Ok(error) => return Err(error.into()), + Err(std::sync::mpsc::TryRecvError::Disconnected) => { + return Err("Worker exited before completion".into()); + } + Err(std::sync::mpsc::TryRecvError::Empty) => {} + } + while received < ids.len() { + match cache.status(ids[received])? { + Some(EditStatus::Published { revision }) => { + println!( + "{}", + json!({"event":if document_ids.contains(&ids[received]) {"document_receipt"} else {"remote_receipt"}, "id":ids[received], "revision":revision.to_string(), "at_us":now()}) + ); + received += 1; + } + Some(_) => break, + None => return Err("Local intent disappeared".into()), + } + } + if Instant::now() >= deadline { + return Err("Offline workload timed out; cache retained".into()); + } + let pending = cache.pending()?; + let capacity = if outage + .as_ref() + .is_some_and(|directory| !directory.join("offline-outage-down").exists()) + { + 1 + } else if documents && outage.is_some() { + 8 * (1 + DOCUMENT_OPERATIONS.len()) + } else { + 8 + }; + if generated < operations && pending.len() < capacity { + let source = cache.snapshot()?; + let target = view(&source)?; + let at = u32::try_from(target.text.encode_utf16().count())?; + let token = format!(" [{}:{}]", args[2], generated); + let started = now(); + match cache.edit_text(&source, target.space, target.object, at..at, &token) { + Ok(Some(id)) => { + println!( + "{}", + json!({"event":"local_commit", "id":id, "operation":generated, "space":target.space.to_string(), "object":target.object.to_string(), "before":target.text, "token":token, "started_us":started, "finished_us":now()}) + ); + ids.push(id); + if documents { + let (parent, added) = queue_document( + &cache, + &args[2], + generated, + document_parent, + deadline, + )?; + document_parent = Some(parent); + document_ids.extend(added); + ids.extend(added); + } + generated += 1; + } + Err(Error::Io(error)) if error.kind() == io::ErrorKind::ResourceBusy => {} + other => return Err(format!("Unexpected local result: {other:?}").into()), + } + } + if let Some(intent) = pending.first() + && matches!(cache.status(intent.id)?, Some(EditStatus::Conflict(_))) + { + let local = cache.snapshot()?; + let remote = cache.remote_snapshot()?; + let current = view(&remote)?; + let notebook::Operation::Text(edit) = &intent.operation else { + return Err(format!( + "Document intent {} requires review: {:?}", + intent.id, + cache.status(intent.id)? + ) + .into()); + }; + let at = append_position(&edit.before, ¤t.text, &edit.replacement) + .ok_or("Append model disagrees with retained history")?; + match cache.rebase_conflict(intent.id, &local, &remote, at..at) { + Ok(()) => println!( + "{}", + json!({"event":"reviewed_append", "id":intent.id, "before":edit.before, "remote":current.text, "token":edit.replacement, "at_us":now()}) + ), + Err(Error::Io(error)) + if [ + io::ErrorKind::ResourceBusy, + io::ErrorKind::WouldBlock, + io::ErrorKind::InvalidInput, + ] + .contains(&error.kind()) => {} + Err(error) => return Err(error.into()), + } + } + if generated == operations && received == ids.len() { + if !cache.pending()?.is_empty() { + return Err("Acknowledged queue did not drain".into()); + } + break; + } + seed = seed + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + thread::sleep(Duration::from_millis(1 + (seed >> 32) % 7)); + } + Ok(()) + })(); + let stopped = worker.stop(); + result?; + stopped?; + drop(cache); + let reopened = Replica::open(&cache_path)?; + if !reopened.pending()?.is_empty() { + return Err("Pending edits reappeared after reopen".into()); + } + for id in ids { + let Some(EditStatus::Published { revision }) = reopened.status(id)? else { + return Err("Receipt did not survive reopen".into()); + }; + println!( + "{}", + json!({"event":if document_ids.contains(&id) {"reopened_document_receipt"} else {"reopened_receipt"}, "id":id, "revision":revision.to_string()}) + ); + } + println!( + "{}", + json!({"event":"done", "operations":operations, "at_us":now()}) + ); + Ok(()) +} + +#[test] +fn append_review_requires_a_unique_ordered_history_and_an_absent_new_token() { + assert_eq!( + append_position( + "Concurrent edits: [w0:0]", + "Concurrent edits: [w1:0] [w0:0]", + " [w0:1]" + ), + Some(31) + ); + for current in [ + "Concurrent edits:", + "Concurrent edits: [w0:0] [w0:0]", + "Concurrent edits: [w0:1] [w0:0]", + "Concurrent edits: changed [w0:0]", + ] { + assert_eq!( + append_position("Concurrent edits: [w0:0]", current, " [w0:1]"), + None + ); + } + assert_eq!( + append_position( + "Concurrent edits: [w0:0] [w1:0]", + "Concurrent edits: [w1:0] [w0:0]", + " [w0:1]" + ), + None + ); +} + +#[cfg(test)] +#[path = "../../onestore/tests/support/disk.rs"] +mod disk; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn document_workload_retains_dependencies_and_receipts_across_reopen() { + struct Server { + disk: disk::Disk, + lost_reply: bool, + } + impl Remote for Server { + fn read(&mut self) -> io::Result> { + Ok(self.disk.visible.clone()) + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + edit.commit(&mut self.disk)?; + if std::mem::take(&mut self.lost_reply) { + Err(CommitError { + state: CommitState::Unknown, + error: io::Error::from(io::ErrorKind::ConnectionAborted), + }) + } else { + Ok(()) + } + } + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + onestore::confirm_snapshot(&mut self.disk, snapshot) + } + } + let source = + onestore::create_section("workload.one", "Concurrent edits:", "Author").unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("cache.sqlite"); + let mut cache = Replica::create(&path, &source).unwrap(); + let mut remote = Traced { + remote: Server { + disk: disk::Disk { + visible: source.clone(), + durable: source.clone(), + operation: 0, + fail_at: None, + write_limit: 97, + random: 1, + }, + lost_reply: false, + }, + before: None, + pause: None, + documents: true, + }; + println!( + "{}", + json!({"event":"ready", "actor":"w0", "offline":true, + "document_operations":true, "document_graph":true, "document_kinds":DOCUMENT_OPERATIONS}) + ); + let deadline = Instant::now() + Duration::from_secs(30); + let mut parent = None; + let mut ids = Vec::new(); + for operation in 0..2 { + let (outline, added) = + queue_document(&cache, "w0", operation, parent, deadline).unwrap(); + parent = Some(outline); + ids.extend(added); + drop(cache); + cache = Replica::open(&path).unwrap(); + } + let local = cache.snapshot().unwrap(); + assert_eq!( + cache.pending().unwrap().len(), + 2 * DOCUMENT_OPERATIONS.len() + ); + for (step, id) in ids.iter().enumerate() { + remote.remote.lost_reply = matches!( + DOCUMENT_OPERATIONS[step % DOCUMENT_OPERATIONS.len()], + "split" | "nest" | "unnest" | "join" | "tail_split" | "delete" + ); + let result = cache.sync_once(&mut remote); + let state = if result.is_err() { + assert!(matches!( + result, + Err(Error::Remote(CommitError { + state: CommitState::Unknown, + .. + })) + )); + drop(cache); + cache = Replica::open(&path).unwrap(); + cache.sync_once(&mut remote).unwrap().unwrap() + } else { + result.unwrap().unwrap() + }; + assert_eq!(state.0, *id); + let EditStatus::Published { revision } = state.1 else { + panic!("{state:?}") + }; + println!( + "{}", + json!({"event":"document_receipt", "id":id, "revision":revision.to_string(), "at_us":now()}) + ); + drop(cache); + cache = Replica::open(&path).unwrap(); + assert_eq!( + cache.status(*id).unwrap(), + Some(EditStatus::Published { revision }) + ); + if step + 1 < ids.len() { + assert_eq!(cache.snapshot().unwrap(), local); + } + } + assert!(cache.pending().unwrap().is_empty()); + for id in ids { + let Some(EditStatus::Published { revision }) = cache.status(id).unwrap() else { + panic!() + }; + println!( + "{}", + json!({"event":"reopened_document_receipt", "id":id, "revision":revision.to_string()}) + ); + } + remote.read().unwrap(); + println!("{}", json!({"event":"done"})); + } + + #[test] + fn publication_differences_retain_removed_text_and_graph_identities() { + let before = DocumentView { + texts: serde_json::from_value(json!({"left":{"text":"ab"}, "right":{"text":"cd"}, "untouched":{"text":"ef"}})).unwrap(), + graph: serde_json::from_value(json!({"outline":{"children":["a","b"]}, "a":{"content":["left"]}, "b":{"content":["right"]}})).unwrap(), + }; + let after = DocumentView { + texts: serde_json::from_value( + json!({"left":{"text":"abcd"}, "untouched":{"text":"ef"}}), + ) + .unwrap(), + graph: serde_json::from_value( + json!({"outline":{"children":["a"]}, "a":{"content":["left"]}}), + ) + .unwrap(), + }; + let changes = after.changes(&before); + assert_eq!( + changes.texts, + serde_json::from_value::>( + json!({"left":{"text":"abcd"}, "right":null}) + ) + .unwrap() + ); + assert_eq!( + changes.graph, + serde_json::from_value::>( + json!({"outline":{"children":["a"]}, "b":null}) + ) + .unwrap() + ); + assert_eq!(before.changes(&after).texts["right"], before.texts["right"]); + assert_eq!(before.changes(&before), DocumentView::default()); + } +} diff --git a/crates/notebook/examples/smb_reconnect_client.rs b/crates/notebook/examples/smb_reconnect_client.rs new file mode 100644 index 0000000000000000000000000000000000000000..d72a42c9c0a88db608811c8e5b08601222c2f8ee --- /dev/null +++ b/crates/notebook/examples/smb_reconnect_client.rs @@ -0,0 +1,267 @@ +#[path = "../../onestore/examples/support/concurrent.rs"] +mod concurrent; + +use notebook::smb::{Client, Credentials}; +use onestore::{ + CommitError, CommitState, ExGuid, RevisionIndex, Store, + document::{Document, Kind}, +}; +use serde_json::json; +use std::{ + cell::RefCell, + env, io, thread, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +fn paragraph( + bytes: &[u8], + space: ExGuid, + object: ExGuid, +) -> Result> { + let store = Store::parse(bytes)?; + let index = RevisionIndex::parse(&store)?; + index.validate_current()?; + let document = Document::parse(&index)?; + let space = &document.spaces[&space]; + let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; + let Kind::RichText { text, .. } = &revision.nodes[&object].kind else { + return Err("The append target is no longer text.".into()); + }; + Ok(text.clone()) +} + +// Only the owned append workload guarantees unique tokens that no writer removes. +fn retained(before: &str, token: &str, current: &str, state: CommitState) -> io::Result { + let count = current.matches(token).count(); + if count == 0 && state != CommitState::Committed && current.starts_with(before) { + return Ok(false); + } + if count == 1 + && state != CommitState::NotCommitted + && current.starts_with(&format!("{before}{token}")) + { + return Ok(true); + } + Err(io::Error::other( + "The append history contradicts the commit outcome.", + )) +} + +fn main() -> Result<(), Box> { + let args: Vec<_> = env::args().skip(1).collect(); + if args + .first() + .is_none_or(|mode| !["read", "write"].contains(&mode.as_str())) + { + return Err("Reconnect testing requires the append-only workload.".into()); + } + let address = env::var("ONESTORE_SMB_LAB")?; + let share = env::var("ONESTORE_SMB_SHARE")?; + let client = RefCell::new(Some(Client::connect( + &address, + &share, + Credentials::default(), + Duration::from_secs(5), + )?)); + let read = |path: &str| { + let mut session = client.borrow_mut(); + if session.is_none() { + match Client::connect( + &address, + &share, + Credentials::default(), + Duration::from_secs(5), + ) { + Ok(fresh) => { + *session = Some(fresh); + println!( + "{}", + json!({"event":"transport_connected", "at_us":SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_micros()}) + ); + } + Err(error) => { + println!( + "{}", + json!({"event":"transport_connect_error","error":error.to_string()}) + ); + return Err(io::ErrorKind::WouldBlock.into()); + } + } + } + let started = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_micros(); + match session.as_ref().unwrap().read(path, 256 * 1024 * 1024) { + Ok(bytes) => { + if args.get(2).is_some_and(|actor| actor == "r0") + && let Some(folder) = env::var_os("ONESTORE_OFFLINE_FORMAT_REPLY_DIR") + { + let folder = std::path::PathBuf::from(folder); + let captured = folder.join("offline-retired.one"); + if !captured.exists() + && let Ok(marker) = std::fs::read(folder.join("offline-paused-w0.isolate")) + && let Ok(watched) = serde_json::from_slice::(&marker) + && watched["after_us"] + .as_u64() + .is_some_and(|after| started > u128::from(after)) + { + let sid: ExGuid = watched["space"] + .as_str() + .ok_or_else(|| io::Error::other("Missing watched space"))? + .parse() + .map_err(io::Error::other)?; + let rid: ExGuid = watched["revision"] + .as_str() + .ok_or_else(|| io::Error::other("Missing watched revision"))? + .parse() + .map_err(io::Error::other)?; + let store = Store::parse(&bytes).map_err(io::Error::other)?; + let index = RevisionIndex::parse(&store).map_err(io::Error::other)?; + if index + .spaces + .get(&sid) + .is_some_and(|space| !space.revisions.contains_key(&rid)) + { + index.validate_current().map_err(io::Error::other)?; + std::fs::write(captured.with_extension("tmp"), &bytes)?; + std::fs::rename(captured.with_extension("tmp"), captured)?; + println!( + "{}", + json!({"event":"revision_retired", "space":sid.to_string(), "revision":rid.to_string(), "started_us":started, "finished_us":SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_micros()}) + ); + } + } + } + Ok(bytes) + } + Err(error) + if matches!( + error.kind(), + io::ErrorKind::Other | io::ErrorKind::TimedOut | io::ErrorKind::NotConnected + ) => + { + println!( + "{}", + json!({"event":"transport_read_error","error":error.to_string()}) + ); + *session = None; + Err(io::ErrorKind::WouldBlock.into()) + } + result => result, + } + }; + concurrent::run( + &args, + read, + |path, source, space, object, range, replacement| { + let outcome = client.borrow().as_ref().unwrap().commit_text( + path, + source, + space, + object, + range, + replacement, + ); + let Err(error) = outcome else { + return Ok(()); + }; + if error.state == CommitState::NotCommitted + && matches!( + error.error.kind(), + io::ErrorKind::WouldBlock + | io::ErrorKind::ResourceBusy + | io::ErrorKind::PermissionDenied + | io::ErrorKind::NotFound + ) + { + return Err(error); + } + println!( + "{}", + json!({"event":"transport_commit_error","state":format!("{:?}",error.state),"token":replacement,"error":error.error.to_string()}) + ); + *client.borrow_mut() = None; + let deadline = Instant::now() + Duration::from_secs(60); + loop { + if Instant::now() >= deadline { + return Err(error); + } + let current = match read(path) { + Ok(bytes) => bytes, + Err(retry) if retry.kind() == io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(100)); + continue; + } + Err(_) => return Err(error), + }; + let published = paragraph(source, space, object) + .and_then(|before| { + Ok(retained( + &before, + replacement, + ¶graph(¤t, space, object)?, + error.state, + )?) + }) + .map_err(|failure| CommitError { + state: error.state, + error: io::Error::other(failure.to_string()), + })?; + if published { + let confirmation = client.borrow().as_ref().unwrap().commit_text( + path, + ¤t, + space, + object, + 0..0, + "", + ); + if let Err(failure) = confirmation + && failure.state != CommitState::Committed + { + println!( + "{}", + json!({"event":"transport_confirmation_error","state":format!("{:?}", failure.state),"error":failure.error.to_string()}) + ); + *client.borrow_mut() = None; + thread::sleep(Duration::from_millis(100)); + continue; + } + println!( + "{}", + json!({"event":"transport_reconciled","token":replacement,"published":true,"flush_confirmed":true}) + ); + return Ok(()); + } + println!( + "{}", + json!({"event":"transport_reconciled","token":replacement,"published":false}) + ); + return Err(CommitError { + state: CommitState::NotCommitted, + error: io::ErrorKind::ResourceBusy.into(), + }); + } + }, + ) +} + +#[test] +fn uncertain_append_requires_one_retained_token_and_its_predecessor() { + for state in [CommitState::Unknown, CommitState::Committed] { + assert!(retained("before", " [w0:0]", "before [w0:0] [w1:0]", state).unwrap()); + } + for state in [CommitState::Unknown, CommitState::NotCommitted] { + assert!(!retained("before", " [w0:0]", "before [w1:0]", state).unwrap()); + } + for (current, state) in [ + ("before [w0:0] [w0:0]", CommitState::Unknown), + ("changed [w0:0]", CommitState::Unknown), + ("befor", CommitState::Unknown), + ("before", CommitState::Committed), + ("before [w0:0]", CommitState::NotCommitted), + ] { + assert!(retained("before", " [w0:0]", current, state).is_err()); + } +} diff --git a/crates/notebook/examples/support/view.rs b/crates/notebook/examples/support/view.rs new file mode 100644 index 0000000000000000000000000000000000000000..280be0443d90acca9f8db8dd3d7f0dc63d1b5ecc --- /dev/null +++ b/crates/notebook/examples/support/view.rs @@ -0,0 +1,57 @@ +use onestore::{ + ExGuid, RevisionIndex, Store, + document::{Document, Kind}, +}; + +pub struct View { + pub space: ExGuid, + pub object: ExGuid, + pub revision: ExGuid, + pub text: String, +} + +pub fn view(bytes: &[u8]) -> Result> { + let store = Store::parse(bytes)?; + if !store.checksum_mismatches.is_empty() { + return Err("Transaction checksum damage".into()); + } + let index = RevisionIndex::parse(&store)?; + index.validate_current()?; + let document = Document::parse(&index)?; + let mut found = Vec::new(); + for (sid, page) in document.pages()? { + let space = &document.spaces[&sid]; + let rid = space.contexts[&ExGuid::default()]; + let revision = &space.revisions[&rid]; + let mut pending = vec![page]; + let mut seen = std::collections::BTreeSet::new(); + while let Some(oid) = pending.pop() { + if !seen.insert(oid) { + continue; + } + let node = &revision.nodes[&oid]; + pending.extend( + node.children + .iter() + .chain(&node.content) + .chain(&node.structure) + .copied(), + ); + if let Kind::RichText { text, .. } = &node.kind + && text.starts_with("Concurrent edits:") + { + revision.text_runs(oid)?; + found.push(View { + space: sid, + object: oid, + revision: rid, + text: text.clone(), + }); + } + } + } + if found.len() != 1 { + return Err("Expected one concurrent-edit paragraph".into()); + } + Ok(found.pop().unwrap()) +} diff --git a/crates/notebook/src/assets.rs b/crates/notebook/src/assets.rs new file mode 100644 index 0000000000000000000000000000000000000000..fd74b47164ef17cd5230e20d8b0aca938eae9c01 --- /dev/null +++ b/crates/notebook/src/assets.rs @@ -0,0 +1,134 @@ +use super::*; +use onestore::FileDataReference; +use rusqlite::OptionalExtension; +use sha2::{Digest, Sha256}; + +impl Replica { + /// Reads a previously downloaded external payload without network access. + /// Absence is distinct from an empty payload; every returned buffer passes its stored checksum. + pub fn cached_asset(&self, filename: &str, limit: usize) -> Result>> { + let key = key(filename)?; + let connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + cached(&connection, &key, limit) + } + + /// Fetches a declared external payload and durably retains it without changing the edit queue. + /// Different bytes for an already cached identity return `AssetChanged`, preserving the cache. + /// Network I/O does not hold the cache mutex; a stale reference fails before local publication. + pub fn fetch_asset( + &self, + source: &mut impl crate::discover::Source, + section: &str, + filename: &str, + limit: usize, + ) -> Result> { + let key = key(filename)?; + { + let connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + if !referenced(&connection, &key)? { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "The retained document images do not reference this external payload", + ) + .into()); + } + } + let bytes = crate::discover::read_external_asset(source, section, filename, limit)?; + let mut connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + if !referenced(&transaction, &key)? { + return Err(io::Error::new( + io::ErrorKind::ResourceBusy, + "The external payload reference changed during download", + ) + .into()); + } + let previous = match cached(&transaction, &key, bytes.len()) { + Err(Error::Io(error)) if error.kind() == io::ErrorKind::FileTooLarge => { + return Err(Error::AssetChanged); + } + other => other?, + }; + if let Some(previous) = previous { + if previous != bytes { + return Err(Error::AssetChanged); + } + } else { + transaction.execute( + "INSERT INTO assets(name,data,sha256) VALUES (?1,?2,?3)", + params![key, &bytes, &Sha256::digest(&bytes)[..]], + )?; + } + transaction.commit()?; + Ok(bytes) + } +} + +pub(crate) fn key(filename: &str) -> Result { + format!("{filename}").parse::()?; + Ok(filename.to_ascii_lowercase()) +} + +fn referenced(connection: &Connection, key: &str) -> Result { + for column in ["working", "base"] { + let image: Vec = connection.query_row( + &format!("SELECT {column} FROM replica WHERE id=1"), + [], + |row| row.get(0), + )?; + let store = Store::parse(&image)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + if document.spaces.values().flat_map(|space| space.revisions.values()) + .flat_map(|revision| revision.nodes.values()).any(|node| { + matches!(&node.kind, Kind::File { reference: FileDataReference::External(name), .. } if name.eq_ignore_ascii_case(key)) + }) { + return Ok(true); + } + } + Ok(false) +} + +pub(crate) fn cached(connection: &Connection, key: &str, limit: usize) -> Result>> { + let version: u32 = connection.pragma_query_value(None, "user_version", |row| row.get(0))?; + if version < 5 { + return Ok(None); + } + let length: Option = connection + .query_row( + "SELECT length(data) FROM assets WHERE name=?1", + [key], + |row| row.get(0), + ) + .optional()?; + let Some(length) = length else { + return Ok(None); + }; + let length = + usize::try_from(length).map_err(|_| io::Error::from(io::ErrorKind::InvalidData))?; + if length > limit { + return Err(io::Error::from(io::ErrorKind::FileTooLarge).into()); + } + let (bytes, expected): (Vec, Vec) = connection.query_row( + "SELECT data,sha256 FROM assets WHERE name=?1", + [key], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + if bytes.len() != length || Sha256::digest(&bytes)[..] != expected { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Cached external payload checksum mismatch", + ) + .into()); + } + Ok(Some(bytes)) +} diff --git a/crates/notebook/src/bin/onestore-diagnostic.rs b/crates/notebook/src/bin/onestore-diagnostic.rs new file mode 100644 index 0000000000000000000000000000000000000000..cb5fbe8ecd9e720d60238cfec56dd06c59a8ad46 --- /dev/null +++ b/crates/notebook/src/bin/onestore-diagnostic.rs @@ -0,0 +1,126 @@ +use onestore::{ExGuid, Insertion, PreparedEdit, TextAttribute}; +use serde::Deserialize; +use serde_json::{Value, json}; +use std::{ + env, fs, + io::{self, Write}, +}; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Edit { + space: ExGuid, + object: ExGuid, + action: Action, +} + +#[derive(Deserialize)] +#[serde(tag = "type", deny_unknown_fields)] +enum Action { + Text { + start: u32, + end: u32, + replacement: String, + }, + Format { + start: u32, + end: u32, + attributes: Vec, + }, + Paragraph { + before: Option, + text: String, + author: String, + }, + Outline { + x: f32, + y: f32, + text: String, + author: String, + }, +} + +fn run(args: &[std::ffi::OsString]) -> Result> { + if let [mode, root] = args + && mode == "catalog" + { + let catalog = notebook::discover::discover( + &mut notebook::discover::Local::open(root)?, + notebook::discover::Limits { + entries: 100_000, + bytes_per_file: 256 * 1024 * 1024, + depth: 64, + }, + )?; + return Ok(json!({"ok": true, "catalog": catalog})); + } + if args.len() != 3 + || !["snapshot", "check", "commit"] + .iter() + .any(|mode| args[0] == *mode) + { + return Err("Usage: onestore-diagnostic catalog ROOT | snapshot FILE NEW_SNAPSHOT | check SNAPSHOT - | commit FILE SNAPSHOT; edits arrive as JSON on stdin".into()); + } + if args[0] == "snapshot" { + let bytes = onestore::read_file(&args[1])?; + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&args[2])?; + file.write_all(&bytes)?; + file.sync_all()?; + return Ok(json!({"ok": true, "bytes": bytes.len()})); + } + let check = args[0] == "check"; + if check && args[2] != "-" { + return Err("The check command requires '-' as its final argument".into()); + } + let bytes = fs::read(if check { &args[1] } else { &args[2] })?; + let edit: Edit = serde_json::from_reader(io::stdin().lock())?; + let sid = edit.space; + let oid = edit.object; + let prepared = match edit.action { + Action::Text { + start, + end, + replacement, + } => PreparedEdit::text(&bytes, sid, oid, start..end, &replacement)?, + Action::Format { + start, + end, + attributes, + } => PreparedEdit::format(&bytes, sid, oid, start..end, &attributes)?, + Action::Paragraph { + before, + text, + author, + } => PreparedEdit::insert( + &bytes, + sid, + &Insertion::paragraph(oid, before, &text, &author)?, + )?, + Action::Outline { x, y, text, author } => { + PreparedEdit::insert(&bytes, sid, &Insertion::outline(oid, x, y, &text, &author)?)? + } + }; + if check { + return Ok(json!({"ok": true})); + } + Ok(match prepared.commit_file(&args[1]) { + Ok(()) => json!({"ok": true, "state": "Committed"}), + Err(error) => json!({"ok": false, "state": format!("{:?}", error.state), + "kind": format!("{:?}", error.error.kind()), "error": error.error.to_string()}), + }) +} + +fn main() { + let args: Vec<_> = env::args_os().skip(1).collect(); + let result = run(&args).unwrap_or_else(|error| { + let mut result = json!({"ok": false, "kind": "Input", "error": error.to_string()}); + if args.first().is_some_and(|mode| mode == "commit") { + result["state"] = json!("NotCommitted"); + } + result + }); + println!("{result}"); +} diff --git a/crates/notebook/src/discover.rs b/crates/notebook/src/discover.rs new file mode 100644 index 0000000000000000000000000000000000000000..57f57db439cb559fa66b96fdc1a1428b2cbcc387 --- /dev/null +++ b/crates/notebook/src/discover.rs @@ -0,0 +1,412 @@ +//! Read-only notebook discovery over a caller-supplied root. + +use onestore::{ + FileType, RevisionIndex, Store, + document::{Document, Kind}, +}; +use serde::Serialize; +use std::{ + collections::{BTreeMap, BTreeSet}, + io, +}; + +mod source; +pub use source::Local; +#[cfg(feature = "smb")] +pub use source::Smb; + +#[derive(Debug, Serialize)] +pub struct Section { + pub path: String, + /// Header.guidFile, also used by FileIdentityGuid in a parent TOC. + pub file_id: [u8; 16], + pub state: SectionState, +} + +#[derive(Debug, Serialize)] +pub enum SectionState { + Readable { + /// An explicit SectionDisplayName; otherwise use the current filename without its extension. + name: Option, + }, + Locked, + Unreadable(onestore::Error), +} + +#[derive(Debug, Serialize)] +pub struct Folder { + pub path: String, + pub toc: Option, + pub sections: Vec
, + pub groups: Vec, +} + +#[derive(Debug, Serialize)] +pub struct Toc { + pub filename: String, + pub file_id: [u8; 16], + /// Stale or unavailable TOC references; these are not inferred active sections. + pub unresolved: Vec, +} + +#[derive(Debug, Serialize)] +pub struct TocReference { + /// Native TOCs can retain an identity after removing its cached filename. + pub filename: Option, + pub file: [u8; 16], + pub order: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum EntryKind { + File, + Directory, + Other, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct Entry { + pub name: String, + pub kind: EntryKind, +} + +/// Rooted file access. Paths are relative, UTF-8, and use `/` between components. +pub trait Source { + /// Return the complete immediate directory or an error, never a truncated success. + fn entries(&mut self, path: &str, limit: usize) -> io::Result>; + /// Return a consistent file snapshot, rejecting images larger than the byte limit. + fn read(&mut self, path: &str, limit: usize) -> io::Result>; + /// Reads an external payload with the same completeness and size guarantees. + fn read_asset(&mut self, path: &str, limit: usize) -> io::Result> { + self.read(path, limit) + } +} + +/// Reads an external file-data reference from the section's sibling `_onefiles` folder. +/// `NotFound` in `Error::Io` is distinct from a successfully read zero-byte payload. +pub fn read_external_asset( + source: &mut impl Source, + section: &str, + filename: &str, + limit: usize, +) -> Result, Error> { + let (stem, extension) = section.rsplit_once('.').ok_or_else(|| Error::Entry { + path: section.into(), + })?; + if !extension.eq_ignore_ascii_case("one") + || !section.split('/').all(component) + || stem.ends_with('/') + || stem.is_empty() + { + return Err(Error::Entry { + path: section.into(), + }); + } + format!("{filename}") + .parse::() + .map_err(|_| Error::Entry { + path: filename.into(), + })?; + let path = format!("{stem}_onefiles/{filename}"); + let bytes = source.read_asset(&path, limit).map_err(|error| Error::Io { + path: path.clone(), + error, + })?; + if bytes.len() > limit { + return Err(Error::Limit { path }); + } + Ok(bytes) +} + +pub struct Limits { + pub entries: usize, + pub bytes_per_file: usize, + pub depth: usize, +} + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("Cannot read {path}: {error}")] + Io { + path: String, + #[source] + error: io::Error, + }, + #[error("Invalid notebook file {path}: {error}")] + Document { + path: String, + #[source] + error: onestore::Error, + }, + #[error("Discovery limit exceeded at {path}")] + Limit { path: String }, + #[error("Directory changed during discovery: {path}")] + Changed { path: String }, + #[error("Invalid or unsupported directory entry: {path}")] + Entry { path: String }, + #[error("File identity occurs at both {first} and {second}")] + DuplicateIdentity { + file: [u8; 16], + first: String, + second: String, + }, +} + +/// Discovers rooted notebook topology within caller-specified work and size limits. +/// Reserved `_onefiles` directories contain payloads, not section groups. +pub fn discover(source: &mut impl Source, limits: Limits) -> Result { + let mut remaining = limits.entries; + let mut identities = BTreeMap::new(); + scan(source, "", &limits, 0, &mut remaining, &mut identities) +} + +fn scan( + source: &mut impl Source, + path: &str, + limits: &Limits, + depth: usize, + remaining: &mut usize, + identities: &mut BTreeMap<[u8; 16], String>, +) -> Result { + if depth > limits.depth { + return Err(Error::Limit { path: path.into() }); + } + let mut listing = source + .entries(path, *remaining) + .map_err(|error| Error::Io { + path: path.into(), + error, + })?; + *remaining = remaining + .checked_sub(listing.len()) + .ok_or_else(|| Error::Limit { path: path.into() })?; + listing.sort(); + let mut names = BTreeSet::new(); + for entry in &listing { + if !component(&entry.name) || !names.insert(&entry.name) { + return Err(Error::Entry { + path: join(path, &entry.name), + }); + } + } + let mut result = Folder { + path: path.into(), + toc: None, + sections: Vec::new(), + groups: Vec::new(), + }; + for entry in &listing { + let child = join(path, &entry.name); + if entry.kind == EntryKind::Directory { + if entry.name.to_ascii_lowercase().ends_with("_onefiles") { + continue; + } + result.groups.push(scan( + source, + &child, + limits, + depth + 1, + remaining, + identities, + )?); + continue; + } + let lower = entry.name.to_ascii_lowercase(); + let expected = if lower.ends_with(".one") { + FileType::Section + } else if lower.ends_with(".onetoc2") { + FileType::TableOfContents + } else { + continue; + }; + if entry.kind != EntryKind::File + || (expected == FileType::TableOfContents && result.toc.is_some()) + { + return Err(Error::Entry { path: child }); + } + let bytes = source + .read(&child, limits.bytes_per_file) + .map_err(|error| Error::Io { + path: child.clone(), + error, + })?; + if bytes.len() > limits.bytes_per_file { + return Err(Error::Limit { path: child }); + } + let store = Store::parse(&bytes).map_err(|error| Error::Document { + path: child.clone(), + error, + })?; + if store.header.file_type != expected || !store.checksum_mismatches.is_empty() { + return Err(Error::Document { + path: child, + error: onestore::Error { + offset: 0, + message: "Unexpected file type or checksum mismatch", + }, + }); + } + let file_id = store.header.file_id; + let parsed = (|| { + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + let revision = document.active(document.root)?; + if expected == FileType::Section { + if revision + .roots + .get(&1) + .and_then(|id| revision.nodes.get(id)) + .is_some_and(|node| matches!(node.kind, Kind::Encrypted { .. })) + { + result.sections.push(Section { + path: child.clone(), + file_id, + state: SectionState::Locked, + }); + return Ok(()); + } + index.validate_current()?; + document.pages()?; + let name = revision + .roots + .get(&2) + .and_then(|id| revision.nodes.get(id)) + .and_then(|node| match &node.kind { + Kind::SectionMetadata { name, .. } => name.clone(), + _ => None, + }); + result.sections.push(Section { + path: child.clone(), + file_id, + state: SectionState::Readable { name }, + }); + } else { + index.validate_current()?; + let node = revision.roots.get(&1).and_then(|id| revision.nodes.get(id)); + let Some(Kind::Toc { entries, .. }) = node.map(|node| &node.kind) else { + return Err(onestore::Error { + offset: 0, + message: "Missing notebook TOC root", + }); + }; + let mut seen = BTreeSet::new(); + let mut unresolved = Vec::new(); + for id in entries { + let Some(Kind::Toc { + filename, + identity: Some(file), + order: Some(order), + .. + }) = revision.nodes.get(id).map(|node| &node.kind) + else { + return Err(onestore::Error { + offset: 0, + message: "Incomplete notebook TOC reference", + }); + }; + if filename.as_deref().is_some_and(|name| !component(name)) + || !seen.insert(*file) + { + return Err(onestore::Error { + offset: 0, + message: "Invalid or duplicated notebook TOC reference", + }); + } + unresolved.push(TocReference { + filename: filename.clone(), + file: *file, + order: *order, + }); + } + result.toc = Some(Toc { + filename: entry.name.clone(), + file_id, + unresolved, + }); + } + Ok(()) + })(); + if let Err(error) = parsed { + if expected == FileType::Section { + result.sections.push(Section { + path: child.clone(), + file_id, + state: SectionState::Unreadable(error), + }); + } else { + return Err(Error::Document { path: child, error }); + } + } + if let Some(first) = identities.insert(file_id, child.clone()) { + return Err(Error::DuplicateIdentity { + file: file_id, + first, + second: child, + }); + } + } + let order: BTreeMap<_, _> = result + .toc + .iter() + .flat_map(|toc| &toc.unresolved) + .map(|entry| (entry.file, entry.order)) + .collect(); + result.sections.sort_by(|a, b| { + (order.get(&a.file_id).copied().unwrap_or(u32::MAX), &a.path) + .cmp(&(order.get(&b.file_id).copied().unwrap_or(u32::MAX), &b.path)) + }); + result.groups.sort_by(|a, b| { + ( + a.toc + .as_ref() + .and_then(|toc| order.get(&toc.file_id).copied()) + .unwrap_or(u32::MAX), + &a.path, + ) + .cmp(&( + b.toc + .as_ref() + .and_then(|toc| order.get(&toc.file_id).copied()) + .unwrap_or(u32::MAX), + &b.path, + )) + }); + let present: BTreeSet<_> = result + .sections + .iter() + .map(|section| section.file_id) + .chain( + result + .groups + .iter() + .filter_map(|group| group.toc.as_ref().map(|toc| toc.file_id)), + ) + .collect(); + if let Some(toc) = &mut result.toc { + toc.unresolved + .retain(|entry| !present.contains(&entry.file)); + } + let mut observed = source + .entries(path, listing.len()) + .map_err(|error| Error::Io { + path: path.into(), + error, + })?; + observed.sort(); + if observed != listing { + return Err(Error::Changed { path: path.into() }); + } + Ok(result) +} + +fn component(name: &str) -> bool { + !name.is_empty() && name != "." && name != ".." && !name.contains(['/', '\\', '\0']) +} + +fn join(parent: &str, name: &str) -> String { + if parent.is_empty() { + name.into() + } else { + format!("{parent}/{name}") + } +} diff --git a/crates/notebook/src/discover/source.rs b/crates/notebook/src/discover/source.rs new file mode 100644 index 0000000000000000000000000000000000000000..47af7fe9e61b680237776e7976303fe3b5269fa9 --- /dev/null +++ b/crates/notebook/src/discover/source.rs @@ -0,0 +1,121 @@ +use super::{Entry, EntryKind, Source, component}; +use std::{ + io, + path::{Path, PathBuf}, +}; + +/// A canonical local root; symbolic-link entries are not traversed. +pub struct Local { + root: PathBuf, +} + +impl Local { + pub fn open(root: impl AsRef) -> io::Result { + let root = root.as_ref().canonicalize()?; + if !root.is_dir() { + return Err(io::ErrorKind::NotADirectory.into()); + } + Ok(Self { root }) + } + + fn path(&self, relative: &str) -> io::Result { + if !relative.is_empty() && !relative.split('/').all(component) { + return Err(io::ErrorKind::InvalidInput.into()); + } + let path = self.root.join(relative).canonicalize()?; + if !path.starts_with(&self.root) { + return Err(io::ErrorKind::PermissionDenied.into()); + } + Ok(path) + } +} + +impl Source for Local { + fn entries(&mut self, path: &str, limit: usize) -> io::Result> { + let mut entries = Vec::new(); + for entry in std::fs::read_dir(self.path(path)?)? { + let entry = entry?; + if entries.len() == limit { + return Err(io::ErrorKind::FileTooLarge.into()); + } + let name = entry + .file_name() + .into_string() + .map_err(|_| io::ErrorKind::InvalidData)?; + let kind = entry.file_type()?; + entries.push(Entry { + name, + kind: if kind.is_dir() { + EntryKind::Directory + } else if kind.is_file() { + EntryKind::File + } else { + EntryKind::Other + }, + }); + } + Ok(entries) + } + + fn read(&mut self, path: &str, limit: usize) -> io::Result> { + onestore::read_file_limited(self.path(path)?, limit) + } +} + +#[cfg(feature = "smb")] +/// A share-relative root on an existing blocking SMB connection. +pub struct Smb<'a> { + client: &'a crate::smb::Client, + root: String, +} + +#[cfg(feature = "smb")] +impl<'a> Smb<'a> { + pub fn new(client: &'a crate::smb::Client, root: &str) -> io::Result { + let root = root.replace('\\', "/"); + if !root.is_empty() && !root.split('/').all(component) { + return Err(io::ErrorKind::InvalidInput.into()); + } + Ok(Self { client, root }) + } + + fn path(&self, relative: &str) -> io::Result { + if !relative.is_empty() && !relative.split('/').all(component) { + return Err(io::ErrorKind::InvalidInput.into()); + } + Ok(if relative.is_empty() { + self.root.clone() + } else { + super::join(&self.root, relative) + }) + } +} + +#[cfg(feature = "smb")] +impl Source for Smb<'_> { + fn entries(&mut self, path: &str, limit: usize) -> io::Result> { + Ok(self + .client + .read_dir(&self.path(path)?, limit)? + .into_iter() + .map(|entry| Entry { + name: entry.name, + kind: if entry.attributes & 0x400 != 0 { + EntryKind::Other + } else if entry.attributes & 0x10 != 0 { + EntryKind::Directory + } else { + EntryKind::File + }, + }) + .collect()) + } + + fn read(&mut self, path: &str, limit: usize) -> io::Result> { + self.client.read_storage(&self.path(path)?, limit) + } + + fn read_asset(&mut self, path: &str, limit: usize) -> io::Result> { + self.client.read_asset(&self.path(path)?, limit) + } +} diff --git a/crates/notebook/src/formatting.rs b/crates/notebook/src/formatting.rs new file mode 100644 index 0000000000000000000000000000000000000000..c229f53032d87d21e919e5ec87d8b665608fd62f --- /dev/null +++ b/crates/notebook/src/formatting.rs @@ -0,0 +1,203 @@ +use super::*; +use onestore::TextAttribute; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::BTreeMap; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Span { + end: u32, + values: Vec, +} + +/// A formatting intent and the text/attribute values observed before local publication. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FormatEdit { + pub object: ExGuid, + pub before: String, + pub range: Range, + pub attributes: Vec, + observed: Vec, +} + +fn fields(attributes: &[TextAttribute]) -> BTreeMap<&'static str, Value> { + let mut fields = BTreeMap::new(); + for attribute in attributes { + let (name, value) = match attribute { + TextAttribute::Bold(v) => ("bold", json!(v)), + TextAttribute::Italic(v) => ("italic", json!(v)), + TextAttribute::Underline(v) => ("underline", json!(v)), + TextAttribute::Strike(v) => ("strike", json!(v)), + TextAttribute::Superscript(v) => { + if *v { + fields.insert("subscript", json!(false)); + } + ("superscript", json!(v)) + } + TextAttribute::Subscript(v) => { + if *v { + fields.insert("superscript", json!(false)); + } + ("subscript", json!(v)) + } + TextAttribute::Font(v) => ("font", json!(v)), + TextAttribute::FontSize(v) => ("font_size", json!(v)), + TextAttribute::Color(v) | TextAttribute::Highlight(v) => ( + if matches!(attribute, TextAttribute::Color(_)) { + "color" + } else { + "highlight" + }, + json!(v.map_or(0xff000000, |[r, g, b]| u32::from_le_bytes([r, g, b, 0]))), + ), + }; + fields.insert(name, value); + } + fields +} + +fn observe( + source: &[u8], + space: ExGuid, + object: ExGuid, + range: Range, + fields: &BTreeMap<&str, Value>, +) -> Result> { + let store = Store::parse(source)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + let revision = document.active(space)?; + let mut spans: Vec = Vec::new(); + let mut start = 0; + for run in revision.text_runs(object)? { + let end = + start + u32::try_from(run.text.encode_utf16().count()).map_err(io::Error::other)?; + if (start < range.end && range.start < end) || (start == 0 && end == 0 && range == (0..0)) { + let format = serde_json::to_value(run.format).map_err(io::Error::other)?; + let values = fields + .iter() + .map(|(name, desired)| { + let value = &format[*name]; + if value.is_null() && desired.is_boolean() { + json!(false) + } else if value.is_null() && matches!(*name, "color" | "highlight") { + json!(0xff000000_u32) + } else { + value.clone() + } + }) + .collect::>(); + let end = end.min(range.end) - range.start; + if let Some(last) = spans.last_mut().filter(|s| s.values == values) { + last.end = end; + } else { + spans.push(Span { end, values }); + } + } + start = end; + } + Ok(spans) +} + +impl Replica { + /// Durably records a visual-formatting change and its observed attribute values. + /// Independent remote attributes can merge; competing values preserve a conflict. + pub fn format( + &self, + source: &[u8], + space: ExGuid, + object: ExGuid, + range: Range, + attributes: &[TextAttribute], + ) -> Result> { + let (edit, prepared) = FormatEdit::capture(source, space, object, range, attributes)?; + self.record(source, space, Operation::Format(edit), &prepared) + } +} + +impl FormatEdit { + pub(crate) fn capture<'a>( + source: &'a [u8], + space: ExGuid, + object: ExGuid, + range: Range, + attributes: &[TextAttribute], + ) -> Result<(Self, PreparedEdit<'a>)> { + let prepared = PreparedEdit::format(source, space, object, range.clone(), attributes)?; + let before = paragraph(source, space, object)?.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "Prepared formatting has no text target", + ) + })?; + let observed = observe(source, space, object, range.clone(), &fields(attributes))?; + let edit = Self { + object, + before, + range, + attributes: attributes.to_vec(), + observed, + }; + Ok((edit, prepared)) + } + + pub(crate) fn prepare<'a>( + &self, + snapshot: &'a [u8], + space: ExGuid, + ) -> Result, ConflictKind>> { + let Some(text) = paragraph(snapshot, space, self.object)? else { + return Ok(Err(ConflictKind::TargetUnavailable)); + }; + let Some(range) = rebase::rebase(&self.before, &text, self.range.clone()) else { + return Ok(Err(ConflictKind::TextChanged)); + }; + let prepared = match PreparedEdit::format( + snapshot, + space, + self.object, + range.clone(), + &self.attributes, + ) { + Ok(prepared) => prepared, + Err(_) => return Ok(Err(ConflictKind::UnsupportedEdit)), + }; + let desired = fields(&self.attributes); + let observed = observe(snapshot, space, self.object, range.clone(), &desired)?; + let wanted: Vec<_> = desired.values().collect(); + for spans in [&self.observed, &observed] { + if spans.is_empty() + || (!range.is_empty() && spans[0].end == 0) + || spans.last().unwrap().end != range.end - range.start + || spans.windows(2).any(|s| s[0].end >= s[1].end) + || spans.iter().any(|s| s.values.len() != wanted.len()) + { + return Ok(Err(ConflictKind::UnsupportedEdit)); + } + } + let (mut before, mut after) = (0, 0); + while before < self.observed.len() && after < observed.len() { + let old = &self.observed[before]; + let current = &observed[after]; + if old + .values + .iter() + .zip(¤t.values) + .zip(&wanted) + .any(|((old, new), wanted)| new != old && new != *wanted) + { + return Ok(Err(ConflictKind::FormattingChanged)); + } + let end = old.end.min(current.end); + if old.end == end { + before += 1; + } + if current.end == end { + after += 1; + } + } + Ok(Ok(prepared)) + } +} diff --git a/crates/notebook/src/lib.rs b/crates/notebook/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..a768226d5ff9fba55f5408b8bd7d3953be5c804f --- /dev/null +++ b/crates/notebook/src/lib.rs @@ -0,0 +1,434 @@ +#![forbid(unsafe_code)] +#![doc = include_str!("../README.md")] + +pub mod discover; +#[cfg(feature = "smb")] +pub mod smb; + +use onestore::{ + ExGuid, Insertion, PageCreation, PreparedEdit, RevisionIndex, Store, + document::{Document, Kind}, +}; +use rusqlite::{Connection, OpenFlags, TransactionBehavior, params}; +use std::{fs::OpenOptions, io, ops::Range, path::Path, sync::Mutex, time::Duration}; + +mod assets; +mod formatting; +mod outline; +mod pages; +mod paragraph; +mod rebase; +mod recovery; +mod schema; +mod tree; +pub use formatting::FormatEdit; +pub use outline::OutlineEdit; +pub use pages::PageEdits; +pub use paragraph::{JoinEdit, SplitEdit}; +pub use recovery::{Recovery, RecoverySummary}; +pub use tree::TreeEdit; +mod sync; +pub use sync::{ConflictKind, EditStatus, Remote}; +mod worker; +#[cfg(feature = "smb")] +pub use smb::SmbRemote; +pub use worker::SyncWorker; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error(transparent)] + Database(#[from] rusqlite::Error), + #[error(transparent)] + Io(#[from] io::Error), + #[error(transparent)] + Document(#[from] onestore::Error), + #[error(transparent)] + Remote(#[from] onestore::CommitError), + #[error(transparent)] + RemoteIo(io::Error), + #[error(transparent)] + Discovery(#[from] discover::Error), + #[error("External payload identity now refers to different bytes")] + AssetChanged, +} + +type Result = std::result::Result; + +const APPLICATION_ID: u32 = 0x4f4e454f; +const SCHEMA_VERSION: u32 = 11; + +/// Text and its observed precondition, retained across cache reopen and rebasing. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TextEdit { + pub object: ExGuid, + pub before: String, + pub range: Range, + pub replacement: String, +} + +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub enum Operation { + CreatePage(PageCreation), + Pages(PageEdits), + Text(TextEdit), + Insert(Insertion), + Format(FormatEdit), + Split(SplitEdit), + Join(JoinEdit), + Outline(OutlineEdit), + Tree(TreeEdit), +} + +/// A locally acknowledged intent; its ID remains stable across cache reopen. +#[derive(Debug, Clone, PartialEq)] +pub struct PendingEdit { + pub id: u64, + pub space: ExGuid, + pub operation: Operation, +} + +/// Owns one local cache. Share this handle between threads; a second open fails busy. +/// SQLite's exclusive connection retains ownership between local transactions. +pub struct Replica { + connection: Mutex, + synchronization: Mutex<()>, + worker: Mutex>, +} + +impl Replica { + /// Seeds a new cache from a validated notebook image, refusing any existing path. + /// An initialization error preserves the created file for inspection. + pub fn create(path: impl AsRef, source: &[u8]) -> Result { + validate(source)?; + let mut options = OpenOptions::new(); + options.read(true).write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + drop(options.open(path.as_ref())?); + Self::connect(path.as_ref(), Some(source)) + } + + /// Reopens an existing cache and its durable pending edits without network access. + /// Unrecognized databases and unsupported journal modes are rejected without conversion. + pub fn open(path: impl AsRef) -> Result { + Self::connect(path.as_ref(), None) + } + + fn connect(path: &Path, source: Option<&[u8]>) -> Result { + let mut connection = cache_connection(path)?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Exclusive)?; + let application: u32 = + transaction.pragma_query_value(None, "application_id", |row| row.get(0))?; + let version: u32 = + transaction.pragma_query_value(None, "user_version", |row| row.get(0))?; + if let Some(source) = source { + let tables: i64 = + transaction + .query_row("SELECT count(*) FROM sqlite_schema", [], |row| row.get(0))?; + if application != 0 || version != 0 || tables != 0 { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "Cache initialization found an existing database", + ) + .into()); + } + transaction.pragma_update(None, "application_id", APPLICATION_ID)?; + transaction.pragma_update(None, "user_version", SCHEMA_VERSION)?; + transaction.execute_batch( + " + CREATE TABLE replica ( + id INTEGER PRIMARY KEY CHECK(id=1), + base BLOB NOT NULL, + working BLOB NOT NULL + ) STRICT; + ", + )?; + schema::create(&transaction)?; + transaction.execute("INSERT INTO replica VALUES (1, ?1, ?1)", [source])?; + } else { + if application != APPLICATION_ID || !(1..=SCHEMA_VERSION).contains(&version) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Unrecognized cache or unsupported schema version", + ) + .into()); + } + validate_images(&transaction)?; + if version < SCHEMA_VERSION { + schema::migrate(&transaction, version)?; + transaction.pragma_update(None, "user_version", SCHEMA_VERSION)?; + } + pending(&transaction)?; + } + transaction.commit()?; + Ok(Self { + connection: Mutex::new(connection), + synchronization: Mutex::new(()), + worker: Mutex::new(std::sync::Weak::new()), + }) + } + + /// Returns the latest complete locally committed image, including pending edits. + pub fn snapshot(&self) -> Result> { + let connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + Ok( + connection.query_row("SELECT working FROM replica WHERE id=1", [], |row| { + row.get(0) + })?, + ) + } + + pub fn pending(&self) -> Result> { + let connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + pending(&connection) + } + + /// Atomically records an intent and its resulting local image; returns its durable ID. + /// Unchanged text returns `None`. A stale image returns `Io(ResourceBusy)`. + /// On synchronization, replacement text inherits the remote style at the rebased start. + /// After a database error, reopen and inspect the cache before retrying the edit. + pub fn edit_text( + &self, + source: &[u8], + space: ExGuid, + object: ExGuid, + range: Range, + replacement: &str, + ) -> Result> { + let edit = PreparedEdit::text(source, space, object, range.clone(), replacement)?; + let before = paragraph(source, space, object)?.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "Prepared edit has no text target", + ) + })?; + self.record( + source, + space, + Operation::Text(TextEdit { + object, + before, + range, + replacement: replacement.to_owned(), + }), + &edit, + ) + } + + /// Durably queues a validated insertion with its stable object identities. + /// Uses the same snapshot and local-acknowledgement contract as `edit_text`. + pub fn insert( + &self, + source: &[u8], + space: ExGuid, + insertion: &Insertion, + ) -> Result> { + let edit = PreparedEdit::insert(source, space, insertion)?; + self.record(source, space, Operation::Insert(insertion.clone()), &edit) + } + + /// Queues a new page and its section entry with stable identities for dependent edits. + pub fn create_page(&self, source: &[u8], page: &PageCreation) -> Result> { + let edit = PreparedEdit::create_page(source, page)?; + self.record( + source, + page.space(), + Operation::CreatePage(page.clone()), + &edit, + ) + } + + fn record( + &self, + source: &[u8], + space: ExGuid, + operation: Operation, + edit: &PreparedEdit<'_>, + ) -> Result> { + let mut connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let current: Vec = + transaction.query_row("SELECT working FROM replica WHERE id=1", [], |row| { + row.get(0) + })?; + if current != source { + return Err(io::Error::new( + io::ErrorKind::ResourceBusy, + "The local snapshot changed before this edit", + ) + .into()); + } + if edit.as_bytes() == source { + return Ok(None); + } + transaction.execute( + "INSERT INTO edits(space, operation) VALUES (?1, ?2)", + params![ + space.to_string(), + serde_json::to_string(&operation).map_err(io::Error::other)? + ], + )?; + let id = u64::try_from(transaction.last_insert_rowid()).map_err(io::Error::other)?; + transaction.execute( + "UPDATE replica SET working=?1 WHERE id=1", + [edit.as_bytes()], + )?; + transaction.commit()?; + drop(connection); + self.wake_sync(); + Ok(Some(id)) + } + + fn wake_sync(&self) { + if let Ok(worker) = self.worker.lock() + && let Some(worker) = worker.upgrade() + { + worker.wake(); + } + } +} + +fn active_paths( + view: &onestore::document::Revision<'_>, + pages: &[ExGuid], + objects: &[ExGuid], +) -> Option>> { + let parents = view.parents(pages).ok()?; + objects + .iter() + .map(|object| { + if !parents.contains_key(object) && !pages.contains(object) { + return None; + } + let mut path = Vec::new(); + let mut at = *object; + while !pages.contains(&at) { + let [parent] = parents.get(&at)?.as_slice() else { + return None; + }; + if path.len() >= view.nodes.len() { + return None; + } + path.push(*parent); + at = *parent; + } + Some(path) + }) + .collect() +} + +fn paragraph(source: &[u8], space: ExGuid, object: ExGuid) -> Result> { + let store = Store::parse(source)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + let node = document + .active(space) + .ok() + .and_then(|revision| revision.nodes.get(&object)); + Ok(match node.map(|node| &node.kind) { + Some(Kind::RichText { text, .. }) => Some(text.clone()), + _ => None, + }) +} + +fn validate(source: &[u8]) -> Result { + let store = Store::parse(source)?; + if !store.checksum_mismatches.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Notebook transaction checksum damage", + ) + .into()); + } + let index = RevisionIndex::parse(&store)?; + index.validate_current()?; + Document::parse(&index)?; + Ok(index.root) +} + +fn pending(connection: &Connection) -> Result> { + let mut query = connection.prepare("SELECT id, space, operation FROM edits ORDER BY id")?; + let mut rows = query.query([])?; + let mut edits = Vec::new(); + while let Some(row) = rows.next()? { + let edit = PendingEdit { + id: u64::try_from(row.get::<_, i64>(0)?).map_err(io::Error::other)?, + space: row.get::<_, String>(1)?.parse()?, + operation: serde_json::from_str(&row.get::<_, String>(2)?) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?, + }; + if matches!(&edit.operation, Operation::CreatePage(page) if page.space() != edit.space) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Cached page identity differs from its creation intent", + ) + .into()); + } + edits.push(edit); + } + Ok(edits) +} + +fn cache_connection(path: &Path) -> Result { + let connection = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_WRITE)?; + connection.busy_timeout(Duration::ZERO)?; + connection.execute_batch( + "PRAGMA locking_mode=EXCLUSIVE; PRAGMA synchronous=EXTRA; PRAGMA fullfsync=ON; PRAGMA foreign_keys=ON;", + )?; + for (name, expected) in [("locking_mode", "exclusive"), ("journal_mode", "delete")] { + let actual: String = connection.pragma_query_value(None, name, |row| row.get(0))?; + if actual != expected { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Unsupported cache locking or journal mode", + ) + .into()); + } + } + for (name, expected) in [("synchronous", 3), ("fullfsync", 1), ("foreign_keys", 1)] { + let actual: i64 = connection.pragma_query_value(None, name, |row| row.get(0))?; + if actual != expected { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "Required cache synchronization is unavailable", + ) + .into()); + } + } + Ok(connection) +} + +fn validate_images(connection: &Connection) -> Result<()> { + let integrity: String = connection.query_row("PRAGMA quick_check", [], |row| row.get(0))?; + if integrity != "ok" { + return Err( + io::Error::new(io::ErrorKind::InvalidData, "Cache integrity check failed").into(), + ); + } + let (base, working): (Vec, Vec) = + connection.query_row("SELECT base, working FROM replica WHERE id=1", [], |row| { + Ok((row.get(0)?, row.get(1)?)) + })?; + if validate(&base)? != validate(&working)? { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Cache images belong to different documents", + ) + .into()); + } + Ok(()) +} diff --git a/crates/notebook/src/outline.rs b/crates/notebook/src/outline.rs new file mode 100644 index 0000000000000000000000000000000000000000..8651735ac02a49b0cb619a15a99b8648b63312d7 --- /dev/null +++ b/crates/notebook/src/outline.rs @@ -0,0 +1,138 @@ +use super::*; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Observed { + path: Vec, + values: Vec, +} + +/// A layout or saved expansion intent with its original ancestry and property values. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OutlineEdit { + pub object: ExGuid, + pub change: onestore::OutlineEdit, + observed: Observed, +} + +fn observe( + source: &[u8], + space: ExGuid, + object: ExGuid, + change: onestore::OutlineEdit, +) -> Result> { + let store = Store::parse(source)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + let Ok(view) = document.active(space) else { + return Ok(None); + }; + let pages = document.pages_in(space)?; + if pages.len() != 1 { + return Ok(None); + } + let Some(mut paths) = active_paths(view, &pages, &[object]) else { + return Ok(None); + }; + let path = paths.pop().unwrap(); + let node = &view.nodes[&object]; + let values = match (change, &node.kind) { + (onestore::OutlineEdit::Position { .. }, Kind::Outline { .. }) => { + vec![json!(node.layout.x), json!(node.layout.y)] + } + (onestore::OutlineEdit::Width { .. }, Kind::Outline { .. }) => { + vec![ + json!(node.layout.max_width), + json!(node.layout.width_set_by_user.unwrap_or(false)), + json!(node.layout.reserved_width), + ] + } + (onestore::OutlineEdit::Collapsed(_), Kind::Paragraph { collapse_state, .. }) => { + vec![json!(collapse_state.unwrap_or(0))] + } + _ => return Ok(None), + }; + Ok(Some(Observed { path, values })) +} + +impl Replica { + /// Durably records layout or saved expansion changes, preserving independent remote edits. + pub fn outline( + &self, + source: &[u8], + space: ExGuid, + object: ExGuid, + change: onestore::OutlineEdit, + ) -> Result> { + let (edit, prepared) = OutlineEdit::capture(source, space, object, change)?; + self.record(source, space, Operation::Outline(edit), &prepared) + } +} + +impl OutlineEdit { + pub(crate) fn capture<'a>( + source: &'a [u8], + space: ExGuid, + object: ExGuid, + change: onestore::OutlineEdit, + ) -> Result<(Self, PreparedEdit<'a>)> { + let prepared = PreparedEdit::outline(source, space, object, change)?; + let observed = observe(source, space, object, change)?.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "Prepared outline edit has no active target", + ) + })?; + Ok(( + Self { + object, + change, + observed, + }, + prepared, + )) + } + + pub(crate) fn prepare<'a>( + &self, + snapshot: &'a [u8], + space: ExGuid, + ) -> Result, ConflictKind>> { + let Some(current) = observe(snapshot, space, self.object, self.change)? else { + return Ok(Err(ConflictKind::TargetUnavailable)); + }; + if current.path != self.observed.path { + return Ok(Err(ConflictKind::StructureChanged)); + } + let prepared = match PreparedEdit::outline(snapshot, space, self.object, self.change) { + Ok(prepared) => prepared, + Err(_) => return Ok(Err(ConflictKind::UnsupportedEdit)), + }; + let wanted = + observe(prepared.as_bytes(), space, self.object, self.change)?.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "Prepared outline edit lost its target", + ) + })?; + if self.observed.values.len() != current.values.len() + || wanted.values.len() != current.values.len() + { + return Ok(Err(ConflictKind::UnsupportedEdit)); + } + if self + .observed + .values + .iter() + .zip(¤t.values) + .zip(&wanted.values) + .any(|((before, current), wanted)| current != before && current != wanted) + { + return Ok(Err(ConflictKind::LayoutChanged)); + } + Ok(Ok(prepared)) + } +} diff --git a/crates/notebook/src/pages.rs b/crates/notebook/src/pages.rs new file mode 100644 index 0000000000000000000000000000000000000000..f77536addde20c68f8e8f0c76a2c315d7f883cbb --- /dev/null +++ b/crates/notebook/src/pages.rs @@ -0,0 +1,168 @@ +use super::*; +use onestore::{PageEdit, PagePosition}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +type PageOrder = Vec<(ExGuid, u32)>; + +/// An atomic page batch with its observed order and indentation. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PageEdits { + pub edits: Vec, + observed: PageOrder, +} + +fn observe(source: &[u8]) -> Result> { + let store = Store::parse(source)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + let mut pages = Vec::new(); + for (sid, _) in document.pages()? { + let view = document.active(sid)?; + let Some(metadata) = view.roots.get(&2).and_then(|id| view.nodes.get(id)) else { + return Ok(None); + }; + let Kind::Metadata { level, .. } = metadata.kind else { + return Ok(None); + }; + pages.push((sid, level.unwrap_or(1))); + } + Ok(Some((document.root, pages))) +} + +fn positions(pages: &[(ExGuid, u32)]) -> Result> { + let mut positions = BTreeMap::new(); + for (at, (sid, level)) in pages.iter().enumerate() { + if sid.guid == [0; 16] + || !(1..=3).contains(level) + || positions.insert(*sid, (at, *level)).is_some() + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Page observations need unique identities and valid indentation", + ) + .into()); + } + } + Ok(positions) +} + +impl Replica { + /// Durably queues the complete page batch using the supplied local snapshot. + pub fn pages(&self, source: &[u8], edits: &[PageEdit]) -> Result> { + let (space, batch, prepared) = PageEdits::capture(source, edits)?; + self.record(source, space, Operation::Pages(batch), &prepared) + } +} + +impl PageEdits { + fn capture<'a>( + source: &'a [u8], + edits: &[PageEdit], + ) -> Result<(ExGuid, Self, PreparedEdit<'a>)> { + let prepared = PreparedEdit::pages(source, edits)?; + let (space, observed) = observe(source)?.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "Prepared page edits have no page metadata", + ) + })?; + positions(&observed)?; + Ok(( + space, + Self { + edits: edits.to_vec(), + observed, + }, + prepared, + )) + } + + pub(crate) fn prepare<'a>( + &self, + source: &'a [u8], + space: ExGuid, + ) -> Result, ConflictKind>> { + let Some((root, current)) = observe(source)? else { + return Ok(Err(ConflictKind::TargetUnavailable)); + }; + if root != space { + return Ok(Err(ConflictKind::TargetUnavailable)); + } + let before = positions(&self.observed)?; + let current = positions(¤t)?; + for edit in &self.edits { + let Some((_, original_level)) = before.get(&edit.space()) else { + return Ok(Err(ConflictKind::TargetUnavailable)); + }; + let Some((_, current_level)) = current.get(&edit.space()) else { + return Ok(Err(ConflictKind::TargetUnavailable)); + }; + if current_level != original_level && *current_level != edit.level() { + return Ok(Err(ConflictKind::StructureChanged)); + } + } + let prepared = match PreparedEdit::pages(source, &self.edits) { + Ok(prepared) => prepared, + Err(_) => return Ok(Err(ConflictKind::StructureChanged)), + }; + let (_, wanted) = observe(prepared.as_bytes())?.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "Prepared page edits lost page metadata", + ) + })?; + let wanted = positions(&wanted)?; + for edit in &self.edits { + if edit.position() == PagePosition::Keep { + continue; + } + let sid = edit.space(); + let peers = before + .keys() + .filter(|id| **id != sid && current.contains_key(id)); + let mut unchanged = true; + let mut satisfied = true; + for id in peers { + let observed = current[id].0 < current[&sid].0; + unchanged &= observed == (before[id].0 < before[&sid].0); + satisfied &= observed == (wanted[id].0 < wanted[&sid].0); + } + if !unchanged && !satisfied { + return Ok(Err(ConflictKind::StructureChanged)); + } + } + Ok(Ok(prepared)) + } + + pub(crate) fn review(&self, source: &[u8], space: ExGuid, edits: &[PageEdit]) -> Result { + let identities_match = edits.len() == self.edits.len() + && edits.iter().all(|edit| { + self.edits + .iter() + .find(|original| original.space() == edit.space()) + .is_some_and(|original| { + original + .reposition(edit.position(), edit.level()) + .is_ok_and(|revised| revised == *edit) + }) + }); + if !identities_match { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Retain the original page and series identities when reviewing a page batch", + ) + .into()); + } + let (root, reviewed, _) = Self::capture(source, edits)?; + if root != space { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Review the original section for page edits", + ) + .into()); + } + Ok(reviewed) + } +} diff --git a/crates/notebook/src/paragraph.rs b/crates/notebook/src/paragraph.rs new file mode 100644 index 0000000000000000000000000000000000000000..f97975e1da365fc07b4d823454389c00e2ce9810 --- /dev/null +++ b/crates/notebook/src/paragraph.rs @@ -0,0 +1,200 @@ +use super::*; +use onestore::{ParagraphJoin, ParagraphSplit}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Observed { + text: String, + structure: Value, +} + +/// A stable split intent and the paragraph state observed before local publication. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SplitEdit { + pub intent: ParagraphSplit, + observed: Observed, +} + +/// A join intent and both paragraphs' observed text, placement and tag ownership. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct JoinEdit { + pub intent: ParagraphJoin, + observed: [Observed; 2], +} + +fn observe(source: &[u8], space: ExGuid, texts: &[ExGuid]) -> Result>> { + let store = Store::parse(source)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + let Ok(view) = document.active(space) else { + return Ok(None); + }; + let pages = document.pages_in(space)?; + let Some(paths) = active_paths(view, &pages, texts) else { + return Ok(None); + }; + let mut observed = Vec::new(); + for (text, path) in texts.iter().zip(paths) { + let node = &view.nodes[text]; + let Kind::RichText { text: content, .. } = &node.kind else { + return Ok(None); + }; + let Some(paragraph) = path.first().and_then(|id| view.nodes.get(id)) else { + return Ok(None); + }; + let Kind::Paragraph { lists, .. } = ¶graph.kind else { + return Ok(None); + }; + if paragraph.content != [*text] { + return Ok(None); + } + let lists: Vec<_> = lists.iter().map(|id| (*id, &view.nodes[id].kind)).collect(); + observed.push(Observed { + text: content.clone(), + structure: json!({ + "path": path, + "children": paragraph.children, + "child_level": paragraph.child_level, + "paragraph": paragraph.kind, + "lists": lists, + "tags": node.tags, + }), + }); + } + Ok(Some(observed)) +} + +impl Replica { + /// Durably queues a split with stable new identities and observed structural preconditions. + pub fn split( + &self, + source: &[u8], + space: ExGuid, + intent: &ParagraphSplit, + ) -> Result> { + let (operation, prepared) = SplitEdit::capture(source, space, intent)?; + self.record(source, space, Operation::Split(operation), &prepared) + } + + /// Durably queues a join; changed placement, children or tags require conflict review. + pub fn join( + &self, + source: &[u8], + space: ExGuid, + intent: &ParagraphJoin, + ) -> Result> { + let (operation, prepared) = JoinEdit::capture(source, space, intent)?; + self.record(source, space, Operation::Join(operation), &prepared) + } +} + +impl SplitEdit { + pub(crate) fn capture<'a>( + source: &'a [u8], + space: ExGuid, + intent: &ParagraphSplit, + ) -> Result<(Self, PreparedEdit<'a>)> { + let prepared = PreparedEdit::split(source, space, intent)?; + let Some(mut observed) = observe(source, space, &[intent.position().0])? else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Select text in an active paragraph", + ) + .into()); + }; + Ok(( + Self { + intent: intent.clone(), + observed: observed.remove(0), + }, + prepared, + )) + } + + pub(crate) fn prepare<'a>( + &self, + source: &'a [u8], + space: ExGuid, + ) -> Result, ConflictKind>> { + let (text, offset) = self.intent.position(); + let Some(observed) = observe(source, space, &[text])? else { + return Ok(Err(ConflictKind::TargetUnavailable)); + }; + if observed[0].structure != self.observed.structure { + return Ok(Err(ConflictKind::StructureChanged)); + } + let Some(range) = rebase::rebase(&self.observed.text, &observed[0].text, offset..offset) + else { + return Ok(Err(ConflictKind::TextChanged)); + }; + Ok( + PreparedEdit::split(source, space, &self.intent.reposition(range.start)) + .map_err(|_| ConflictKind::UnsupportedEdit), + ) + } +} + +impl JoinEdit { + pub(crate) fn review(&self, source: &[u8], space: ExGuid) -> Result { + let (reviewed, _) = Self::capture(source, space, &self.intent)?; + if reviewed.observed[0].text.is_empty() != self.observed[0].text.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "The join would change its surviving text identity", + ) + .into()); + } + Ok(reviewed) + } + + pub(crate) fn capture<'a>( + source: &'a [u8], + space: ExGuid, + intent: &ParagraphJoin, + ) -> Result<(Self, PreparedEdit<'a>)> { + let prepared = PreparedEdit::join(source, space, intent)?; + let Some(observed) = observe(source, space, &intent.texts())? else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Select text in two active paragraphs", + ) + .into()); + }; + Ok(( + Self { + intent: intent.clone(), + observed: observed.try_into().unwrap(), + }, + prepared, + )) + } + + pub(crate) fn prepare<'a>( + &self, + source: &'a [u8], + space: ExGuid, + ) -> Result, ConflictKind>> { + let Some(observed) = observe(source, space, &self.intent.texts())? else { + return Ok(Err(ConflictKind::TargetUnavailable)); + }; + for (i, (old, new)) in self.observed.iter().zip(&observed).enumerate() { + if old.structure != new.structure { + return Ok(Err(ConflictKind::StructureChanged)); + } + let at = if i == 0 { + u32::try_from(old.text.encode_utf16().count()).map_err(io::Error::other)? + } else { + 0 + }; + if rebase::rebase(&old.text, &new.text, at..at).is_none() { + return Ok(Err(ConflictKind::TextChanged)); + } + } + Ok(PreparedEdit::join(source, space, &self.intent) + .map_err(|_| ConflictKind::UnsupportedEdit)) + } +} diff --git a/crates/notebook/src/rebase.rs b/crates/notebook/src/rebase.rs new file mode 100644 index 0000000000000000000000000000000000000000..9a6dce0a8b16201774a6482925ad04659b6ab1b6 --- /dev/null +++ b/crates/notebook/src/rebase.rs @@ -0,0 +1,297 @@ +use std::ops::Range; + +const INFINITY: u32 = 1 << 30; +const MAX_CELLS: usize = 4_000_000; + +struct Matrix { + band: usize, + values: Vec, +} + +impl Matrix { + fn get(&self, row: usize, column: usize) -> u32 { + let Some(column) = column + .checked_add(self.band) + .and_then(|at| at.checked_sub(row)) + else { + return INFINITY; + }; + let width = self.band * 2 + 1; + if column >= width { + return INFINITY; + } + self.values + .get(row * width + column) + .copied() + .unwrap_or(INFINITY) + } + + fn build(before: &[char], after: &[char], band: usize) -> Self { + let width = band * 2 + 1; + let mut matrix = Self { + band, + values: vec![INFINITY; (before.len() + 1) * width], + }; + for row in 0..=before.len() { + for column in row.saturating_sub(band)..=after.len().min(row + band) { + let mut cost = if row == 0 && column == 0 { 0 } else { INFINITY }; + if row > 0 { + cost = cost.min(matrix.get(row - 1, column) + 1); + } + if column > 0 { + cost = cost.min(matrix.get(row, column - 1) + 1); + } + if row > 0 && column > 0 && before[row - 1] == after[column - 1] { + cost = cost.min(matrix.get(row - 1, column - 1)); + } + matrix.values[row * width + column + band - row] = cost; + } + } + matrix + } +} + +/// Requires the same mapping in every minimum insertion/deletion alignment. +pub(crate) fn rebase(before: &str, after: &str, range: Range) -> Option> { + if range.start > range.end { + return None; + } + let mut a: Vec<_> = before.chars().collect(); + let offsets: Vec<_> = std::iter::once(0) + .chain(a.iter().scan(0_u32, |at, character| { + *at = at.checked_add(character.len_utf16() as u32)?; + Some(*at) + })) + .collect(); + let local = + offsets.binary_search(&range.start).ok()?..offsets.binary_search(&range.end).ok()?; + if before == after { + return Some(range); + } + let mut b: Vec<_> = after.chars().collect(); + let (n, m) = (a.len(), b.len()); + let mut band = n.abs_diff(m).max(1); + let forward = loop { + let width = band.checked_mul(2)?.checked_add(1)?; + if (n + 1).checked_mul(width)? > MAX_CELLS { + return None; + } + let matrix = Matrix::build(&a, &b, band); + if matrix.get(n, m) <= band as u32 { + break matrix; + } + band *= 2; + }; + let distance = forward.get(n, m); + a.reverse(); + b.reverse(); + let backward = Matrix::build(&a, &b, band); + a.reverse(); + b.reverse(); + let position = |index: usize| { + let mut matched = None; + for column in index.saturating_sub(band)..=m.min(index + band) { + let cost = forward.get(index, column); + if cost + 1 + backward.get(n - index - 1, m - column) == distance { + return None; + } + if b.get(column) == Some(&a[index]) + && cost + backward.get(n - index - 1, m - column - 1) == distance + { + if matched.is_some() { + return None; + } + matched = Some(column); + } + } + matched + }; + let mapped = if local.is_empty() { + if local.start > 0 { + position(local.start - 1)?; + } + if local.start < n { + position(local.start)?; + } + let mut boundary = None; + for column in local.start.saturating_sub(band)..=m.min(local.start + band) { + if forward.get(local.start, column) + backward.get(n - local.start, m - column) + == distance + { + if boundary.is_some() { + return None; + } + boundary = Some(column); + } + } + let at = boundary?; + at..at + } else { + let start = position(local.start)?; + for index in local.start + 1..local.end { + if position(index)? != start + index - local.start { + return None; + } + } + start..start + local.len() + }; + let start = b[..mapped.start] + .iter() + .map(|character| character.len_utf16()) + .sum::(); + let end = start + + b[mapped] + .iter() + .map(|character| character.len_utf16()) + .sum::(); + Some(u32::try_from(start).ok()?..u32::try_from(end).ok()?) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn optimal_paths(a: &[char], b: &[char]) -> Vec> { + fn visit( + a: &[char], + b: &[char], + path: &mut Vec<(usize, usize)>, + cost: usize, + best: &mut usize, + found: &mut Vec>, + ) { + if cost > *best { + return; + } + let (i, j) = *path.last().unwrap(); + if (i, j) == (a.len(), b.len()) { + if cost < *best { + found.clear(); + *best = cost; + } + found.push(path.clone()); + return; + } + for (next_i, next_j, charge) in [(i + 1, j + 1, 0), (i + 1, j, 1), (i, j + 1, 1)] { + if next_i > a.len() || next_j > b.len() || (charge == 0 && a[i] != b[j]) { + continue; + } + path.push((next_i, next_j)); + visit(a, b, path, cost + charge, best, found); + path.pop(); + } + } + let mut found = Vec::new(); + let mut best = usize::MAX; + visit(a, b, &mut vec![(0, 0)], 0, &mut best, &mut found); + found + } + + #[test] + fn bounded_alignment_agrees_with_exhaustive_paths_for_every_small_unicode_edit() { + let mut words = vec![String::new()]; + for length in 1..=4 { + for bits in 0..1 << length { + words.push( + (0..length) + .map(|bit| if bits & (1 << bit) == 0 { 'a' } else { '🦀' }) + .collect(), + ); + } + } + let mut cases = 0; + for before in &words { + let a: Vec<_> = before.chars().collect(); + for after in &words { + let b: Vec<_> = after.chars().collect(); + let paths = optimal_paths(&a, &b); + for start in 0..=a.len() { + for end in start..=a.len() { + let mut expected = None; + let mut valid = true; + for path in &paths { + let mapping: Vec<_> = (0..a.len()) + .map(|i| { + path.windows(2).find_map(|edge| { + (edge[0].0 == i && edge[1] == (i + 1, edge[0].1 + 1)) + .then_some(edge[0].1) + }) + }) + .collect(); + let candidate = if start == end { + let vertices: Vec<_> = path + .iter() + .filter(|(i, _)| *i == start) + .map(|(_, j)| *j) + .collect(); + if (start > 0 && mapping[start - 1].is_none()) + || (start < a.len() && mapping[start].is_none()) + || vertices.len() != 1 + { + None + } else { + Some(vertices[0]..vertices[0]) + } + } else if let Some(first) = mapping[start] { + (start..end) + .all(|i| mapping[i] == Some(first + i - start)) + .then_some(first..first + end - start) + } else { + None + }; + let Some(candidate) = candidate else { + valid = false; + break; + }; + if expected.as_ref().is_some_and(|old| *old != candidate) { + valid = false; + break; + } + expected = Some(candidate); + } + let expected = if valid { + expected.map(|range| { + b[..range.start].iter().map(|c| c.len_utf16() as u32).sum() + ..b[..range.end].iter().map(|c| c.len_utf16() as u32).sum() + }) + } else { + None + }; + let range = a[..start].iter().map(|c| c.len_utf16() as u32).sum() + ..a[..end].iter().map(|c| c.len_utf16() as u32).sum(); + assert_eq!( + rebase(before, after, range), + expected, + "{before:?} -> {after:?}, characters {start}..{end}" + ); + cases += 1; + } + } + } + } + assert_eq!(cases, 10881); + } + + #[test] + fn maps_disjoint_unicode_edits_and_rejects_ambiguous_or_overlapping_changes() { + assert_eq!(rebase("ab🦀cd", "Xab🦀cYd", 2..4), Some(3..5)); + assert_eq!(rebase("abc", "XabcY", 1..2), Some(2..3)); + assert_eq!(rebase("abc", "XabcY", 1..1), Some(2..2)); + assert_eq!(rebase("abc", "abcX", 3..3), None); + assert_eq!(rebase("abc", "ac", 1..2), None); + assert_eq!(rebase("abc", "", 1..1), None); + assert_eq!(rebase("aaaa", "aaaaa", 1..2), None); + assert_eq!(rebase("ab🦀cd", "ab🦀cd", 3..4), None); + assert_eq!(rebase("abc", "abc", 4..4), None); + } + + #[test] + fn long_paragraphs_with_small_remote_changes_use_a_narrow_band() { + let text = format!("{}🦀{}", "a".repeat(8192), "b".repeat(8192)); + assert_eq!( + rebase(&text, &format!("X{text}Y"), 8192..8194), + Some(8193..8195) + ); + assert_eq!(rebase(&"a".repeat(8192), &"b".repeat(8192), 1..2), None); + } +} diff --git a/crates/notebook/src/recovery.rs b/crates/notebook/src/recovery.rs new file mode 100644 index 0000000000000000000000000000000000000000..5b87b1c733d53acaba966f24c4fe60ba5df548e7 --- /dev/null +++ b/crates/notebook/src/recovery.rs @@ -0,0 +1,190 @@ +use super::*; +use rusqlite::backup::{Backup, StepResult}; +use std::{collections::BTreeMap, fs::File}; + +const RECOVERY_ID: u32 = 0x4f4e4552; + +/// Counts and image sizes without notebook text, paths, authors or credentials. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +pub struct RecoverySummary { + pub queued_edits: u64, + pub conflicts: u64, + pub uncertain_edits: u64, + pub published_receipts: u64, + pub working_bytes: u64, + pub remote_bytes: u64, + pub cached_assets: u64, + pub cached_asset_bytes: u64, +} + +/// Read-only recovery evidence; it cannot publish or acknowledge an edit. +pub struct Recovery { + connection: Connection, +} + +impl Recovery { + /// Opens an exported archive without migration or conversion into a writable replica. + pub fn open(path: impl AsRef) -> Result { + let connection = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; + connection.busy_timeout(Duration::ZERO)?; + connection.execute_batch("BEGIN")?; + let application: u32 = + connection.pragma_query_value(None, "application_id", |row| row.get(0))?; + let version: u32 = connection.pragma_query_value(None, "user_version", |row| row.get(0))?; + if application != RECOVERY_ID || !(4..=SCHEMA_VERSION).contains(&version) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Unrecognized recovery archive or unsupported schema version", + ) + .into()); + } + validate_images(&connection)?; + for edit in pending(&connection)? { + sync::status(&connection, edit.id)?; + } + receipts(&connection)?; + Ok(Self { connection }) + } + + pub fn summary(&self) -> Result { + summary(&self.connection) + } + + pub fn snapshot(&self) -> Result> { + Ok(self + .connection + .query_row("SELECT working FROM replica WHERE id=1", [], |row| { + row.get(0) + })?) + } + + pub fn remote_snapshot(&self) -> Result> { + Ok(self + .connection + .query_row("SELECT base FROM replica WHERE id=1", [], |row| row.get(0))?) + } + + pub fn pending(&self) -> Result> { + pending(&self.connection) + } + + pub fn status(&self, id: u64) -> Result> { + sync::status(&self.connection, id) + } + + pub fn receipts(&self) -> Result> { + receipts(&self.connection) + } + + /// Reads a previously downloaded external payload without accessing its former server. + pub fn cached_asset(&self, filename: &str, limit: usize) -> Result>> { + assets::cached(&self.connection, &assets::key(filename)?, limit) + } +} + +impl Replica { + /// Summarizes durable state without exposing notebook content. + pub fn recovery_summary(&self) -> Result { + let connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + summary(&connection) + } + + /// Exports a consistent archive to a new local path without changing the live queue. + /// Archives contain notebook content and cannot be opened as writable replicas. + /// An error after publication can leave a complete archive at the destination. + pub fn export_recovery(&self, path: impl AsRef) -> Result<()> { + let path = path.as_ref(); + if path.file_name().is_none() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Provide a new recovery archive filename", + ) + .into()); + } + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let temporary = tempfile::Builder::new() + .prefix(".onestore-recovery-") + .tempfile_in(parent)?; + let mut destination = cache_connection(temporary.path())?; + { + let source = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + let backup = Backup::new(&source, &mut destination)?; + if backup.step(-1)? != StepResult::Done { + return Err(io::Error::new( + io::ErrorKind::WouldBlock, + "Recovery export could not acquire the database snapshot", + ) + .into()); + } + } + destination.pragma_update(None, "application_id", RECOVERY_ID)?; + destination.close().map_err(|(_, error)| error)?; + temporary.as_file().sync_all()?; + temporary + .persist_noclobber(path) + .map_err(|error| error.error)?; + File::open(parent)?.sync_all()?; + Ok(()) + } +} + +fn summary(connection: &Connection) -> Result { + let version: u32 = connection.pragma_query_value(None, "user_version", |row| row.get(0))?; + let (cached_assets, cached_asset_bytes) = if version < 5 { + (0, 0) + } else { + connection.query_row( + "SELECT count(*),coalesce(sum(length(data)),0) FROM assets", + [], + |row| Ok((unsigned(row, 0)?, unsigned(row, 1)?)), + )? + }; + Ok(connection.query_row( + "SELECT (SELECT count(*) FROM edits), (SELECT count(*) FROM conflicts), + (SELECT count(*) FROM attempt), (SELECT count(*) FROM receipts), + length(working), length(base) FROM replica WHERE id=1", + [], + |row| { + Ok(RecoverySummary { + queued_edits: unsigned(row, 0)?, + conflicts: unsigned(row, 1)?, + uncertain_edits: unsigned(row, 2)?, + published_receipts: unsigned(row, 3)?, + working_bytes: unsigned(row, 4)?, + remote_bytes: unsigned(row, 5)?, + cached_assets, + cached_asset_bytes, + }) + }, + )?) +} + +fn receipts(connection: &Connection) -> Result> { + let mut statement = + connection.prepare("SELECT edit_id, revision FROM receipts ORDER BY edit_id")?; + let mut rows = statement.query([])?; + let mut receipts = BTreeMap::new(); + while let Some(row) = rows.next()? { + receipts.insert(unsigned(row, 0)?, row.get::<_, String>(1)?.parse()?); + } + Ok(receipts) +} + +fn unsigned(row: &rusqlite::Row<'_>, column: usize) -> rusqlite::Result { + u64::try_from(row.get::<_, i64>(column)?).map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + column, + rusqlite::types::Type::Integer, + Box::new(error), + ) + }) +} diff --git a/crates/notebook/src/schema.rs b/crates/notebook/src/schema.rs new file mode 100644 index 0000000000000000000000000000000000000000..b392c200c66b01dc09c0a76dc128b06f03f2766b --- /dev/null +++ b/crates/notebook/src/schema.rs @@ -0,0 +1,143 @@ +use super::*; +use rusqlite::{OptionalExtension, Transaction}; + +const CONFLICTS: &str = "CREATE TABLE conflicts ( + edit_id INTEGER PRIMARY KEY REFERENCES edits(id) ON DELETE CASCADE, + kind INTEGER NOT NULL CHECK(kind BETWEEN 0 AND 6) +) STRICT;"; + +const ASSETS: &str = "CREATE TABLE assets ( + name TEXT PRIMARY KEY NOT NULL, + data BLOB NOT NULL, + sha256 BLOB NOT NULL CHECK(length(sha256)=32) +) STRICT;"; + +pub(crate) fn create(transaction: &Transaction<'_>) -> Result<()> { + transaction.execute_batch( + "CREATE TABLE edits ( + id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(id>0), + space TEXT NOT NULL, + operation TEXT NOT NULL + ) STRICT; + CREATE TABLE attempt ( + id INTEGER PRIMARY KEY CHECK(id=1), + edit_id INTEGER NOT NULL UNIQUE REFERENCES edits(id) ON DELETE CASCADE, + revisions TEXT NOT NULL + ) STRICT; + CREATE TABLE receipts ( + edit_id INTEGER PRIMARY KEY CHECK(edit_id>0), + revision TEXT NOT NULL + ) STRICT;", + )?; + transaction.execute_batch(CONFLICTS)?; + transaction.execute_batch(ASSETS)?; + Ok(()) +} + +pub(crate) fn migrate(transaction: &Transaction<'_>, version: u32) -> Result<()> { + if version >= 10 { + return Ok(()); + } + if version >= 3 { + transaction.execute_batch("ALTER TABLE conflicts RENAME TO old_conflicts;")?; + transaction.execute_batch(CONFLICTS)?; + transaction.execute_batch( + "INSERT INTO conflicts SELECT * FROM old_conflicts; DROP TABLE old_conflicts;", + )?; + if version < 5 { + transaction.execute_batch(ASSETS)?; + } + transaction.execute_batch("ALTER TABLE attempt RENAME COLUMN revision TO revisions;")?; + migrate_attempts(transaction)?; + return Ok(()); + } + + let mut query = transaction.prepare( + "SELECT id, space, object, before_text, start, end, replacement FROM edits ORDER BY id", + )?; + let mut rows = query.query([])?; + let mut edits = Vec::new(); + while let Some(row) = rows.next()? { + edits.push(PendingEdit { + id: u64::try_from(row.get::<_, i64>(0)?).map_err(io::Error::other)?, + space: row.get::<_, String>(1)?.parse()?, + operation: Operation::Text(TextEdit { + object: row.get::<_, String>(2)?.parse()?, + before: row.get(3)?, + range: row.get(4)?..row.get(5)?, + replacement: row.get(6)?, + }), + }); + } + drop(rows); + drop(query); + let sequence: i64 = transaction + .query_row( + "SELECT seq FROM sqlite_sequence WHERE name='edits'", + [], + |row| row.get(0), + ) + .optional()? + .unwrap_or(0); + let (mut attempts, mut conflicts, mut receipts) = (Vec::new(), Vec::new(), Vec::new()); + if version == 2 { + let mut query = transaction.prepare("SELECT edit_id, revision FROM attempt")?; + attempts = query + .query_map([], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) + })? + .collect::>()?; + let mut query = transaction.prepare("SELECT edit_id, kind FROM conflicts")?; + conflicts = query + .query_map([], |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)))? + .collect::>()?; + let mut query = transaction.prepare("SELECT edit_id, revision FROM receipts")?; + receipts = query + .query_map([], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) + })? + .collect::>()?; + transaction + .execute_batch("DROP TABLE attempt; DROP TABLE conflicts; DROP TABLE receipts;")?; + } + transaction.execute_batch("DROP TABLE edits;")?; + create(transaction)?; + for edit in edits { + transaction.execute( + "INSERT INTO edits(id,space,operation) VALUES (?1,?2,?3)", + params![ + i64::try_from(edit.id).map_err(io::Error::other)?, + edit.space.to_string(), + serde_json::to_string(&edit.operation).map_err(io::Error::other)? + ], + )?; + } + // A drained queue must not reuse IDs belonging to existing durable receipts. + transaction.execute("DELETE FROM sqlite_sequence WHERE name='edits'", [])?; + transaction.execute( + "INSERT INTO sqlite_sequence(name,seq) VALUES ('edits',?1)", + [sequence], + )?; + for (id, revision) in attempts { + transaction.execute( + "INSERT INTO attempt VALUES (1,?1,?2)", + params![id, revision], + )?; + } + for (id, kind) in conflicts { + transaction.execute("INSERT INTO conflicts VALUES (?1,?2)", params![id, kind])?; + } + for (id, revision) in receipts { + transaction.execute("INSERT INTO receipts VALUES (?1,?2)", params![id, revision])?; + } + migrate_attempts(transaction)?; + Ok(()) +} + +fn migrate_attempts(transaction: &Transaction<'_>) -> Result<()> { + transaction.execute_batch( + "UPDATE attempt SET revisions=json_object( + (SELECT space FROM edits WHERE edits.id=attempt.edit_id), revisions);", + )?; + Ok(()) +} diff --git a/crates/notebook/src/smb/directory.rs b/crates/notebook/src/smb/directory.rs new file mode 100644 index 0000000000000000000000000000000000000000..4997431a09200a47462e6573c515ead57de50eb9 --- /dev/null +++ b/crates/notebook/src/smb/directory.rs @@ -0,0 +1,219 @@ +use super::*; +use smb2::msg::query_directory::{ + FileInformationClass, QueryDirectoryFlags, QueryDirectoryRequest, QueryDirectoryResponse, +}; +use std::collections::BTreeMap; + +/// Observed directory metadata, not a stable notebook identity or a file snapshot. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DirectoryEntry { + pub name: String, + pub size: u64, + /// MS-FSCC file attributes; directory is 0x10 and reparse point is 0x400. + pub attributes: u32, +} + +impl Client { + /// Enumerates a share-relative directory completely or returns an error without a partial list. + /// An empty path selects the share root. The limit excludes `.` and `..`. + /// Concurrent directory changes are not an atomic snapshot; repeated names return ResourceBusy. + pub fn read_dir(&self, path: &str, limit: usize) -> io::Result> { + if path.contains('\0') || path.encode_utf16().count() > 32767 { + return Err(io::ErrorKind::InvalidInput.into()); + } + let response: CreateResponse = self.request( + Command::Create, + CreateRequest { + requested_oplock_level: OplockLevel::None, + impersonation_level: ImpersonationLevel::Impersonation, + desired_access: FileAccessMask::new(0x80000000), + file_attributes: 0, + share_access: ShareAccess(7), + create_disposition: CreateDisposition::FileOpen, + create_options: 0x1, + name: smb2::encode_path(&path.replace('\\', "/")), + create_contexts: Vec::new(), + }, + )?; + let file = File { + client: self, + id: Some(response.file_id), + }; + let mut entries = BTreeMap::new(); + loop { + let response: io::Result = + self.request_with(Command::QueryDirectory, |connection| { + ( + QueryDirectoryRequest { + file_information_class: FileInformationClass::FileDirectoryInformation, + flags: QueryDirectoryFlags(if entries.is_empty() { 1 } else { 0 }), + file_index: 0, + file_id: file.id.unwrap(), + output_buffer_length: connection + .params() + .expect("connected SMB session is negotiated") + .max_transact_size + .min(65536), + file_name: "*".into(), + }, + CreditCharge(1), + ) + }); + let response = match response { + Ok(response) => response, + Err(error) + if matches!( + error.get_ref().and_then(|error| error.downcast_ref::()), + Some(smb2::Error::Protocol { status, command: Command::QueryDirectory }) + if status.0 == 0x80000006 || (entries.is_empty() && status.0 == 0xc000000f) + ) => + { + break; + } + Err(error) => return Err(error), + }; + for entry in decode(&response.output_buffer)? { + if entries + .insert(entry.name, (entry.size, entry.attributes)) + .is_some() + { + return Err(io::ErrorKind::ResourceBusy.into()); + } + if entries.len() + - usize::from(entries.contains_key(".")) + - usize::from(entries.contains_key("..")) + > limit + { + return Err(io::ErrorKind::FileTooLarge.into()); + } + } + } + file.close()?; + Ok(entries + .into_iter() + .filter(|(name, _)| name != "." && name != "..") + .map(|(name, (size, attributes))| DirectoryEntry { + name, + size, + attributes, + }) + .collect()) + } +} + +fn decode(mut bytes: &[u8]) -> io::Result> { + if bytes.len() > 65536 { + return Err(io::ErrorKind::InvalidData.into()); + } + let mut entries = Vec::new(); + loop { + if bytes.len() < 64 { + return Err(io::ErrorKind::InvalidData.into()); + } + let next = u32::from_le_bytes(bytes[..4].try_into().unwrap()) as usize; + let length = u32::from_le_bytes(bytes[60..64].try_into().unwrap()) as usize; + let end = 64usize + .checked_add(length) + .ok_or(io::ErrorKind::InvalidData)?; + if length == 0 + || !length.is_multiple_of(2) + || end > bytes.len() + || (next != 0 && (next < end || !next.is_multiple_of(8) || next >= bytes.len())) + || (next == 0 && bytes.len() - end > 7) + { + return Err(io::ErrorKind::InvalidData.into()); + } + let units: Vec<_> = bytes[64..end] + .chunks_exact(2) + .map(|unit| u16::from_le_bytes([unit[0], unit[1]])) + .collect(); + let name = String::from_utf16(&units).map_err(|_| io::ErrorKind::InvalidData)?; + if name.contains(['\0', '/', '\\']) { + return Err(io::ErrorKind::InvalidData.into()); + } + let size = i64::from_le_bytes(bytes[40..48].try_into().unwrap()); + entries.push(DirectoryEntry { + name, + size: size.try_into().map_err(|_| io::ErrorKind::InvalidData)?, + attributes: u32::from_le_bytes(bytes[56..60].try_into().unwrap()), + }); + if next == 0 { + return Ok(entries); + } + bytes = &bytes[next..]; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn record(name: &str, size: u64, attributes: u32) -> Vec { + let mut bytes = vec![0; 64]; + bytes[40..48].copy_from_slice(&size.to_le_bytes()); + bytes[56..60].copy_from_slice(&attributes.to_le_bytes()); + let name: Vec<_> = name.encode_utf16().flat_map(u16::to_le_bytes).collect(); + bytes[60..64].copy_from_slice(&(name.len() as u32).to_le_bytes()); + bytes.extend(name); + bytes + } + + #[test] + fn directory_records_preserve_names_sizes_and_attributes() { + let mut bytes = Vec::new(); + let mut expected = Vec::new(); + for i in 0..300u32 { + let name = format!("Section {i} 🦀 e\u{301}.one"); + let size = u64::from(i) * 100_000_000; + let attributes = if i % 3 == 0 { 0x410 } else { 0x20 }; + let mut entry = record(&name, size, attributes); + if i != 299 { + entry.resize(entry.len().next_multiple_of(8), 0xa5); + let length = entry.len() as u32; + entry[..4].copy_from_slice(&length.to_le_bytes()); + } + bytes.extend(entry); + expected.push(DirectoryEntry { + name, + size, + attributes, + }); + } + assert_eq!(decode(&bytes).unwrap(), expected); + for end in 0..bytes.len() { + assert!(decode(&bytes[..end]).is_err(), "accepted prefix {end}"); + } + } + + #[test] + fn invalid_directory_records_never_become_partial_results() { + assert!(decode(&vec![0; 65537]).is_err()); + for name in ["", "bad\0name", "a/b", "a\\b"] { + assert!(decode(&record(name, 0, 0)).is_err()); + } + let valid = record("a.one", 12, 0x20); + for (at, value) in [ + (0, 8), + (0, 65), + (0, 72), + (0, u32::MAX), + (60, 0), + (60, 1), + (60, u32::MAX), + (44, u32::MAX), + ] { + let mut bytes = valid.clone(); + bytes[at..at + 4].copy_from_slice(&value.to_le_bytes()); + assert!(decode(&bytes).is_err(), "offset={at} value={value}"); + } + let mut bytes = valid.clone(); + bytes[64..66].copy_from_slice(&0xd800u16.to_le_bytes()); + assert!(decode(&bytes).is_err()); + let mut bytes = valid; + bytes.extend_from_slice(&[0; 8]); + assert!(decode(&bytes).is_err()); + for name in [".", ".."] { + assert_eq!(decode(&record(name, 0, 0x10)).unwrap()[0].name, name); + } + } +} diff --git a/crates/notebook/src/smb/mod.rs b/crates/notebook/src/smb/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..e8ccdff841bd0e37acf6d4c57eb1198a76d7c859 --- /dev/null +++ b/crates/notebook/src/smb/mod.rs @@ -0,0 +1,518 @@ +//! Blocking SMB access to OneNote sections, table-of-contents files and payloads. + +use onestore::{CommitError, CommitIo, CommitState, ExGuid}; +use smb2::{ + Session, Tree, + client::connection::{Connection, NegotiatedParams}, + msg::{ + close::{CloseRequest, CloseResponse}, + create::{ + CreateDisposition, CreateRequest, CreateResponse, ImpersonationLevel, ShareAccess, + }, + flush::{FlushRequest, FlushResponse}, + lock::{LockElement, LockRequest, LockResponse}, + query_info::{InfoType, QueryInfoRequest, QueryInfoResponse}, + read::{ReadRequest, ReadResponse}, + write::{WriteRequest, WriteResponse}, + }, + pack::{Pack, ReadCursor, Unpack}, + types::{ + Command, CreditCharge, Dialect, FileId, OplockLevel, + flags::{Capabilities, FileAccessMask}, + }, +}; +use std::{io, ops::Range, sync::Mutex, time::Duration}; +use tokio::runtime::{Handle, Runtime}; + +mod directory; +mod remote; +pub use directory::DirectoryEntry; +pub use remote::SmbRemote; + +#[derive(Default)] +pub struct Credentials<'a> { + pub username: &'a str, + pub password: &'a str, + pub domain: &'a str, +} + +/// Blocking connection with no automatic request replay or cached file contents. +/// Call from a background thread outside a Tokio runtime. +/// Paths are relative to the share; both `/` and `\` are separators. +pub struct Client { + connection: Mutex>, + tree: Tree, + timeout: Duration, + runtime: Mutex>, +} + +impl Client { + pub fn connect( + address: &str, + share: &str, + credentials: Credentials<'_>, + timeout: Duration, + ) -> io::Result { + if Handle::try_current().is_ok() || timeout.is_zero() { + return Err(io::ErrorKind::InvalidInput.into()); + } + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build()?; + let (connection, tree) = runtime + .block_on(async { + tokio::time::timeout(timeout, async { + let mut connection = Connection::connect(address, timeout).await?; + connection.set_compression_requested(false); + connection.negotiate().await?; + Session::setup( + &mut connection, + credentials.username, + credentials.password, + credentials.domain, + ) + .await?; + let tree = Tree::connect(&mut connection, share).await?; + Ok::<_, smb2::Error>((connection, tree)) + }) + .await + }) + .map_err(|_| io::Error::from(io::ErrorKind::TimedOut))? + .map_err(io::Error::other)?; + Ok(Self { + connection: Mutex::new(Some(connection)), + tree, + timeout, + runtime: Mutex::new(Some(runtime)), + }) + } + + fn retire(&self) { + if let Some(connection) = self + .connection + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take() + { + connection.mark_dead(); + } + if let Some(runtime) = self + .runtime + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take() + { + runtime.shutdown_background(); + } + } + + fn request(&self, command: Command, body: impl Pack) -> io::Result { + self.request_with(command, |_| (body, CreditCharge(1))) + } + + fn request_with( + &self, + command: Command, + prepare: impl FnOnce(&Connection) -> (B, CreditCharge), + ) -> io::Result { + if Handle::try_current().is_ok() { + return Err(io::ErrorKind::InvalidInput.into()); + } + let connection = self + .connection + .lock() + .map_err(|_| io::ErrorKind::Other)? + .clone() + .ok_or(io::ErrorKind::NotConnected)?; + let frame = { + let runtime = self.runtime.lock().map_err(|_| io::ErrorKind::Other)?; + let (body, charge) = prepare(&connection); + runtime + .as_ref() + .ok_or(io::ErrorKind::NotConnected)? + .block_on(async { + tokio::time::timeout( + self.timeout, + connection.execute_with_credits( + command, + &body, + Some(self.tree.tree_id), + charge, + ), + ) + .await + }) + }; + let frame = frame + .map_err(|_| io::Error::from(io::ErrorKind::TimedOut)) + .and_then(|result| result.map_err(io::Error::other)) + .inspect_err(|_| self.retire())?; + if frame.header.command != command { + self.retire(); + return Err(io::ErrorKind::InvalidData.into()); + } + if frame.header.status.0 != 0 { + let kind = match frame.header.status.0 { + 0xc0000043 | 0xc0000054 | 0xc0000055 => io::ErrorKind::WouldBlock, + 0xc0000011 => io::ErrorKind::UnexpectedEof, + 0xc0000034 | 0xc000003a => io::ErrorKind::NotFound, + 0xc0000022 => io::ErrorKind::PermissionDenied, + 0xc0000103 => io::ErrorKind::NotADirectory, + _ => io::ErrorKind::Other, + }; + return Err(io::Error::new( + kind, + smb2::Error::Protocol { + status: frame.header.status, + command, + }, + )); + } + T::unpack(&mut ReadCursor::new(&frame.body)).map_err(|error| { + self.retire(); + io::Error::new(io::ErrorKind::InvalidData, error) + }) + } + + fn open(&self, path: &str, write: bool) -> io::Result> { + self.open_shared(path, write, if write { 5 } else { 7 }) + } + + fn open_shared(&self, path: &str, write: bool, sharing: u32) -> io::Result> { + if path.is_empty() || path.contains('\0') || path.encode_utf16().count() > 32767 { + return Err(io::ErrorKind::InvalidInput.into()); + } + let response: CreateResponse = self.request( + Command::Create, + CreateRequest { + requested_oplock_level: OplockLevel::None, + impersonation_level: ImpersonationLevel::Impersonation, + desired_access: FileAccessMask::new(if write { 0xc0000000 } else { 0x80000000 }), + file_attributes: 0, + share_access: ShareAccess(sharing), + create_disposition: CreateDisposition::FileOpen, + create_options: 0x42, + name: smb2::encode_path(&path.replace('\\', "/")), + create_contexts: Vec::new(), + }, + )?; + Ok(File { + client: self, + id: Some(response.file_id), + }) + } + + /// Reads a bounded external payload while denying concurrent writes and deletion. + /// Empty files succeed; limits, sharing contention and failed close return no payload. + pub fn read_asset(&self, path: &str, limit: usize) -> io::Result> { + let mut file = self.open_shared(path, false, 1)?; + let mut bytes = Vec::new(); + let mut block = [0; 65536]; + loop { + let count = (limit - bytes.len()).min(block.len() - 1) + 1; + let read = file.read_at( + u64::try_from(bytes.len()).map_err(|_| io::ErrorKind::InvalidInput)?, + &mut block[..count], + )?; + if read == 0 { + break; + } + bytes.extend_from_slice(&block[..read]); + if bytes.len() > limit { + return Err(io::ErrorKind::FileTooLarge.into()); + } + } + file.close()?; + Ok(bytes) + } + + /// Reads one bounded, consistent snapshot; contention returns WouldBlock. + pub fn read(&self, path: &str, limit: usize) -> io::Result> { + self.read_with(path, |file| { + onestore::read_snapshot(|offset, output| file.read_at(offset, output), limit) + }) + } + + /// Reads consistent storage, including encrypted or incomplete document graphs. + /// Storage validation alone does not establish edit readiness. + pub fn read_storage(&self, path: &str, limit: usize) -> io::Result> { + self.read_with(path, |file| { + onestore::read_storage_snapshot(|offset, output| file.read_at(offset, output), limit) + }) + } + + fn read_with( + &self, + path: &str, + snapshot: impl FnOnce(&mut File<'_>) -> io::Result>>, + ) -> io::Result> { + let mut file = self.open(path, false)?.coordinate(path, false)?; + let result = snapshot(&mut file) + .and_then(|snapshot| snapshot.ok_or_else(|| io::ErrorKind::WouldBlock.into())); + let closed = file.close(); + let snapshot = result?; + closed?; + Ok(snapshot) + } + + pub fn commit_text( + &self, + path: &str, + source: &[u8], + space: ExGuid, + object: ExGuid, + range: Range, + replacement: &str, + ) -> Result<(), CommitError> { + self.commit(path, |file| { + onestore::commit_text(file, source, space, object, range, replacement) + }) + } + + pub fn commit_property_bytes( + &self, + path: &str, + source: &[u8], + space: ExGuid, + object: ExGuid, + property: u32, + value: &[u8], + ) -> Result<(), CommitError> { + self.commit(path, |file| { + onestore::commit_property_bytes(file, source, space, object, property, value) + }) + } + + /// Publishes a prepared edit using the same native writer coordination as text commits. + pub fn commit_prepared( + &self, + path: &str, + edit: &onestore::PreparedEdit<'_>, + ) -> Result<(), CommitError> { + self.commit(path, |file| edit.commit(file)) + } + + /// Confirms an observed snapshot's durability under native writer coordination. + pub fn confirm_snapshot(&self, path: &str, source: &[u8]) -> Result<(), CommitError> { + self.commit(path, |file| onestore::confirm_snapshot(file, source)) + } + + fn commit( + &self, + path: &str, + operation: impl FnOnce(&mut File<'_>) -> Result<(), CommitError>, + ) -> Result<(), CommitError> { + let mut file = self + .open(path, true) + .and_then(|file| file.coordinate(path, true)) + .map_err(|error| CommitError { + state: CommitState::NotCommitted, + error, + })?; + let result = operation(&mut file); + let closed = file.close(); + result?; + closed.map_err(|error| CommitError { + state: CommitState::Committed, + error, + }) + } +} + +impl Drop for Client { + fn drop(&mut self) { + self.retire(); + } +} + +struct File<'a> { + client: &'a Client, + id: Option, +} +impl File<'_> { + fn coordinate(self, path: &str, write: bool) -> io::Result { + self.lock(0xfffffffb, 0x11)?; + if write { + self.lock(0xfffffffd, 0x12)?; + } + let current = self.client.open(path, false)?; + let same = self.identity()? == current.identity()?; + current.close()?; + if !same { + return Err(io::ErrorKind::ResourceBusy.into()); + } + Ok(self) + } + + fn lock(&self, offset: u64, flags: u32) -> io::Result<()> { + let _: LockResponse = self.client.request( + Command::Lock, + LockRequest { + file_id: self.id.unwrap(), + lock_sequence: 0, + locks: vec![LockElement { + offset, + length: 1, + flags, + }], + }, + )?; + Ok(()) + } + + fn identity(&self) -> io::Result<(u64, u32)> { + let index: QueryInfoResponse = self.client.request( + Command::QueryInfo, + QueryInfoRequest { + info_type: InfoType::File, + file_info_class: 6, + output_buffer_length: 8, + additional_information: 0, + flags: 0, + file_id: self.id.unwrap(), + input_buffer: Vec::new(), + }, + )?; + let index = u64::from_le_bytes( + index + .output_buffer + .try_into() + .map_err(|_| io::ErrorKind::InvalidData)?, + ); + if index == 0 { + return Err(io::ErrorKind::Unsupported.into()); + } + let volume: QueryInfoResponse = self.client.request( + Command::QueryInfo, + QueryInfoRequest { + info_type: InfoType::Filesystem, + file_info_class: 1, + output_buffer_length: 1024, + additional_information: 0, + flags: 0, + file_id: self.id.unwrap(), + input_buffer: Vec::new(), + }, + )?; + let serial = volume + .output_buffer + .get(8..12) + .ok_or(io::ErrorKind::InvalidData)?; + Ok((index, u32::from_le_bytes(serial.try_into().unwrap()))) + } + + fn close(mut self) -> io::Result<()> { + self.release() + } + + fn release(&mut self) -> io::Result<()> { + if let Some(file_id) = self.id.take() { + let result: io::Result = self + .client + .request(Command::Close, CloseRequest { file_id, flags: 0 }); + if result.is_err() { + self.client.retire(); + } + result?; + } + Ok(()) + } +} +impl Drop for File<'_> { + fn drop(&mut self) { + let _ = self.release(); + } +} +fn read_size(params: &NegotiatedParams, credits: u16, requested: usize) -> usize { + let budget = if params.dialect != Dialect::Smb2_0_2 + && params.capabilities.contains(Capabilities::LARGE_MTU) + { + usize::from(credits.max(1)) * 65536 + } else { + 65536 + }; + requested + .min(params.max_read_size as usize) + .min(budget) + .min(1024 * 1024) +} + +impl CommitIo for File<'_> { + fn read_at(&mut self, offset: u64, output: &mut [u8]) -> io::Result { + if output.is_empty() { + return Ok(0); + } + let mut size = 0; + let response: io::Result = + self.client.request_with(Command::Read, |connection| { + size = read_size( + &connection + .params() + .expect("connected SMB session is negotiated"), + connection.credits(), + output.len(), + ); + ( + ReadRequest { + file_id: self.id.unwrap(), + offset, + length: size as u32, + minimum_count: 1, + flags: 0, + padding: 0, + channel: 0, + remaining_bytes: 0, + read_channel_info: Vec::new(), + }, + CreditCharge(size.div_ceil(65536) as u16), + ) + }); + let response = match response { + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(0), + result => result?, + }; + if response.data.len() > size { + self.client.retire(); + return Err(io::ErrorKind::InvalidData.into()); + } + output[..response.data.len()].copy_from_slice(&response.data); + Ok(response.data.len()) + } + fn write_at(&mut self, offset: u64, bytes: &[u8]) -> io::Result { + if bytes.is_empty() { + return Ok(0); + } + let size = bytes.len().min(65536); + let response: WriteResponse = self.client.request( + Command::Write, + WriteRequest { + file_id: self.id.unwrap(), + offset, + data: bytes[..size].to_vec(), + data_offset: 112, + flags: 1, + channel: 0, + remaining_bytes: 0, + write_channel_info_offset: 0, + write_channel_info_length: 0, + }, + )?; + if response.count as usize > size { + return Err(io::ErrorKind::InvalidData.into()); + } + Ok(response.count as usize) + } + fn flush(&mut self) -> io::Result<()> { + let _: FlushResponse = self.client.request( + Command::Flush, + FlushRequest { + file_id: self.id.unwrap(), + }, + )?; + Ok(()) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/notebook/src/smb/remote.rs b/crates/notebook/src/smb/remote.rs new file mode 100644 index 0000000000000000000000000000000000000000..7dc3c146d32c21ada33ebedc6e4123232b953a3c --- /dev/null +++ b/crates/notebook/src/smb/remote.rs @@ -0,0 +1,36 @@ +use super::Client; +use crate::Remote; +use onestore::{CommitError, PreparedEdit}; +use std::io; + +/// Binds every reconciliation operation to one share-relative file and read limit. +/// Connection loss retires the client; reconnect before subsequent sync attempts. +pub struct SmbRemote { + client: Client, + path: String, + limit: usize, +} + +impl SmbRemote { + pub fn new(client: Client, path: impl Into, limit: usize) -> Self { + Self { + client, + path: path.into(), + limit, + } + } +} + +impl Remote for SmbRemote { + fn read(&mut self) -> io::Result> { + self.client.read(&self.path, self.limit) + } + + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + self.client.commit_prepared(&self.path, edit) + } + + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + self.client.confirm_snapshot(&self.path, snapshot) + } +} diff --git a/crates/notebook/src/smb/tests.rs b/crates/notebook/src/smb/tests.rs new file mode 100644 index 0000000000000000000000000000000000000000..94b3dbd9638288fc2ea2c0fc69b6a8a6f13a3f22 --- /dev/null +++ b/crates/notebook/src/smb/tests.rs @@ -0,0 +1,507 @@ +use super::*; +use onestore::{ + RevisionIndex, Store, + document::{Document, Kind}, +}; +use std::{ + fs, + time::{Instant, SystemTime, UNIX_EPOCH}, +}; + +mod faults; + +#[test] +#[ignore = "requires an owned Samba directory and ONESTORE_SMB_DIRECTORY_ORACLE from its local filesystem"] +fn live_directory() { + let client = client(); + let bytes = fs::read(std::env::var("ONESTORE_SMB_DIRECTORY_ORACLE").unwrap()).unwrap(); + let oracle: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + let root = oracle["path"].as_str().unwrap(); + let expected = oracle["entries"].as_array().unwrap(); + let entries = client.read_dir(root, expected.len()).unwrap(); + assert_eq!(entries.len(), expected.len()); + for expected in expected { + let entry = entries + .iter() + .find(|entry| entry.name == expected["name"]) + .unwrap(); + let directory = expected["directory"].as_bool().unwrap(); + assert_eq!(entry.attributes & 0x10 != 0, directory, "{}", entry.name); + if !directory { + assert_eq!( + entry.size, + expected["size"].as_u64().unwrap(), + "{}", + entry.name + ); + } + } + assert_eq!( + client.read_dir(root, entries.len() - 1).unwrap_err().kind(), + io::ErrorKind::FileTooLarge + ); + assert_eq!(client.read_dir(root, entries.len()).unwrap(), entries); + assert!( + client + .read_dir(&format!("{root}/empty"), 0) + .unwrap() + .is_empty() + ); + assert_eq!( + client + .read_dir(&format!("{root}/missing"), 1) + .unwrap_err() + .kind(), + io::ErrorKind::NotFound + ); + assert_eq!( + client + .read_dir(&format!("{root}/file.one"), 1) + .unwrap_err() + .kind(), + io::ErrorKind::NotADirectory + ); + assert_eq!( + client + .read_dir(&format!("{root}/denied"), 1) + .unwrap_err() + .kind(), + io::ErrorKind::PermissionDenied + ); + assert!( + client + .read_dir("", 10_000) + .unwrap() + .iter() + .any(|entry| entry.name == root) + ); +} + +#[test] +#[ignore = "requires an owned Samba directory through a proxy that interrupts a later directory page or close"] +fn live_directory_interruption() { + let client = client(); + let root = std::env::var("ONESTORE_SMB_DIRECTORY").unwrap(); + let started = Instant::now(); + assert!(client.read_dir(&root, 10_000).is_err()); + assert!(started.elapsed() < Duration::from_secs(10)); + assert_eq!( + client.read_dir(&root, 10_000).unwrap_err().kind(), + io::ErrorKind::NotConnected + ); +} + +fn client() -> Client { + Client::connect( + &std::env::var("ONESTORE_SMB_LAB").unwrap(), + "agent", + Credentials::default(), + Duration::from_secs(5), + ) + .unwrap() +} +fn create(client: &Client, path: &str, bytes: &[u8]) { + let response: CreateResponse = client + .request( + Command::Create, + CreateRequest { + requested_oplock_level: OplockLevel::None, + impersonation_level: ImpersonationLevel::Impersonation, + desired_access: FileAccessMask::new(0xc0000000), + file_attributes: 0, + share_access: ShareAccess(7), + create_disposition: CreateDisposition::FileCreate, + create_options: 0x42, + name: path.to_owned(), + create_contexts: Vec::new(), + }, + ) + .unwrap(); + let mut file = File { + client, + id: Some(response.file_id), + }; + let mut offset = 0; + while offset < bytes.len() { + offset += file.write_at(offset as u64, &bytes[offset..]).unwrap(); + } + file.flush().unwrap(); + file.close().unwrap(); +} +fn text(bytes: &[u8]) -> (ExGuid, ExGuid, String) { + let store = Store::parse(bytes).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + let doc = Document::parse(&index).unwrap(); + doc.spaces + .iter() + .find_map(|(sid, space)| { + let revision = space.active().unwrap(); + revision + .nodes + .iter() + .find_map(|(oid, node)| match &node.kind { + Kind::RichText { text, .. } => Some((*sid, *oid, text.clone())), + _ => None, + }) + }) + .unwrap() +} +#[test] +#[ignore = "requires disposable Samba and an existing ONESTORE_SMB_EVIDENCE directory"] +fn live_coordination() { + let output = std::env::var("ONESTORE_SMB_EVIDENCE").unwrap(); + let writer = client(); + let path = format!( + "adapter-{}.one", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + let source = onestore::create_section(&path, "Before café 🦀", "Fixture").unwrap(); + create(&writer, &path, &source); + assert_eq!(writer.read(&path, 1 << 20).unwrap(), source); + let readers: Vec<_> = (0..12).map(|_| client()).collect(); + let mut held: Vec<_> = readers + .iter() + .map(|client| { + client + .open(&path, false) + .unwrap() + .coordinate(&path, false) + .unwrap() + }) + .collect(); + let (sid, oid, before) = text(&source); + let replacement = "After café 🦀"; + writer + .commit_text( + &path, + &source, + sid, + oid, + 0..before.encode_utf16().count() as u32, + replacement, + ) + .unwrap(); + let after = writer.read(&path, 1 << 20).unwrap(); + assert_eq!(text(&after).2, replacement); + for file in &mut held { + let snapshot = onestore::read_snapshot(|offset, out| file.read_at(offset, out), 1 << 20) + .unwrap() + .unwrap(); + assert_eq!(snapshot, after); + } + let error = writer + .commit_text(&path, &source, sid, oid, 0..1, "X") + .unwrap_err(); + assert_eq!(error.state, CommitState::NotCommitted); + assert_eq!(error.error.kind(), io::ErrorKind::ResourceBusy); + assert_eq!(writer.read(&path, 1 << 20).unwrap(), after); + drop(held); + + let stale = writer.open(&path, true).unwrap(); + let maintenance = client(); + let guard = maintenance.open(&path, false).unwrap(); + let _: LockResponse = maintenance + .request( + Command::Lock, + LockRequest { + file_id: guard.id.unwrap(), + lock_sequence: 0, + locks: vec![ + LockElement { + offset: 0xfffffffc, + length: 1, + flags: 0x12, + }, + LockElement { + offset: 0xffffeffc, + length: 4096, + flags: 0x12, + }, + ], + }, + ) + .unwrap(); + let replacement_path = format!("{path}.replacement"); + create(&maintenance, &replacement_path, &source); + let mut connection = maintenance + .connection + .lock() + .unwrap() + .as_ref() + .unwrap() + .clone(); + maintenance + .runtime + .lock() + .unwrap() + .as_ref() + .unwrap() + .block_on( + maintenance + .tree + .rename(&mut connection, &path, &format!("{path}.old")), + ) + .unwrap(); + maintenance + .runtime + .lock() + .unwrap() + .as_ref() + .unwrap() + .block_on( + maintenance + .tree + .rename(&mut connection, &replacement_path, &path), + ) + .unwrap(); + drop(connection); + guard.close().unwrap(); + let error = stale.coordinate(&path, true).err().unwrap(); + assert_eq!(error.kind(), io::ErrorKind::ResourceBusy); + assert_eq!(writer.read(&path, 1 << 20).unwrap(), source); + + let retiring = client(); + let file = retiring + .open(&path, true) + .unwrap() + .coordinate(&path, true) + .unwrap(); + retiring.retire(); + assert_eq!( + retiring.read(&path, 1 << 20).unwrap_err().kind(), + io::ErrorKind::NotConnected + ); + drop(file); + let deadline = Instant::now() + Duration::from_secs(5); + loop { + match writer + .open(&path, true) + .and_then(|file| file.coordinate(&path, true)) + { + Ok(file) => { + file.close().unwrap(); + break; + } + Err(error) + if error.kind() == io::ErrorKind::WouldBlock && Instant::now() < deadline => + { + std::thread::sleep(Duration::from_millis(10)) + } + Err(error) => panic!("retired connection retained locks: {error}"), + } + } + + let mut unfinished = writer + .open(&path, true) + .unwrap() + .coordinate(&path, true) + .unwrap(); + assert_eq!( + unfinished + .write_at(source.len() as u64, b"unpublished") + .unwrap(), + 11 + ); + unfinished.flush().unwrap(); + unfinished.close().unwrap(); + let snapshot = writer.read(&path, 1 << 20).unwrap(); + assert_eq!(snapshot.len(), source.len() + 11); + assert_eq!(text(&snapshot).2, before); + writer + .commit_text( + &path, + &snapshot, + sid, + oid, + 0..before.encode_utf16().count() as u32, + "Recovered café 🦀", + ) + .unwrap(); + let recovered = writer.read(&path, 1 << 20).unwrap(); + assert_eq!(text(&recovered).2, "Recovered café 🦀"); + let spare = client(); + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() + .block_on(async { + drop(spare); + }); + fs::write(std::path::Path::new(&output).join("source.one"), &source).unwrap(); + fs::write(std::path::Path::new(&output).join("committed.one"), &after).unwrap(); + fs::write( + std::path::Path::new(&output).join("recovered.one"), + &recovered, + ) + .unwrap(); + println!( + "{}", + serde_json::json!({"path":path,"readers":12,"committed_while_readers_held":true,"fresh_reads":12,"stale_snapshot_rejected":true,"replaced_handle_rejected":true,"retirement_releases_locks":true,"unpublished_tail_recovered":true,"drop_inside_runtime":true}) + ); +} + +#[test] +#[ignore = "requires ONESTORE_SMB_LAB pointing to disposable Samba"] +fn live_storage_inspection() { + let reader = client(); + for (index, fixture) in [ + "native-encrypted/encrypted-01/notebook/synthetic.one", + "native-encrypted/cold-encrypted-02/notebook/Open Notebook.one", + ] + .into_iter() + .enumerate() + { + let source = fs::read(format!("../../corpus/{fixture}")).unwrap(); + let path = format!( + "inspection-{index}-{}.one", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + create(&reader, &path, &source); + assert_eq!(reader.read_storage(&path, source.len()).unwrap(), source); + assert_eq!( + reader.read(&path, source.len()).unwrap_err().kind(), + io::ErrorKind::WouldBlock + ); + let maintenance = client(); + let guard = maintenance.open(&path, false).unwrap(); + guard.lock(0xfffffffb, 0x12).unwrap(); + assert_eq!( + reader.read_storage(&path, source.len()).unwrap_err().kind(), + io::ErrorKind::WouldBlock + ); + guard.close().unwrap(); + assert_eq!(reader.read_storage(&path, source.len()).unwrap(), source); + } +} + +#[test] +#[ignore = "requires ONESTORE_SMB_LAB pointing to disposable Samba"] +fn live_assets() { + let reader = client(); + let writer = client(); + for size in [0, 1, 65535, 65536, 65537, 1048577] { + let bytes: Vec<_> = (0..size).map(|index| (index % 251) as u8).collect(); + let path = format!( + "asset-{size}-{}.onebin", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + create(&writer, &path, &bytes); + assert_eq!(reader.read_asset(&path, size).unwrap(), bytes); + if size != 0 { + assert_eq!( + reader.read_asset(&path, size - 1).unwrap_err().kind(), + io::ErrorKind::FileTooLarge + ); + assert_eq!(reader.read_asset(&path, size).unwrap(), bytes); + } + let writing = writer.open(&path, true).unwrap(); + assert_eq!( + reader.read_asset(&path, size).unwrap_err().kind(), + io::ErrorKind::WouldBlock + ); + writing.close().unwrap(); + let asset = reader.open_shared(&path, false, 1).unwrap(); + assert_eq!( + writer.open(&path, true).err().unwrap().kind(), + io::ErrorKind::WouldBlock + ); + let other = writer.open_shared(&path, false, 1).unwrap(); + other.close().unwrap(); + asset.close().unwrap(); + writer.open(&path, true).unwrap().close().unwrap(); + assert_eq!(reader.read_asset(&path, size).unwrap(), bytes); + } + assert_eq!( + reader + .read_asset("absent-payload.onebin", 0) + .unwrap_err() + .kind(), + io::ErrorKind::NotFound + ); +} + +#[test] +#[ignore = "requires an owned Samba fixture and maintenance controller"] +fn live_reader_hold() { + let client = client(); + let path = std::env::var("ONESTORE_SMB_PATH").unwrap(); + let output = std::path::PathBuf::from(std::env::var("ONESTORE_SMB_HOLD").unwrap()); + assert!(!output.join("ready").exists() && !output.join("release").exists()); + let mut file = client + .open(&path, false) + .unwrap() + .coordinate(&path, false) + .unwrap(); + fs::write(output.join("ready"), b"held").unwrap(); + let deadline = Instant::now() + Duration::from_secs(300); + let mut accepted = 0; + let mut retries = 0; + while !output.join("release").exists() { + assert!( + Instant::now() < deadline, + "maintenance controller timed out" + ); + match onestore::read_snapshot(|offset, out| file.read_at(offset, out), 256 << 20).unwrap() { + Some(_) => accepted += 1, + None => retries += 1, + } + std::thread::sleep(Duration::from_millis(5)); + } + file.close().unwrap(); + assert!(accepted > 0); + fs::write( + output.join("released.json"), + serde_json::to_vec(&serde_json::json!({"accepted": accepted, "retries": retries})).unwrap(), + ) + .unwrap(); +} + +#[test] +fn read_limits_respect_negotiation_and_available_credits() { + for dialect in Dialect::ALL { + for large_mtu in [false, true] { + for max_read_size in [65536, 65537, 131072, 1048576, u32::MAX] { + let params = NegotiatedParams { + dialect: *dialect, + max_read_size, + max_write_size: 65536, + max_transact_size: 65536, + server_guid: Default::default(), + signing_required: false, + capabilities: Capabilities(if large_mtu { + Capabilities::LARGE_MTU + } else { + 0 + }), + gmac_negotiated: false, + cipher: None, + compression_supported: false, + }; + for credits in [0, 1, 2, 3, 15, 16, 17, u16::MAX] { + for requested in [1, 1024, 65535, 65536, 65537, 131072, 1048576, usize::MAX] { + let size = read_size(¶ms, credits, requested); + assert!(size > 0 && size <= requested && size <= max_read_size as usize); + assert!(size <= 1048576); + assert!(size.div_ceil(65536) <= usize::from(credits.max(1))); + if *dialect == Dialect::Smb2_0_2 || !large_mtu { + assert!(size <= 65536); + } else if credits >= 16 && max_read_size >= 1048576 && requested >= 1048576 + { + assert_eq!(size, 1048576); + } + } + } + } + } + } +} diff --git a/crates/notebook/src/smb/tests/faults.rs b/crates/notebook/src/smb/tests/faults.rs new file mode 100644 index 0000000000000000000000000000000000000000..99641ee062cb06e9a4dad56f284ca954293b65fc --- /dev/null +++ b/crates/notebook/src/smb/tests/faults.rs @@ -0,0 +1,511 @@ +use super::*; +use serde_json::{Value, json}; +use std::{collections::BTreeMap, path::Path, process::Child}; + +#[derive(Clone)] +struct Snapshot { + content: BTreeMap, + modified: BTreeMap<(ExGuid, ExGuid), u32>, +} + +fn snapshot(bytes: &[u8]) -> Snapshot { + let store = Store::parse(bytes).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + Document::parse(&index).unwrap(); + let mut content = BTreeMap::new(); + let mut modified = BTreeMap::new(); + for (sid, space) in &index.spaces { + let revision = index + .resolve(*sid, space.labels[&(ExGuid::default(), 1)]) + .unwrap(); + let mut objects = BTreeMap::new(); + for (oid, object) in &revision.objects { + if let Some(onestore::FileDataReference::Internal(guid)) = + object.file_reference().unwrap() + { + store.file_data(guid).unwrap(); + } + let data = match object.data { + onestore::ObjectData::Properties(bytes) => { + let mut properties = onestore::PropertySets::parse(bytes).unwrap(); + for property in &mut properties.sets[0] { + if property.id == 0x14001d7a { + let onestore::Value::Bytes(value) = property.value else { + panic!() + }; + assert!( + modified + .insert( + (*sid, *oid), + u32::from_le_bytes(value.try_into().unwrap()) + ) + .is_none() + ); + property.value = onestore::Value::Bytes(&[0; 4]); + } + } + format!("{properties:?}") + } + other => format!("{other:?}"), + }; + objects.insert( + *oid, + ( + object.jcid, + object.reference_count, + data, + format!("{:?}", object.references().unwrap()), + ), + ); + } + content.insert(*sid, format!("{:?} {objects:?}", revision.roots)); + } + Snapshot { content, modified } +} + +fn stamp() -> u32 { + (SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() + - 315532800) + .try_into() + .unwrap() +} + +fn published( + actual: &Snapshot, + old: &Snapshot, + new: &Snapshot, + time: std::ops::RangeInclusive, +) -> bool { + if actual.content == old.content { + assert_eq!( + actual.modified, old.modified, + "Unpublished modification time changed" + ); + return false; + } + assert_eq!( + actual.content, new.content, + "Partial or unexpected publication" + ); + assert!(actual.modified.keys().eq(new.modified.keys())); + for (key, value) in &actual.modified { + if old.modified.get(key) != new.modified.get(key) { + assert!( + time.contains(value), + "Modification time outside the commit interval" + ); + } else { + assert_eq!( + *value, new.modified[key], + "Unrelated modification time changed" + ); + } + } + true +} + +#[test] +fn publication_oracle_bounds_changed_timestamps_and_preserves_others() { + let id = ExGuid::default(); + let other = ExGuid { n: 1, ..id }; + let old = Snapshot { + content: BTreeMap::from([(id, "old".into())]), + modified: BTreeMap::from([((id, id), 10), ((id, other), 7)]), + }; + let new = Snapshot { + content: BTreeMap::from([(id, "new".into())]), + modified: BTreeMap::from([((id, id), 20), ((id, other), 7)]), + }; + let mut actual = new.clone(); + actual.modified.insert((id, id), 30); + assert!(published(&actual, &old, &new, 25..=35)); + assert!(!published(&old, &old, &new, 25..=35)); + for (key, value) in [((id, id), 24), ((id, id), 36), ((id, other), 30)] { + let mut changed = actual.clone(); + changed.modified.insert(key, value); + assert!(std::panic::catch_unwind(|| published(&changed, &old, &new, 25..=35)).is_err()); + } + actual.content = old.content.clone(); + assert!(std::panic::catch_unwind(|| published(&actual, &old, &new, 25..=35)).is_err()); +} + +struct Proxy(Child); +impl Drop for Proxy { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +fn records(output: &Path) -> Vec { + let data = fs::read_to_string(output.join("proxy.jsonl")).unwrap(); + data.rsplit_once('\n') + .map_or("", |(complete, _)| complete) + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() +} + +fn configure(output: &Path, state: Value) -> usize { + fs::write(output.join("control.tmp"), state.to_string()).unwrap(); + fs::rename(output.join("control.tmp"), output.join("control.json")).unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let events = records(output); + if let Some(index) = events + .iter() + .rposition(|event| event.get("control") == Some(&state)) + { + return index + 1; + } + assert!( + Instant::now() < deadline, + "proxy did not acknowledge its control state" + ); + std::thread::sleep(Duration::from_millis(5)); + } +} + +fn proxy(output: &Path) -> (Proxy, String) { + let address = std::env::var("ONESTORE_SMB_LAB").unwrap(); + let (host, port) = address.rsplit_once(':').unwrap(); + let mut proxy = Proxy( + std::process::Command::new("python3") + .arg(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tools/smb-proxy.py")) + .arg(output.join("control.json")) + .args(["--port", "0", "--server", host, "--server-port", port]) + .stdout(fs::File::create(output.join("proxy.jsonl")).unwrap()) + .stderr(fs::File::create(output.join("proxy.stderr")).unwrap()) + .spawn() + .unwrap(), + ); + let deadline = Instant::now() + Duration::from_secs(5); + let port = loop { + if let Some(port) = records(output) + .iter() + .find_map(|event| event["listening"].as_u64()) + { + break port; + } + assert!(proxy.0.try_wait().unwrap().is_none()); + assert!(Instant::now() < deadline, "proxy did not start"); + std::thread::sleep(Duration::from_millis(5)); + }; + (proxy, format!("127.0.0.1:{port}")) +} + +#[test] +#[ignore = "requires an owned Samba share and a new ONESTORE_SMB_EVIDENCE directory"] +fn live_asset_loss() { + let output = std::path::PathBuf::from(std::env::var("ONESTORE_SMB_EVIDENCE").unwrap()); + fs::create_dir(&output).unwrap(); + let (_proxy, address) = proxy(&output); + let observer = client(); + let bytes: Vec<_> = (0..1048577).map(|index| (index % 251) as u8).collect(); + let path = format!( + "asset-loss-{}.onebin", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + create(&observer, &path, &bytes); + for (command, occurrence) in [(8, 1), (8, 9), (8, 17), (8, 18), (6, 1)] { + for direction in ["request", "response"] { + let reader = Client::connect( + &address, + "agent", + Credentials::default(), + Duration::from_secs(5), + ) + .unwrap(); + // Response occurrences count only replies with the selected status. + let begin = configure( + &output, + json!({"cut":command,"occurrence":if command == 8 && occurrence == 18 && direction == "response" { 1 } else { occurrence }, + "direction":direction,"status":if command == 8 && occurrence == 18 { "0xc0000011" } else { "0x0" }}), + ); + assert!( + reader.read_asset(&path, bytes.len()).is_err(), + "{command}/{occurrence}/{direction}" + ); + assert_eq!( + reader.read_asset(&path, bytes.len()).unwrap_err().kind(), + io::ErrorKind::NotConnected + ); + assert_eq!( + records(&output)[begin..] + .iter() + .filter(|row| row.get("cut").is_some()) + .count(), + 1 + ); + configure( + &output, + json!({"phase":format!("{command}-{occurrence}-{direction}-reconnected")}), + ); + let reconnected = Client::connect( + &address, + "agent", + Credentials::default(), + Duration::from_secs(5), + ) + .unwrap(); + assert_eq!(reconnected.read_asset(&path, bytes.len()).unwrap(), bytes); + } + } + assert_eq!(observer.read_asset(&path, bytes.len()).unwrap(), bytes); +} + +#[test] +#[ignore = "requires an owned Samba share and a new ONESTORE_SMB_EVIDENCE directory"] +fn live_message_loss() { + let output = std::path::PathBuf::from(std::env::var("ONESTORE_SMB_EVIDENCE").unwrap()); + fs::create_dir(&output).unwrap(); + fs::create_dir(output.join("interrupted")).unwrap(); + fs::create_dir(output.join("recovered")).unwrap(); + fs::create_dir(output.join("source")).unwrap(); + let (_proxy, proxied) = proxy(&output); + let observer = client(); + let prefix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let fixtures = [ + ( + "chunked", + onestore::create_section("fault.one", "Before café 🦀", "Fault test").unwrap(), + ), + ( + "carry", + fs::read( + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../corpus/append/round-01/tx-255/notebook/synthetic.one"), + ) + .unwrap(), + ), + ]; + let mut results = Vec::new(); + for (fixture, source) in fixtures { + fs::write( + output.join("source").join(format!("{fixture}.one")), + &source, + ) + .unwrap(); + let (sid, oid, before) = text(&source); + let after = if fixture == "chunked" { + "After café 🦀 ".repeat(6000) + } else { + "After café 🦀".to_owned() + }; + let suffix = " [reconnected]"; + fs::write( + output.join(format!("{fixture}-intent.json")), + serde_json::to_vec(&json!({"before":before,"after":after,"suffix":suffix})).unwrap(), + ) + .unwrap(); + let old = snapshot(&source); + let deadline = Instant::now() + Duration::from_secs(2); + while old.modified.values().any(|value| *value >= stamp()) { + assert!( + Instant::now() < deadline, + "source timestamps did not precede the test" + ); + std::thread::sleep(Duration::from_millis(5)); + } + let expected = onestore::replace_text( + &source, + sid, + oid, + 0..before.encode_utf16().count() as u32, + &after, + ) + .unwrap(); + let new = snapshot(&expected); + let baseline_path = format!("fault-{prefix}-{fixture}-baseline.one"); + create(&observer, &baseline_path, &source); + configure(&output, json!({"phase":format!("{fixture}-setup")})); + let baseline = Client::connect( + &proxied, + "agent", + Credentials::default(), + Duration::from_secs(5), + ) + .unwrap(); + let start = configure(&output, json!({"phase":format!("{fixture}-baseline")})); + let began = stamp(); + baseline + .commit_text( + &baseline_path, + &source, + sid, + oid, + 0..before.encode_utf16().count() as u32, + &after, + ) + .unwrap(); + let events = records(&output); + let mut occurrences = BTreeMap::new(); + let mut cuts = Vec::new(); + for event in &events[start..] { + let Some(direction) = event["direction"].as_str() else { + continue; + }; + if event["status"] == "0x103" { + continue; + } + let command = event["command"].as_u64().unwrap(); + let status = event["status"].as_str().map(str::to_owned); + let count = occurrences + .entry((direction.to_owned(), command, status.clone())) + .or_insert(0); + *count += 1; + cuts.push((direction.to_owned(), command, status, *count)); + } + assert!( + cuts.iter() + .any(|(direction, command, _, count)| direction == "response" + && *command == 7 + && *count == 3) + ); + assert!(published( + &snapshot(&observer.read(&baseline_path, 1 << 20).unwrap()), + &old, + &new, + began..=stamp() + )); + drop(baseline); + for (case, (direction, command, status, occurrence)) in cuts.iter().enumerate() { + let name = format!("{fixture}-{case:03}"); + let path = format!("fault-{prefix}-{name}.one"); + create(&observer, &path, &source); + configure(&output, json!({"phase":format!("{name}-setup")})); + let interrupted = Client::connect( + &proxied, + "agent", + Credentials::default(), + Duration::from_secs(5), + ) + .unwrap(); + let start = configure( + &output, + json!({"phase":name, "direction":direction, "cut":command, "status":status, "occurrence":occurrence}), + ); + let began = stamp(); + let error = interrupted + .commit_text( + &path, + &source, + sid, + oid, + 0..before.encode_utf16().count() as u32, + &after, + ) + .unwrap_err(); + assert_eq!( + interrupted.read(&path, 1 << 20).unwrap_err().kind(), + io::ErrorKind::NotConnected + ); + let events = records(&output); + assert_eq!( + events[start..] + .iter() + .filter(|event| event.get("cut").is_some()) + .count(), + 1 + ); + assert!( + !events + .iter() + .any(|event| event.get("trace_error").is_some()) + ); + let fresh = client(); + let deadline = Instant::now() + Duration::from_secs(5); + loop { + match fresh + .open(&path, true) + .and_then(|file| file.coordinate(&path, true)) + { + Ok(file) => { + file.close().unwrap(); + break; + } + Err(error) + if error.kind() == io::ErrorKind::WouldBlock + && Instant::now() < deadline => + { + std::thread::sleep(Duration::from_millis(5)) + } + Err(error) => panic!("{name}: retired session retained exclusion: {error}"), + } + } + let saved = fresh.read(&path, 1 << 20).unwrap(); + fs::write( + output.join("interrupted").join(format!("{name}.one")), + &saved, + ) + .unwrap(); + let visible = published(&snapshot(&saved), &old, &new, began..=stamp()); + match error.state { + CommitState::NotCommitted => assert!(!visible, "{name}"), + CommitState::Committed => assert!(visible, "{name}"), + CommitState::Unknown => {} + } + if !visible { + fresh + .commit_text( + &path, + &saved, + sid, + oid, + 0..before.encode_utf16().count() as u32, + &after, + ) + .unwrap(); + } + let saved = fresh.read(&path, 1 << 20).unwrap(); + assert!( + published(&snapshot(&saved), &old, &new, began..=stamp()), + "{name}: replay did not publish" + ); + let end = after.encode_utf16().count() as u32; + fresh + .commit_text(&path, &saved, sid, oid, end..end, suffix) + .unwrap(); + let recovered = fresh.read(&path, 1 << 20).unwrap(); + assert_eq!(text(&recovered).2, format!("{after}{suffix}")); + fs::write( + output.join("recovered").join(format!("{name}.one")), + recovered, + ) + .unwrap(); + results.push(json!({"case":name,"path":path,"direction":direction,"command":command,"status":status,"occurrence":occurrence, + "state":format!("{:?}",error.state),"visible":if visible {"after"} else {"before"}, + "retired_session_rejected":true,"exclusion_released":true,"fresh_commit_succeeded":true})); + fs::write( + output.join("results.json"), + serde_json::to_vec_pretty(&results).unwrap(), + ) + .unwrap(); + } + } + for state in ["NotCommitted", "Unknown", "Committed"] { + assert!(results.iter().any(|result| result["state"] == state)); + } + assert!( + results + .iter() + .any(|result| result["state"] == "Unknown" && result["visible"] == "before") + ); + assert!( + results + .iter() + .any(|result| result["state"] == "Unknown" && result["visible"] == "after") + ); + println!("{} message-loss cases passed", results.len()); +} diff --git a/crates/notebook/src/sync.rs b/crates/notebook/src/sync.rs new file mode 100644 index 0000000000000000000000000000000000000000..199845ff7184d5d3ca9d7b468fcde94d8d599468 --- /dev/null +++ b/crates/notebook/src/sync.rs @@ -0,0 +1,671 @@ +use super::*; +use onestore::{CommitError, CommitState}; +use rusqlite::OptionalExtension; +use std::{ + collections::BTreeMap, + sync::{MutexGuard, TryLockError}, +}; + +/// A single remote file with fresh reads and native-compatible guarded publication. +/// Errors retain publication state; confirmation compares, flushes, and notifies cached readers. +pub trait Remote { + fn read(&mut self) -> io::Result>; + fn publish(&mut self, edit: &PreparedEdit<'_>) -> std::result::Result<(), CommitError>; + fn confirm(&mut self, snapshot: &[u8]) -> std::result::Result<(), CommitError>; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i64)] +pub enum ConflictKind { + TextChanged = 0, + TargetUnavailable = 1, + UnsupportedEdit = 2, + FormattingChanged = 3, + StructureChanged = 4, + LayoutChanged = 5, + ContentChanged = 6, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EditStatus { + Pending, + /// Retained publication attempt; this revision alone may be insufficient to confirm it. + AwaitingConfirmation { + revision: ExGuid, + }, + Conflict(ConflictKind), + /// Revision of the intent's space when its complete effect was confirmed. + Published { + revision: ExGuid, + }, +} + +impl Replica { + /// Returns a durable receipt or the persisted state of a locally acknowledged edit. + pub fn status(&self, id: u64) -> Result> { + let connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + status(&connection, id) + } + + /// The last observed remote image, retained alongside the complete local working image. + /// Observation alone does not acknowledge any pending edit's remote durability. + pub fn remote_snapshot(&self) -> Result> { + let connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + Ok(connection.query_row("SELECT base FROM replica WHERE id=1", [], |row| row.get(0))?) + } + + /// Reconciles one pending edit, or refreshes the working image when the queue is empty. + /// Network I/O holds synchronization ownership without holding the cache mutex. + /// Uncertain edits are never replayed; retired formatting may confirm its complete observed effect. + pub fn sync_once(&self, remote: &mut impl Remote) -> Result> { + let _owner = self.sync_owner()?; + let snapshot = remote.read().map_err(Error::RemoteIo)?; + let identity = validate(&snapshot)?; + let (intent, attempted) = { + let mut connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + let transaction = + connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let base: Vec = + transaction + .query_row("SELECT base FROM replica WHERE id=1", [], |row| row.get(0))?; + let base_store = Store::parse(&base)?; + if RevisionIndex::parse(&base_store)?.root != identity { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Remote snapshot belongs to another document", + ) + .into()); + } + let Some(intent) = pending(&transaction)?.into_iter().next() else { + transaction.execute( + "UPDATE replica SET base=?1, working=?1 WHERE id=1", + [&snapshot], + )?; + transaction.commit()?; + return Ok(None); + }; + let attempted = transaction + .query_row( + "SELECT revisions FROM attempt WHERE edit_id=?1", + [i64::try_from(intent.id).map_err(io::Error::other)?], + |row| row.get::<_, String>(0), + ) + .optional()?; + transaction.execute("UPDATE replica SET base=?1 WHERE id=1", [&snapshot])?; + transaction.commit()?; + (intent, attempted) + }; + if let Some(encoded) = attempted { + let revisions = attempted_revisions(&encoded, intent.space)?; + let mut revision = revisions[&intent.space]; + let store = Store::parse(&snapshot)?; + let index = RevisionIndex::parse(&store)?; + if !revisions.iter().all(|(space, revision)| { + index + .spaces + .get(space) + .is_some_and(|space| space.revisions.contains_key(revision)) + }) { + let satisfied = match &intent.operation { + Operation::Format(edit) if revisions.len() == 1 => edit + .prepare(&snapshot, intent.space)? + .is_ok_and(|prepared| prepared.as_bytes() == snapshot), + _ => false, + }; + if !satisfied { + return Ok(Some(( + intent.id, + EditStatus::AwaitingConfirmation { revision }, + ))); + } + revision = index.active(intent.space)?; + } + if let Err(error) = remote.confirm(&snapshot) { + if error.state == CommitState::Committed { + self.acknowledge(intent.id, revision, &snapshot)?; + } + return Err(error.into()); + } + self.acknowledge(intent.id, revision, &snapshot)?; + return Ok(Some((intent.id, EditStatus::Published { revision }))); + } + let candidate = match &intent.operation { + Operation::CreatePage(page) => PreparedEdit::create_page(&snapshot, page) + .map_err(|_| ConflictKind::StructureChanged), + Operation::Pages(edit) => edit.prepare(&snapshot, intent.space)?, + Operation::Format(edit) => edit.prepare(&snapshot, intent.space)?, + Operation::Split(edit) => edit.prepare(&snapshot, intent.space)?, + Operation::Join(edit) => edit.prepare(&snapshot, intent.space)?, + Operation::Outline(edit) => edit.prepare(&snapshot, intent.space)?, + Operation::Tree(edit) => edit.prepare(&snapshot, intent.space)?, + Operation::Insert(insertion) => { + PreparedEdit::insert(&snapshot, intent.space, insertion) + .map_err(|_| ConflictKind::UnsupportedEdit) + } + Operation::Text(edit) => paragraph(&snapshot, intent.space, edit.object)? + .ok_or(ConflictKind::TargetUnavailable) + .and_then(|text| { + crate::rebase::rebase(&edit.before, &text, edit.range.clone()) + .ok_or(ConflictKind::TextChanged) + }) + .and_then(|range| { + PreparedEdit::text( + &snapshot, + intent.space, + edit.object, + range, + &edit.replacement, + ) + .map_err(|_| ConflictKind::UnsupportedEdit) + }), + }; + let prepared = match candidate { + Ok(prepared) => prepared, + Err(kind) => { + let connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + connection.execute("INSERT INTO conflicts(edit_id, kind) VALUES (?1, ?2) ON CONFLICT(edit_id) DO UPDATE SET kind=excluded.kind", params![i64::try_from(intent.id).map_err(io::Error::other)?, kind as i64])?; + return Ok(Some((intent.id, EditStatus::Conflict(kind)))); + } + }; + if prepared.as_bytes() == snapshot { + let store = Store::parse(&snapshot)?; + let index = RevisionIndex::parse(&store)?; + let revision = index.active(intent.space)?; + if let Err(error) = remote.confirm(&snapshot) { + if error.state == CommitState::Committed { + self.acknowledge(intent.id, revision, &snapshot)?; + } + return Err(error.into()); + } + self.acknowledge(intent.id, revision, &snapshot)?; + return Ok(Some((intent.id, EditStatus::Published { revision }))); + } + let store = Store::parse(prepared.as_bytes())?; + let index = RevisionIndex::parse(&store)?; + let revision = index.active(intent.space)?; + let before_store = Store::parse(&snapshot)?; + let before = RevisionIndex::parse(&before_store)?; + let mut revisions: BTreeMap<_, _> = index + .spaces + .iter() + .filter_map(|(sid, space)| { + let revision = *space.labels.get(&(ExGuid::default(), 1))?; + (before + .spaces + .get(sid) + .and_then(|s| s.labels.get(&(ExGuid::default(), 1))) + != Some(&revision)) + .then_some((*sid, revision)) + }) + .collect(); + // The receipt's space can be unchanged in a multi-space edit. + revisions.insert(intent.space, revision); + { + let mut connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + let transaction = + connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + transaction.execute( + "DELETE FROM conflicts WHERE edit_id=?1", + [i64::try_from(intent.id).map_err(io::Error::other)?], + )?; + transaction.execute( + "INSERT INTO attempt(id, edit_id, revisions) VALUES (1, ?1, ?2)", + params![ + i64::try_from(intent.id).map_err(io::Error::other)?, + serde_json::to_string(&revisions).map_err(io::Error::other)? + ], + )?; + transaction.commit()?; + } + match remote.publish(&prepared) { + Ok(()) => {} + Err(error) if error.state == CommitState::NotCommitted => { + let connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + connection.execute( + "DELETE FROM attempt WHERE edit_id=?1", + [i64::try_from(intent.id).map_err(io::Error::other)?], + )?; + return Err(error.into()); + } + Err(error) if error.state == CommitState::Committed => { + self.acknowledge(intent.id, revision, prepared.as_bytes())?; + return Err(error.into()); + } + Err(error) => return Err(error.into()), + } + self.acknowledge(intent.id, revision, prepared.as_bytes())?; + Ok(Some((intent.id, EditStatus::Published { revision }))) + } + + /// Places the oldest text, formatting or split conflict at a reviewed remote UTF-16 range. + /// The requested replacement/attributes, local image, intent ID and later edits are preserved. + /// Both supplied images must match `snapshot` and `remote_snapshot`; stale review + /// returns `Io(ResourceBusy)`. Uncertain publication attempts cannot be rebased. + pub fn rebase_conflict( + &self, + id: u64, + local: &[u8], + remote: &[u8], + range: Range, + ) -> Result<()> { + self.resolve_conflict(id, local, remote, |intent| { + Ok(match intent.operation { + Operation::Text(mut edit) => { + let prepared = PreparedEdit::text( + remote, + intent.space, + edit.object, + range.clone(), + &edit.replacement, + )?; + if prepared.as_bytes() == remote { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "The selected range already contains the replacement", + ) + .into()); + } + edit.before = + paragraph(remote, intent.space, edit.object)?.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "The remote text target is unavailable", + ) + })?; + edit.range = range; + Operation::Text(edit) + } + Operation::Format(edit) => Operation::Format( + FormatEdit::capture( + remote, + intent.space, + edit.object, + range, + &edit.attributes, + )? + .0, + ), + Operation::Split(edit) => { + if !range.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Choose a single split boundary", + ) + .into()); + } + Operation::Split( + crate::paragraph::SplitEdit::capture( + remote, + intent.space, + &edit.intent.reposition(range.start), + )? + .0, + ) + } + Operation::Join(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Review the paragraph join using rebase_join_conflict", + ) + .into()); + } + Operation::Outline(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Review layout changes using rebase_layout_conflict", + ) + .into()); + } + Operation::Tree(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Review the subtree edit using rebase_tree_conflict", + ) + .into()); + } + Operation::Insert(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Insertion conflicts require a placement, not a text range", + ) + .into()); + } + Operation::CreatePage(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Review page placement using rebase_page_creation_conflict", + ) + .into()); + } + Operation::Pages(_) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Review page edits using rebase_pages_conflict", + ) + .into()); + } + }) + }) + } + + /// Repositions an unattempted page-creation conflict, retaining dependent object identities. + pub fn rebase_page_creation_conflict( + &self, + id: u64, + local: &[u8], + remote: &[u8], + before: Option, + ) -> Result<()> { + self.resolve_conflict(id, local, remote, |intent| { + let Operation::CreatePage(page) = intent.operation else { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Select a page-creation conflict", + ) + .into()); + }; + let page = page.reposition(before)?; + PreparedEdit::create_page(remote, &page)?; + Ok(Operation::CreatePage(page)) + }) + } + + /// Reviews a page batch against both cache images, retaining its page and series identities. + pub fn rebase_pages_conflict( + &self, + id: u64, + local: &[u8], + remote: &[u8], + edits: &[onestore::PageEdit], + ) -> Result<()> { + self.resolve_conflict(id, local, remote, |intent| { + let Operation::Pages(batch) = intent.operation else { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Select a page movement or indentation conflict", + ) + .into()); + }; + Ok(Operation::Pages(batch.review( + remote, + intent.space, + edits, + )?)) + }) + } + + /// Reviews a join against both current cache images, retaining its original text identities. + /// The surviving text identity must remain the same for dependent edits. + pub fn rebase_join_conflict(&self, id: u64, local: &[u8], remote: &[u8]) -> Result<()> { + self.resolve_conflict(id, local, remote, |intent| { + let Operation::Join(edit) = intent.operation else { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Select a paragraph join conflict", + ) + .into()); + }; + Ok(Operation::Join(edit.review(remote, intent.space)?)) + }) + } + + /// Repositions the oldest paragraph insertion conflict against reviewed cache images. + /// Object identities and dependent edits are retained; uncertain attempts cannot be moved. + pub fn rebase_paragraph_conflict( + &self, + id: u64, + local: &[u8], + remote: &[u8], + parent: ExGuid, + before: Option, + ) -> Result<()> { + self.resolve_conflict(id, local, remote, |intent| { + let Operation::Insert(insertion) = intent.operation else { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Select a paragraph insertion conflict", + ) + .into()); + }; + let insertion = insertion.reposition_paragraph(parent, before)?; + PreparedEdit::insert(remote, intent.space, &insertion)?; + Ok(Operation::Insert(insertion)) + }) + } + + /// Repositions the oldest outline insertion conflict against reviewed cache images. + /// Object identities and dependent edits are retained; uncertain attempts cannot be moved. + pub fn rebase_outline_conflict( + &self, + id: u64, + local: &[u8], + remote: &[u8], + page: ExGuid, + x: f32, + y: f32, + ) -> Result<()> { + self.resolve_conflict(id, local, remote, |intent| { + let Operation::Insert(insertion) = intent.operation else { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Select an outline insertion conflict", + ) + .into()); + }; + let insertion = insertion.reposition_outline(page, x, y)?; + PreparedEdit::insert(remote, intent.space, &insertion)?; + Ok(Operation::Insert(insertion)) + }) + } + + /// Reviews the original layout intent against both current cache images. + /// Competing property values are replaced only after this explicit review. + pub fn rebase_layout_conflict(&self, id: u64, local: &[u8], remote: &[u8]) -> Result<()> { + self.resolve_conflict(id, local, remote, |intent| { + let Operation::Outline(edit) = intent.operation else { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Select an outline layout conflict", + ) + .into()); + }; + Ok(Operation::Outline( + OutlineEdit::capture(remote, intent.space, edit.object, edit.change)?.0, + )) + }) + } + + /// Reviews the original subtree intent against both current cache images. + /// Replacement paragraph identities must remain unchanged for dependent edits. + pub fn rebase_tree_conflict(&self, id: u64, local: &[u8], remote: &[u8]) -> Result<()> { + self.resolve_conflict(id, local, remote, |intent| { + let Operation::Tree(edit) = intent.operation else { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Select a subtree move or deletion conflict", + ) + .into()); + }; + Ok(Operation::Tree(edit.review(remote, intent.space)?)) + }) + } + + fn resolve_conflict( + &self, + id: u64, + local: &[u8], + remote: &[u8], + update: impl FnOnce(PendingEdit) -> Result, + ) -> Result<()> { + let owner = self.sync_owner()?; + let intent = self + .pending()? + .into_iter() + .next() + .filter(|intent| intent.id == id) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "Only the oldest conflict can be rebased", + ) + })?; + let operation = update(intent)?; + let mut connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let (base, working): (Vec, Vec) = + transaction.query_row("SELECT base, working FROM replica WHERE id=1", [], |row| { + Ok((row.get(0)?, row.get(1)?)) + })?; + if working != local || base != remote { + return Err(io::Error::new( + io::ErrorKind::ResourceBusy, + "The reviewed cache images changed", + ) + .into()); + } + let id = i64::try_from(id).map_err(io::Error::other)?; + let eligible: bool = transaction.query_row( + "SELECT EXISTS(SELECT 1 FROM conflicts WHERE edit_id=?1) AND NOT EXISTS(SELECT 1 FROM attempt)", + [id], |row| row.get(0), + )?; + if !eligible { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "The edit is not an unattempted conflict", + ) + .into()); + } + transaction.execute( + "UPDATE edits SET operation=?1 WHERE id=?2", + params![ + serde_json::to_string(&operation).map_err(io::Error::other)?, + id + ], + )?; + transaction.execute("DELETE FROM conflicts WHERE edit_id=?1", [id])?; + transaction.commit()?; + drop(connection); + drop(owner); + self.wake_sync(); + Ok(()) + } + + fn sync_owner(&self) -> Result> { + Ok(self + .synchronization + .try_lock() + .map_err(|error| match error { + TryLockError::WouldBlock => io::Error::from(io::ErrorKind::WouldBlock), + TryLockError::Poisoned(_) => { + io::Error::other("Synchronization owner panicked; reopen the cache") + } + })?) + } + + fn acknowledge(&self, id: u64, revision: ExGuid, snapshot: &[u8]) -> Result<()> { + let id = i64::try_from(id).map_err(io::Error::other)?; + let mut connection = self + .connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked"))?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + transaction.execute( + "INSERT INTO receipts(edit_id, revision) VALUES (?1, ?2)", + params![id, revision.to_string()], + )?; + transaction.execute("DELETE FROM edits WHERE id=?1", [id])?; + transaction.execute("UPDATE replica SET base=?1, working=CASE WHEN EXISTS(SELECT 1 FROM edits) THEN working ELSE ?1 END WHERE id=1", [snapshot])?; + transaction.commit()?; + Ok(()) + } +} + +pub(crate) fn status(connection: &Connection, id: u64) -> Result> { + let id = i64::try_from(id).map_err(io::Error::other)?; + if let Some(revision) = connection + .query_row( + "SELECT revision FROM receipts WHERE edit_id=?1", + [id], + |row| row.get::<_, String>(0), + ) + .optional()? + { + return Ok(Some(EditStatus::Published { + revision: revision.parse()?, + })); + } + let version: u32 = connection.pragma_query_value(None, "user_version", |row| row.get(0))?; + let column = if version < 10 { + "revision" + } else { + "revisions" + }; + let record: Option<(Option, Option, String)> = connection.query_row( + &format!("SELECT attempt.{column}, conflicts.kind, edits.space FROM edits LEFT JOIN attempt ON attempt.edit_id=edits.id LEFT JOIN conflicts ON conflicts.edit_id=edits.id WHERE edits.id=?1"), [id], |row| Ok((row.get(0)?,row.get(1)?, row.get(2)?))).optional()?; + Ok(match record { + None => None, + Some((Some(revision), _, space)) => Some(EditStatus::AwaitingConfirmation { + revision: if version < 10 { + revision.parse()? + } else { + let space = space.parse()?; + attempted_revisions(&revision, space)?[&space] + }, + }), + Some((None, Some(kind), _)) => Some(EditStatus::Conflict(match kind { + 0 => ConflictKind::TextChanged, + 1 => ConflictKind::TargetUnavailable, + 2 => ConflictKind::UnsupportedEdit, + 3 => ConflictKind::FormattingChanged, + 4 => ConflictKind::StructureChanged, + 5 => ConflictKind::LayoutChanged, + 6 => ConflictKind::ContentChanged, + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Unknown cached conflict kind", + ) + .into()); + } + })), + Some((None, None, _)) => Some(EditStatus::Pending), + }) +} + +fn attempted_revisions(encoded: &str, space: ExGuid) -> Result> { + let revisions: BTreeMap = serde_json::from_str(encoded) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + if !revisions.contains_key(&space) + || revisions + .iter() + .any(|(sid, rid)| sid.guid == [0; 16] || rid.guid == [0; 16]) + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Cached publication evidence is incomplete", + ) + .into()); + } + Ok(revisions) +} diff --git a/crates/notebook/src/tree.rs b/crates/notebook/src/tree.rs new file mode 100644 index 0000000000000000000000000000000000000000..83282746bbf20c1b3fb92b862b8ed02b3b4c35a5 --- /dev/null +++ b/crates/notebook/src/tree.rs @@ -0,0 +1,541 @@ +use super::*; +use onestore::{FileDataReference, IdStream, ObjectData, PropertySets, ResolvedRevision, Value}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged, deny_unknown_fields)] +enum Content { + Legacy([u8; 32]), + Semantic { sha256: [u8; 32] }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Observed { + path: Vec<(ExGuid, u8)>, + siblings: BTreeMap, + destination: Vec<(ExGuid, u8)>, + content: Option, +} + +/// A subtree intent with observed placement, deletion content and replacement identities. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TreeEdit { + pub intent: onestore::TreeEdit, + observed: Observed, + created: BTreeSet, +} + +fn fingerprint( + store: &Store<'_>, + raw: &ResolvedRevision<'_>, + root: ExGuid, + legacy: bool, +) -> Result<[u8; 32]> { + let mut objects = BTreeMap::new(); + let mut visiting = BTreeSet::new(); + let mut pending = vec![(root, false)]; + while let Some((id, complete)) = pending.pop() { + if objects.contains_key(&id) { + continue; + } + let object = &raw.objects[&id]; + let references = object.references()?; + if !complete { + if !visiting.insert(id) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Subtree property references form a cycle", + ) + .into()); + } + pending.push((id, true)); + pending.extend(references.objects.iter().map(|id| (*id, false))); + continue; + } + visiting.remove(&id); + let mut hash = Sha256::new(); + hash.update(object.jcid.to_le_bytes()); + match object.data { + ObjectData::Properties(bytes) => { + hash.update([0]); + let properties = PropertySets::parse(bytes)?; + let mut object_ids = references.objects.into_iter(); + let mut spaces = references.object_spaces.into_iter(); + let mut contexts = references.contexts.into_iter(); + let mut fields = Vec::new(); + for set in &properties.sets { + let mut values = BTreeMap::new(); + for property in set { + let mut value = Sha256::new(); + match &property.value { + Value::NoData => {} + Value::Bytes(bytes) => value.update(bytes), + Value::References { + stream, + compact_ids, + } => { + let references = match stream { + IdStream::Objects => &mut object_ids, + IdStream::ObjectSpaces => &mut spaces, + IdStream::Contexts => &mut contexts, + }; + for reference in references.take(compact_ids.len() / 4) { + if !legacy && *stream == IdStream::Objects { + let immutable = + raw.objects[&reference].jcid & 0x100000 != 0; + value.update([u8::from(immutable)]); + if !immutable { + value.update(reference.guid); + value.update(reference.n.to_le_bytes()); + } + value.update(objects[&reference]); + } else { + value.update(reference.guid); + value.update(reference.n.to_le_bytes()); + } + } + } + Value::Sets(_) => {} + } + if property.id == 0x14001d7a + || (!legacy + && property.id == 0x24001c20 + && matches!(&property.value, Value::References { compact_ids, .. } if compact_ids.is_empty())) + { + continue; + } + values.insert(property.id, value); + } + fields.push(values); + } + // Arena indices and CompactIDs can change without changing property content. + let mut sets = vec![[0; 32]; properties.sets.len()]; + for at in (0..properties.sets.len()).rev() { + for property in &properties.sets[at] { + if let Value::Sets(children) = &property.value { + let value = fields[at].get_mut(&property.id).unwrap(); + for child in children.clone() { + value.update(sets[child]); + } + } + } + let mut set = Sha256::new(); + for (id, value) in &fields[at] { + set.update(id.to_le_bytes()); + set.update(value.clone().finalize()); + } + sets[at] = set.finalize().into(); + } + hash.update(sets[0]); + } + ObjectData::File { + reference, + extension, + } => { + hash.update([1]); + hash.update(Sha256::digest(reference)); + hash.update(Sha256::digest(extension)); + if let Some(FileDataReference::Internal(guid)) = object.file_reference()? { + hash.update(Sha256::digest(store.file_data(guid)?)); + } + } + ObjectData::Encrypted(_) => unreachable!("references rejected encrypted content"), + } + objects.insert(id, <[u8; 32]>::from(hash.finalize())); + } + if !legacy { + return Ok(objects[&root]); + } + let mut hash = Sha256::new(); + for (id, value) in objects { + hash.update(id.guid); + hash.update(id.n.to_le_bytes()); + hash.update(value); + } + Ok(hash.finalize().into()) +} + +fn observe( + source: &[u8], + space: ExGuid, + intent: &onestore::TreeEdit, + legacy: bool, +) -> Result> { + let store = Store::parse(source)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + let Ok(view) = document.active(space) else { + return Ok(None); + }; + let pages = document.pages_in(space)?; + if pages.len() != 1 { + return Ok(None); + } + let object = intent.object(); + let mut targets = vec![object]; + if let Some((parent, _)) = intent.destination() { + targets.push(parent); + } + let Some(paths) = active_paths(view, &pages, &targets) else { + return Ok(None); + }; + let Some(parent) = paths[0].first() else { + return Ok(None); + }; + let children = &view.nodes[parent].children; + let Some(position) = children.iter().position(|id| *id == object) else { + return Ok(None); + }; + let levels = |path: &[ExGuid]| { + path.iter() + .map(|id| (*id, view.nodes[id].child_level.unwrap_or(1))) + .collect::>() + }; + let destination = if let Some((parent, _)) = intent.destination() { + levels(&[&[parent], paths[1].as_slice()].concat()) + } else { + Vec::new() + }; + Ok(Some(Observed { + path: levels(&paths[0]), + siblings: children + .iter() + .enumerate() + .filter_map(|(at, id)| (*id != object).then_some((*id, at < position))) + .collect(), + destination, + content: if intent.destination().is_none() { + let sha256 = fingerprint(&store, &index.resolve_active(space)?, object, legacy)?; + Some(if legacy { + Content::Legacy(sha256) + } else { + Content::Semantic { sha256 } + }) + } else { + None + }, + })) +} + +fn mutable_objects(source: &[u8], space: ExGuid) -> Result> { + let store = Store::parse(source)?; + let index = RevisionIndex::parse(&store)?; + let raw = index.resolve_active(space)?; + Ok(raw + .reachable()? + .into_iter() + .filter(|id| raw.objects[id].jcid & 0x100000 == 0) + .collect()) +} + +impl Replica { + /// Queues a move or deletion, retaining content changes for explicit deletion review. + pub fn tree( + &self, + source: &[u8], + space: ExGuid, + intent: &onestore::TreeEdit, + ) -> Result> { + let (edit, prepared) = TreeEdit::capture(source, space, intent)?; + self.record(source, space, Operation::Tree(edit), &prepared) + } +} + +impl TreeEdit { + pub(crate) fn capture<'a>( + source: &'a [u8], + space: ExGuid, + intent: &onestore::TreeEdit, + ) -> Result<(Self, PreparedEdit<'a>)> { + let prepared = PreparedEdit::tree(source, space, intent)?; + let observed = observe(source, space, intent, false)?.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "Prepared subtree edit has no active target", + ) + })?; + let before = mutable_objects(source, space)?; + let after = mutable_objects(prepared.as_bytes(), space)?; + Ok(( + Self { + intent: intent.clone(), + observed, + created: &after - &before, + }, + prepared, + )) + } + + pub(crate) fn prepare<'a>( + &self, + source: &'a [u8], + space: ExGuid, + ) -> Result, ConflictKind>> { + let Some(current) = observe( + source, + space, + &self.intent, + matches!(self.observed.content, Some(Content::Legacy(_))), + )? + else { + return Ok(Err(ConflictKind::TargetUnavailable)); + }; + let prepared = match PreparedEdit::tree(source, space, &self.intent) { + Ok(prepared) => prepared, + Err(_) => return Ok(Err(ConflictKind::UnsupportedEdit)), + }; + let after = mutable_objects(prepared.as_bytes(), space)?; + if prepared.as_bytes() == source { + return Ok(if self.created.is_subset(&after) { + Ok(prepared) + } else { + Err(ConflictKind::StructureChanged) + }); + } + if current.path != self.observed.path + || current.destination != self.observed.destination + || current.siblings.iter().any(|(id, before)| { + self.observed + .siblings + .get(id) + .is_some_and(|was_before| before != was_before) + }) + || &after - &mutable_objects(source, space)? != self.created + { + return Ok(Err(ConflictKind::StructureChanged)); + } + if current.content != self.observed.content { + return Ok(Err(ConflictKind::ContentChanged)); + } + Ok(Ok(prepared)) + } + + pub(crate) fn review(&self, source: &[u8], space: ExGuid) -> Result { + let (reviewed, _) = Self::capture(source, space, &self.intent)?; + if reviewed.created != self.created { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "The subtree edit would change its replacement paragraph identities", + ) + .into()); + } + Ok(reviewed) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use onestore::Object; + use std::sync::Arc; + + fn set(fields: &[(u32, Vec)]) -> Vec { + let mut bytes = u16::try_from(fields.len()).unwrap().to_le_bytes().to_vec(); + for (id, _) in fields { + bytes.extend(id.to_le_bytes()); + } + for (_, value) in fields { + bytes.extend(value); + } + bytes + } + + #[test] + fn deletion_fingerprint_resolves_references_and_nested_sets_without_storage_order() { + let source = onestore::create_section("fingerprint.one", "Original", "Author").unwrap(); + let store = Store::parse(&source).unwrap(); + let root = ExGuid { + guid: [1; 16], + n: 1, + }; + let child = ExGuid { + guid: [2; 16], + n: 7, + }; + let hash = |reverse: bool, global: u32, timestamp: u32, change: usize, immutable: bool| { + let mut fields = vec![ + (0x14001d7a, timestamp.to_le_bytes().to_vec()), + (0x20000001, vec![]), + ( + 0x44000002, + set(&[(0x14000003, (u32::from(change == 1)).to_le_bytes().to_vec())]), + ), + ( + 0x44000004, + set(&[(if change == 2 { 0x08000005 } else { 0x88000005 }, vec![])]), + ), + ]; + if change != 5 { + fields.push((0x24001c20, 0_u32.to_le_bytes().to_vec())); + } + if change == 6 { + fields.push((0x24000007, 0_u32.to_le_bytes().to_vec())); + } + if reverse { + fields.reverse(); + } + let mut bytes = 0x80000001_u32.to_le_bytes().to_vec(); + bytes.extend(((global << 8) | child.n).to_le_bytes()); + bytes.extend(set(&fields)); + let mut child_bytes = 0x80000000_u32.to_le_bytes().to_vec(); + child_bytes.extend(set(&[( + 0x14000006, + (u32::from(change == 3)).to_le_bytes().to_vec(), + )])); + let ids = Arc::new(BTreeMap::from([( + global, + if change == 4 { [3; 16] } else { child.guid }, + )])); + let target = ExGuid { + guid: ids[&global], + ..child + }; + let raw = ResolvedRevision { + roots: BTreeMap::from([(1, root)]), + objects: BTreeMap::from([ + ( + root, + Object { + jcid: 0x6000d, + reference_count: 1, + data: ObjectData::Properties(&bytes), + global_ids: ids.clone(), + }, + ), + ( + target, + Object { + jcid: if immutable { 0x12004d } else { 0x6000e }, + reference_count: 1, + data: ObjectData::Properties(&child_bytes), + global_ids: ids, + }, + ), + ]), + }; + fingerprint(&store, &raw, root, false).unwrap() + }; + let original = hash(false, 0, 1, 0, false); + assert_eq!(original, hash(true, 137, 2, 0, false)); + assert_eq!(original, hash(true, 137, 2, 5, false)); + assert_ne!(original, hash(true, 137, 2, 6, false)); + for change in 1..=4 { + assert_ne!(original, hash(true, 137, 2, change, false)); + } + let immutable = hash(false, 0, 1, 0, true); + assert_eq!(immutable, hash(true, 137, 2, 4, true)); + for change in 1..=3 { + assert_ne!(immutable, hash(true, 137, 2, change, true)); + } + } + + #[test] + fn version_eight_deletion_observations_survive_migration_and_upgrade_only_after_review() { + struct Server(Vec); + impl Remote for Server { + fn read(&mut self) -> io::Result> { + Ok(self.0.clone()) + } + fn publish( + &mut self, + edit: &PreparedEdit<'_>, + ) -> std::result::Result<(), onestore::CommitError> { + self.0 = edit.as_bytes().to_vec(); + Ok(()) + } + fn confirm( + &mut self, + snapshot: &[u8], + ) -> std::result::Result<(), onestore::CommitError> { + assert!(self.0 == snapshot); + Ok(()) + } + } + let source = onestore::create_section("legacy.one", "AlphaOmega", "Author").unwrap(); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let (sid, _) = document.pages().unwrap()[0]; + let view = document.active(sid).unwrap(); + let text = *view.nodes.iter().find(|(_, node)| matches!(&node.kind, Kind::RichText { text, .. } if text == "AlphaOmega")).unwrap().0; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("legacy.sqlite"); + let cache = Replica::create(&path, &source).unwrap(); + cache + .format( + &source, + sid, + text, + 0..10, + &[onestore::TextAttribute::Bold(true)], + ) + .unwrap(); + let split = onestore::ParagraphSplit::new(text, 5, "Author").unwrap(); + cache + .split(&cache.snapshot().unwrap(), sid, &split) + .unwrap(); + let before = cache.snapshot().unwrap(); + let intent = onestore::TreeEdit::delete(split.object(), "Author").unwrap(); + let deletion = cache.tree(&before, sid, &intent).unwrap().unwrap(); + let dependent = cache + .edit_text(&cache.snapshot().unwrap(), sid, text, 0..0, "Local ") + .unwrap() + .unwrap(); + let mut queue = cache.pending().unwrap(); + let Operation::Tree(edit) = &mut queue[2].operation else { + panic!() + }; + edit.observed = observe(&before, sid, &intent, true).unwrap().unwrap(); + let serialized = serde_json::to_string(&queue[2].operation).unwrap(); + let local = cache.snapshot().unwrap(); + drop(cache); + let db = Connection::open(&path).unwrap(); + db.execute( + "UPDATE edits SET operation=?1 WHERE id=?2", + params![serialized, i64::try_from(deletion).unwrap()], + ) + .unwrap(); + db.pragma_update(None, "user_version", 8).unwrap(); + db.execute_batch("ALTER TABLE attempt RENAME COLUMN revisions TO revision;") + .unwrap(); + drop(db); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), queue); + assert!(cache.snapshot().unwrap() == local); + let mut server = Server(source); + for _ in 0..2 { + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Published { .. })) + )); + } + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((deletion, EditStatus::Conflict(ConflictKind::ContentChanged))) + ); + cache + .rebase_tree_conflict(deletion, &local, &server.0) + .unwrap(); + let queue = cache.pending().unwrap(); + let Operation::Tree(edit) = &queue[0].operation else { + panic!() + }; + assert!(matches!( + edit.observed.content, + Some(Content::Semantic { .. }) + )); + for id in [deletion, dependent] { + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) + ); + } + assert_eq!( + paragraph(&server.0, sid, text).unwrap().as_deref(), + Some("Local Alpha") + ); + } +} diff --git a/crates/notebook/src/worker.rs b/crates/notebook/src/worker.rs new file mode 100644 index 0000000000000000000000000000000000000000..d506a60361e4adc11ead3d22c4b73a052ea07268 --- /dev/null +++ b/crates/notebook/src/worker.rs @@ -0,0 +1,167 @@ +use super::*; +use std::{ + collections::hash_map::RandomState, + hash::BuildHasher, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + mpsc::{self, SyncSender}, + }, + thread::{self, JoinHandle}, + time::Instant, +}; + +pub(super) struct Signal { + stopped: AtomicBool, + sender: SyncSender<()>, +} + +impl Signal { + pub(super) fn wake(&self) { + // One retained notification covers edits that arrive during network I/O. + let _ = self.sender.try_send(()); + } +} + +/// Owns automatic reconciliation. Dropping requests cancellation without blocking. +/// The in-flight sync step finishes before ownership is released; `stop` waits for it. +pub struct SyncWorker { + signal: Arc, + thread: Option>>, +} + +impl SyncWorker { + /// Requests a retry, for example after a network reachability change. + /// A pending contention backoff finishes before processing the notification. + pub fn wake(&self) { + self.signal.wake(); + } + + /// Cancels future steps and waits for the current step and callback to finish. + /// A stopped worker leaves pending edits and uncertain attempts in the cache. + /// Call outside the worker's own callback, which cannot join its calling thread. + pub fn stop(mut self) -> Result<()> { + self.signal.stopped.store(true, Ordering::Release); + self.signal.wake(); + self.thread + .take() + .expect("Worker owns its thread") + .join() + .map_err(|_| io::Error::other("Synchronization worker panicked"))? + } +} + +impl Drop for SyncWorker { + fn drop(&mut self) { + self.signal.stopped.store(true, Ordering::Release); + self.signal.wake(); + } +} + +impl Replica { + /// Starts one worker, reconnecting through `connect` after transport failures. + /// Local edits wake it; `interval` controls idle polling and transport retries. + /// Contended operations returning `NotCommitted` also back off by up to one second. + /// `observe` runs on the worker after each attempt, including connection errors. + /// Cache/document errors stop the worker; inspect them through `observe` or `stop`. + /// Remote calls and callbacks must be bounded for `stop` to have bounded latency. + pub fn start_sync( + self: &Arc, + interval: Duration, + mut connect: F, + mut observe: O, + ) -> io::Result + where + R: Remote + 'static, + F: FnMut() -> io::Result + Send + 'static, + O: FnMut(&Result>) + Send + 'static, + { + if interval.is_zero() || Instant::now().checked_add(interval).is_none() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Synchronization interval must be positive and representable", + )); + } + let mut owner = self + .worker + .lock() + .map_err(|_| io::Error::other("Synchronization worker registration panicked"))?; + if owner.upgrade().is_some() { + return Err(io::ErrorKind::WouldBlock.into()); + } + let (sender, receiver) = mpsc::sync_channel(1); + let signal = Arc::new(Signal { + stopped: AtomicBool::new(false), + sender, + }); + let replica = Arc::clone(self); + let worker_signal = Arc::clone(&signal); + let thread = thread::Builder::new() + .name("onestore-sync".into()) + .spawn(move || { + let mut remote = None; + let jitter = RandomState::new(); + let mut contention = 0_u32; + while !worker_signal.stopped.load(Ordering::Acquire) { + let result = match remote.as_mut() { + Some(remote) => replica.sync_once(remote), + None => match connect() { + Ok(connected) => { + remote = Some(connected); + continue; + } + Err(error) => Err(Error::RemoteIo(error)), + }, + }; + observe(&result); + match result { + Ok(Some((_, EditStatus::Published { .. }))) => { + contention = 0; + continue; + } + Ok(None) => contention = 0, + Ok(_) => {} + Err(Error::Remote(onestore::CommitError { + state: onestore::CommitState::NotCommitted, + ref error, + })) if matches!( + error.kind(), + io::ErrorKind::WouldBlock | io::ErrorKind::ResourceBusy + ) => + { + contention = contention.saturating_add(1); + let ceiling = (50_u64 << contention.min(5)).min(1000); + let until = Instant::now() + + Duration::from_millis(jitter.hash_one(contention) % ceiling); + // Local wakes must not keep competing writers in the same retry phase. + while !worker_signal.stopped.load(Ordering::Acquire) { + let Some(remaining) = until.checked_duration_since(Instant::now()) + else { + break; + }; + let _ = receiver.recv_timeout(remaining); + } + continue; + } + Err(Error::RemoteIo(ref error)) + if matches!( + error.kind(), + io::ErrorKind::WouldBlock | io::ErrorKind::ResourceBusy + ) => {} + Err(Error::RemoteIo(_) | Error::Remote(_)) => remote = None, + Err(Error::Io(ref error)) if error.kind() == io::ErrorKind::WouldBlock => {} + Err(error) => return Err(error), + } + if !worker_signal.stopped.load(Ordering::Acquire) { + let _ = receiver.recv_timeout(interval); + } + } + Ok(()) + })?; + *owner = Arc::downgrade(&signal); + Ok(SyncWorker { + signal, + thread: Some(thread), + }) + } +} diff --git a/crates/notebook/tests/assets.rs b/crates/notebook/tests/assets.rs new file mode 100644 index 0000000000000000000000000000000000000000..378c08b225879aaf6a246ca0e3b745c242030c98 --- /dev/null +++ b/crates/notebook/tests/assets.rs @@ -0,0 +1,449 @@ +use notebook::{EditStatus, Error, Operation, Recovery, Replica}; +use std::{ + fs, io, + path::Path, + sync::{Barrier, mpsc}, + time::Duration, +}; + +const FIXTURE: &str = "../../corpus/native-external-assets/notebook"; +const LEGACY: &str = "../../corpus/offline-v4/live.sqlite"; + +fn copied_cache(root: &Path) -> Replica { + let path = root.join("cache.sqlite"); + assert!(!path.exists()); + fs::copy(LEGACY, &path).unwrap(); + Replica::open(path).unwrap() +} + +fn payload(size: usize) -> (String, Vec) { + fs::read_dir(Path::new(FIXTURE).join("synthetic_onefiles")) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find_map(|path| { + let bytes = fs::read(&path).unwrap(); + (bytes.len() == size) + .then(|| (path.file_name().unwrap().to_str().unwrap().into(), bytes)) + }) + .unwrap() +} + +struct Payload(Option>>); +impl notebook::discover::Source for Payload { + fn entries(&mut self, _: &str, _: usize) -> io::Result> { + panic!("Unexpected enumeration") + } + fn read(&mut self, _: &str, _: usize) -> io::Result> { + self.0.take().expect("Unexpected repeated payload read") + } +} + +#[test] +fn downloaded_media_survives_migration_reopen_and_recovery_with_the_queue_intact() { + let directory = tempfile::tempdir().unwrap(); + let original = fs::read(LEGACY).unwrap(); + let cache = copied_cache(directory.path()); + let working = cache.snapshot().unwrap(); + let remote = cache.remote_snapshot().unwrap(); + let pending = cache.pending().unwrap(); + assert_eq!( + pending.iter().map(|edit| edit.id).collect::>(), + [1, 2] + ); + let uncertain = cache.status(1).unwrap(); + assert!(matches!( + uncertain, + Some(EditStatus::AwaitingConfirmation { .. }) + )); + let mut source = notebook::discover::Local::open(FIXTURE).unwrap(); + for size in [0, 1024] { + let (name, bytes) = payload(size); + assert!(cache.cached_asset(&name, size).unwrap().is_none()); + assert_eq!( + cache + .fetch_asset(&mut source, "synthetic.one", &name, size) + .unwrap(), + bytes + ); + assert_eq!( + cache + .cached_asset(&name.to_ascii_lowercase(), size) + .unwrap(), + Some(bytes) + ); + } + let summary = cache.recovery_summary().unwrap(); + assert_eq!( + (summary.cached_assets, summary.cached_asset_bytes), + (2, 1024) + ); + assert_eq!(cache.snapshot().unwrap(), working); + assert_eq!(cache.remote_snapshot().unwrap(), remote); + assert_eq!(cache.pending().unwrap(), pending); + assert_eq!(cache.status(1).unwrap(), uncertain); + cache + .export_recovery(directory.path().join("recovery.sqlite")) + .unwrap(); + drop(cache); + let cache = Replica::open(directory.path().join("cache.sqlite")).unwrap(); + let recovery = Recovery::open(directory.path().join("recovery.sqlite")).unwrap(); + assert_eq!(recovery.summary().unwrap(), summary); + assert_eq!(recovery.pending().unwrap(), pending); + assert_eq!(recovery.status(1).unwrap(), uncertain); + assert!(recovery.receipts().unwrap().is_empty()); + for size in [0, 1024] { + let (name, bytes) = payload(size); + assert_eq!( + cache.cached_asset(&name, size).unwrap(), + Some(bytes.clone()) + ); + assert_eq!(recovery.cached_asset(&name, size).unwrap(), Some(bytes)); + } + assert_eq!(fs::read(LEGACY).unwrap(), original); +} + +#[test] +fn an_original_version_four_archive_remains_read_only_and_has_no_cached_media() { + let path = "../../corpus/offline-v4/recovery.sqlite"; + let before = fs::read(path).unwrap(); + let archive = Recovery::open(path).unwrap(); + assert_eq!(archive.pending().unwrap().len(), 2); + assert!(matches!( + archive.status(1).unwrap(), + Some(EditStatus::AwaitingConfirmation { .. }) + )); + assert_eq!(archive.summary().unwrap().cached_assets, 0); + assert_eq!(archive.summary().unwrap().cached_asset_bytes, 0); + assert!(archive.cached_asset(&payload(0).0, 0).unwrap().is_none()); + drop(archive); + assert_eq!(fs::read(path).unwrap(), before); +} + +#[test] +fn unsuccessful_refreshes_and_changed_identity_data_preserve_the_downloaded_payload() { + let directory = tempfile::tempdir().unwrap(); + let cache = copied_cache(directory.path()); + let (name, bytes) = payload(1024); + let mut source = Payload(Some(Ok(bytes.clone()))); + cache + .fetch_asset(&mut source, "synthetic.one", &name, 1024) + .unwrap(); + let before = fs::read(directory.path().join("cache.sqlite")).unwrap(); + let mut unavailable = Payload(Some(Err(io::ErrorKind::ConnectionReset.into()))); + assert!( + matches!(cache.fetch_asset(&mut unavailable, "synthetic.one", &name, 1024), + Err(Error::Discovery(notebook::discover::Error::Io { error, .. })) if error.kind() == io::ErrorKind::ConnectionReset) + ); + for size in [0, 1024, 2048] { + let mut changed = Payload(Some(Ok(vec![9; size]))); + assert!(matches!( + cache.fetch_asset(&mut changed, "synthetic.one", &name, 2048), + Err(Error::AssetChanged) + )); + } + assert!( + matches!(cache.cached_asset(&name, 1023), Err(Error::Io(error)) if error.kind() == io::ErrorKind::FileTooLarge) + ); + assert_eq!(cache.cached_asset(&name, 1024).unwrap(), Some(bytes)); + assert_eq!( + fs::read(directory.path().join("cache.sqlite")).unwrap(), + before + ); +} + +#[test] +fn unreferenced_payloads_are_rejected_before_io_or_local_changes() { + let directory = tempfile::tempdir().unwrap(); + let cache = copied_cache(directory.path()); + let mut unused = Payload(None); + assert!( + matches!(cache.fetch_asset(&mut unused, "synthetic.one", "00000000-0000-0000-0000-000000000001.onebin", 100), + Err(Error::Io(error)) if error.kind() == io::ErrorKind::InvalidInput) + ); + assert!( + cache + .fetch_asset(&mut unused, "synthetic.one", "../payload.onebin", 100) + .is_err() + ); + assert_eq!(cache.recovery_summary().unwrap().cached_assets, 0); +} + +#[test] +fn download_network_wait_does_not_block_local_edits() { + struct Waiting { + entered: mpsc::Sender<()>, + released: mpsc::Receiver<()>, + bytes: Vec, + } + impl notebook::discover::Source for Waiting { + fn entries(&mut self, _: &str, _: usize) -> io::Result> { + panic!("Unexpected enumeration") + } + fn read(&mut self, _: &str, _: usize) -> io::Result> { + self.entered.send(()).unwrap(); + self.released.recv_timeout(Duration::from_secs(5)).unwrap(); + Ok(self.bytes.clone()) + } + } + let directory = tempfile::tempdir().unwrap(); + let cache = copied_cache(directory.path()); + let (name, bytes) = payload(1024); + let (entered, waiting) = mpsc::channel(); + let (release, released) = mpsc::channel(); + let mut source = Waiting { + entered, + released, + bytes: bytes.clone(), + }; + std::thread::scope(|scope| { + let download = scope.spawn(|| { + cache + .fetch_asset(&mut source, "synthetic.one", &name, 1024) + .unwrap() + }); + waiting.recv_timeout(Duration::from_secs(5)).unwrap(); + let pending = cache.pending().unwrap(); + let Operation::Text(edit) = &pending[0].operation else { + panic!() + }; + let id = cache + .edit_text( + &cache.snapshot().unwrap(), + pending[0].space, + edit.object, + 0..0, + "during download ", + ) + .unwrap() + .unwrap(); + release.send(()).unwrap(); + assert_eq!(download.join().unwrap(), bytes); + assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); + }); +} + +#[test] +fn concurrent_downloads_publish_one_immutable_cache_entry() { + let directory = tempfile::tempdir().unwrap(); + let cache = copied_cache(directory.path()); + let (name, bytes) = payload(1024); + let ready = Barrier::new(8); + std::thread::scope(|scope| { + for _ in 0..8 { + scope.spawn(|| { + let mut source = notebook::discover::Local::open(FIXTURE).unwrap(); + ready.wait(); + assert_eq!( + cache + .fetch_asset(&mut source, "synthetic.one", &name, 1024) + .unwrap(), + bytes + ); + }); + } + }); + assert_eq!(cache.recovery_summary().unwrap().cached_assets, 1); +} + +#[test] +fn a_native_refresh_removing_the_reference_rejects_an_inflight_download() { + struct Native; + impl notebook::Remote for Native { + fn read(&mut self) -> io::Result> { + fs::read("../../corpus/native-external-assets/native/synthetic.one") + } + fn publish(&mut self, _: &onestore::PreparedEdit<'_>) -> Result<(), onestore::CommitError> { + panic!("Unexpected publication") + } + fn confirm(&mut self, _: &[u8]) -> Result<(), onestore::CommitError> { + panic!("Unexpected confirmation") + } + } + struct Refresh<'a>(&'a Replica); + impl notebook::discover::Source for Refresh<'_> { + fn entries(&mut self, _: &str, _: usize) -> io::Result> { + panic!("Unexpected enumeration") + } + fn read(&mut self, _: &str, _: usize) -> io::Result> { + assert_eq!(self.0.sync_once(&mut Native).unwrap(), None); + Ok(payload(1024).1) + } + } + let root = tempfile::tempdir().unwrap(); + let source = fs::read(Path::new(FIXTURE).join("synthetic.one")).unwrap(); + let cache = Replica::create(root.path().join("cache.sqlite"), &source).unwrap(); + let (name, _) = payload(1024); + assert!( + matches!(cache.fetch_asset(&mut Refresh(&cache), "synthetic.one", &name, 1024), + Err(Error::Io(error)) if error.kind() == io::ErrorKind::ResourceBusy) + ); + assert!(cache.cached_asset(&name, 1024).unwrap().is_none()); + assert_eq!( + cache.snapshot().unwrap(), + fs::read("../../corpus/native-external-assets/native/synthetic.one").unwrap() + ); +} + +#[test] +fn local_failure_and_bad_cached_bytes_never_become_successful_downloads() { + let directory = tempfile::tempdir().unwrap(); + drop(copied_cache(directory.path())); + let path = directory.path().join("cache.sqlite"); + let connection = rusqlite::Connection::open(&path).unwrap(); + connection.execute_batch("CREATE TRIGGER fail_asset BEFORE INSERT ON assets BEGIN SELECT RAISE(ABORT,'Test asset failure'); END").unwrap(); + drop(connection); + let (name, bytes) = payload(1024); + let cache = Replica::open(&path).unwrap(); + let mut source = notebook::discover::Local::open(FIXTURE).unwrap(); + assert!(matches!( + cache.fetch_asset(&mut source, "synthetic.one", &name, 1024), + Err(Error::Database(_)) + )); + assert!(cache.cached_asset(&name, 1024).unwrap().is_none()); + assert_eq!(cache.pending().unwrap().len(), 2); + drop(cache); + let connection = rusqlite::Connection::open(&path).unwrap(); + connection.execute_batch("DROP TRIGGER fail_asset").unwrap(); + drop(connection); + let cache = Replica::open(&path).unwrap(); + cache + .fetch_asset(&mut source, "synthetic.one", &name, 1024) + .unwrap(); + drop(cache); + let connection = rusqlite::Connection::open(&path).unwrap(); + let mut damaged = bytes; + damaged[0] ^= 1; + connection + .execute("UPDATE assets SET data=?1", [damaged]) + .unwrap(); + drop(connection); + let cache = Replica::open(&path).unwrap(); + assert!( + matches!(cache.cached_asset(&name, 1024), Err(Error::Io(error)) if error.kind() == io::ErrorKind::InvalidData) + ); + assert!( + matches!(cache.fetch_asset(&mut source, "synthetic.one", &name, 1024), Err(Error::Io(error)) if error.kind() == io::ErrorKind::InvalidData) + ); + cache + .export_recovery(directory.path().join("damaged.sqlite")) + .unwrap(); + let archive = Recovery::open(directory.path().join("damaged.sqlite")).unwrap(); + assert!(archive.cached_asset(&name, 1024).is_err()); + assert_eq!(archive.pending().unwrap().len(), 2); +} + +#[test] +fn abrupt_process_exit_retains_only_completed_downloads_and_archives() { + const CHILD: &str = "ONESTORE_ASSET_EXIT_CASE"; + if let Ok(phase) = std::env::var(CHILD) { + struct ExitDuringRead; + impl notebook::discover::Source for ExitDuringRead { + fn entries(&mut self, _: &str, _: usize) -> io::Result> { + panic!("Unexpected enumeration") + } + fn read(&mut self, _: &str, _: usize) -> io::Result> { + std::process::exit(83) + } + } + let root = std::env::var("ONESTORE_ASSET_EXIT_ROOT").unwrap(); + let cache = copied_cache(Path::new(&root)); + let (name, bytes) = payload(1024); + if phase == "download" { + cache + .fetch_asset(&mut ExitDuringRead, "synthetic.one", &name, bytes.len()) + .unwrap(); + panic!("Read returned after process exit"); + } + cache + .fetch_asset( + &mut notebook::discover::Local::open(FIXTURE).unwrap(), + "synthetic.one", + &name, + bytes.len(), + ) + .unwrap(); + if phase == "archive" { + cache + .export_recovery(Path::new(&root).join("recovery.sqlite")) + .unwrap(); + } + std::process::exit(83); + } + for phase in ["download", "cached", "archive"] { + let root = tempfile::tempdir().unwrap(); + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "abrupt_process_exit_retains_only_completed_downloads_and_archives", + ]) + .env(CHILD, phase) + .env("ONESTORE_ASSET_EXIT_ROOT", root.path()) + .output() + .unwrap(); + assert_eq!( + output.status.code(), + Some(83), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let cache = Replica::open(root.path().join("cache.sqlite")).unwrap(); + let (name, bytes) = payload(1024); + let expected = (phase != "download").then_some(bytes); + assert_eq!(cache.cached_asset(&name, 1024).unwrap(), expected); + assert_eq!(cache.pending().unwrap().len(), 2); + assert!(matches!( + cache.status(1).unwrap(), + Some(EditStatus::AwaitingConfirmation { .. }) + )); + if phase == "archive" { + let recovery = Recovery::open(root.path().join("recovery.sqlite")).unwrap(); + assert_eq!(recovery.cached_asset(&name, 1024).unwrap(), expected); + assert_eq!(recovery.pending().unwrap(), cache.pending().unwrap()); + assert_eq!(recovery.status(1).unwrap(), cache.status(1).unwrap()); + } + } +} + +#[test] +#[cfg(feature = "smb")] +#[ignore = "requires a disposable Samba mirror at ONESTORE_SMB_NOTEBOOK"] +fn live_smb_downloads_survive_disconnect_and_cache_reopen() { + let root = tempfile::tempdir().unwrap(); + let cache = copied_cache(root.path()); + let client = notebook::smb::Client::connect( + &std::env::var("ONESTORE_SMB_LAB").unwrap(), + "agent", + notebook::smb::Credentials::default(), + Duration::from_secs(5), + ) + .unwrap(); + let notebook = std::env::var("ONESTORE_SMB_NOTEBOOK").unwrap(); + let mut source = notebook::discover::Smb::new(&client, ¬ebook).unwrap(); + for size in [0, 771, 1024] { + let (name, bytes) = payload(size); + assert_eq!( + cache + .fetch_asset(&mut source, "synthetic.one", &name, size) + .unwrap(), + bytes + ); + } + drop(client); + cache + .export_recovery(root.path().join("recovery.sqlite")) + .unwrap(); + drop(cache); + let cache = Replica::open(root.path().join("cache.sqlite")).unwrap(); + let recovery = Recovery::open(root.path().join("recovery.sqlite")).unwrap(); + for size in [0, 771, 1024] { + let (name, bytes) = payload(size); + assert_eq!( + cache.cached_asset(&name, size).unwrap(), + Some(bytes.clone()) + ); + assert_eq!(recovery.cached_asset(&name, size).unwrap(), Some(bytes)); + } + assert_eq!(cache.pending().unwrap(), recovery.pending().unwrap()); + assert_eq!(cache.status(1).unwrap(), recovery.status(1).unwrap()); + assert_eq!(cache.recovery_summary().unwrap().cached_assets, 3); +} diff --git a/crates/notebook/tests/cache.rs b/crates/notebook/tests/cache.rs new file mode 100644 index 0000000000000000000000000000000000000000..36d48874caa846b77725c861db4260ac59e626f5 --- /dev/null +++ b/crates/notebook/tests/cache.rs @@ -0,0 +1,931 @@ +use notebook::{Error, Replica}; +use onestore::{ + ExGuid, RevisionIndex, Store, + document::{Document, Kind}, +}; +use std::{ + collections::BTreeSet, + fs, + io::ErrorKind, + sync::Barrier, + time::{Duration, Instant}, +}; + +fn target(source: &[u8]) -> (ExGuid, ExGuid, String) { + let store = Store::parse(source).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + let doc = Document::parse(&index).unwrap(); + doc.spaces + .iter() + .find_map(|(sid, space)| { + let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; + revision + .nodes + .iter() + .find_map(|(oid, node)| match &node.kind { + Kind::RichText { text, .. } => Some((*sid, *oid, text.clone())), + _ => None, + }) + }) + .unwrap() +} + +#[test] +fn cache_reopen_preserves_exact_images_and_intents() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("section.sqlite"); + let source = onestore::create_section("section.one", "café 🦀", "Fixture").unwrap(); + let replica = Replica::create(&path, &source).unwrap(); + assert_eq!(replica.snapshot().unwrap(), source); + assert!(replica.pending().unwrap().is_empty()); + let (sid, oid, _) = target(&source); + assert_eq!( + replica.edit_text(&source, sid, oid, 0..0, "").unwrap(), + None + ); + assert_eq!( + replica.edit_text(&source, sid, oid, 0..4, "café").unwrap(), + None + ); + let first = replica + .edit_text(&source, sid, oid, 5..7, "🐈 日本語") + .unwrap() + .unwrap(); + let edited = replica.snapshot().unwrap(); + assert_eq!(target(&edited).2, "café 🐈 日本語"); + let intents = replica.pending().unwrap(); + assert_eq!(intents.len(), 1); + assert_eq!(intents[0].id, first); + let notebook::Operation::Text(first_edit) = &intents[0].operation else { + panic!() + }; + assert_eq!(first_edit.before, "café 🦀"); + assert_eq!(first_edit.range, 5..7); + assert_eq!(first_edit.replacement, "🐈 日本語"); + assert_eq!((intents[0].space, first_edit.object), (sid, oid)); + drop(replica); + let replica = Replica::open(&path).unwrap(); + assert_eq!(replica.snapshot().unwrap(), edited); + assert_eq!(replica.pending().unwrap(), intents); + let second = replica + .edit_text(&edited, sid, oid, 0..0, "Recovered ") + .unwrap() + .unwrap(); + assert!(second > first); + let notebook::Operation::Text(second_edit) = &replica.pending().unwrap()[1].operation else { + panic!() + }; + assert_eq!(second_edit.before, "café 🐈 日本語"); +} + +#[test] +fn failed_edits_preserve_both_intent_queue_and_working_image() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("section.sqlite"); + let source = onestore::create_section("section.one", "café 🦀", "Fixture").unwrap(); + let replica = Replica::create(&path, &source).unwrap(); + let (sid, oid, _) = target(&source); + for (range, replacement) in [(6..7, "X"), (0..u32::MAX, "X"), (0..1, "\n")] { + assert!( + replica + .edit_text(&source, sid, oid, range, replacement) + .is_err() + ); + assert_eq!(replica.snapshot().unwrap(), source); + assert!(replica.pending().unwrap().is_empty()); + } + drop(replica); + let connection = rusqlite::Connection::open(&path).unwrap(); + connection.execute_batch("CREATE TRIGGER fail_image BEFORE UPDATE ON replica BEGIN SELECT RAISE(ABORT, 'Injected image update failure'); END;").unwrap(); + drop(connection); + let replica = Replica::open(&path).unwrap(); + assert!(matches!( + replica.edit_text(&source, sid, oid, 0..0, "lost? "), + Err(Error::Database(_)) + )); + drop(replica); + let replica = Replica::open(&path).unwrap(); + assert_eq!(replica.snapshot().unwrap(), source); + assert!(replica.pending().unwrap().is_empty()); +} + +#[test] +fn concurrent_recovery_exports_capture_one_complete_acknowledged_queue() { + use notebook::{Operation, Recovery}; + + let directory = tempfile::tempdir().unwrap(); + let source = onestore::create_section("recovery.one", "base", "Fixture").unwrap(); + let (space, object, _) = target(&source); + let replica = Replica::create(directory.path().join("live.sqlite"), &source).unwrap(); + let start = Barrier::new(4); + std::thread::scope(|scope| { + for writer in 0..3 { + let (replica, start) = (&replica, &start); + scope.spawn(move || { + start.wait(); + for edit in 0..20 { + loop { + let source = replica.snapshot().unwrap(); + match replica.edit_text( + &source, + space, + object, + 0..0, + &format!("[{writer}:{edit}] "), + ) { + Ok(Some(_)) => break, + Err(Error::Io(error)) if error.kind() == ErrorKind::ResourceBusy => { + continue; + } + other => panic!("Unexpected local edit: {other:?}"), + } + } + } + }); + } + start.wait(); + for n in 0..12 { + let path = directory.path().join(format!("recovery-{n}.sqlite")); + replica.export_recovery(&path).unwrap(); + let archive = Recovery::open(path).unwrap(); + let pending = archive.pending().unwrap(); + let mut expected = String::new(); + for edit in pending.iter().rev() { + let Operation::Text(edit) = &edit.operation else { + panic!() + }; + assert_eq!(edit.range, 0..0); + expected.push_str(&edit.replacement); + } + expected.push_str("base"); + assert_eq!(target(&archive.snapshot().unwrap()).2, expected); + assert_eq!(archive.remote_snapshot().unwrap(), source); + assert_eq!( + archive.summary().unwrap().queued_edits, + pending.len() as u64 + ); + assert!(archive.receipts().unwrap().is_empty()); + } + }); + assert_eq!(replica.pending().unwrap().len(), 60); +} + +#[test] +fn ownership_and_foreign_file_rejection_preserve_existing_data() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("section.sqlite"); + assert!(Replica::open(&path).is_err()); + assert!(!path.exists()); + assert!(Replica::create(&path, b"invalid").is_err()); + assert!(!path.exists()); + let source = onestore::create_section("section.one", "Owned", "Fixture").unwrap(); + let replica = Replica::create(&path, &source).unwrap(); + for _ in 0..3 { + assert!( + matches!(Replica::open(&path), Err(Error::Database(error)) if error.sqlite_error_code() == Some(rusqlite::ErrorCode::DatabaseBusy)) + ); + assert!( + matches!(Replica::create(&path, &source), Err(Error::Io(error)) if error.kind() == ErrorKind::AlreadyExists) + ); + assert_eq!(replica.snapshot().unwrap(), source); + } + drop(replica); + for sql in [ + "PRAGMA application_id=0", + "PRAGMA application_id=1330529615; PRAGMA user_version=99", + ] { + let connection = rusqlite::Connection::open(&path).unwrap(); + connection.execute_batch(sql).unwrap(); + drop(connection); + let before = fs::read(&path).unwrap(); + assert!(Replica::open(&path).is_err()); + assert_eq!(fs::read(&path).unwrap(), before); + } + let foreign = dir.path().join("foreign.sqlite"); + let connection = rusqlite::Connection::open(&foreign).unwrap(); + connection + .execute_batch( + "CREATE TABLE unrelated (value TEXT); INSERT INTO unrelated VALUES ('preserve');", + ) + .unwrap(); + drop(connection); + let before = fs::read(&foreign).unwrap(); + assert!(Replica::open(&foreign).is_err()); + assert_eq!(fs::read(&foreign).unwrap(), before); + let incomplete = dir.path().join("incomplete.sqlite"); + fs::write(&incomplete, []).unwrap(); + assert!(Replica::open(&incomplete).is_err()); + assert_eq!(fs::read(&incomplete).unwrap(), b""); +} + +#[test] +fn twelve_local_editors_reject_stale_ranges_and_preserve_every_acknowledgement() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("section.sqlite"); + let source = onestore::create_section("section.one", "Shared café 🦀", "Fixture").unwrap(); + let replica = Replica::create(&path, &source).unwrap(); + let (sid, oid, _) = target(&source); + let barrier = Barrier::new(12); + let deadline = Instant::now() + Duration::from_secs(60); + let outcomes = std::thread::scope(|scope| { + let handles: Vec<_> = (0..12) + .map(|writer| { + let (replica, source, barrier) = (&replica, &source, &barrier); + scope.spawn(move || { + barrier.wait(); + let first = + replica.edit_text(source, sid, oid, 0..0, &format!("[initial-{writer}] ")); + let mut ids = Vec::new(); + let first_won = match first { + Ok(Some(id)) => { + ids.push(id); + true + } + Err(Error::Io(error)) if error.kind() == ErrorKind::ResourceBusy => false, + other => panic!("Unexpected first edit: {other:?}"), + }; + for edit in 0..20 { + loop { + assert!( + Instant::now() < deadline, + "Writer {writer} stopped progressing at {edit}" + ); + let source = replica.snapshot().unwrap(); + match replica.edit_text( + &source, + sid, + oid, + 0..0, + &format!("[{writer}-{edit}] "), + ) { + Ok(Some(id)) => { + ids.push(id); + break; + } + Err(Error::Io(error)) + if error.kind() == ErrorKind::ResourceBusy => {} + other => panic!("Unexpected edit: {other:?}"), + } + } + } + (first_won, ids) + }) + }) + .collect(); + handles + .into_iter() + .map(|handle| handle.join().unwrap()) + .collect::>() + }); + assert_eq!(outcomes.iter().filter(|(won, _)| *won).count(), 1); + let ids: BTreeSet<_> = outcomes.into_iter().flat_map(|(_, ids)| ids).collect(); + assert_eq!(ids.len(), 241); + let final_bytes = replica.snapshot().unwrap(); + let content = target(&final_bytes).2; + let mut expected = "Shared café 🦀".to_owned(); + for pending in replica.pending().unwrap() { + let notebook::Operation::Text(edit) = pending.operation else { + panic!() + }; + assert_eq!(edit.before, expected); + assert_eq!(edit.range, 0..0); + expected.insert_str(0, &edit.replacement); + } + assert_eq!(content, expected); + for writer in 0..12 { + for edit in 0..20 { + assert_eq!(content.matches(&format!("[{writer}-{edit}] ")).count(), 1); + } + } + assert_eq!( + replica + .pending() + .unwrap() + .iter() + .map(|edit| edit.id) + .collect::>(), + ids + ); + assert!( + matches!(replica.edit_text(&source, sid, oid, 0..0, ""), Err(Error::Io(error)) if error.kind() == ErrorKind::ResourceBusy) + ); + drop(replica); + let reopened = Replica::open(&path).unwrap(); + assert_eq!(reopened.snapshot().unwrap(), final_bytes); + assert_eq!(reopened.pending().unwrap().len(), 241); +} + +#[test] +fn seeded_unicode_edits_and_restarts_match_an_independent_text_model() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("model.sqlite"); + let mut text = "ab🚀ab🦀 é repeated repeated".to_owned(); + let mut source = onestore::create_section("model.one", &text, "Fixture").unwrap(); + let mut replica = Replica::create(&path, &source).unwrap(); + let (space, object, _) = target(&source); + let mut random = 911_u64; + let mut next = || { + random ^= random << 13; + random ^= random >> 7; + random ^= random << 17; + random + }; + let mut intents = Vec::new(); + for step in 0..1024 { + let boundaries: Vec<_> = text + .char_indices() + .map(|(at, _)| at) + .chain([text.len()]) + .collect(); + let first = boundaries[next() as usize % boundaries.len()]; + let last = boundaries[next() as usize % boundaries.len()]; + let bytes = first.min(last)..first.max(last); + let range = u32::try_from(text[..bytes.start].encode_utf16().count()).unwrap() + ..u32::try_from(text[..bytes.end].encode_utf16().count()).unwrap(); + let replacement = ["", "🐈", "日本語", "repeated", "é", "ab🦀ab"][next() as usize % 6]; + let mut expected = text.clone(); + expected.replace_range(bytes, replacement); + let acknowledgement = replica + .edit_text(&source, space, object, range.clone(), replacement) + .unwrap(); + if let Some(id) = acknowledgement { + assert_ne!(text, expected); + assert!( + intents + .last() + .is_none_or(|edit: ¬ebook::PendingEdit| edit.id < id) + ); + intents.push(notebook::PendingEdit { + id, + space, + operation: notebook::Operation::Text(notebook::TextEdit { + object, + before: text, + range, + replacement: replacement.into(), + }), + }); + assert!( + matches!(replica.edit_text(&source, space, object, 0..0, "stale"), Err(Error::Io(error)) if error.kind() == ErrorKind::ResourceBusy) + ); + } else { + assert_eq!(text, expected); + } + text = expected; + source = replica.snapshot().unwrap(); + assert_eq!(target(&source).2, text, "seed 911, step {step}"); + if step % 37 == 0 { + drop(replica); + replica = Replica::open(&path).unwrap(); + assert_eq!(replica.snapshot().unwrap(), source); + assert_eq!(replica.pending().unwrap(), intents); + } + } + assert!( + intents.len() > 512, + "The history checkpoint boundary was not exercised" + ); + drop(replica); + let replica = Replica::open(&path).unwrap(); + assert_eq!(replica.pending().unwrap(), intents); + assert_eq!(replica.snapshot().unwrap(), source); +} + +#[test] +#[ignore = "migrates a fresh copy of a retained version-two or version-three cache"] +fn migrate_retained_cache_copy() { + use notebook::{EditStatus, Operation, PendingEdit, TextEdit}; + let source = std::path::PathBuf::from(std::env::var_os("ONESTORE_MIGRATION_SOURCE").unwrap()); + let output = std::path::PathBuf::from(std::env::var_os("ONESTORE_MIGRATION_OUTPUT").unwrap()); + assert!(source.is_absolute() && output.is_absolute()); + let mut original = fs::File::open(&source).unwrap(); + let mut destination = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&output) + .unwrap(); + std::io::copy(&mut original, &mut destination).unwrap(); + destination.sync_all().unwrap(); + drop(destination); + let db = rusqlite::Connection::open(&output).unwrap(); + let version = db + .pragma_query_value(None, "user_version", |r| r.get::<_, u32>(0)) + .unwrap(); + assert!(matches!(version, 2 | 3)); + if std::env::var_os("ONESTORE_MIGRATION_ROLLBACK").is_some() { + assert_eq!(version, 2); + db.pragma_update(None, "foreign_keys", false).unwrap(); + db.execute( + "INSERT INTO attempt VALUES (1,999999,'{00000001-0000-0000-0000-000000000000},1')", + [], + ) + .unwrap(); + drop(db); + let before = fs::read(&output).unwrap(); + assert!( + matches!(Replica::open(&output), Err(Error::Database(rusqlite::Error::SqliteFailure(error, _))) if error.extended_code == 787) + ); + assert_eq!(fs::read(&output).unwrap(), before); + let db = rusqlite::Connection::open(&output).unwrap(); + assert_eq!( + db.pragma_query_value(None, "user_version", |r| r.get::<_, u32>(0)) + .unwrap(), + 2 + ); + println!("migration: rollback preserved {} bytes", before.len()); + return; + } + if std::env::var_os("ONESTORE_MIGRATION_CONFLICT").is_some() { + db.execute("INSERT INTO conflicts SELECT min(id),0 FROM edits", []) + .unwrap(); + } + let (base, working): (Vec, Vec) = db + .query_row("SELECT base,working FROM replica", [], |r| { + Ok((r.get(0)?, r.get(1)?)) + }) + .unwrap(); + let sequence: i64 = db + .query_row( + "SELECT seq FROM sqlite_sequence WHERE name='edits'", + [], + |r| r.get(0), + ) + .unwrap(); + let pending: Vec = + if version == 2 { + let mut query = db + .prepare("SELECT id,space,object,before_text,start,end,replacement FROM edits ORDER BY id") + .unwrap(); + let pending: Vec = query + .query_map([], |r| { + Ok(PendingEdit { + id: u64::try_from(r.get::<_, i64>(0)?).unwrap(), + space: r.get::<_, String>(1)?.parse().unwrap(), + operation: Operation::Text(TextEdit { + object: r.get::<_, String>(2)?.parse().unwrap(), + before: r.get(3)?, + range: r.get(4)?..r.get(5)?, + replacement: r.get(6)?, + }), + }) + }) + .unwrap() + .collect::>() + .unwrap(); + drop(query); + pending + } else { + let mut query = db + .prepare("SELECT id,space,operation FROM edits ORDER BY id") + .unwrap(); + query + .query_map([], |r| { + Ok(PendingEdit { + id: u64::try_from(r.get::<_, i64>(0)?).unwrap(), + space: r.get::<_, String>(1)?.parse().unwrap(), + operation: serde_json::from_str(&r.get::<_, String>(2)?).unwrap(), + }) + }) + .unwrap() + .collect::>() + .unwrap() + }; + let mut states = std::collections::BTreeMap::new(); + for edit in &pending { + states.insert(edit.id, EditStatus::Pending); + } + for table in ["receipts", "attempt"] { + let mut query = db + .prepare(&format!("SELECT edit_id,revision FROM {table}")) + .unwrap(); + for row in query + .query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?))) + .unwrap() + { + let (id, rid) = row.unwrap(); + let revision = rid.parse().unwrap(); + states.insert( + u64::try_from(id).unwrap(), + if table == "receipts" { + EditStatus::Published { revision } + } else { + EditStatus::AwaitingConfirmation { revision } + }, + ); + } + } + let mut query = db.prepare("SELECT edit_id,kind FROM conflicts").unwrap(); + for row in query + .query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, u32>(1)?))) + .unwrap() + { + let (id, kind) = row.unwrap(); + assert_eq!(kind, 0); + states.insert( + u64::try_from(id).unwrap(), + EditStatus::Conflict(notebook::ConflictKind::TextChanged), + ); + } + drop(query); + drop(db); + let cache = Replica::open(&output).unwrap(); + assert_eq!(cache.snapshot().unwrap(), working); + assert_eq!(cache.remote_snapshot().unwrap(), base); + assert_eq!(cache.pending().unwrap(), pending); + for (id, status) in &states { + assert_eq!(cache.status(*id).unwrap(), Some(*status)); + } + drop(cache); + let cache = Replica::open(&output).unwrap(); + assert_eq!(cache.pending().unwrap(), pending); + let store = Store::parse(&working).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let (sid, page) = doc.pages().unwrap()[0]; + let insertion = onestore::Insertion::outline( + page, + 144.0, + 720.0, + "After cache migration", + "Migration author", + ) + .unwrap(); + let next = cache.insert(&working, sid, &insertion).unwrap().unwrap(); + assert_eq!(next, u64::try_from(sequence + 1).unwrap()); + let updated = cache.snapshot().unwrap(); + drop(cache); + let cache = Replica::open(&output).unwrap(); + assert_eq!(cache.snapshot().unwrap(), updated); + assert_eq!( + cache.pending().unwrap().last().unwrap().operation, + Operation::Insert(insertion) + ); + println!( + "migration: {}", + serde_json::json!({"pending":pending.len(),"retained_statuses":states.len(),"next_id":next,"base_bytes":base.len(),"working_bytes":working.len()}) + ); +} + +#[test] +fn twelve_local_clients_preserve_inserted_identities_and_dependent_edits() { + use onestore::Insertion; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("parallel.sqlite"); + let source = onestore::create_section("parallel.one", "Original", "Author").unwrap(); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let (sid, page) = doc.pages().unwrap()[0]; + let cache = Replica::create(&path, &source).unwrap(); + let barrier = Barrier::new(12); + let deadline = Instant::now() + Duration::from_secs(90); + let all = std::thread::scope(|scope| { + let handles: Vec<_> = (0..12) + .map(|client| { + let (cache, barrier) = (&cache, &barrier); + scope.spawn(move || { + let outline = Insertion::outline( + page, + 72.0, + 144.0 + client as f32 * 72.0, + &format!("Client {client}"), + "Author", + ) + .unwrap(); + let mut ids = Vec::new(); + let mut objects = Vec::new(); + barrier.wait(); + for sequence in 0..4 { + let insertion = if sequence == 0 { + outline.clone() + } else { + Insertion::paragraph( + outline.object(), + None, + &format!("Paragraph {client}:{sequence}"), + "Author", + ) + .unwrap() + } + .with_formatting(0..6, &[onestore::TextAttribute::Italic(true)]) + .unwrap(); + loop { + assert!( + Instant::now() < deadline, + "Client {client} stopped at insertion {sequence}" + ); + let snapshot = cache.snapshot().unwrap(); + match cache.insert(&snapshot, sid, &insertion) { + Ok(Some(id)) => { + ids.push(id); + objects.push(insertion.text_object()); + break; + } + Err(Error::Io(e)) if e.kind() == ErrorKind::ResourceBusy => {} + other => panic!("{other:?}"), + } + } + if sequence == 0 { + continue; + } + loop { + assert!( + Instant::now() < deadline, + "Client {client} stopped at text {sequence}" + ); + let snapshot = cache.snapshot().unwrap(); + match cache.edit_text( + &snapshot, + sid, + insertion.text_object(), + 0..0, + "Edited ", + ) { + Ok(Some(id)) => { + ids.push(id); + break; + } + Err(Error::Io(e)) if e.kind() == ErrorKind::ResourceBusy => {} + other => panic!("{other:?}"), + } + } + } + (ids, objects) + }) + }) + .collect(); + handles + .into_iter() + .map(|h| h.join().unwrap()) + .collect::>() + }); + let ids: BTreeSet<_> = all + .iter() + .flat_map(|(ids, _)| ids.iter().copied()) + .collect(); + assert_eq!(ids.len(), 84); + let snapshot = cache.snapshot().unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.snapshot().unwrap(), snapshot); + assert_eq!( + cache + .pending() + .unwrap() + .iter() + .map(|e| e.id) + .collect::>(), + ids + ); + let store = Store::parse(&snapshot).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let s = &doc.spaces[&sid]; + let v = &s.revisions[&s.contexts[&ExGuid::default()]]; + for (client, (_, objects)) in all.iter().enumerate() { + for (sequence, id) in objects.iter().enumerate() { + let wanted = if sequence == 0 { + format!("Client {client}") + } else { + format!("Edited Paragraph {client}:{sequence}") + }; + assert!(matches!(&v.nodes[id].kind,Kind::RichText{text,..} if *text==wanted)); + let runs = v.text_runs(*id).unwrap(); + assert_eq!(runs[0].format.italic, Some(true)); + assert_eq!( + runs[0].text, + if sequence == 0 { + "Client" + } else { + "Edited Paragr" + } + ); + assert!(runs[1..].iter().all(|run| run.format.italic != Some(true))); + } + } +} + +#[test] +fn unrecognized_persisted_operations_are_rejected_without_dropping_fields() { + for operation in ["Text", "Insert", "Format", "Split", "Join", "Outline"] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("unknown.sqlite"); + let mut source = onestore::create_section("unknown.one", "Original", "Author").unwrap(); + let (sid, oid, _) = target(&source); + let split = onestore::ParagraphSplit::new(oid, 3, "Author").unwrap(); + if operation == "Join" { + source = onestore::PreparedEdit::split(&source, sid, &split) + .unwrap() + .as_bytes() + .to_vec(); + } + let cache = Replica::create(&path, &source).unwrap(); + if operation == "Insert" { + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let (_, page) = doc.pages().unwrap()[0]; + let insertion = + onestore::Insertion::outline(page, 144.0, 144.0, "Inserted", "Author").unwrap(); + cache.insert(&source, sid, &insertion).unwrap(); + } else if operation == "Outline" { + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let space = &doc.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let paragraph = *view + .nodes + .iter() + .find(|(_, node)| node.content == [oid]) + .unwrap() + .0; + cache + .outline( + &source, + sid, + paragraph, + onestore::OutlineEdit::Collapsed(true), + ) + .unwrap(); + } else if operation == "Text" { + cache.edit_text(&source, sid, oid, 0..0, "New ").unwrap(); + } else if operation == "Split" { + cache.split(&source, sid, &split).unwrap(); + } else if operation == "Join" { + cache + .join( + &source, + sid, + &onestore::ParagraphJoin::new(oid, split.text_object(), "Author").unwrap(), + ) + .unwrap(); + } else { + cache + .format( + &source, + sid, + oid, + 0..4, + &[onestore::TextAttribute::Bold(true)], + ) + .unwrap(); + } + drop(cache); + let db = rusqlite::Connection::open(&path).unwrap(); + let encoded: String = db + .query_row("SELECT operation FROM edits", [], |r| r.get(0)) + .unwrap(); + let mut value: serde_json::Value = serde_json::from_str(&encoded).unwrap(); + value[operation]["future_option"] = true.into(); + db.execute("UPDATE edits SET operation=?1", [value.to_string()]) + .unwrap(); + if matches!(operation, "Text" | "Insert") { + db.execute_batch("ALTER TABLE attempt RENAME COLUMN revisions TO revision; DROP TABLE assets; DROP TABLE conflicts; CREATE TABLE conflicts (edit_id INTEGER PRIMARY KEY REFERENCES edits(id) ON DELETE CASCADE, kind INTEGER NOT NULL CHECK(kind BETWEEN 0 AND 2)) STRICT; PRAGMA user_version=3;").unwrap(); + } + drop(db); + let before = fs::read(&path).unwrap(); + assert!( + matches!(Replica::open(&path),Err(Error::Io(error))if error.kind()==ErrorKind::InvalidData) + ); + assert_eq!(fs::read(&path).unwrap(), before); + } +} + +#[test] +fn prior_schema_migrations_retain_queue_evidence_assets_and_enable_content_conflicts() { + for (version, ceiling) in [(5, 3), (6, 4), (7, 5), (8, 6), (9, 6)] { + use notebook::{ConflictKind, EditStatus, Recovery}; + use sha2::{Digest, Sha256}; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("previous.sqlite"); + let source = onestore::create_section("migration.one", "Original", "Author").unwrap(); + let (sid, oid, _) = target(&source); + let cache = Replica::create(&path, &source).unwrap(); + let first = cache + .edit_text(&source, sid, oid, 0..0, "New ") + .unwrap() + .unwrap(); + let second = cache + .format( + &cache.snapshot().unwrap(), + sid, + oid, + 0..3, + &[onestore::TextAttribute::Bold(true)], + ) + .unwrap() + .unwrap(); + let queue = cache.pending().unwrap(); + let local = cache.snapshot().unwrap(); + let store = Store::parse(&local).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let revision = index.spaces[&sid].labels[&(ExGuid::default(), 1)]; + drop(cache); + let db = rusqlite::Connection::open(&path).unwrap(); + let current_version: u32 = db + .pragma_query_value(None, "user_version", |row| row.get(0)) + .unwrap(); + let asset = "00112233-4455-6677-8899-aabbccddeeff.onebin"; + let data = b"retained media"; + db.execute( + "INSERT INTO assets VALUES (?1,?2,?3)", + rusqlite::params![asset, data.as_slice(), Sha256::digest(data).as_slice()], + ) + .unwrap(); + db.execute_batch(&format!( + "ALTER TABLE attempt RENAME COLUMN revisions TO revision; + DROP TABLE conflicts; CREATE TABLE conflicts ( + edit_id INTEGER PRIMARY KEY REFERENCES edits(id) ON DELETE CASCADE, + kind INTEGER NOT NULL CHECK(kind BETWEEN 0 AND {ceiling})) STRICT; + PRAGMA user_version={version}; + UPDATE sqlite_sequence SET seq=1000 WHERE name='edits';" + )) + .unwrap(); + db.execute( + "INSERT INTO conflicts VALUES (?1,3)", + [i64::try_from(second).unwrap()], + ) + .unwrap(); + db.execute( + "INSERT INTO attempt VALUES (1,?1,?2)", + rusqlite::params![i64::try_from(first).unwrap(), revision.to_string()], + ) + .unwrap(); + db.execute( + "INSERT INTO receipts VALUES (999,?1)", + [revision.to_string()], + ) + .unwrap(); + drop(db); + let archive = directory.path().join("legacy-recovery.sqlite"); + std::fs::copy(&path, &archive).unwrap(); + let archive_db = rusqlite::Connection::open(&archive).unwrap(); + archive_db + .pragma_update(None, "application_id", 0x4f4e4552) + .unwrap(); + drop(archive_db); + let original_archive = std::fs::read(&archive).unwrap(); + let recovery = Recovery::open(&archive).unwrap(); + assert_eq!( + recovery.status(first).unwrap(), + Some(EditStatus::AwaitingConfirmation { revision }) + ); + assert_eq!(recovery.pending().unwrap(), queue); + drop(recovery); + assert!(std::fs::read(&archive).unwrap() == original_archive); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), queue); + assert_eq!( + cache.cached_asset(asset, 1024).unwrap(), + Some(data.to_vec()) + ); + assert!(cache.snapshot().unwrap() == local); + assert!(cache.remote_snapshot().unwrap() == source); + assert_eq!( + cache.status(first).unwrap(), + Some(EditStatus::AwaitingConfirmation { revision }) + ); + assert_eq!( + cache.status(second).unwrap(), + Some(EditStatus::Conflict(ConflictKind::FormattingChanged)) + ); + assert_eq!( + cache.status(999).unwrap(), + Some(EditStatus::Published { revision }) + ); + let split = onestore::ParagraphSplit::new(oid, 3, "Author").unwrap(); + assert_eq!(cache.split(&local, sid, &split).unwrap(), Some(1001)); + let pending = cache.pending().unwrap(); + let archive = directory.path().join("recovery.sqlite"); + cache.export_recovery(&archive).unwrap(); + let recovery = Recovery::open(&archive).unwrap(); + assert_eq!(recovery.pending().unwrap(), pending); + assert_eq!( + recovery.cached_asset(asset, 1024).unwrap(), + Some(data.to_vec()) + ); + assert_eq!( + recovery.status(first).unwrap(), + cache.status(first).unwrap() + ); + assert_eq!(recovery.receipts().unwrap().get(&999), Some(&revision)); + drop(cache); + let db = rusqlite::Connection::open(&path).unwrap(); + assert_eq!( + db.pragma_query_value(None, "user_version", |row| row.get::<_, u32>(0)) + .unwrap(), + current_version + ); + db.execute("INSERT INTO conflicts VALUES (1001,6)", []) + .unwrap(); + drop(db); + let cache = Replica::open(&path).unwrap(); + assert_eq!( + cache.status(1001).unwrap(), + Some(EditStatus::Conflict(ConflictKind::ContentChanged)) + ); + } +} diff --git a/crates/notebook/tests/discovery.rs b/crates/notebook/tests/discovery.rs new file mode 100644 index 0000000000000000000000000000000000000000..f61babffc362c846cda2e541593ec5a9ae584352 --- /dev/null +++ b/crates/notebook/tests/discovery.rs @@ -0,0 +1,280 @@ +use notebook::discover::{Entry, Error, Limits, Local, Source, discover}; +use std::{fs, io, path::Path}; + +fn limits() -> Limits { + Limits { + entries: 1000, + bytes_per_file: 16 * 1024 * 1024, + depth: 16, + } +} + +fn fixture(root: &Path) -> Vec { + let section = onestore::create_section("one.one", "Retained 🦀", "Fixture").unwrap(); + fs::write(root.join("one.one"), §ion).unwrap(); + let identity = onestore::Store::parse(§ion).unwrap().header.file_id; + let toc = onestore::create_table_of_contents("Open Notebook.onetoc2", &[("one.one", identity)]) + .unwrap(); + fs::write(root.join("Open Notebook.onetoc2"), toc).unwrap(); + section +} + +#[test] +fn rename_preserves_identity_and_does_not_trust_a_stale_cached_filename() { + let root = tempfile::tempdir().unwrap(); + let section = fixture(root.path()); + let mut source = Local::open(root.path()).unwrap(); + let before = discover(&mut source, limits()).unwrap(); + fs::rename( + root.path().join("one.one"), + root.path().join("Renamed 🦀.ONE"), + ) + .unwrap(); + let after = discover(&mut source, limits()).unwrap(); + assert_eq!(before.sections[0].file_id, after.sections[0].file_id); + assert_eq!(after.sections[0].path, "Renamed 🦀.ONE"); + assert!(after.toc.as_ref().unwrap().unresolved.is_empty()); + assert_eq!( + fs::read(root.path().join("Renamed 🦀.ONE")).unwrap(), + section + ); + fs::write( + root.path().join("one.one"), + onestore::create_section("one.one", "Different", "Fixture").unwrap(), + ) + .unwrap(); + let with_replacement = discover(&mut source, limits()).unwrap(); + assert_eq!( + with_replacement.sections[0].file_id, + before.sections[0].file_id + ); + assert_eq!(with_replacement.sections[0].path, "Renamed 🦀.ONE"); + assert_ne!( + with_replacement.sections[1].file_id, + before.sections[0].file_id + ); + fs::remove_file(root.path().join("Renamed 🦀.ONE")).unwrap(); + let absent = discover(&mut source, limits()).unwrap(); + assert_eq!(absent.toc.as_ref().unwrap().unresolved.len(), 1); + assert_eq!( + absent.toc.as_ref().unwrap().unresolved[0].file, + before.sections[0].file_id + ); + assert_ne!( + absent.sections[0].file_id, + absent.toc.as_ref().unwrap().unresolved[0].file + ); +} + +#[test] +fn copied_identities_are_ambiguous_even_across_different_groups() { + let root = tempfile::tempdir().unwrap(); + fixture(root.path()); + fs::create_dir(root.path().join("group")).unwrap(); + fs::copy( + root.path().join("one.one"), + root.path().join("group/other.one"), + ) + .unwrap(); + assert!( + matches!(discover(&mut Local::open(root.path()).unwrap(), limits()), + Err(Error::DuplicateIdentity { first, second, .. }) + if [first.as_str(), second.as_str()].contains(&"one.one") + && [first.as_str(), second.as_str()].contains(&"group/other.one")) + ); +} + +#[test] +fn locked_and_unreadable_sections_retain_their_storage_identity() { + let root = tempfile::tempdir().unwrap(); + for (name, fixture) in [ + ( + "locked.one", + "native-encrypted/encrypted-01/notebook/synthetic.one", + ), + ( + "stub.one", + "native-encrypted/cold-encrypted-02/notebook/Open Notebook.one", + ), + ] { + fs::copy( + Path::new("../../corpus").join(fixture), + root.path().join(name), + ) + .unwrap(); + } + let catalog = discover(&mut Local::open(root.path()).unwrap(), limits()).unwrap(); + assert!(catalog.toc.is_none()); + assert!(matches!( + catalog.sections[0].state, + notebook::discover::SectionState::Locked + )); + assert!(matches!( + catalog.sections[1].state, + notebook::discover::SectionState::Unreadable(_) + )); + for section in catalog.sections { + let bytes = fs::read(root.path().join(section.path)).unwrap(); + assert_eq!( + section.file_id, + onestore::Store::parse(&bytes).unwrap().header.file_id + ); + } +} + +#[test] +fn a_group_rename_retains_toc_identity_and_parent_order() { + let root = tempfile::tempdir().unwrap(); + let native = Path::new("../../corpus/m6/native-features-01/notebook"); + for relative in [ + "Open Notebook.onetoc2", + "Group A/Open Notebook.onetoc2", + "Group A/Duplicate.one", + "Group B/Open Notebook.onetoc2", + "Group B/Nested/Open Notebook.onetoc2", + "Group B/Nested/Duplicate.one", + ] { + let target = root.path().join(relative); + fs::create_dir_all(target.parent().unwrap()).unwrap(); + fs::copy(native.join(relative), target).unwrap(); + } + let before = discover(&mut Local::open(root.path()).unwrap(), limits()).unwrap(); + assert_eq!(before.groups[0].path, "Group A"); + fs::rename(root.path().join("Group A"), root.path().join("Renamed 🦀")).unwrap(); + let catalog = discover(&mut Local::open(root.path()).unwrap(), limits()).unwrap(); + assert_eq!(catalog.groups[0].path, "Renamed 🦀"); + assert_eq!( + catalog.groups[0].toc.as_ref().unwrap().file_id, + before.groups[0].toc.as_ref().unwrap().file_id + ); + assert_eq!(catalog.groups[1].path, "Group B"); + assert_eq!( + catalog.groups[0].sections[0].path, + "Renamed 🦀/Duplicate.one" + ); + assert_eq!( + serde_json::to_value(&catalog.toc.unwrap().unresolved).unwrap(), + serde_json::to_value(&before.toc.unwrap().unresolved).unwrap() + ); +} + +#[test] +fn limits_and_incomplete_files_do_not_produce_a_catalog() { + let root = tempfile::tempdir().unwrap(); + fixture(root.path()); + let mut source = Local::open(root.path()).unwrap(); + assert!( + discover( + &mut source, + Limits { + entries: 1, + ..limits() + } + ) + .is_err() + ); + assert!( + discover( + &mut source, + Limits { + bytes_per_file: 8, + ..limits() + } + ) + .is_err() + ); + fs::create_dir(root.path().join("group")).unwrap(); + assert!(matches!( + discover( + &mut source, + Limits { + depth: 0, + ..limits() + } + ), + Err(Error::Limit { .. }) + )); + fs::write(root.path().join("one.one"), b"unfinished").unwrap(); + assert!( + matches!(discover(&mut source, limits()), Err(Error::Document { path, .. }) if path == "one.one") + ); +} + +#[test] +fn a_failed_refresh_retains_the_previous_catalog_as_an_independent_value() { + struct Changing { + source: Local, + reads: usize, + fail: bool, + } + impl Source for Changing { + fn entries(&mut self, path: &str, limit: usize) -> io::Result> { + self.reads += 1; + let mut entries = self.source.entries(path, limit)?; + if self.reads == 2 { + if self.fail { + return Err(io::ErrorKind::ConnectionAborted.into()); + } + entries[0].name.push_str(".renamed"); + } + Ok(entries) + } + fn read(&mut self, path: &str, limit: usize) -> io::Result> { + self.source.read(path, limit) + } + } + let root = tempfile::tempdir().unwrap(); + fixture(root.path()); + let mut source = Local::open(root.path()).unwrap(); + let catalog = discover(&mut source, limits()).unwrap(); + let before = serde_json::to_value(&catalog).unwrap(); + for fail in [false, true] { + let mut changing = Changing { + source: Local::open(root.path()).unwrap(), + reads: 0, + fail, + }; + let error = discover(&mut changing, limits()).unwrap_err(); + if fail { + assert!( + matches!(error, Error::Io { error, .. } if error.kind() == io::ErrorKind::ConnectionAborted) + ); + } else { + assert!(matches!(error, Error::Changed { .. })); + } + assert_eq!(serde_json::to_value(&catalog).unwrap(), before); + } +} + +#[test] +fn local_access_stays_inside_the_selected_root() { + let root = tempfile::tempdir().unwrap(); + fixture(root.path()); + let mut source = Local::open(root.path()).unwrap(); + for path in [ + "../one.one", + "/one.one", + "x/../one.one", + "one.one\0", + "x\\one.one", + ] { + assert_eq!( + source.read(path, 100).unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + } + #[cfg(unix)] + { + let other = tempfile::tempdir().unwrap(); + fs::write(other.path().join("other.one"), b"outside").unwrap(); + std::os::unix::fs::symlink(other.path().join("other.one"), root.path().join("link.one")) + .unwrap(); + assert_eq!( + source.read("link.one", 100).unwrap_err().kind(), + io::ErrorKind::PermissionDenied + ); + assert!( + matches!(discover(&mut source, limits()), Err(Error::Entry { path }) if path == "link.one") + ); + } +} diff --git a/crates/notebook/tests/external_assets.rs b/crates/notebook/tests/external_assets.rs new file mode 100644 index 0000000000000000000000000000000000000000..8b7dc36e86a0757ab4a72903adfada28c4a724ea --- /dev/null +++ b/crates/notebook/tests/external_assets.rs @@ -0,0 +1,162 @@ +use notebook::discover::{Error, Local, read_external_asset}; +use std::{fs, io}; + +#[test] +fn nested_external_payloads_are_exact_and_empty_is_distinct_from_missing() { + let root = tempfile::tempdir().unwrap(); + let directory = root.path().join("Group 🦀/Section é_onefiles"); + fs::create_dir_all(&directory).unwrap(); + let filename = "2a83ae62-6754-4383-8e2b-4033ff3cfba1.onebin"; + let mut source = Local::open(root.path()).unwrap(); + let section = "Group 🦀/Section é.ONE"; + assert!( + matches!(read_external_asset(&mut source, section, filename, 0), + Err(Error::Io { error, .. }) if error.kind() == io::ErrorKind::NotFound) + ); + fs::write(directory.join(filename), []).unwrap(); + assert!( + read_external_asset(&mut source, section, filename, 0) + .unwrap() + .is_empty() + ); + let bytes: Vec<_> = (0..65537).map(|index| (index % 251) as u8).collect(); + fs::write(directory.join(filename), &bytes).unwrap(); + assert!( + matches!(read_external_asset(&mut source, section, filename, bytes.len()-1), + Err(Error::Io { error, .. }) if error.kind() == io::ErrorKind::FileTooLarge) + ); + assert_eq!( + read_external_asset(&mut source, section, filename, bytes.len()).unwrap(), + bytes + ); + assert_eq!(fs::read(directory.join(filename)).unwrap(), bytes); +} + +#[test] +fn invalid_asset_locations_fail_before_accessing_the_source() { + struct Unused; + impl notebook::discover::Source for Unused { + fn entries(&mut self, _: &str, _: usize) -> io::Result> { + panic!("Unexpected enumeration") + } + fn read(&mut self, _: &str, _: usize) -> io::Result> { + panic!("Unexpected read") + } + } + let filename = "2a83ae62-6754-4383-8e2b-4033ff3cfba1.onebin"; + for section in [ + "", + ".one", + "Group/.one", + "/Section.one", + "../Section.one", + "Group/../Section.one", + "Group\\Section.one", + "Section.onetoc2", + ] { + assert!(matches!( + read_external_asset(&mut Unused, section, filename, 100), + Err(Error::Entry { .. }) + )); + } + for name in [ + "", + "attachment.png", + "../2a83ae62-6754-4383-8e2b-4033ff3cfba1.onebin", + "2a83ae62-6754-4383-8e2b-4033ff3cfba1.onebin:other", + "", + ] { + assert!(matches!( + read_external_asset(&mut Unused, "Section.one", name, 100), + Err(Error::Entry { .. }) + )); + } +} + +#[test] +fn reserved_payload_directories_are_not_notebook_groups() { + let root = tempfile::tempdir().unwrap(); + fs::write( + root.path().join("Section.ONE"), + onestore::create_section("Section.one", "Fixture", "Author").unwrap(), + ) + .unwrap(); + for name in ["section_onefiles", "orphan_onefiles", "ordinary"] { + fs::create_dir(root.path().join(name)).unwrap(); + } + fs::write( + root.path().join("section_onefiles/unfinished.onebin"), + b"payload", + ) + .unwrap(); + let catalog = notebook::discover::discover( + &mut Local::open(root.path()).unwrap(), + notebook::discover::Limits { + entries: 4, + bytes_per_file: 1 << 20, + depth: 1, + }, + ) + .unwrap(); + assert_eq!(catalog.sections.len(), 1); + assert_eq!( + catalog + .groups + .iter() + .map(|group| group.path.as_str()) + .collect::>(), + ["ordinary"] + ); +} + +#[test] +#[cfg(feature = "smb")] +#[ignore = "requires a disposable Samba mirror of corpus/native-external-assets/notebook at ONESTORE_SMB_NOTEBOOK"] +fn live_external_payloads() { + use onestore::{ + FileDataReference, RevisionIndex, Store, + document::{Document, Kind}, + }; + let root = std::path::Path::new("../../corpus/native-external-assets/notebook"); + let bytes = fs::read(root.join("synthetic.one")).unwrap(); + let store = Store::parse(&bytes).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let client = notebook::smb::Client::connect( + &std::env::var("ONESTORE_SMB_LAB").unwrap(), + "agent", + notebook::smb::Credentials::default(), + std::time::Duration::from_secs(5), + ) + .unwrap(); + let mut remote = + notebook::discover::Smb::new(&client, &std::env::var("ONESTORE_SMB_NOTEBOOK").unwrap()) + .unwrap(); + let mut local = Local::open(root).unwrap(); + let mut seen = std::collections::BTreeSet::new(); + for node in document + .spaces + .values() + .flat_map(|space| space.revisions.values()) + .flat_map(|revision| revision.nodes.values()) + { + if let Kind::File { + reference: FileDataReference::External(filename), + .. + } = &node.kind + && seen.insert(filename) + { + let expected = fs::read(root.join("synthetic_onefiles").join(filename)).unwrap(); + assert_eq!( + read_external_asset(&mut local, "synthetic.one", filename, expected.len()).unwrap(), + expected + ); + assert_eq!( + read_external_asset(&mut remote, "synthetic.one", filename, expected.len()) + .unwrap(), + expected + ); + } + } + assert_eq!(seen.len(), 3); +} diff --git a/crates/notebook/tests/support/page_schedule.rs b/crates/notebook/tests/support/page_schedule.rs new file mode 100644 index 0000000000000000000000000000000000000000..af00a40b0cf1d0b5bf3cc9865783fcd16c290cdb --- /dev/null +++ b/crates/notebook/tests/support/page_schedule.rs @@ -0,0 +1,254 @@ +use crate::disk; +use notebook::{EditStatus, Operation, Remote, Replica}; +use onestore::{ + CommitError, ExGuid, Insertion, PageEdit, PagePosition, PreparedEdit, RevisionIndex, Store, + document::{Document, Kind}, +}; +use std::{io, sync::LazyLock}; + +#[path = "../../../onestore/tests/support/current.rs"] +mod current; + +static SOURCE: LazyLock> = LazyLock::new(|| { + let mut source = onestore::create_section("page-schedule.one", "Original", "Author").unwrap(); + for _ in 0..3 { + let page = onestore::PageCreation::new(None, Some("Same title"), "Author").unwrap(); + source = PreparedEdit::create_page(&source, &page) + .unwrap() + .as_bytes() + .to_vec(); + } + source +}); + +fn pages(source: &[u8]) -> Vec<(ExGuid, ExGuid, u32)> { + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + let document = Document::parse(&index).unwrap(); + document + .pages() + .unwrap() + .into_iter() + .map(|(sid, page)| { + let space = &document.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let Kind::Metadata { level, .. } = view.nodes[&view.roots[&2]].kind else { + panic!() + }; + (sid, page, level.unwrap_or(1)) + }) + .collect() +} + +struct Session<'a> { + disk: &'a mut disk::Disk, + operation: Option<&'a Operation>, +} + +impl Remote for Session<'_> { + fn read(&mut self) -> io::Result> { + Ok(self.disk.visible.clone()) + } + + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + let mut expected = pages(&self.disk.visible); + match self.operation.unwrap() { + Operation::CreatePage(page) => expected.push((page.space(), page.object(), 1)), + Operation::Pages(batch) => { + for change in &batch.edits { + let at = expected.iter().position(|p| p.0 == change.space()).unwrap(); + expected[at].2 = change.level(); + if let PagePosition::Before(before) = change.position() { + let page = expected.remove(at); + let at = before.map_or(expected.len(), |before| { + expected.iter().position(|p| p.0 == before).unwrap() + }); + expected.insert(at, page); + } + } + } + Operation::Insert(insertion) => { + let store = Store::parse(edit.as_bytes()).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let matching: Vec<_> = document + .spaces + .values() + .filter_map(|space| { + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + view.nodes.get(&insertion.text_object()) + }) + .collect(); + assert_eq!(matching.len(), 1); + assert!( + matches!(&matching[0].kind, Kind::RichText { text, .. } if text == "Body 🦋 é") + ); + } + _ => panic!(), + } + assert_eq!(pages(edit.as_bytes()), expected); + let old = current::current(&self.disk.durable); + let new = current::current(edit.as_bytes()); + let result = edit.commit(self.disk); + let observed = current::current(&self.disk.durable); + assert!(observed == old || observed == new); + if result.is_ok() { + assert_eq!(observed, new); + } + result + } + + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + onestore::confirm_snapshot(self.disk, snapshot) + } +} + +pub fn run(input: &[u8]) { + let directory = tempfile::tempdir().unwrap(); + let mut replicas: [Option; 12] = std::array::from_fn(|_| None); + let mut owned: [Vec<(ExGuid, ExGuid)>; 12] = std::array::from_fn(|_| Vec::new()); + let mut disk = disk::Disk { + visible: SOURCE.clone(), + durable: SOURCE.clone(), + operation: 0, + fail_at: None, + write_limit: 4096, + random: 1, + }; + for step in input.chunks_exact(8).take(48) { + let actor = usize::from(step[0]) % replicas.len(); + let path = directory.path().join(format!("{actor}.sqlite")); + let cache = replicas[actor].get_or_insert_with(|| Replica::create(&path, &SOURCE).unwrap()); + let snapshot = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + match step[1] % 8 { + 0 | 1 => { + let title = (step[2] & 1 != 0).then_some("Same 🦋 é"); + let page = onestore::PageCreation::new(None, title, "Author").unwrap(); + cache.create_page(&snapshot, &page).unwrap().unwrap(); + owned[actor].push((page.space(), page.object())); + if step[1] % 8 == 1 { + let insertion = + Insertion::outline(page.object(), 36.0, 36.0, "Body 🦋 é", "Author") + .unwrap(); + cache + .insert(&cache.snapshot().unwrap(), page.space(), &insertion) + .unwrap() + .unwrap(); + } + } + 2 => { + let statuses: Vec<_> = pending + .iter() + .map(|edit| cache.status(edit.id).unwrap()) + .collect(); + replicas[actor] = None; + let cache = Replica::open(&path).unwrap(); + assert!(cache.snapshot().unwrap() == snapshot); + assert_eq!(cache.pending().unwrap(), pending); + assert_eq!( + pending + .iter() + .map(|edit| cache.status(edit.id).unwrap()) + .collect::>(), + statuses + ); + replicas[actor] = Some(cache); + } + 3 | 4 => { + disk.operation = 0; + disk.write_limit = if step[3] & 1 == 0 { 17 } else { 4096 }; + disk.fail_at = + (step[2] != 0).then_some(usize::from(u16::from_le_bytes([step[2], step[3]]))); + disk.random = u64::from(step[4]) + 1; + let prior = pending + .first() + .and_then(|edit| cache.status(edit.id).unwrap()); + let mut session = Session { + disk: &mut disk, + operation: pending.first().map(|edit| &edit.operation), + }; + let result = cache.sync_once(&mut session); + if matches!(prior, Some(EditStatus::AwaitingConfirmation { .. })) { + assert!( + result.is_err() + || !matches!(result.unwrap(), Some((_, EditStatus::Conflict(_)))) + ); + } + } + 5 => { + disk.visible.clone_from(&disk.durable); + disk.fail_at = None; + } + 6 => { + let order = pages(&snapshot); + let count = 1 + usize::from(step[3] & 1 != 0); + let selected: Vec<_> = (0..count) + .map(|i| order[(usize::from(step[2]) + i) % order.len()].0) + .collect(); + let anchor = (step[4] & 1 != 0) + .then_some(order[usize::from(step[5]) % order.len()].0) + .filter(|sid| !selected.contains(sid)); + let edits: Vec<_> = selected + .iter() + .enumerate() + .map(|(i, sid)| { + let level = u32::from(step[6 + i]) % 3 + 1; + if step[4] & 2 == 0 { + PageEdit::set_level(*sid, level).unwrap() + } else { + PageEdit::move_to(*sid, anchor, level).unwrap() + } + }) + .collect(); + let expected = PreparedEdit::pages(&snapshot, &edits); + match cache.pages(&snapshot, &edits) { + Ok(id) => { + let expected = expected.unwrap(); + assert_eq!( + current::current(&cache.snapshot().unwrap()), + current::current(expected.as_bytes()) + ); + assert_eq!(id.is_some(), expected.as_bytes() != snapshot); + } + Err(_) => { + assert!(expected.is_err()); + assert_eq!(cache.snapshot().unwrap(), snapshot); + assert_eq!(cache.pending().unwrap(), pending); + } + } + } + 7 => { + if let Some(edit) = pending.first() + && matches!( + cache.status(edit.id).unwrap(), + Some(EditStatus::Conflict(_)) + ) + && let Operation::Pages(batch) = &edit.operation + { + let remote = cache.remote_snapshot().unwrap(); + let order = pages(&remote); + let anchor = (step[2] & 1 != 0) + .then_some(order[usize::from(step[3]) % order.len()].0) + .filter(|sid| batch.edits.iter().all(|e| e.space() != *sid)); + let edits: Vec<_> = batch + .edits + .iter() + .map(|edit| edit.reposition(PagePosition::Before(anchor), 1).unwrap()) + .collect(); + let result = cache.rebase_pages_conflict(edit.id, &snapshot, &remote, &edits); + assert_eq!(cache.snapshot().unwrap(), snapshot); + if result.is_err() { + assert_eq!(cache.pending().unwrap(), pending); + } + } + } + _ => unreachable!(), + } + let local = pages(&replicas[actor].as_ref().unwrap().snapshot().unwrap()); + for page in &owned[actor] { + assert!(local.iter().any(|(sid, oid, _)| (*sid, *oid) == *page)); + } + } +} diff --git a/crates/notebook/tests/support/tree_schedule.rs b/crates/notebook/tests/support/tree_schedule.rs new file mode 100644 index 0000000000000000000000000000000000000000..170b19af59783786ee119301b7900e869d7536b4 --- /dev/null +++ b/crates/notebook/tests/support/tree_schedule.rs @@ -0,0 +1,271 @@ +use crate::disk; +use notebook::{EditStatus, Operation, Remote, Replica}; +use onestore::{ + CommitError, ExGuid, Insertion, PreparedEdit, RevisionIndex, Store, TreeEdit, + document::{Document, Kind}, +}; +use std::{io, sync::LazyLock}; + +static SOURCE: LazyLock<(Vec, ExGuid, ExGuid)> = LazyLock::new(|| { + let mut source = + onestore::create_section("tree-schedule.one", "Original 🦀 é 0", "Author").unwrap(); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let (sid, page) = document.pages().unwrap()[0]; + let space = &document.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let outline = *view.nodes[&page] + .children + .iter() + .find(|id| matches!(view.nodes[id].kind, Kind::Outline { .. })) + .unwrap(); + for at in 1..12 { + let insert = + Insertion::paragraph(outline, None, &format!("Original 🦀 é {at}"), "Author").unwrap(); + source = PreparedEdit::insert(&source, sid, &insert) + .unwrap() + .as_bytes() + .to_vec(); + } + (source, sid, outline) +}); + +fn rows(source: &[u8]) -> Vec<(ExGuid, ExGuid, String)> { + let (_, sid, outline) = &*SOURCE; + let store = Store::parse(source).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + let document = Document::parse(&index).unwrap(); + let space = &document.spaces[sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let (_, page) = document + .pages() + .unwrap() + .into_iter() + .find(|(space, _)| space == sid) + .unwrap(); + if !view.nodes[&page].children.contains(outline) { + return Vec::new(); + } + let node = &view.nodes[outline]; + let mut rows = Vec::new(); + for paragraph in &node.children { + let node = &view.nodes[paragraph]; + assert!(node.children.is_empty()); + let [text] = node.content.as_slice() else { + panic!() + }; + let Kind::RichText { text: value, .. } = &view.nodes[text].kind else { + panic!() + }; + rows.push((*paragraph, *text, value.clone())); + } + rows +} + +struct Session<'a> { + disk: &'a mut disk::Disk, + operation: Option<&'a Operation>, +} + +impl Remote for Session<'_> { + fn read(&mut self) -> io::Result> { + Ok(self.disk.visible.clone()) + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + let before = rows(&self.disk.visible); + let mut expected = before.clone(); + match self.operation.unwrap() { + Operation::Tree(edit) => { + let at = expected + .iter() + .position(|(id, _, _)| *id == edit.intent.object()) + .unwrap(); + let row = expected.remove(at); + if let Some((parent, anchor)) = edit.intent.destination() { + assert_eq!(parent, SOURCE.2); + let at = anchor + .map(|anchor| { + expected + .iter() + .position(|(id, _, _)| *id == anchor) + .unwrap() + }) + .unwrap_or(expected.len()); + expected.insert(at, row); + } + } + Operation::Text(edit) => { + assert_eq!(edit.range, 0..1); + let (_, _, text) = expected + .iter_mut() + .find(|(_, id, _)| *id == edit.object) + .unwrap(); + text.replace_range(0..1, &edit.replacement); + } + Operation::Format(format) => { + let [onestore::TextAttribute::FontSize(size)] = format.attributes.as_slice() else { + panic!() + }; + let store = Store::parse(edit.as_bytes()).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let space = &document.spaces[&SOURCE.1]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + assert_eq!( + view.text_runs(format.object).unwrap()[0].format.font_size, + Some(*size) + ); + } + _ => panic!(), + } + assert_eq!(rows(edit.as_bytes()), expected); + let result = edit.commit(self.disk); + let durable = rows(&self.disk.durable); + assert!(durable == before || durable == expected); + if result.is_ok() { + assert_eq!(durable, expected); + } + result + } + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + onestore::confirm_snapshot(self.disk, snapshot) + } +} + +pub fn run(input: &[u8]) { + let (source, sid, outline) = &*SOURCE; + let directory = tempfile::tempdir().unwrap(); + let mut replicas: [Option; 12] = std::array::from_fn(|_| None); + let mut disk = disk::Disk { + visible: source.clone(), + durable: source.clone(), + operation: 0, + fail_at: None, + write_limit: 4096, + random: 1, + }; + for step in input.chunks_exact(8).take(48) { + let actor = usize::from(step[0]) % replicas.len(); + let path = directory.path().join(format!("{actor}.sqlite")); + let cache = replicas[actor].get_or_insert_with(|| Replica::create(&path, source).unwrap()); + let snapshot = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + let local = rows(&snapshot); + match step[1] % 8 { + 0..=2 if !local.is_empty() => { + let target = usize::from(step[2]) % local.len(); + let id = match step[1] % 8 { + 0 => { + let before = + (step[3] & 1 == 0).then(|| local[usize::from(step[4]) % local.len()].0); + cache + .tree( + &snapshot, + *sid, + &TreeEdit::move_to(local[target].0, *outline, before, "Offline") + .unwrap(), + ) + .unwrap() + } + 1 => cache + .tree( + &snapshot, + *sid, + &TreeEdit::delete(local[target].0, "Offline").unwrap(), + ) + .unwrap(), + _ if step[3] & 1 == 0 => cache + .edit_text( + &snapshot, + *sid, + local[target].1, + 0..1, + &char::from(b'A' + step[3] % 26).to_string(), + ) + .unwrap(), + _ => cache + .format( + &snapshot, + *sid, + local[target].1, + 0..1, + &[onestore::TextAttribute::FontSize( + 12.0 + f32::from(step[3] % 20), + )], + ) + .unwrap(), + }; + let next = cache.pending().unwrap(); + assert_eq!(next[..pending.len()], pending); + assert_eq!(next.len(), pending.len() + usize::from(id.is_some())); + } + 3 | 4 => { + disk.operation = 0; + disk.fail_at = (step[4] != 0) + .then_some(usize::from(u16::from_le_bytes([step[4], step[5]])) % 2048 + 1); + disk.write_limit = if step[6] & 1 == 0 { 17 } else { 4096 }; + disk.random = u64::from(step[7]) + 1; + let _ = cache.sync_once(&mut Session { + disk: &mut disk, + operation: pending.first().map(|p| &p.operation), + }); + let next = cache.pending().unwrap(); + if next.len() == pending.len() { + assert_eq!(next, pending); + } else { + assert_eq!(next, pending[1..]); + let Some(EditStatus::Published { revision }) = + cache.status(pending[0].id).unwrap() + else { + panic!() + }; + let store = Store::parse(&disk.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + assert!(index.spaces[sid].revisions.contains_key(&revision)); + } + if !next.is_empty() { + assert!(cache.snapshot().unwrap() == snapshot); + } + disk.visible.clone_from(&disk.durable); + } + 5 => { + let mut expected = rows(&disk.visible); + if !expected.is_empty() { + let target = usize::from(step[2]) % expected.len(); + let (_, text, content) = &mut expected[target]; + let at = u32::try_from(content.encode_utf16().count()).unwrap(); + content.push('R'); + let snapshot = disk.visible.clone(); + let edit = PreparedEdit::text(&snapshot, *sid, *text, at..at, "R").unwrap(); + disk.fail_at = None; + edit.commit(&mut disk).unwrap(); + assert_eq!(rows(&disk.durable), expected); + } + } + _ => { + if step[1] % 8 == 6 + && let Some(first) = pending.first() + && matches!(first.operation, Operation::Tree(_)) + && matches!( + cache.status(first.id).unwrap(), + Some(EditStatus::Conflict(_)) + ) + { + let remote = cache.remote_snapshot().unwrap(); + let _ = cache.rebase_tree_conflict(first.id, &snapshot, &remote); + } + let pending = cache.pending().unwrap(); + drop(replicas[actor].take()); + let cache = Replica::open(&path).unwrap(); + assert!(cache.snapshot().unwrap() == snapshot); + assert_eq!(cache.pending().unwrap(), pending); + replicas[actor] = Some(cache); + } + } + rows(&disk.durable); + rows(&replicas[actor].as_ref().unwrap().snapshot().unwrap()); + } +} diff --git a/crates/notebook/tests/sync.rs b/crates/notebook/tests/sync.rs new file mode 100644 index 0000000000000000000000000000000000000000..f03a39611443d7f5649768e4b993db46df84a3ac --- /dev/null +++ b/crates/notebook/tests/sync.rs @@ -0,0 +1,3931 @@ +use notebook::{ConflictKind, EditStatus, Error, Remote, Replica}; +use onestore::{ + CommitError, CommitIo, CommitState, ExGuid, PreparedEdit, RevisionIndex, Store, + document::{Document, Kind}, +}; +use std::io; + +#[path = "../../onestore/tests/support/disk.rs"] +mod disk; +#[path = "sync/outline.rs"] +mod outline; +#[path = "sync/page.rs"] +mod page; +#[path = "support/page_schedule.rs"] +mod page_schedule; +#[path = "sync/pages.rs"] +mod pages; +#[path = "sync/tree.rs"] +mod tree; +#[path = "support/tree_schedule.rs"] +mod tree_schedule; + +mod paragraph { + use super::*; + use onestore::{Insertion, ParagraphJoin, ParagraphSplit, TextAttribute}; + + fn fixture() -> (Vec, ExGuid, ExGuid, ExGuid) { + let source = onestore::create_section("paragraph.one", "ab🦀cd", "Fixture").unwrap(); + let (sid, left, _) = text(&source); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let space = &doc.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let parent = *view + .nodes + .iter() + .find(|(_, node)| node.content == [left]) + .unwrap() + .0; + (source, sid, left, parent) + } + + fn native_reconciliation(keyboard: bool) -> (Vec, Vec) { + let source = include_bytes!( + "../../../corpus/paragraph-edit/reconciliation/before/notebook/synthetic.one" + ); + let native: &[u8] = if keyboard { + include_bytes!( + "../../../corpus/paragraph-edit/reconciliation/keyboard/notebook/synthetic.one" + ) + } else { + include_bytes!( + "../../../corpus/paragraph-edit/reconciliation/remote/notebook/synthetic.one" + ) + }; + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let mut server = Server::new(native); + let mut recorded = Vec::new(); + let mut counts = [0; 3]; + for (sid, page) in document.pages().unwrap() { + let space = &document.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let Kind::Metadata { + title: Some(name), .. + } = &view.nodes[&view.roots[&2]].kind + else { + continue; + }; + if !name.starts_with("Split ") && !name.starts_with("Join ") { + continue; + } + let outline = view.nodes[&page] + .children + .iter() + .find(|id| matches!(view.nodes[id].kind, Kind::Outline { .. })) + .unwrap(); + let children = &view.nodes[outline].children; + let left = view.nodes[&children[0]].content[0]; + let split = name.starts_with("Split "); + let adoption = name == "Join empty adoption"; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("native-reconciliation.sqlite"); + let mut cache = Replica::create(&path, source).unwrap(); + let (id, target, intent) = if split { + let intent = ParagraphSplit::new(left, 2, "Offline author").unwrap(); + ( + cache.split(source, sid, &intent).unwrap().unwrap(), + intent.text_object(), + serde_json::to_value(intent).unwrap(), + ) + } else { + let right = view.nodes[&children[1]].content[0]; + let intent = ParagraphJoin::new(left, right, "Offline author").unwrap(); + ( + cache.join(source, sid, &intent).unwrap().unwrap(), + if adoption { right } else { left }, + serde_json::to_value(intent).unwrap(), + ) + }; + drop(cache); + cache = Replica::open(&path).unwrap(); + let at = if split { + 2 + } else if adoption { + 1 + } else { + 4 + }; + let dependent = cache + .edit_text( + &cache.snapshot().unwrap(), + sid, + target, + at..at + 1, + if adoption { "I" } else { "C" }, + ) + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + drop(cache); + cache = Replica::open(&path).unwrap(); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.pending().unwrap(), pending); + let previous = server.publications; + let remote = server.visible.clone(); + let (actual, status) = cache.sync_once(&mut server).unwrap().unwrap(); + assert_eq!(actual, id); + assert_eq!(cache.snapshot().unwrap(), local); + if !keyboard { + counts[2] += 1; + assert_eq!( + status, + EditStatus::Conflict(ConflictKind::TargetUnavailable), + "{name}" + ); + assert_eq!(server.publications, previous); + assert_eq!(server.visible, remote); + drop(cache); + cache = Replica::open(&path).unwrap(); + assert_eq!(cache.status(id).unwrap(), Some(status)); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.pending().unwrap(), pending); + continue; + } + let automatic = ["prefix", "format", "sibling"] + .iter() + .any(|suffix| name.ends_with(suffix)); + if automatic { + counts[0] += 1; + assert!( + matches!(status, EditStatus::Published { .. }), + "{name}: {status:?}" + ); + } else { + let expected = if name.ends_with("boundary") || adoption { + ConflictKind::TextChanged + } else { + ConflictKind::StructureChanged + }; + assert_eq!(status, EditStatus::Conflict(expected), "{name}"); + assert_eq!(server.publications, previous); + assert_eq!(server.visible, remote); + assert_eq!(cache.pending().unwrap(), pending); + drop(cache); + cache = Replica::open(&path).unwrap(); + assert_eq!(cache.status(id).unwrap(), Some(status)); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.pending().unwrap(), pending); + if adoption || name == "Join remote list" { + counts[2] += 1; + assert!(cache.rebase_join_conflict(id, &local, &remote).is_err()); + assert_eq!(cache.pending().unwrap(), pending); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.status(id).unwrap(), Some(status)); + recorded.push(serde_json::json!({"case": name, "space": sid, "intent": intent, "outcome": "retained_conflict"})); + continue; + } + counts[1] += 1; + if split { + let offset = if name.ends_with("boundary") { 3 } else { 2 }; + cache + .rebase_conflict(id, &local, &remote, offset..offset) + .unwrap(); + let notebook::Operation::Split(edit) = &cache.pending().unwrap()[0].operation + else { + panic!() + }; + assert_eq!( + edit.intent, + serde_json::from_value::(intent.clone()) + .unwrap() + .reposition(offset) + ); + } else { + cache.rebase_join_conflict(id, &local, &remote).unwrap(); + } + drop(cache); + cache = Replica::open(&path).unwrap(); + let (actual, status) = cache.sync_once(&mut server).unwrap().unwrap(); + assert_eq!(actual, id); + assert!( + matches!(status, EditStatus::Published { .. }), + "{name}: {status:?}" + ); + } + drop(cache); + cache = Replica::open(&path).unwrap(); + let first_receipt = cache.status(id).unwrap().unwrap(); + let (actual, status) = cache.sync_once(&mut server).unwrap().unwrap(); + assert_eq!(actual, dependent); + assert!( + matches!(status, EditStatus::Published { .. }), + "{name} dependent: {status:?}" + ); + assert_eq!(server.publications, previous + 2); + drop(cache); + cache = Replica::open(&path).unwrap(); + assert_eq!(cache.status(id).unwrap(), Some(first_receipt)); + assert_eq!(cache.status(dependent).unwrap(), Some(status)); + assert!(cache.pending().unwrap().is_empty()); + assert_eq!(cache.snapshot().unwrap(), server.durable); + let EditStatus::Published { revision } = first_receipt else { + panic!() + }; + let EditStatus::Published { + revision: dependent_revision, + } = status + else { + panic!() + }; + recorded.push(serde_json::json!({"case": name, "space": sid, "intent": intent, + "outcome": if automatic { "automatic" } else { "reviewed" }, "receipt": revision, "dependent_receipt": dependent_revision})); + } + assert_eq!(counts, if keyboard { [6, 8, 2] } else { [0, 0, 16] }); + assert_eq!(server.publications, if keyboard { 28 } else { 0 }); + (server.durable, recorded) + } + + #[test] + fn native_com_replacement_preserves_every_local_paragraph_branch() { + native_reconciliation(false); + } + + #[test] + fn native_changes_commute_or_retain_reviewable_paragraph_dependencies() { + native_reconciliation(true); + } + + #[test] + #[ignore = "exports native-controlled reconciliation for cold OneNote validation"] + fn export_native_reconciliation() { + let output = std::path::PathBuf::from( + std::env::var_os("ONESTORE_NATIVE_RECONCILIATION_OUTPUT").unwrap(), + ); + assert!(output.is_absolute()); + std::fs::create_dir(&output).unwrap(); + let (bytes, cases) = native_reconciliation(true); + std::fs::create_dir(output.join("candidate")).unwrap(); + std::fs::write(output.join("candidate/synthetic.one"), bytes).unwrap(); + std::fs::write( + output.join("manifest.json"), + serde_json::to_vec_pretty(&cases).unwrap(), + ) + .unwrap(); + } + + #[test] + fn native_splits_retain_independent_local_identities_and_changed_boundaries() { + let source = include_bytes!("../../../corpus/paragraph-edit/before/notebook/synthetic.one"); + let native = include_bytes!("../../../corpus/paragraph-edit/split/notebook/synthetic.one"); + let manifest: serde_json::Value = + serde_json::from_str(include_str!("../../../corpus/paragraph-edit/manifest.json")) + .unwrap(); + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let mut accepted = 0; + let mut conflicts = 0; + for case in manifest["cases"].as_array().unwrap() { + if case["case"] == "Split before hyperlink" { + continue; + } + let left: ExGuid = serde_json::from_value(case["original_text"].clone()).unwrap(); + let native_right: ExGuid = serde_json::from_value(case["new_text"].clone()).unwrap(); + let (sid, _) = document + .pages() + .unwrap() + .into_iter() + .find(|(sid, _)| { + let space = &document.spaces[sid]; + space.revisions[&space.contexts[&ExGuid::default()]] + .nodes + .contains_key(&left) + }) + .unwrap(); + let split = ParagraphSplit::new( + left, + u32::try_from(case["offset_utf16"].as_u64().unwrap()).unwrap(), + "Offline author", + ) + .unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("native-split.sqlite"); + let mut cache = Replica::create(&path, source).unwrap(); + let id = cache.split(source, sid, &split).unwrap().unwrap(); + let current = cache.snapshot().unwrap(); + let dependent = cache + .edit_text( + ¤t, + sid, + split.text_object(), + 0..0, + "Local dependent edit: ", + ) + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + let mut server = Server::new(native); + let (published, status) = cache.sync_once(&mut server).unwrap().unwrap(); + assert_eq!(published, id); + assert_eq!(cache.snapshot().unwrap(), local); + if matches!(case["case"].as_str().unwrap(), "Split end" | "Split empty") { + accepted += 1; + assert!( + matches!(status, EditStatus::Published { .. }), + "{}: {status:?}", + case["case"] + ); + drop(cache); + cache = Replica::open(&path).unwrap(); + assert_eq!(cache.sync_once(&mut server).unwrap().unwrap().0, dependent); + assert_eq!(server.publications, 2); + let store = Store::parse(&server.visible).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let space = &document.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let original: ExGuid = + serde_json::from_value(case["original_paragraph"].clone()).unwrap(); + let native_paragraph: ExGuid = + serde_json::from_value(case["new_paragraph"].clone()).unwrap(); + let parent = view + .nodes + .values() + .find(|node| node.children.contains(&original)) + .unwrap(); + let position = parent + .children + .iter() + .position(|id| *id == original) + .unwrap(); + assert_eq!( + &parent.children[position..position + 3], + &[original, split.object(), native_paragraph] + ); + assert!( + matches!(&view.nodes[&native_right].kind, Kind::RichText {text,..} if text.is_empty()) + ); + assert!( + matches!(&view.nodes[&split.text_object()].kind, Kind::RichText {text,..} if text == "Local dependent edit: ") + ); + } else { + conflicts += 1; + assert!( + matches!( + status, + EditStatus::Conflict( + ConflictKind::TextChanged | ConflictKind::StructureChanged + ) + ), + "{}: {status:?}", + case["case"] + ); + assert_eq!(server.publications, 0); + assert_eq!(cache.pending().unwrap(), pending); + drop(cache); + cache = Replica::open(&path).unwrap(); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.pending().unwrap(), pending); + assert_eq!(cache.status(id).unwrap(), Some(status)); + } + } + assert_eq!((accepted, conflicts), (2, 10)); + } + + #[test] + fn native_joins_do_not_acknowledge_or_discard_an_independent_local_branch() { + let source = include_bytes!( + "../../../corpus/paragraph-edit/join-tags/before/notebook/synthetic.one" + ); + let native = include_bytes!( + "../../../corpus/paragraph-edit/join-tags/joined/notebook/synthetic.one" + ); + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let mut cases = 0; + for (sid, page) in document.pages().unwrap() { + let space = &document.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let Kind::Metadata { + title: Some(name), .. + } = &view.nodes[&view.roots[&2]].kind + else { + continue; + }; + if !name.starts_with("Join ") { + continue; + } + cases += 1; + let outline = view.nodes[&page] + .children + .iter() + .find(|id| matches!(view.nodes[id].kind, Kind::Outline { .. })) + .unwrap(); + let children = &view.nodes[outline].children; + let left = view.nodes[&children[0]].content[0]; + let right = view.nodes[&children[1]].content[0]; + let Kind::RichText { + text: left_text, .. + } = &view.nodes[&left].kind + else { + panic!() + }; + let survivor = if left_text.is_empty() { right } else { left }; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("native-join.sqlite"); + let mut cache = Replica::create(&path, source).unwrap(); + let join = ParagraphJoin::new(left, right, "Offline author").unwrap(); + let id = cache.join(source, sid, &join).unwrap().unwrap(); + let current = cache.snapshot().unwrap(); + cache + .edit_text(¤t, sid, survivor, 0..0, "Local dependent edit: ") + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + let mut server = Server::new(native); + let (conflicted, status) = cache.sync_once(&mut server).unwrap().unwrap(); + assert_eq!(conflicted, id); + assert!( + matches!( + status, + EditStatus::Conflict(ConflictKind::TargetUnavailable) + ), + "{name}: {status:?}" + ); + assert_eq!(server.publications, 0); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.pending().unwrap(), pending); + drop(cache); + cache = Replica::open(&path).unwrap(); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.pending().unwrap(), pending); + assert_eq!(cache.status(id).unwrap(), Some(status)); + } + assert_eq!(cases, 3); + } + + #[test] + fn split_join_dependencies_reopen_and_rebase_with_remote_text_and_styles() { + let (source, sid, left, parent) = fixture(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("paragraph.sqlite"); + let mut cache = Replica::create(&path, &source).unwrap(); + let split = ParagraphSplit::new(left, 2, "Local").unwrap(); + let child = Insertion::paragraph(split.object(), None, "Child", "Local").unwrap(); + let join = ParagraphJoin::new(left, split.text_object(), "Local").unwrap(); + let mut ids = Vec::new(); + for step in 0..6 { + let current = cache.snapshot().unwrap(); + let id = match step { + 0 => cache.split(¤t, sid, &split), + 1 => cache.edit_text(¤t, sid, split.text_object(), 0..2, "🦋"), + 2 => cache.format( + ¤t, + sid, + split.text_object(), + 2..3, + &[TextAttribute::Bold(true)], + ), + 3 => cache.insert(¤t, sid, &child), + 4 => cache.join(¤t, sid, &join), + 5 => cache.edit_text(¤t, sid, left, 5..6, "D"), + _ => unreachable!(), + } + .unwrap() + .unwrap(); + ids.push(id); + let saved = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + drop(cache); + cache = Replica::open(&path).unwrap(); + assert_eq!(cache.snapshot().unwrap(), saved); + assert_eq!(cache.pending().unwrap(), pending); + } + let remote = onestore::replace_text(&source, sid, left, 0..0, "Z").unwrap(); + let remote = + PreparedEdit::format(&remote, sid, left, 1..2, &[TextAttribute::Italic(true)]).unwrap(); + let mut server = Server::new(remote.as_bytes()); + let local = cache.snapshot().unwrap(); + for (i, id) in ids.iter().enumerate() { + let (published, state) = cache.sync_once(&mut server).unwrap().unwrap(); + assert_eq!(published, *id); + assert!(matches!(state, EditStatus::Published { .. }), "{state:?}"); + drop(cache); + cache = Replica::open(&path).unwrap(); + assert_eq!(cache.status(*id).unwrap(), Some(state)); + if i != ids.len() - 1 { + assert_eq!(cache.snapshot().unwrap(), local); + } + } + assert_eq!(server.publications, ids.len()); + assert_eq!(server.visible, server.durable); + assert_eq!(cache.snapshot().unwrap(), server.durable); + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + let doc = Document::parse(&index).unwrap(); + let space = &doc.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + assert!( + matches!(&view.nodes[&left].kind, Kind::RichText { text, .. } if text == "Zab🦋cD") + ); + assert_eq!(view.nodes[&parent].children, [child.object()]); + assert_eq!(view.nodes[&parent].content, [left]); + let runs = view.text_runs(left).unwrap(); + let chars: Vec<_> = runs + .iter() + .flat_map(|run| { + run.text + .chars() + .map(|c| (c, run.format.bold, run.format.italic)) + }) + .collect(); + assert_eq!(chars.iter().find(|v| v.0 == 'a').unwrap().2, Some(true)); + assert_eq!(chars.iter().find(|v| v.0 == 'c').unwrap().1, Some(true)); + } + + #[test] + fn changed_split_children_require_fresh_review_without_reallocating_dependents() { + let (source, sid, left, parent) = fixture(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("paragraph.sqlite"); + let cache = Replica::create(&path, &source).unwrap(); + let split = ParagraphSplit::new(left, 2, "Local").unwrap(); + let id = cache.split(&source, sid, &split).unwrap().unwrap(); + let dependent = cache + .edit_text( + &cache.snapshot().unwrap(), + sid, + split.text_object(), + 0..2, + "🦋", + ) + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let child = Insertion::paragraph(parent, None, "Remote child", "Remote").unwrap(); + let remote = PreparedEdit::insert(&source, sid, &child).unwrap(); + let mut server = Server::new(remote.as_bytes()); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, EditStatus::Conflict(ConflictKind::StructureChanged))) + ); + assert_eq!(server.publications, 0); + assert_eq!(cache.snapshot().unwrap(), local); + assert!( + cache + .rebase_conflict(id, &source, &server.visible, 2..2) + .is_err() + ); + assert!( + cache + .rebase_conflict(id, &local, &server.visible, 2..3) + .is_err() + ); + cache + .rebase_conflict(id, &local, &server.visible, 2..2) + .unwrap(); + let pending = cache.pending().unwrap(); + let notebook::Operation::Split(edit) = &pending[0].operation else { + panic!() + }; + assert_eq!(edit.intent, split); + assert_eq!(pending[1].id, dependent); + drop(cache); + let cache = Replica::open(&path).unwrap(); + for expected in [id, dependent] { + let (actual, state) = cache.sync_once(&mut server).unwrap().unwrap(); + assert_eq!(actual, expected); + assert!(matches!(state, EditStatus::Published { .. })); + } + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let space = &doc.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + assert_eq!(view.nodes[&split.object()].children, [child.object()]); + assert!( + matches!(&view.nodes[&split.text_object()].kind, Kind::RichText { text, .. } if text == "🦋cd") + ); + } + + #[test] + fn uncertain_paragraph_operations_keep_the_original_attempt_across_reopen() { + for join in [false, true] { + for fault in [ + Fault::Before, + Fault::UnknownBefore, + Fault::UnknownAfter, + Fault::Committed, + ] { + let (source, sid, left, _) = fixture(); + let split = ParagraphSplit::new(left, 2, "Local").unwrap(); + let source = if join { + PreparedEdit::split(&source, sid, &split) + .unwrap() + .as_bytes() + .to_vec() + } else { + source + }; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("paragraph.sqlite"); + let cache = Replica::create(&path, &source).unwrap(); + let id = if join { + cache.join( + &source, + sid, + &ParagraphJoin::new(left, split.text_object(), "Local").unwrap(), + ) + } else { + cache.split(&source, sid, &split) + } + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let intent = cache.pending().unwrap(); + let mut server = Server::new(&source); + server.fault = fault; + assert!(cache.sync_once(&mut server).is_err()); + let state = cache.status(id).unwrap().unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.status(id).unwrap(), Some(state)); + if matches!(fault, Fault::UnknownBefore | Fault::UnknownAfter) { + assert!(matches!(state, EditStatus::AwaitingConfirmation { .. })); + assert_eq!(cache.pending().unwrap(), intent); + assert_eq!(cache.snapshot().unwrap(), local); + if matches!(fault, Fault::UnknownBefore) { + assert_eq!(cache.sync_once(&mut server).unwrap(), Some((id, state))); + assert_eq!(server.publications, 1); + continue; + } + server.fault = Fault::Confirm; + assert!(cache.sync_once(&mut server).is_err()); + assert_eq!(cache.status(id).unwrap(), Some(state)); + let (_, state) = cache.sync_once(&mut server).unwrap().unwrap(); + assert!(matches!(state, EditStatus::Published { .. })); + assert_eq!(server.publications, 1); + } else if matches!(fault, Fault::Before) { + assert_eq!(state, EditStatus::Pending); + assert!(matches!( + cache.sync_once(&mut server).unwrap().unwrap().1, + EditStatus::Published { .. } + )); + assert_eq!(server.publications, 2); + } else { + assert!(matches!(state, EditStatus::Published { .. })); + assert_eq!(server.publications, 1); + } + assert!(cache.pending().unwrap().is_empty()); + let saved = cache.snapshot().unwrap(); + if matches!(fault, Fault::UnknownAfter) { + // Confirmation refreshes version metadata after reading the acknowledged snapshot. + assert!(saved[..212] == server.durable[..212]); + assert!(saved[252..] == server.durable[252..]); + assert_eq!( + Store::parse(&server.durable).unwrap().header.generation, + Store::parse(&saved).unwrap().header.generation + 1 + ); + } else { + assert!(saved == server.durable); + } + assert_eq!(cache.sync_once(&mut server).unwrap(), None); + assert!(cache.snapshot().unwrap() == server.durable); + } + } + } + + #[test] + fn join_reviews_preserve_survivors_and_require_current_child_placement() { + for empty in [false, true] { + let (source, sid, left, _) = fixture(); + let split = ParagraphSplit::new(left, if empty { 0 } else { 2 }, "Local").unwrap(); + let source = PreparedEdit::split(&source, sid, &split) + .unwrap() + .as_bytes() + .to_vec(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("join.sqlite"); + let cache = Replica::create(&path, &source).unwrap(); + let intent = ParagraphJoin::new(left, split.text_object(), "Local").unwrap(); + let id = cache.join(&source, sid, &intent).unwrap().unwrap(); + let survivor = if empty { split.text_object() } else { left }; + let dependent = cache + .edit_text(&cache.snapshot().unwrap(), sid, survivor, 0..0, "Local ") + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let child = + Insertion::paragraph(split.object(), None, "Remote child", "Remote").unwrap(); + let remote = PreparedEdit::insert(&source, sid, &child) + .unwrap() + .as_bytes() + .to_vec(); + let mut server = Server::new(&remote); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, EditStatus::Conflict(ConflictKind::StructureChanged))) + ); + assert!(cache.rebase_join_conflict(id, &source, &remote).is_err()); + assert_eq!(server.publications, 0); + cache.rebase_join_conflict(id, &local, &remote).unwrap(); + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) + ); + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == dependent) + ); + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let space = &doc.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let parent = view + .nodes + .values() + .find(|node| node.content == [survivor]) + .unwrap(); + assert_eq!(parent.children, [child.object()]); + assert!( + matches!(&view.nodes[&survivor].kind, Kind::RichText { text, .. } if text == "Local ab🦀cd") + ); + } + + let (source, sid, left, _) = fixture(); + let split = ParagraphSplit::new(left, 0, "Local").unwrap(); + let source = PreparedEdit::split(&source, sid, &split) + .unwrap() + .as_bytes() + .to_vec(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("identity.sqlite"); + let cache = Replica::create(&path, &source).unwrap(); + let intent = ParagraphJoin::new(left, split.text_object(), "Local").unwrap(); + let id = cache.join(&source, sid, &intent).unwrap().unwrap(); + let local = cache.snapshot().unwrap(); + let remote = onestore::replace_text(&source, sid, left, 0..0, "Remote").unwrap(); + let mut server = Server::new(&remote); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, EditStatus::Conflict(ConflictKind::TextChanged))) + ); + assert!(cache.rebase_join_conflict(id, &local, &remote).is_err()); + assert!(cache.snapshot().unwrap() == local); + assert_eq!(server.publications, 0); + assert_eq!(cache.pending().unwrap().len(), 1); + } + + #[test] + fn overlapping_clients_retain_each_local_branch_without_replaying_removed_targets() { + let (source, sid, left, _) = fixture(); + let split = ParagraphSplit::new(left, 2, "Seed").unwrap(); + let source = PreparedEdit::split(&source, sid, &split) + .unwrap() + .as_bytes() + .to_vec(); + let directory = tempfile::tempdir().unwrap(); + let first = Replica::create(directory.path().join("first.sqlite"), &source).unwrap(); + let second_path = directory.path().join("second.sqlite"); + let second = Replica::create(&second_path, &source).unwrap(); + let join = ParagraphJoin::new(left, split.text_object(), "First").unwrap(); + first.join(&source, sid, &join).unwrap(); + let right_split = ParagraphSplit::new(split.text_object(), 2, "Second").unwrap(); + let id = second.split(&source, sid, &right_split).unwrap().unwrap(); + second + .edit_text( + &second.snapshot().unwrap(), + sid, + right_split.text_object(), + 0..0, + "Second ", + ) + .unwrap(); + let local = second.snapshot().unwrap(); + let pending = second.pending().unwrap(); + let mut server = Server::new(&source); + assert!(matches!( + first.sync_once(&mut server).unwrap().unwrap().1, + EditStatus::Published { .. } + )); + assert_eq!( + second.sync_once(&mut server).unwrap(), + Some((id, EditStatus::Conflict(ConflictKind::TargetUnavailable))) + ); + drop(second); + let second = Replica::open(&second_path).unwrap(); + assert_eq!(second.pending().unwrap(), pending); + assert!(second.snapshot().unwrap() == local); + assert_eq!(server.publications, 1); + assert_eq!( + second.status(id).unwrap(), + Some(EditStatus::Conflict(ConflictKind::TargetUnavailable)) + ); + } + + #[test] + #[ignore = "exports reconciled paragraph edits for independent native validation"] + fn export_native_offline_paragraphs() { + use std::{fs, path::PathBuf}; + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../corpus/paragraph-edit"); + let output = PathBuf::from(std::env::var_os("ONESTORE_OFFLINE_PARAGRAPH_OUTPUT").unwrap()); + assert!(output.is_absolute()); + fs::create_dir(&output).unwrap(); + for (name, source, manifest, split) in [ + ( + "splits", + "before/notebook/synthetic.one", + "rust-split/manifest.json", + true, + ), + ( + "joins", + "split/notebook/synthetic.one", + "rust-join/split/manifest.json", + false, + ), + ( + "inheritance", + "join-edges/before/notebook/synthetic.one", + "rust-join/inheritance/manifest.json", + false, + ), + ( + "tags", + "join-tags/before/notebook/synthetic.one", + "rust-join/tags/manifest.json", + false, + ), + ] { + let source = fs::read(root.join(source)).unwrap(); + let folder = output.join(name); + fs::create_dir(&folder).unwrap(); + let path = folder.join("cache.sqlite"); + let mut cache = Replica::create(&path, &source).unwrap(); + let mut server = Server::new(&source); + let manifest: serde_json::Value = + serde_json::from_slice(&fs::read(root.join(manifest)).unwrap()).unwrap(); + let cases = if split { &manifest["cases"] } else { &manifest }; + let mut recorded = Vec::new(); + for case in cases + .as_array() + .unwrap() + .iter() + .filter(|case| case.get("intent").is_some()) + { + let local = cache.snapshot().unwrap(); + let (text, offset) = if split { + serde_json::from_value::(case["intent"].clone()) + .unwrap() + .position() + } else { + ( + serde_json::from_value::(case["intent"].clone()) + .unwrap() + .texts()[0], + 1, + ) + }; + let store = Store::parse(&local).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let (sid, space) = doc + .spaces + .iter() + .find(|(_, space)| { + space.revisions[&space.contexts[&ExGuid::default()]] + .nodes + .contains_key(&text) + }) + .unwrap(); + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let Kind::RichText { text: content, .. } = &view.nodes[&text].kind else { + panic!() + }; + let remote_prefix = offset > 0 && !content.is_empty(); + assert!(!content.contains('☂')); + if remote_prefix { + server.visible = + onestore::replace_text(&server.visible, *sid, text, 0..0, "☂").unwrap(); + server.durable.clone_from(&server.visible); + } + let id = if split { + cache.split( + &local, + *sid, + &serde_json::from_value(case["intent"].clone()).unwrap(), + ) + } else { + cache.join( + &local, + *sid, + &serde_json::from_value(case["intent"].clone()).unwrap(), + ) + } + .unwrap() + .unwrap(); + recorded.push(serde_json::json!({"case": case["case"], "id": id, + "space": sid, "intent": case["intent"], "remote_prefix": remote_prefix})); + drop(cache); + cache = Replica::open(&path).unwrap(); + } + for case in &mut recorded { + let id = case["id"].as_u64().unwrap(); + if id % 2 == 0 { + server.fault = Fault::UnknownAfter; + assert!(cache.sync_once(&mut server).is_err()); + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::AwaitingConfirmation { .. }) + )); + drop(cache); + cache = Replica::open(&path).unwrap(); + } + let (actual, status) = cache.sync_once(&mut server).unwrap().unwrap(); + assert_eq!(actual, id); + let EditStatus::Published { revision } = status else { + panic!("{name}: {status:?}") + }; + case["revision"] = serde_json::to_value(revision).unwrap(); + } + assert_eq!(server.publications, recorded.len()); + assert_eq!(cache.sync_once(&mut server).unwrap(), None); + assert!(cache.snapshot().unwrap() == server.durable); + cache + .export_recovery(folder.join("recovery.sqlite")) + .unwrap(); + fs::create_dir(folder.join("candidate")).unwrap(); + fs::write(folder.join("candidate/synthetic.one"), &server.durable).unwrap(); + fs::write( + folder.join("manifest.json"), + serde_json::to_vec_pretty(&recorded).unwrap(), + ) + .unwrap(); + } + } +} + +#[derive(Clone, Copy, Default)] +enum Fault { + #[default] + None, + Before, + UnknownBefore, + UnknownAfter, + Committed, + PanicBefore, + PanicAfter, + Confirm, + ConfirmCommitted, +} + +struct Server { + visible: Vec, + durable: Vec, + fault: Fault, + publications: usize, + confirmations: usize, +} + +impl Server { + fn new(source: &[u8]) -> Self { + Self { + visible: source.to_vec(), + durable: source.to_vec(), + fault: Fault::None, + publications: 0, + confirmations: 0, + } + } +} + +fn failure(state: CommitState) -> CommitError { + CommitError { + state, + error: io::Error::from(io::ErrorKind::ConnectionAborted), + } +} + +impl CommitIo for Server { + fn read_at(&mut self, offset: u64, output: &mut [u8]) -> io::Result { + let offset = usize::try_from(offset).unwrap(); + let size = output.len().min(self.visible.len().saturating_sub(offset)); + if size > 0 { + output[..size].copy_from_slice(&self.visible[offset..offset + size]); + } + Ok(size) + } + fn write_at(&mut self, offset: u64, bytes: &[u8]) -> io::Result { + let offset = usize::try_from(offset).unwrap(); + self.visible + .resize(self.visible.len().max(offset + bytes.len()), 0); + self.visible[offset..offset + bytes.len()].copy_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + self.durable.clone_from(&self.visible); + Ok(()) + } +} + +impl Remote for Server { + fn read(&mut self) -> io::Result> { + Ok(self.visible.clone()) + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + self.publications += 1; + let fault = std::mem::take(&mut self.fault); + match fault { + Fault::Before => return Err(failure(CommitState::NotCommitted)), + Fault::UnknownBefore => return Err(failure(CommitState::Unknown)), + Fault::PanicBefore => panic!("Terminated before remote I/O"), + _ => {} + } + let old = self.durable.clone(); + edit.commit(self)?; + match fault { + Fault::UnknownAfter => { + self.durable = old; + Err(failure(CommitState::Unknown)) + } + Fault::Committed => Err(failure(CommitState::Committed)), + Fault::PanicAfter => panic!("Terminated after remote publication"), + _ => Ok(()), + } + } + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + self.confirmations += 1; + if matches!(self.fault, Fault::Confirm) { + self.fault = Fault::None; + return Err(failure(CommitState::Unknown)); + } + onestore::confirm_snapshot(self, snapshot)?; + if matches!(self.fault, Fault::ConfirmCommitted) { + self.fault = Fault::None; + return Err(failure(CommitState::Committed)); + } + Ok(()) + } +} + +fn text(source: &[u8]) -> (ExGuid, ExGuid, String) { + let store = Store::parse(source).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + let doc = Document::parse(&index).unwrap(); + doc.spaces + .iter() + .find_map(|(sid, space)| { + space.revisions[&space.contexts[&ExGuid::default()]] + .nodes + .iter() + .find_map(|(oid, node)| match &node.kind { + Kind::RichText { text, .. } => Some((*sid, *oid, text.clone())), + _ => None, + }) + }) + .unwrap() +} + +#[test] +fn recovery_archive_preserves_typed_queue_uncertainty_and_receipts_without_becoming_a_writer() { + use notebook::{Recovery, RecoverySummary}; + use onestore::{Insertion, TextAttribute}; + + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("live.sqlite"); + let archive_path = directory.path().join("recovery.sqlite"); + let source = onestore::create_section("recovery.one", "Original", "Fixture").unwrap(); + let (space, object, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let published = cache + .edit_text(&source, space, object, 0..0, "Published ") + .unwrap() + .unwrap(); + let mut server = Server::new(&source); + let (_, receipt) = cache.sync_once(&mut server).unwrap().unwrap(); + let source = cache.snapshot().unwrap(); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let (page_space, page) = document.pages().unwrap()[0]; + let outline = Insertion::outline(page, 100.0, 200.0, "Outline", "Fixture").unwrap(); + let inserted = cache + .insert(&source, page_space, &outline) + .unwrap() + .unwrap(); + let paragraph = Insertion::paragraph(outline.object(), None, "Recovery 🦀", "Fixture").unwrap(); + cache + .insert(&cache.snapshot().unwrap(), page_space, ¶graph) + .unwrap(); + cache + .format( + &cache.snapshot().unwrap(), + page_space, + paragraph.text_object(), + 0..3, + &[TextAttribute::Bold(true)], + ) + .unwrap(); + cache + .edit_text( + &cache.snapshot().unwrap(), + page_space, + paragraph.text_object(), + 0..0, + "Pending ", + ) + .unwrap(); + server.fault = Fault::UnknownBefore; + assert!(matches!( + cache.sync_once(&mut server), + Err(Error::Remote(CommitError { + state: CommitState::Unknown, + .. + })) + )); + let working = cache.snapshot().unwrap(); + let remote = cache.remote_snapshot().unwrap(); + let pending = cache.pending().unwrap(); + let uncertain = cache.status(inserted).unwrap(); + let source_file = std::fs::read(&path).unwrap(); + let summary = RecoverySummary { + queued_edits: 4, + conflicts: 0, + uncertain_edits: 1, + published_receipts: 1, + working_bytes: working.len() as u64, + remote_bytes: remote.len() as u64, + cached_assets: 0, + cached_asset_bytes: 0, + }; + assert_eq!(cache.recovery_summary().unwrap(), summary); + cache.export_recovery(&archive_path).unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), source_file); + let archive_bytes = std::fs::read(&archive_path).unwrap(); + assert!(Replica::open(&archive_path).is_err()); + assert_eq!(std::fs::read(&archive_path).unwrap(), archive_bytes); + assert!(Recovery::open(&path).is_err()); + let archive = Recovery::open(&archive_path).unwrap(); + assert_eq!(archive.summary().unwrap(), summary); + assert_eq!(archive.snapshot().unwrap(), working); + assert_eq!(archive.remote_snapshot().unwrap(), remote); + assert_eq!(archive.pending().unwrap(), pending); + assert_eq!(archive.status(published).unwrap(), Some(receipt)); + assert_eq!(archive.status(inserted).unwrap(), uncertain); + for edit in &pending { + assert_eq!( + archive.status(edit.id).unwrap(), + cache.status(edit.id).unwrap() + ); + } + let EditStatus::Published { revision } = receipt else { + panic!() + }; + assert_eq!(archive.receipts().unwrap(), [(published, revision)].into()); + assert_eq!( + archive.status(u64::MAX - 1).unwrap_err().to_string(), + cache.status(u64::MAX - 1).unwrap_err().to_string() + ); + for existing in [&path, &archive_path] { + assert!( + matches!(cache.export_recovery(existing), Err(Error::Io(error)) if error.kind() == io::ErrorKind::AlreadyExists) + ); + } + assert_eq!(std::fs::read(&path).unwrap(), source_file); + assert_eq!(std::fs::read(&archive_path).unwrap(), archive_bytes); + cache + .edit_text(&working, space, object, 0..0, "Later ") + .unwrap(); + assert_eq!(archive.snapshot().unwrap(), working); + assert_eq!(archive.pending().unwrap(), pending); + drop(archive); + assert_eq!( + Recovery::open(&archive_path).unwrap().summary().unwrap(), + summary + ); + assert_eq!(std::fs::read(&archive_path).unwrap(), archive_bytes); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + std::fs::metadata(&archive_path) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); + } + let remaining: Vec<_> = std::fs::read_dir(directory.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert!( + remaining + .iter() + .all(|name| !name.to_string_lossy().starts_with(".onestore-recovery-")), + "Temporary recovery files remain: {remaining:?}" + ); +} + +#[test] +fn recovery_archive_retains_conflict_images_and_rejects_foreign_or_future_archives() { + use notebook::Recovery; + + let directory = tempfile::tempdir().unwrap(); + let source = onestore::create_section("recovery.one", "Original", "Fixture").unwrap(); + let (space, object, _) = text(&source); + let cache = Replica::create(directory.path().join("live.sqlite"), &source).unwrap(); + let id = cache + .edit_text(&source, space, object, 0..8, "Local") + .unwrap() + .unwrap(); + let changed = onestore::replace_text(&source, space, object, 0..8, "Remote").unwrap(); + let mut server = Server::new(&changed); + let outcome = cache.sync_once(&mut server).unwrap().unwrap(); + assert_eq!( + outcome, + (id, EditStatus::Conflict(ConflictKind::TextChanged)) + ); + let path = directory.path().join("conflict.sqlite"); + cache.export_recovery(&path).unwrap(); + let archive = Recovery::open(&path).unwrap(); + assert_eq!(text(&archive.snapshot().unwrap()).2, "Local"); + assert_eq!(archive.remote_snapshot().unwrap(), changed); + assert_eq!(archive.status(id).unwrap(), Some(outcome.1)); + assert_eq!(archive.summary().unwrap().conflicts, 1); + assert_eq!(archive.summary().unwrap().uncertain_edits, 0); + drop(archive); + for sql in [ + "PRAGMA user_version=99", + "PRAGMA user_version=4; PRAGMA application_id=0", + ] { + let connection = rusqlite::Connection::open(&path).unwrap(); + connection.execute_batch(sql).unwrap(); + drop(connection); + let before = std::fs::read(&path).unwrap(); + assert!(Recovery::open(&path).is_err()); + assert_eq!(std::fs::read(&path).unwrap(), before); + } + assert_eq!(cache.status(id).unwrap(), Some(outcome.1)); + assert_eq!(text(&cache.snapshot().unwrap()).2, "Local"); +} + +#[test] +fn rebases_multiple_disjoint_remote_changes_and_persists_the_remote_receipt() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("sync.one", "ab🦀cd", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 2..4, "🐈") + .unwrap() + .unwrap(); + let remote = onestore::replace_text(&source, sid, oid, 0..6, "Xab🦀cYd").unwrap(); + let mut server = Server::new(&remote); + let outcome = cache.sync_once(&mut server).unwrap().unwrap(); + assert_eq!(outcome.0, id); + assert!(matches!(outcome.1, EditStatus::Published { .. })); + assert_eq!(text(&server.durable).2, "Xab🐈cYd"); + assert_eq!(cache.snapshot().unwrap(), server.durable); + assert!(cache.pending().unwrap().is_empty()); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.status(id).unwrap(), Some(outcome.1)); + assert_eq!(cache.sync_once(&mut server).unwrap(), None); + assert_eq!(server.publications, 1); + assert_eq!(cache.status(id + 1).unwrap(), None); + let source = cache.snapshot().unwrap(); + let next = cache + .edit_text(&source, sid, oid, 0..0, "Later ") + .unwrap() + .unwrap(); + assert!(next > id); +} + +#[test] +fn overlapping_changes_preserve_both_images_and_survive_restart() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 1..2, "L") + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let remote = onestore::replace_text(&source, sid, oid, 1..2, "R").unwrap(); + let mut server = Server::new(&remote); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, EditStatus::Conflict(ConflictKind::TextChanged))) + ); + assert_eq!(server.publications, 0); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.remote_snapshot().unwrap(), remote); + assert_eq!(cache.pending().unwrap().len(), 1); + assert_eq!( + cache.status(id).unwrap(), + Some(EditStatus::Conflict(ConflictKind::TextChanged)) + ); +} + +#[test] +fn lost_replies_and_process_termination_never_blindly_replay_an_attempt() { + for fault in [ + Fault::UnknownBefore, + Fault::UnknownAfter, + Fault::PanicBefore, + Fault::PanicAfter, + ] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 0..0, "Once ") + .unwrap() + .unwrap(); + let mut server = Server::new(&source); + server.fault = fault; + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + cache.sync_once(&mut server) + })); + let attempted = cache.status(id).unwrap().unwrap(); + assert!(matches!(attempted, EditStatus::AwaitingConfirmation { .. })); + drop(cache); + let cache = Replica::open(&path).unwrap(); + let result = cache.sync_once(&mut server).unwrap().unwrap(); + if matches!(fault, Fault::UnknownAfter | Fault::PanicAfter) { + assert!(matches!(result.1, EditStatus::Published { .. })); + assert_eq!(text(&server.durable).2, "Once abc"); + assert_eq!(server.confirmations, 1); + assert!(cache.pending().unwrap().is_empty()); + } else { + assert_eq!(result.1, attempted); + assert_eq!(cache.pending().unwrap().len(), 1); + assert_eq!(server.confirmations, 0); + assert_eq!(server.durable, source); + } + assert_eq!(server.publications, 1); + } +} + +#[test] +fn failed_confirmation_does_not_promote_visible_bytes_to_a_receipt() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 0..0, "Once ") + .unwrap() + .unwrap(); + let mut server = Server::new(&source); + server.fault = Fault::UnknownAfter; + assert!(cache.sync_once(&mut server).is_err()); + server.fault = Fault::Confirm; + assert!(cache.sync_once(&mut server).is_err()); + assert_eq!(server.durable, source); + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::AwaitingConfirmation { .. }) + )); + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Published { .. })) + )); + assert_eq!(server.publications, 1); + assert_eq!(server.confirmations, 2); + assert_eq!(server.visible, server.durable); +} + +#[test] +fn confirmation_cleanup_failure_still_records_a_durable_receipt() { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(dir.path().join("cache.sqlite"), &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 0..0, "Once ") + .unwrap() + .unwrap(); + let mut server = Server::new(&source); + server.fault = Fault::UnknownAfter; + assert!(cache.sync_once(&mut server).is_err()); + server.fault = Fault::ConfirmCommitted; + assert!( + matches!(cache.sync_once(&mut server), Err(Error::Remote(error)) if error.state == CommitState::Committed) + ); + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::Published { .. }) + )); + assert!(cache.pending().unwrap().is_empty()); + assert_eq!(server.visible, server.durable); + assert_eq!(server.publications, 1); + assert_eq!(server.confirmations, 1); +} + +#[test] +fn proven_unpublished_attempts_retry_and_committed_cleanup_errors_keep_receipts() { + for fault in [Fault::Before, Fault::Committed] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 0..0, "Once ") + .unwrap() + .unwrap(); + let mut server = Server::new(&source); + server.fault = fault; + assert!(matches!( + cache.sync_once(&mut server), + Err(Error::Remote(_)) + )); + if matches!(fault, Fault::Before) { + assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Published { .. })) + )); + assert_eq!(server.publications, 2); + } else { + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::Published { .. }) + )); + assert_eq!(cache.sync_once(&mut server).unwrap(), None); + assert_eq!(server.publications, 1); + } + assert_eq!(text(&server.durable).2, "Once abc"); + } +} + +#[test] +fn database_failures_before_and_after_publication_preserve_recovery_state() { + for table in ["attempt", "receipts"] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 0..0, "Once ") + .unwrap() + .unwrap(); + drop(cache); + let db = rusqlite::Connection::open(&path).unwrap(); + db.execute_batch(&format!("CREATE TRIGGER interrupted BEFORE INSERT ON {table} BEGIN SELECT RAISE(ABORT,'Injected cache failure'); END;")).unwrap(); + drop(db); + let cache = Replica::open(&path).unwrap(); + let mut server = Server::new(&source); + assert!(matches!( + cache.sync_once(&mut server), + Err(Error::Database(_)) + )); + assert_eq!(server.publications, usize::from(table == "receipts")); + assert_eq!(cache.pending().unwrap().len(), 1); + drop(cache); + let db = rusqlite::Connection::open(&path).unwrap(); + db.execute_batch("DROP TRIGGER interrupted").unwrap(); + drop(db); + let cache = Replica::open(&path).unwrap(); + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Published { .. })) + )); + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::Published { .. }) + )); + assert_eq!(server.publications, 1); + assert_eq!(server.confirmations, usize::from(table == "receipts")); + assert_eq!(text(&server.durable).2, "Once abc"); + } +} + +#[test] +fn twelve_local_editors_progress_during_remote_reads_publication_and_confirmation() { + struct Paused { + server: Server, + phase: &'static str, + entered: std::sync::mpsc::Sender<()>, + resume: std::sync::mpsc::Receiver<()>, + } + impl Paused { + fn wait(&self, phase: &str) { + if self.phase == phase { + self.entered.send(()).unwrap(); + self.resume + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + } + } + } + impl Remote for Paused { + fn read(&mut self) -> io::Result> { + self.wait("read"); + self.server.read() + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + self.wait("publish"); + self.server.publish(edit) + } + fn confirm(&mut self, source: &[u8]) -> Result<(), CommitError> { + self.wait("confirm"); + self.server.confirm(source) + } + } + for phase in ["read", "publish", "confirm"] { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(dir.path().join("cache.sqlite"), &source).unwrap(); + let first = cache + .edit_text(&source, sid, oid, 0..0, "First ") + .unwrap() + .unwrap(); + let mut server = Server::new(&source); + if phase == "confirm" { + server.fault = Fault::UnknownAfter; + assert!(cache.sync_once(&mut server).is_err()); + } + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (resume_tx, resume_rx) = std::sync::mpsc::channel(); + let mut paused = Paused { + server, + phase, + entered: entered_tx, + resume: resume_rx, + }; + let mut server = std::thread::scope(|scope| { + let running = scope.spawn(|| { + let result = cache.sync_once(&mut paused); + assert!( + matches!(result, Ok(Some((id, EditStatus::Published { .. }))) if id == first) + ); + paused.server + }); + entered_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + assert!( + matches!(cache.sync_once(&mut Server::new(&source)),Err(Error::Io(error)) if error.kind()==io::ErrorKind::WouldBlock) + ); + assert!( + matches!(cache.rebase_conflict(first, &[], &[], 0..0), Err(Error::Io(error)) if error.kind() == io::ErrorKind::WouldBlock) + ); + let started = std::time::Instant::now(); + let handles: Vec<_> = (0..12) + .map(|writer| { + let cache = &cache; + scope.spawn(move || { + loop { + let snapshot = cache.snapshot().unwrap(); + match cache.edit_text( + &snapshot, + sid, + oid, + 0..0, + &format!("[{writer}] "), + ) { + Ok(Some(id)) => break id, + Err(Error::Io(error)) + if error.kind() == io::ErrorKind::ResourceBusy => {} + other => panic!("Unexpected local outcome {other:?}"), + } + } + }) + }) + .collect(); + let ids: std::collections::BTreeSet<_> = handles + .into_iter() + .map(|handle| handle.join().unwrap()) + .collect(); + assert_eq!(ids.len(), 12); + assert!( + started.elapsed() < std::time::Duration::from_secs(2), + "Local edits waited for remote {phase}" + ); + resume_tx.send(()).unwrap(); + running.join().unwrap() + }); + assert_eq!(cache.pending().unwrap().len(), 12); + let expected = text(&cache.snapshot().unwrap()).2; + for _ in 0..12 { + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Published { .. })) + )); + } + assert!(cache.pending().unwrap().is_empty()); + assert_eq!(server.publications, 13); + assert_eq!(text(&server.durable).2, expected); + assert_eq!(cache.snapshot().unwrap(), server.durable); + } +} + +#[test] +fn unrelated_remote_files_never_replace_a_local_cache() { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(dir.path().join("cache.sqlite"), &source).unwrap(); + let other = onestore::create_section("other.one", "abc", "Fixture").unwrap(); + let mut server = Server::new(&other); + for pending in [false, true] { + if pending { + cache.edit_text(&source, sid, oid, 0..0, "Local ").unwrap(); + } + let before = cache.snapshot().unwrap(); + assert!( + matches!(cache.sync_once(&mut server),Err(Error::Io(error)) if error.kind()==io::ErrorKind::InvalidInput) + ); + assert_eq!(cache.snapshot().unwrap(), before); + assert_eq!(cache.remote_snapshot().unwrap(), source); + assert_eq!(server.publications, 0); + } +} + +#[test] +fn version_one_cache_migration_preserves_images_intents_and_local_ids() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 0..0, "Local ") + .unwrap() + .unwrap(); + let snapshot = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + drop(cache); + let db = rusqlite::Connection::open(&path).unwrap(); + let current_version: u32 = db + .pragma_query_value(None, "user_version", |row| row.get(0)) + .unwrap(); + db.execute_batch( + "DROP TABLE attempt; DROP TABLE conflicts; DROP TABLE receipts; DROP TABLE edits; DROP TABLE assets; + CREATE TABLE edits ( + id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(id>0), space TEXT NOT NULL, + object TEXT NOT NULL, before_text TEXT NOT NULL, + start INTEGER NOT NULL CHECK(start BETWEEN 0 AND 4294967295), + end INTEGER NOT NULL CHECK(end BETWEEN start AND 4294967295), replacement TEXT NOT NULL + ) STRICT; PRAGMA user_version=1;", + ) + .unwrap(); + db.execute("INSERT INTO edits(id,space,object,before_text,start,end,replacement) VALUES (?1,?2,?3,'abc',0,0,'Local ')",rusqlite::params![i64::try_from(id).unwrap(),sid.to_string(),oid.to_string()]).unwrap(); + drop(db); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.snapshot().unwrap(), snapshot); + assert_eq!(cache.pending().unwrap(), pending); + assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); + let mut server = Server::new(&source); + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Published { .. })) + )); + drop(cache); + let db = rusqlite::Connection::open(&path).unwrap(); + assert_eq!( + db.pragma_query_value(None, "user_version", |row| row.get::<_, u32>(0)) + .unwrap(), + current_version + ); +} + +#[test] +fn reviewed_conflict_rebase_preserves_twelve_dependent_edits_and_survives_reopen() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("resolve.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let first = cache + .edit_text(&source, sid, oid, 1..2, "L") + .unwrap() + .unwrap(); + for n in 0..12 { + cache + .edit_text( + &cache.snapshot().unwrap(), + sid, + oid, + 0..0, + &format!("[{n}] "), + ) + .unwrap(); + } + let local = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + let remote = onestore::replace_text(&source, sid, oid, 0..3, "aRcZ").unwrap(); + let mut server = Server::new(&remote); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((first, EditStatus::Conflict(ConflictKind::TextChanged))) + ); + cache.rebase_conflict(first, &local, &remote, 1..2).unwrap(); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.remote_snapshot().unwrap(), remote); + let rebased = cache.pending().unwrap(); + assert_eq!(&rebased[1..], &pending[1..]); + assert_eq!(rebased[0].id, first); + let notebook::Operation::Text(updated) = &rebased[0].operation else { + panic!() + }; + let notebook::Operation::Text(previous) = &pending[0].operation else { + panic!() + }; + assert_eq!(updated.replacement, previous.replacement); + assert_eq!(updated.before, "aRcZ"); + assert_eq!(cache.status(first).unwrap(), Some(EditStatus::Pending)); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), rebased); + assert_eq!(cache.snapshot().unwrap(), local); + for intent in &pending { + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((id, EditStatus::Published { .. })) if id == intent.id) + ); + } + assert_eq!(server.publications, 13); + assert_eq!(text(&server.durable).2, text(&local).2 + "Z"); + assert_eq!(cache.snapshot().unwrap(), server.durable); + assert!(cache.pending().unwrap().is_empty()); +} + +#[test] +fn conflict_review_rejects_stale_images_invalid_ranges_and_nonconflicting_states() { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("resolve.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(dir.path().join("cache.sqlite"), &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 1..2, "L") + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + assert!( + matches!(cache.rebase_conflict(id, &local, &source, 1..2), Err(Error::Io(error)) if error.kind() == io::ErrorKind::InvalidInput) + ); + let remote = onestore::replace_text(&source, sid, oid, 0..3, "🦀Rc").unwrap(); + let mut server = Server::new(&remote); + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Conflict(_))) + )); + let pending = cache.pending().unwrap(); + for range in [1..2, 100..101] { + assert!(matches!( + cache.rebase_conflict(id, &local, &remote, range), + Err(Error::Document(_)) + )); + assert_eq!(cache.pending().unwrap(), pending); + } + assert!( + matches!(cache.rebase_conflict(id + 1, &local, &remote, 2..3), Err(Error::Io(error)) if error.kind() == io::ErrorKind::InvalidInput) + ); + cache.edit_text(&local, sid, oid, 0..0, "Later ").unwrap(); + let changed = cache.snapshot().unwrap(); + assert!( + matches!(cache.rebase_conflict(id, &local, &remote, 2..3), Err(Error::Io(error)) if error.kind() == io::ErrorKind::ResourceBusy) + ); + let new_remote = onestore::replace_text(&remote, sid, oid, 2..3, "Q").unwrap(); + server.visible = new_remote.clone(); + server.durable = new_remote; + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Conflict(_))) + )); + assert!( + matches!(cache.rebase_conflict(id, &changed, &remote, 2..3), Err(Error::Io(error)) if error.kind() == io::ErrorKind::ResourceBusy) + ); + assert_eq!(cache.snapshot().unwrap(), changed); + assert_eq!(cache.pending().unwrap()[0], pending[0]); + assert_eq!(server.publications, 0); + + let current = cache.remote_snapshot().unwrap(); + cache.rebase_conflict(id, &changed, ¤t, 2..3).unwrap(); + server.fault = Fault::UnknownBefore; + assert!( + matches!(cache.sync_once(&mut server), Err(Error::Remote(error)) if error.state == CommitState::Unknown) + ); + let attempted = cache.status(id).unwrap(); + let pending = cache.pending().unwrap(); + assert!( + matches!(cache.rebase_conflict(id, &changed, ¤t, 2..3), Err(Error::Io(error)) if error.kind() == io::ErrorKind::InvalidInput) + ); + assert_eq!(cache.status(id).unwrap(), attempted); + assert_eq!(cache.pending().unwrap(), pending); + assert_eq!(server.publications, 1); +} + +#[test] +fn failure_between_rebase_and_conflict_clear_rolls_back_the_entire_resolution() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("resolve.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 1..2, "L") + .unwrap() + .unwrap(); + let remote = onestore::replace_text(&source, sid, oid, 0..3, "XaRc").unwrap(); + let mut server = Server::new(&remote); + cache.sync_once(&mut server).unwrap(); + let local = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + let status = cache.status(id).unwrap(); + drop(cache); + let connection = rusqlite::Connection::open(&path).unwrap(); + connection.execute_batch("CREATE TRIGGER fail_clear BEFORE DELETE ON conflicts BEGIN SELECT RAISE(ABORT, 'test conflict clear failure'); END;").unwrap(); + drop(connection); + let cache = Replica::open(&path).unwrap(); + assert!(matches!( + cache.rebase_conflict(id, &local, &remote, 2..3), + Err(Error::Database(_)) + )); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), pending); + assert_eq!(cache.status(id).unwrap(), status); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.remote_snapshot().unwrap(), remote); + assert_eq!(server.publications, 0); +} + +#[test] +fn seeded_reviewed_ranges_preserve_unicode_and_edits_inside_the_original_replacement() { + let dir = tempfile::tempdir().unwrap(); + let original: Vec<_> = "abcdefghij🦀klmnop".chars().collect(); + let mut seed = 911_u64; + for case in 0..64 { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + let at = seed as usize % original.len(); + let prefix = "[".repeat((seed >> 8) as usize % 5); + let suffix = "]".repeat((seed >> 16) as usize % 5); + let start: u32 = original[..at].iter().map(|ch| ch.len_utf16() as u32).sum(); + let end = start + original[at].len_utf16() as u32; + let source = onestore::create_section( + "resolve.one", + &original.iter().collect::(), + "Fixture", + ) + .unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(dir.path().join(format!("{case}.sqlite")), &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, start..end, "λ🦊μ") + .unwrap() + .unwrap(); + let dependent = cache + .edit_text( + &cache.snapshot().unwrap(), + sid, + oid, + start + 1..start + 3, + "🐕", + ) + .unwrap() + .unwrap(); + let mut remote_text = original.clone(); + remote_text.splice(at..at + 1, "Ω🐈π".chars()); + let remote_text = prefix.clone() + &remote_text.iter().collect::() + &suffix; + let remote = onestore::replace_text( + &source, + sid, + oid, + 0..original.iter().map(|ch| ch.len_utf16() as u32).sum(), + &remote_text, + ) + .unwrap(); + let mut server = Server::new(&remote); + assert!( + matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Conflict(_))) + ), + "case {case}, seed {seed}" + ); + let at_remote = start + prefix.len() as u32; + cache + .rebase_conflict( + id, + &cache.snapshot().unwrap(), + &remote, + at_remote..at_remote + 4, + ) + .unwrap(); + for expected in [id, dependent] { + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == expected), + "case {case}, seed {seed}" + ); + } + let mut expected = original.clone(); + expected.splice(at..at + 1, "λ🐕μ".chars()); + let expected = prefix + &expected.iter().collect::() + &suffix; + assert_eq!( + text(&server.durable).2, + expected, + "case {case}, seed {seed}" + ); + assert_eq!(server.publications, 2); + assert!(cache.pending().unwrap().is_empty()); + } +} + +#[test] +fn remote_changes_after_review_cannot_be_overwritten_by_the_reviewed_placement() { + struct ChangedAfterRead { + server: Server, + change: Option>, + } + impl Remote for ChangedAfterRead { + fn read(&mut self) -> io::Result> { + let snapshot = self.server.read()?; + if let Some(changed) = self.change.take() { + self.server.visible = changed.clone(); + self.server.durable = changed; + } + Ok(snapshot) + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + self.server.publish(edit) + } + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + self.server.confirm(snapshot) + } + } + for after_read in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("resolve.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(dir.path().join("cache.sqlite"), &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 1..2, "L") + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let remote = onestore::replace_text(&source, sid, oid, 1..2, "R").unwrap(); + let mut remote = ChangedAfterRead { + server: Server::new(&remote), + change: None, + }; + assert!(matches!( + cache.sync_once(&mut remote).unwrap(), + Some((_, EditStatus::Conflict(_))) + )); + cache + .rebase_conflict(id, &local, &cache.remote_snapshot().unwrap(), 1..2) + .unwrap(); + let changed = onestore::replace_text(&remote.server.visible, sid, oid, 1..2, "Q").unwrap(); + if after_read { + remote.change = Some(changed.clone()); + } else { + remote.server.visible = changed.clone(); + remote.server.durable = changed.clone(); + } + if after_read { + assert!( + matches!(cache.sync_once(&mut remote), Err(Error::Remote(error)) if error.state == CommitState::NotCommitted) + ); + assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); + } + assert_eq!( + cache.sync_once(&mut remote).unwrap(), + Some((id, EditStatus::Conflict(ConflictKind::TextChanged))) + ); + assert_eq!(remote.server.durable, changed); + assert_eq!(remote.server.visible, changed); + assert_eq!(remote.server.publications, usize::from(after_read)); + assert_eq!(cache.snapshot().unwrap(), local); + let notebook::Operation::Text(edit) = &cache.pending().unwrap()[0].operation else { + panic!() + }; + assert_eq!(edit.replacement, "L"); + } +} + +mod worker { + use super::*; + use std::{ + sync::{Arc, Mutex, mpsc}, + time::{Duration, Instant}, + }; + + #[derive(Clone)] + struct Shared(Arc>); + + impl Remote for Shared { + fn read(&mut self) -> io::Result> { + self.0.lock().unwrap().read() + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + self.0.lock().unwrap().publish(edit) + } + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + self.0.lock().unwrap().confirm(snapshot) + } + } + + #[test] + fn reconnects_after_connect_read_and_uncertain_publish_without_replaying() { + struct Session { + shared: Shared, + fail_read: bool, + } + impl Remote for Session { + fn read(&mut self) -> io::Result> { + if self.fail_read { + return Err(io::ErrorKind::ConnectionReset.into()); + } + self.shared.read() + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + self.shared.publish(edit) + } + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + self.shared.confirm(snapshot) + } + } + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Arc::new(Replica::create(&path, &source).unwrap()); + let id = cache + .edit_text(&source, sid, oid, 1..2, "🦀") + .unwrap() + .unwrap(); + let mut server = Server::new(&source); + server.fault = Fault::UnknownAfter; + let server = Arc::new(Mutex::new(server)); + let shared = Shared(Arc::clone(&server)); + let (connected_tx, connected_rx) = mpsc::channel(); + let (observed_tx, observed_rx) = mpsc::channel(); + let mut connections = 0; + let worker = cache + .start_sync( + Duration::from_millis(10), + move || { + connections += 1; + connected_tx.send(connections).unwrap(); + if connections <= 2 { + return Err(io::ErrorKind::ConnectionRefused.into()); + } + Ok(Session { + shared: shared.clone(), + fail_read: connections == 3, + }) + }, + move |result| { + observed_tx + .send(result.as_ref().copied().map_err(|error| error.to_string())) + .unwrap(); + }, + ) + .unwrap(); + let mut errors = 0; + let published = loop { + match observed_rx.recv_timeout(Duration::from_secs(5)).unwrap() { + Err(_) => errors += 1, + Ok(Some((actual, status @ EditStatus::Published { .. }))) => { + assert_eq!(actual, id); + break status; + } + other => panic!("Unexpected result: {other:?}"), + } + }; + worker.stop().unwrap(); + assert_eq!(errors, 4); + assert_eq!(connected_rx.try_iter().collect::>(), [1, 2, 3, 4, 5]); + let server = server.lock().unwrap(); + assert_eq!(server.publications, 1); + assert_eq!(server.confirmations, 1); + assert_eq!(text(&server.durable).2, "a🦀c"); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.status(id).unwrap(), Some(published)); + let cached = cache.snapshot().unwrap(); + // Confirmation changes reader-notification fields without changing the committed graph. + assert_eq!(cached[..212], server.durable[..212]); + assert_eq!(cached[252..], server.durable[252..]); + let cached_generation = Store::parse(&cached).unwrap().header.generation; + let remote_generation = Store::parse(&server.durable).unwrap().header.generation; + if cached_generation == remote_generation { + assert_eq!(cached, server.durable); + } else { + assert_eq!(cached_generation.checked_add(1), Some(remote_generation)); + } + } + + #[test] + fn local_edits_wake_an_idle_worker_and_coalesced_notifications_drain_twelve_writers() { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); + let server = Arc::new(Mutex::new(Server::new(&source))); + let shared = Shared(Arc::clone(&server)); + let (observed_tx, observed_rx) = mpsc::channel(); + let (resume_tx, resume_rx) = mpsc::channel(); + let mut first = true; + let worker = cache + .start_sync( + Duration::from_secs(3600), + move || Ok(shared.clone()), + move |result| { + observed_tx.send(*result.as_ref().unwrap()).unwrap(); + if first { + first = false; + resume_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + } + }, + ) + .unwrap(); + assert_eq!( + observed_rx.recv_timeout(Duration::from_secs(5)).unwrap(), + None + ); + let started = Instant::now(); + let edits = std::thread::scope(|scope| { + (0..12) + .map(|writer| { + let cache = &cache; + scope.spawn(move || { + let replacement = format!("[{writer}] "); + loop { + let snapshot = cache.snapshot().unwrap(); + match cache.edit_text(&snapshot, sid, oid, 0..0, &replacement) { + Ok(Some(id)) => break (id, replacement), + Err(Error::Io(error)) + if error.kind() == io::ErrorKind::ResourceBusy => {} + other => panic!("Unexpected local outcome: {other:?}"), + } + } + }) + }) + .collect::>() + .into_iter() + .map(|join| join.join().unwrap()) + .collect::>() + }); + resume_tx.send(()).unwrap(); + let mut published = std::collections::BTreeSet::new(); + while published.len() < 12 { + if let Some((id, status)) = observed_rx.recv_timeout(Duration::from_secs(5)).unwrap() { + assert!(matches!(status, EditStatus::Published { .. }), "{status:?}"); + assert!(published.insert(id)); + } + } + assert!( + started.elapsed() < Duration::from_secs(5), + "Edits waited for the hourly poll" + ); + worker.stop().unwrap(); + assert_eq!(published, edits.keys().copied().collect()); + let expected = edits.values().rev().cloned().collect::() + "abc"; + let server = server.lock().unwrap(); + assert_eq!(text(&server.durable).2, expected); + assert_eq!(server.publications, 12); + assert_eq!(cache.snapshot().unwrap(), server.durable); + assert!(cache.pending().unwrap().is_empty()); + } + + #[test] + fn dropping_during_publication_is_nonblocking_and_retains_ownership_until_recovery_is_recorded() + { + struct Paused { + shared: Shared, + entered: mpsc::Sender<()>, + resume: mpsc::Receiver<()>, + } + impl Remote for Paused { + fn read(&mut self) -> io::Result> { + self.shared.read() + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + self.entered.send(()).unwrap(); + self.resume.recv_timeout(Duration::from_secs(5)).unwrap(); + self.shared.publish(edit) + } + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + self.shared.confirm(snapshot) + } + } + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Arc::new(Replica::create(&path, &source).unwrap()); + let id = cache + .edit_text(&source, sid, oid, 0..0, "L ") + .unwrap() + .unwrap(); + let mut server = Server::new(&source); + server.fault = Fault::UnknownAfter; + let server = Arc::new(Mutex::new(server)); + let shared = Shared(Arc::clone(&server)); + let (entered_tx, entered_rx) = mpsc::channel(); + let (resume_tx, resume_rx) = mpsc::channel(); + let mut session = Some(Paused { + shared, + entered: entered_tx, + resume: resume_rx, + }); + let worker = cache + .start_sync( + Duration::from_secs(3600), + move || Ok(session.take().unwrap()), + |_| {}, + ) + .unwrap(); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + let stopped = Instant::now(); + drop(worker); + assert!(stopped.elapsed() < Duration::from_millis(500)); + let shared = Shared(Arc::clone(&server)); + let start = cache.start_sync( + Duration::from_secs(3600), + move || Ok(shared.clone()), + |_| {}, + ); + assert!(matches!(start, Err(error) if error.kind() == io::ErrorKind::WouldBlock)); + let weak = Arc::downgrade(&cache); + drop(cache); + resume_tx.send(()).unwrap(); + while weak.upgrade().is_some() { + assert!(stopped.elapsed() < Duration::from_secs(5)); + std::thread::sleep(Duration::from_millis(1)); + } + let cache = Arc::new(Replica::open(&path).unwrap()); + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::AwaitingConfirmation { .. }) + )); + assert_eq!(server.lock().unwrap().publications, 1); + let shared = Shared(Arc::clone(&server)); + let (tx, rx) = mpsc::channel(); + let worker = cache + .start_sync( + Duration::from_secs(3600), + move || Ok(shared.clone()), + move |result| { + tx.send(*result.as_ref().unwrap()).unwrap(); + }, + ) + .unwrap(); + assert!( + matches!(rx.recv_timeout(Duration::from_secs(5)).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) + ); + worker.stop().unwrap(); + let server = server.lock().unwrap(); + assert_eq!(server.publications, 1); + assert_eq!(server.confirmations, 1); + assert_eq!(text(&server.durable).2, "L abc"); + } + + #[test] + fn cache_failures_stop_retries_and_return_the_error_without_remote_publication() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 0..0, "L ") + .unwrap() + .unwrap(); + drop(cache); + let connection = rusqlite::Connection::open(&path).unwrap(); + connection.execute_batch("CREATE TRIGGER fail_attempt BEFORE INSERT ON attempt BEGIN SELECT RAISE(ABORT, 'test cache write failure'); END;").unwrap(); + drop(connection); + let cache = Arc::new(Replica::open(&path).unwrap()); + let server = Arc::new(Mutex::new(Server::new(&source))); + let shared = Shared(Arc::clone(&server)); + let (tx, rx) = mpsc::channel(); + let worker = cache + .start_sync( + Duration::from_millis(1), + move || Ok(shared.clone()), + move |result| { + tx.send(matches!(result, Err(Error::Database(_)))).unwrap(); + }, + ) + .unwrap(); + assert!(rx.recv_timeout(Duration::from_secs(5)).unwrap()); + assert!(matches!(worker.stop(), Err(Error::Database(_)))); + assert!(rx.try_iter().next().is_none()); + assert_eq!(server.lock().unwrap().publications, 0); + assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); + assert_eq!(text(&cache.snapshot().unwrap()).2, "L abc"); + } + + #[test] + fn polling_preserves_conflicts_and_absent_uncertain_revisions_without_replay() { + for uncertain in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = + Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); + let id = cache + .edit_text(&source, sid, oid, 1..2, "L") + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let mut server = if uncertain { + Server::new(&source) + } else { + Server::new(&onestore::replace_text(&source, sid, oid, 1..2, "R").unwrap()) + }; + if uncertain { + server.fault = Fault::UnknownBefore; + } + let server = Arc::new(Mutex::new(server)); + let shared = Shared(Arc::clone(&server)); + let (tx, rx) = mpsc::channel(); + let worker = cache + .start_sync( + Duration::from_millis(10), + move || Ok(shared.clone()), + move |result| { + tx.send(result.as_ref().copied().map_err(|_| ())).unwrap(); + }, + ) + .unwrap(); + if uncertain { + assert!(rx.recv_timeout(Duration::from_secs(5)).unwrap().is_err()); + } + let mut previous = None; + let started = Instant::now(); + for _ in 0..5 { + let (actual, status) = rx + .recv_timeout(Duration::from_secs(5)) + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(actual, id); + if uncertain { + assert!(matches!(status, EditStatus::AwaitingConfirmation { .. })); + } else { + assert_eq!(status, EditStatus::Conflict(ConflictKind::TextChanged)); + } + if let Some(previous) = previous { + assert_eq!(previous, status); + } + previous = Some(status); + } + assert!( + started.elapsed() >= Duration::from_millis(30), + "Worker spun instead of waiting between retries" + ); + worker.stop().unwrap(); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.pending().unwrap().len(), 1); + let server = server.lock().unwrap(); + assert_eq!(server.publications, usize::from(uncertain)); + assert_eq!(server.confirmations, 0); + } + } + + #[test] + fn reachability_notification_retries_without_waiting_for_the_poll() { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); + let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); + let (tx, rx) = mpsc::channel(); + let worker = cache + .start_sync( + Duration::from_secs(3600), + || -> io::Result { Err(io::ErrorKind::NotConnected.into()) }, + move |result| { + tx.send(matches!(result, Err(Error::RemoteIo(_)))).unwrap(); + }, + ) + .unwrap(); + assert!(rx.recv_timeout(Duration::from_secs(5)).unwrap()); + worker.wake(); + assert!(rx.recv_timeout(Duration::from_secs(5)).unwrap()); + worker.stop().unwrap(); + assert!(rx.try_iter().next().is_none()); + } + + #[test] + fn cancellation_during_connect_does_not_read_or_report_a_false_refresh() { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); + let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); + let (entered_tx, entered_rx) = mpsc::channel(); + let (resume_tx, resume_rx) = mpsc::channel(); + let (tx, rx) = mpsc::channel(); + // An invalid image makes any unexpected read observable as an error callback. + let shared = Shared(Arc::new(Mutex::new(Server::new(&[])))); + let worker = cache + .start_sync( + Duration::from_secs(3600), + move || { + entered_tx.send(()).unwrap(); + resume_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + Ok(shared.clone()) + }, + move |_| { + tx.send(()).unwrap(); + }, + ) + .unwrap(); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + drop(worker); + let weak = Arc::downgrade(&cache); + drop(cache); + resume_tx.send(()).unwrap(); + let started = Instant::now(); + while weak.upgrade().is_some() { + assert!(started.elapsed() < Duration::from_secs(5)); + std::thread::sleep(Duration::from_millis(1)); + } + assert_eq!( + rx.recv_timeout(Duration::from_secs(5)), + Err(mpsc::RecvTimeoutError::Disconnected) + ); + } + + #[test] + fn invalid_intervals_and_callback_panics_leave_worker_ownership_recoverable() { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); + let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); + for interval in [Duration::ZERO, Duration::MAX] { + assert!( + matches!(cache.start_sync(interval, || -> io::Result { panic!("Unexpected connection") }, |_| {}), Err(error) if error.kind() == io::ErrorKind::InvalidInput) + ); + } + let shared = Shared(Arc::new(Mutex::new(Server::new(&source)))); + let first = shared.clone(); + let (tx, rx) = mpsc::channel(); + let worker = cache + .start_sync( + Duration::from_secs(3600), + move || Ok(first.clone()), + move |_| { + tx.send(()).unwrap(); + panic!("Test observer panic"); + }, + ) + .unwrap(); + rx.recv_timeout(Duration::from_secs(5)).unwrap(); + assert!(matches!(worker.stop(), Err(Error::Io(_)))); + let (tx, rx) = mpsc::channel(); + let worker = cache + .start_sync( + Duration::from_secs(3600), + move || Ok(shared.clone()), + move |result| { + tx.send(*result.as_ref().unwrap()).unwrap(); + }, + ) + .unwrap(); + assert_eq!(rx.recv_timeout(Duration::from_secs(5)).unwrap(), None); + worker.stop().unwrap(); + assert_eq!(cache.snapshot().unwrap(), source); + } + + #[test] + fn reviewed_conflict_wakes_the_worker_and_publishes_the_original_intent_once() { + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("resolve.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); + let id = cache + .edit_text(&source, sid, oid, 1..2, "L") + .unwrap() + .unwrap(); + let remote = onestore::replace_text(&source, sid, oid, 1..2, "R").unwrap(); + let server = Arc::new(Mutex::new(Server::new(&remote))); + let shared = Shared(Arc::clone(&server)); + let (tx, rx) = mpsc::channel(); + let (resume_tx, resume_rx) = mpsc::channel(); + let mut first = true; + let worker = cache + .start_sync( + Duration::from_secs(3600), + move || Ok(shared.clone()), + move |result| { + tx.send(*result.as_ref().unwrap()).unwrap(); + if first { + first = false; + resume_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + } + }, + ) + .unwrap(); + assert_eq!( + rx.recv_timeout(Duration::from_secs(5)).unwrap(), + Some((id, EditStatus::Conflict(ConflictKind::TextChanged))) + ); + cache + .rebase_conflict( + id, + &cache.snapshot().unwrap(), + &cache.remote_snapshot().unwrap(), + 1..2, + ) + .unwrap(); + assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); + resume_tx.send(()).unwrap(); + assert!( + matches!(rx.recv_timeout(Duration::from_secs(5)).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) + ); + worker.stop().unwrap(); + assert!(cache.pending().unwrap().is_empty()); + let server = server.lock().unwrap(); + assert_eq!(server.publications, 1); + assert_eq!(text(&server.durable).2, "aLc"); + assert_eq!(cache.snapshot().unwrap(), server.durable); + } + + #[test] + fn ordinary_read_and_unpublished_write_contention_reuse_the_connection() { + struct Busy { + server: Server, + reads: usize, + writes: usize, + } + impl Remote for Busy { + fn read(&mut self) -> io::Result> { + self.reads += 1; + match self.reads { + 1 => Err(io::ErrorKind::WouldBlock.into()), + 2 => Err(io::ErrorKind::ResourceBusy.into()), + _ => self.server.read(), + } + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + self.writes += 1; + match self.writes { + 1 | 2 => Err(CommitError { + state: CommitState::NotCommitted, + error: if self.writes == 1 { + io::ErrorKind::WouldBlock + } else { + io::ErrorKind::ResourceBusy + } + .into(), + }), + _ => self.server.publish(edit), + } + } + fn confirm(&mut self, source: &[u8]) -> Result<(), CommitError> { + self.server.confirm(source) + } + } + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); + let id = cache + .edit_text(&source, sid, oid, 0..0, "L ") + .unwrap() + .unwrap(); + let mut remote = Some(Busy { + server: Server::new(&source), + reads: 0, + writes: 0, + }); + let (tx, rx) = mpsc::channel(); + let worker = cache + .start_sync( + Duration::from_millis(1), + move || Ok(remote.take().expect("Contention caused a reconnect")), + move |result| { + tx.send(result.as_ref().copied().map_err(|error| error.to_string())) + .unwrap(); + }, + ) + .unwrap(); + for _ in 0..4 { + assert!(rx.recv_timeout(Duration::from_secs(5)).unwrap().is_err()); + } + assert!( + matches!(rx.recv_timeout(Duration::from_secs(5)).unwrap().unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) + ); + worker.stop().unwrap(); + assert_eq!(text(&cache.snapshot().unwrap()).2, "L abc"); + assert!(cache.pending().unwrap().is_empty()); + } + #[test] + fn publication_backoff_drains_local_wakes_without_waiting_for_the_idle_poll() { + struct BusyOnce(Server); + impl Remote for BusyOnce { + fn read(&mut self) -> io::Result> { + self.0.read() + } + fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { + let server = &mut self.0; + if server.publications == 0 { + server.publications += 1; + return Err(CommitError { + state: CommitState::NotCommitted, + error: io::ErrorKind::ResourceBusy.into(), + }); + } + server.publish(edit) + } + fn confirm(&mut self, source: &[u8]) -> Result<(), CommitError> { + self.0.confirm(source) + } + } + let dir = tempfile::tempdir().unwrap(); + let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); + let first = cache + .edit_text(&source, sid, oid, 0..0, "L ") + .unwrap() + .unwrap(); + let mut remote = Some(BusyOnce(Server::new(&source))); + let observed = Arc::clone(&cache); + let (tx, rx) = mpsc::channel(); + let mut failed = false; + let worker = cache + .start_sync( + Duration::from_secs(3600), + move || Ok(remote.take().expect("Contention must retain the session")), + move |result| match result { + Err(Error::Remote(error)) if error.state == CommitState::NotCommitted => { + assert!(!failed); + failed = true; + let source = observed.snapshot().unwrap(); + let second = observed + .edit_text(&source, sid, oid, 0..0, "Q ") + .unwrap() + .unwrap(); + tx.send((second, None)).unwrap(); + } + Ok(Some((id, status))) => tx.send((*id, Some(*status))).unwrap(), + Ok(None) => {} + other => panic!("Unexpected worker result: {other:?}"), + }, + ) + .unwrap(); + let (second, status) = rx.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!(status, None); + for expected in [first, second] { + let (id, status) = rx.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!(id, expected); + assert!(matches!(status, Some(EditStatus::Published { .. }))); + } + worker.stop().unwrap(); + assert!(cache.pending().unwrap().is_empty()); + assert_eq!(text(&cache.snapshot().unwrap()).2, "Q L abc"); + } +} + +#[test] +fn offline_insertions_survive_reopen_rebase_and_dependent_text_edits() { + use onestore::Insertion; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("insert.sqlite"); + let source = onestore::create_section("insert.one", "Original", "Author").unwrap(); + let (sid, original, _) = text(&source); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let (_, page) = doc.pages().unwrap()[0]; + let cache = Replica::create(&path, &source).unwrap(); + let outline = Insertion::outline(page, 72.0, 144.0, "Offline outline", "Offline author") + .unwrap() + .with_formatting(0..7, &[onestore::TextAttribute::Italic(true)]) + .unwrap(); + let first = cache.insert(&source, sid, &outline).unwrap().unwrap(); + let snapshot = cache.snapshot().unwrap(); + let paragraph = Insertion::paragraph( + outline.object(), + None, + "Offline paragraph 🦀", + "Offline author", + ) + .unwrap() + .with_formatting(8..17, &[onestore::TextAttribute::Bold(true)]) + .unwrap(); + let second = cache.insert(&snapshot, sid, ¶graph).unwrap().unwrap(); + let third = cache + .edit_text( + &cache.snapshot().unwrap(), + sid, + paragraph.text_object(), + 0..0, + "Edited ", + ) + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.pending().unwrap(), pending); + let remote = onestore::replace_text(&source, sid, original, 0..0, "Remote ").unwrap(); + let mut server = Server::new(&remote); + for id in [first, second, third] { + assert!( + matches!(cache.sync_once(&mut server).unwrap(),Some((observed,EditStatus::Published{..})) if observed==id) + ); + } + assert_eq!(server.publications, 3); + assert!(cache.pending().unwrap().is_empty()); + assert_eq!(cache.snapshot().unwrap(), server.durable); + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let s = &doc.spaces[&sid]; + let v = &s.revisions[&s.contexts[&ExGuid::default()]]; + for (id, wanted) in [ + (original, "Remote Original"), + (outline.text_object(), "Offline outline"), + (paragraph.text_object(), "Edited Offline paragraph 🦀"), + ] { + assert!(matches!(&v.nodes[&id].kind,Kind::RichText{text,..} if text==wanted)); + } + let outline_runs = v.text_runs(outline.text_object()).unwrap(); + assert_eq!(outline_runs[0].text, "Offline"); + assert_eq!(outline_runs[0].format.italic, Some(true)); + let runs = v.text_runs(paragraph.text_object()).unwrap(); + assert_eq!(runs[1].text, "paragraph"); + assert_eq!(runs[1].format.bold, Some(true)); + assert_eq!( + v.nodes[&outline.object()].children.last(), + Some(¶graph.object()) + ); +} + +#[test] +fn uncertain_insertions_reconcile_the_original_revision_without_duplicate_objects() { + use onestore::Insertion; + for fault in [ + Fault::Before, + Fault::UnknownBefore, + Fault::UnknownAfter, + Fault::PanicBefore, + Fault::PanicAfter, + Fault::Committed, + ] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("uncertain-insert.sqlite"); + let source = onestore::create_section("insert.one", "Original", "Author").unwrap(); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let (sid, page) = doc.pages().unwrap()[0]; + let cache = Replica::create(&path, &source).unwrap(); + let insertion = Insertion::outline(page, 144.0, 144.0, "Uncertain insertion", "Author") + .unwrap() + .with_formatting(0..9, &[onestore::TextAttribute::Bold(true)]) + .unwrap() + .with_formatting(10..19, &[onestore::TextAttribute::Italic(true)]) + .unwrap(); + let id = cache.insert(&source, sid, &insertion).unwrap().unwrap(); + let local = cache.snapshot().unwrap(); + let mut server = Server::new(&source); + server.fault = fault; + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + cache.sync_once(&mut server) + })); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(server.publications, 1); + if matches!(fault, Fault::UnknownBefore | Fault::PanicBefore) { + let status = cache.status(id).unwrap(); + assert!(matches!( + status, + Some(EditStatus::AwaitingConfirmation { .. }) + )); + for _ in 0..5 { + assert_eq!( + cache.sync_once(&mut server).unwrap(), + status.map(|state| (id, state)) + ); + } + assert_eq!(server.publications, 1); + assert_eq!(server.durable, source); + assert_eq!(cache.snapshot().unwrap(), local); + } else { + if !matches!(fault, Fault::Committed) { + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Published { .. })) + )); + } + assert_eq!( + server.publications, + if matches!(fault, Fault::Before) { 2 } else { 1 } + ); + assert_eq!( + server.confirmations, + usize::from(matches!(fault, Fault::UnknownAfter | Fault::PanicAfter)) + ); + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let s = &doc.spaces[&sid]; + let v = &s.revisions[&s.contexts[&ExGuid::default()]]; + assert_eq!(v.nodes.values().filter(|node|matches!(&node.kind,Kind::RichText{text,..} if text=="Uncertain insertion")).count(),1); + assert!(v.nodes.contains_key(&insertion.object())); + let runs = v.text_runs(insertion.text_object()).unwrap(); + assert_eq!(runs.len(), 3); + assert_eq!(runs[0].text, "Uncertain"); + assert_eq!(runs[0].format.bold, Some(true)); + assert_eq!(runs[1].text, " "); + assert_ne!(runs[1].format.bold, Some(true)); + assert_ne!(runs[1].format.italic, Some(true)); + assert_eq!(runs[2].text, "insertion"); + assert_eq!(runs[2].format.italic, Some(true)); + assert!(cache.pending().unwrap().is_empty()); + } + } +} + +#[test] +fn cross_run_text_rebases_preserve_remote_styles_and_survive_lost_replies() { + use onestore::TextAttribute as A; + for fault in [Fault::None, Fault::UnknownAfter, Fault::PanicAfter] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("cross-run.sqlite"); + let plain = onestore::create_section("cross.one", "ab🦀cde\u{301}fg", "Author").unwrap(); + let (sid, oid, _) = text(&plain); + let bold = PreparedEdit::format(&plain, sid, oid, 1..4, &[A::Bold(true)]).unwrap(); + let source = PreparedEdit::format(bold.as_bytes(), sid, oid, 4..7, &[A::Italic(true)]) + .unwrap() + .as_bytes() + .to_vec(); + let cache = Replica::create(&path, &source).unwrap(); + let first = cache + .edit_text(&source, sid, oid, 2..7, "日本語") + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let second = cache + .edit_text(&local, sid, oid, 3..4, "🐈") + .unwrap() + .unwrap(); + let pending = cache.pending().unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), pending); + let prefix = PreparedEdit::text(&source, sid, oid, 0..0, "Prefix ").unwrap(); + let remote = + PreparedEdit::format(prefix.as_bytes(), sid, oid, 9..11, &[A::Underline(true)]) + .unwrap(); + let mut server = Server::new(remote.as_bytes()); + server.fault = fault; + let attempted = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + cache.sync_once(&mut server) + })); + if matches!(fault, Fault::None) { + assert!( + matches!(attempted.unwrap().unwrap(), Some((id, EditStatus::Published { .. })) if id == first) + ); + } else { + assert!(matches!( + cache.status(first).unwrap(), + Some(EditStatus::AwaitingConfirmation { .. }) + )); + } + drop(cache); + let cache = Replica::open(&path).unwrap(); + while let Some((_, status)) = cache.sync_once(&mut server).unwrap() { + assert!(matches!(status, EditStatus::Published { .. })); + } + assert_eq!(server.publications, 2); + assert!(matches!( + cache.status(second).unwrap(), + Some(EditStatus::Published { .. }) + )); + assert_eq!(text(&server.durable).2, "Prefix ab日🐈語\u{301}fg"); + assert_eq!(cache.snapshot().unwrap(), server.durable); + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let space = &document.spaces[&sid]; + let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; + let runs = revision.text_runs(oid).unwrap(); + let replacement = runs.iter().find(|run| run.text == "日🐈語").unwrap(); + assert_eq!(replacement.format.bold, Some(true)); + assert_eq!(replacement.format.underline, Some(true)); + assert_ne!(replacement.format.italic, Some(true)); + let suffix = runs.last().unwrap(); + assert_eq!(suffix.text, "\u{301}fg"); + assert_ne!(suffix.format.bold, Some(true)); + assert_ne!(suffix.format.underline, Some(true)); + assert_ne!(suffix.format.italic, Some(true)); + } +} + +#[test] +fn offline_formatting_rebases_text_and_merges_independent_attributes() { + use onestore::TextAttribute as A; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("format.sqlite"); + let source = onestore::create_section("format.one", "ab🦀cd", "Author").unwrap(); + let (sid, id, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let local = cache + .format(&source, sid, id, 2..4, &[A::Bold(true)]) + .unwrap() + .unwrap(); + let pending = cache.pending().unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), pending); + let moved = PreparedEdit::text(&source, sid, id, 0..0, "Prefix ").unwrap(); + let styled = + PreparedEdit::format(moved.as_bytes(), sid, id, 9..11, &[A::Italic(true)]).unwrap(); + let mut server = Server::new(styled.as_bytes()); + assert!( + matches!(cache.sync_once(&mut server).unwrap(),Some((observed,EditStatus::Published{..}))if observed==local) + ); + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let space = &doc.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let runs = view.text_runs(id).unwrap(); + let selected = runs.iter().find(|r| r.text == "🦀").unwrap(); + assert_eq!(selected.format.bold, Some(true)); + assert_eq!(selected.format.italic, Some(true)); + assert_eq!( + runs.iter().map(|r| r.text).collect::(), + "Prefix ab🦀cd" + ); +} + +#[test] +fn competing_font_changes_remain_preserved_conflicts() { + use onestore::TextAttribute as A; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("conflict.sqlite"); + let source = onestore::create_section("format.one", "abcdef", "Author").unwrap(); + let (sid, id, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let local_id = cache + .format(&source, sid, id, 1..5, &[A::FontSize(14.0)]) + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + let remote = PreparedEdit::format(&source, sid, id, 2..4, &[A::FontSize(18.0)]).unwrap(); + let mut server = Server::new(remote.as_bytes()); + for _ in 0..3 { + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some(( + local_id, + EditStatus::Conflict(ConflictKind::FormattingChanged) + )) + ); + } + assert_eq!(server.publications, 0); + assert_eq!(server.confirmations, 0); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.pending().unwrap(), pending); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!( + cache.status(local_id).unwrap(), + Some(EditStatus::Conflict(ConflictKind::FormattingChanged)) + ); + assert_eq!(cache.snapshot().unwrap(), local); +} + +#[test] +fn independently_satisfied_formatting_requires_confirmation_before_a_receipt() { + use onestore::TextAttribute as A; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("satisfied.sqlite"); + let source = onestore::create_section("format.one", "abcdef", "Author").unwrap(); + let (sid, id, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let local_id = cache + .format(&source, sid, id, 1..5, &[A::Bold(true)]) + .unwrap() + .unwrap(); + let remote = PreparedEdit::format(&source, sid, id, 1..5, &[A::Bold(true)]).unwrap(); + let mut server = Server::new(remote.as_bytes()); + server.durable = source.clone(); + server.fault = Fault::Confirm; + assert!( + matches!(cache.sync_once(&mut server),Err(Error::Remote(e))if e.state==CommitState::Unknown) + ); + assert_eq!(cache.status(local_id).unwrap(), Some(EditStatus::Pending)); + assert_eq!(server.publications, 0); + assert_eq!(server.durable, source); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert!( + matches!(cache.sync_once(&mut server).unwrap(),Some((id,EditStatus::Published{..}))if id==local_id) + ); + assert_eq!(server.publications, 0); + assert_eq!(server.confirmations, 2); + assert_ne!(server.durable, source); + assert!(cache.pending().unwrap().is_empty()); +} + +#[test] +fn retired_format_attempts_require_the_complete_durable_effect_without_replay() { + use onestore::TextAttribute as A; + for (complete, fault) in [ + (true, Fault::None), + (true, Fault::Confirm), + (true, Fault::ConfirmCommitted), + (false, Fault::None), + ] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("retired-format.sqlite"); + let source = onestore::create_section("format.one", "abcdef", "Author").unwrap(); + let (sid, object, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .format( + &source, + sid, + object, + 1..5, + &[A::Bold(true), A::FontSize(18.0)], + ) + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let mut server = Server::new(&source); + server.fault = Fault::UnknownAfter; + assert!(cache.sync_once(&mut server).is_err()); + let attempted = cache.status(id).unwrap(); + let Some(EditStatus::AwaitingConfirmation { revision: retired }) = attempted else { + panic!() + }; + drop(cache); + let prefix = PreparedEdit::text(&source, sid, object, 0..0, "prefix ").unwrap(); + let mut attributes = vec![A::Bold(true), A::Italic(true)]; + if complete { + attributes.push(A::FontSize(18.0)); + } + server.visible = PreparedEdit::format(prefix.as_bytes(), sid, object, 8..12, &attributes) + .unwrap() + .as_bytes() + .to_vec(); + let store = Store::parse(&server.visible).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let current = index.spaces[&sid].labels[&(ExGuid::default(), 1)]; + assert!(!index.spaces[&sid].revisions.contains_key(&retired)); + let cache = Replica::open(&path).unwrap(); + server.fault = fault; + let result = cache.sync_once(&mut server); + if !complete { + assert_eq!(result.unwrap(), attempted.map(|s| (id, s))); + assert_eq!(server.confirmations, 0); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(server.durable, source); + } else { + if matches!(fault, Fault::Confirm) { + assert!(matches!(result, Err(Error::Remote(e)) if e.state == CommitState::Unknown)); + assert_eq!(cache.status(id).unwrap(), attempted); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(server.durable, source); + let complete = server.visible.clone(); + server.visible = + PreparedEdit::format(&complete, sid, object, 8..12, &[A::Bold(false)]) + .unwrap() + .as_bytes() + .to_vec(); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + attempted.map(|s| (id, s)) + ); + assert_eq!(server.confirmations, 1); + assert_eq!(cache.snapshot().unwrap(), local); + server.visible = complete; + cache.sync_once(&mut server).unwrap(); + } else if matches!(fault, Fault::ConfirmCommitted) { + assert!( + matches!(result, Err(Error::Remote(e)) if e.state == CommitState::Committed) + ); + } else { + assert_eq!( + result.unwrap(), + Some((id, EditStatus::Published { revision: current })) + ); + } + assert_ne!(current, retired); + assert_eq!( + cache.status(id).unwrap(), + Some(EditStatus::Published { revision: current }) + ); + assert!(cache.pending().unwrap().is_empty()); + assert_eq!(text(&server.durable).2, "prefix abcdef"); + assert_ne!(server.durable, source); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!( + cache.status(id).unwrap(), + Some(EditStatus::Published { revision: current }) + ); + } + assert_eq!(server.publications, 1); + } +} + +#[test] +fn retired_text_with_equal_plaintext_but_a_different_surviving_run_remains_uncertain() { + use onestore::TextAttribute; + + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("repeated.sqlite"); + let plain = onestore::create_section("repeated.one", "aa", "Fixture").unwrap(); + let (space, object, _) = text(&plain); + let source = PreparedEdit::format(&plain, space, object, 0..1, &[TextAttribute::Bold(true)]) + .unwrap() + .as_bytes() + .to_vec(); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .edit_text(&source, space, object, 0..1, "") + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let mut server = Server::new(&source); + server.fault = Fault::UnknownAfter; + assert!(cache.sync_once(&mut server).is_err()); + let attempt = cache.status(id).unwrap(); + server.visible = onestore::replace_text(&source, space, object, 1..2, "").unwrap(); + assert_eq!(text(&server.visible).2, text(&local).2); + let bold = |bytes: &[u8]| { + let store = Store::parse(bytes).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let space = &document.spaces[&space]; + space.revisions[&space.contexts[&ExGuid::default()]] + .text_runs(object) + .unwrap() + .into_iter() + .find(|run| !run.text.is_empty()) + .unwrap() + .format + .bold + }; + assert_eq!(bold(&server.visible), Some(true)); + assert_ne!(bold(&server.visible), bold(&local)); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + attempt.map(|state| (id, state)) + ); + assert_eq!(server.publications, 1); + assert_eq!(server.confirmations, 0); + assert_eq!(cache.snapshot().unwrap(), local); +} + +#[test] +fn uncertain_formatting_keeps_the_original_attempt_and_never_replays() { + use onestore::TextAttribute as A; + for fault in [Fault::UnknownBefore, Fault::UnknownAfter] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("unknown-format.sqlite"); + let source = onestore::create_section("format.one", "abcdef", "Author").unwrap(); + let (sid, id, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let local_id = cache + .format(&source, sid, id, 1..5, &[A::Bold(true)]) + .unwrap() + .unwrap(); + let mut server = Server::new(&source); + server.fault = fault; + assert!( + matches!(cache.sync_once(&mut server),Err(Error::Remote(e))if e.state==CommitState::Unknown) + ); + let status = cache.status(local_id).unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + if matches!(fault, Fault::UnknownBefore) { + for _ in 0..4 { + assert_eq!( + cache.sync_once(&mut server).unwrap(), + status.map(|s| (local_id, s)) + ); + } + assert_eq!(server.confirmations, 0); + } else { + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Published { .. })) + )); + assert_eq!(server.confirmations, 1); + } + assert_eq!(server.publications, 1); + } +} + +#[test] +fn reviewed_format_conflicts_preserve_dependent_edits_and_recheck_later_remote_changes() { + use onestore::TextAttribute as A; + for changed_again in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("review-format.sqlite"); + let source = onestore::create_section("format.one", "abcdef", "Author").unwrap(); + let (sid, id, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let first = cache + .format(&source, sid, id, 1..5, &[A::FontSize(14.0)]) + .unwrap() + .unwrap(); + let second = cache + .edit_text(&cache.snapshot().unwrap(), sid, id, 2..2, "X") + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + let remote = PreparedEdit::format(&source, sid, id, 1..5, &[A::FontSize(18.0)]).unwrap(); + let mut server = Server::new(remote.as_bytes()); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((first, EditStatus::Conflict(ConflictKind::FormattingChanged))) + ); + assert!( + matches!(cache.rebase_conflict(first,&source,remote.as_bytes(),1..5),Err(Error::Io(e))if e.kind()==io::ErrorKind::ResourceBusy) + ); + cache + .rebase_conflict(first, &local, remote.as_bytes(), 1..5) + .unwrap(); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.pending().unwrap()[1], pending[1]); + drop(cache); + let cache = Replica::open(&path).unwrap(); + if changed_again { + let newer = + PreparedEdit::format(&server.visible, sid, id, 1..5, &[A::FontSize(20.0)]).unwrap(); + server = Server::new(newer.as_bytes()); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((first, EditStatus::Conflict(ConflictKind::FormattingChanged))) + ); + assert_eq!(server.publications, 0); + assert_eq!(cache.snapshot().unwrap(), local); + } else { + for expected in [first, second] { + assert!( + matches!(cache.sync_once(&mut server).unwrap(),Some((id,EditStatus::Published{..}))if id==expected) + ); + } + assert_eq!(text(&server.durable).2, "abXcdef"); + assert!(cache.pending().unwrap().is_empty()); + } + } +} + +#[test] +fn superscript_reconciliation_checks_the_implicit_subscript_change() { + use onestore::TextAttribute as A; + let directory = tempfile::tempdir().unwrap(); + let source = onestore::create_section("script.one", "abc", "Author").unwrap(); + let (sid, id, _) = text(&source); + let cache = Replica::create(directory.path().join("script.sqlite"), &source).unwrap(); + let local = cache + .format(&source, sid, id, 0..3, &[A::Superscript(true)]) + .unwrap() + .unwrap(); + let remote = PreparedEdit::format(&source, sid, id, 0..3, &[A::Subscript(true)]).unwrap(); + let mut server = Server::new(remote.as_bytes()); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((local, EditStatus::Conflict(ConflictKind::FormattingChanged))) + ); + assert_eq!(server.publications, 0); +} + +#[test] +fn seeded_formatting_reconciliation_matches_a_character_model() { + use onestore::TextAttribute as A; + #[derive(Clone, Debug, PartialEq)] + struct Style { + size: f32, + color: u32, + bold: bool, + italic: bool, + } + let mut conflicts = 0; + let mut published = 0; + let mut satisfied_parts = 0; + for seed in 1_u64..=128 { + let mut random = seed; + let mut next = || { + random = random + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + random >> 32 + }; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("model.sqlite"); + let mut source = + onestore::create_section("model.one", "abcdefghijklmnopqrstuvwxyz012345", "Author") + .unwrap(); + let (sid, object, original_text) = text(&source); + let mut baseline = vec![ + Style { + size: 11.0, + color: 0xff000000, + bold: false, + italic: false + }; + 32 + ]; + for _ in 0..6 { + let start = next() as usize % 32; + let end = start + 1 + next() as usize % (32 - start); + let size = [12.0, 14.0, 18.0][next() as usize % 3]; + let color: u32 = [0xabcdef, 0x987654, 0xff000000][next() as usize % 3]; + let bold = next() % 2 == 0; + source = PreparedEdit::format( + &source, + sid, + object, + start as u32..end as u32, + &[ + A::FontSize(size), + A::Color((color != 0xff000000).then(|| { + let b = color.to_le_bytes(); + [b[0], b[1], b[2]] + })), + A::Bold(bold), + ], + ) + .unwrap() + .as_bytes() + .to_vec(); + for style in &mut baseline[start..end] { + style.size = size; + style.color = color; + style.bold = bold; + } + } + let start = next() as usize % 24; + let end = start + 2 + next() as usize % (31 - start); + let attribute = match seed % 3 { + 0 => A::FontSize(21.0), + 1 => A::Color(Some([0x12, 0x34, 0x56])), + _ => A::Bold(!baseline[start].bold), + }; + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .format( + &source, + sid, + object, + start as u32..end as u32, + std::slice::from_ref(&attribute), + ) + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let mut remote = source.clone(); + let mut expected = baseline.clone(); + for step in 0..8 { + let left = next() as usize % 32; + let right = left + 1 + next() as usize % (32 - left); + let update = if step % 3 == 0 { + attribute.clone() + } else { + match next() % 4 { + 0 => A::FontSize([11.0, 14.0, 18.0, 21.0][next() as usize % 4]), + 1 => A::Color(Some([0x99, 0x88, 0x77])), + 2 => A::Bold(next() % 2 == 0), + _ => A::Italic(true), + } + }; + remote = PreparedEdit::format( + &remote, + sid, + object, + left as u32..right as u32, + std::slice::from_ref(&update), + ) + .unwrap() + .as_bytes() + .to_vec(); + for style in &mut expected[left..right] { + match update { + A::FontSize(value) => style.size = value, + A::Color(Some([r, g, b])) => style.color = u32::from_le_bytes([r, g, b, 0]), + A::Bold(value) => style.bold = value, + A::Italic(value) => style.italic = value, + _ => unreachable!(), + } + } + } + let conflict = (start..end).any(|i| match attribute { + A::FontSize(value) => expected[i].size != baseline[i].size && expected[i].size != value, + A::Color(_) => expected[i].color != baseline[i].color && expected[i].color != 0x563412, + A::Bold(value) => expected[i].bold != baseline[i].bold && expected[i].bold != value, + _ => unreachable!(), + }); + drop(cache); + let cache = Replica::open(&path).unwrap(); + let mut server = Server::new(&remote); + let result = cache.sync_once(&mut server).unwrap().unwrap(); + assert_eq!(result.0, id, "seed {seed}"); + if conflict { + conflicts += 1; + assert_eq!( + result.1, + EditStatus::Conflict(ConflictKind::FormattingChanged), + "seed {seed}" + ); + assert_eq!(server.publications, 0, "seed {seed}"); + assert_eq!(cache.snapshot().unwrap(), local, "seed {seed}"); + continue; + } + published += 1; + assert!( + matches!(result.1, EditStatus::Published { .. }), + "seed {seed}: {result:?}" + ); + for style in &mut expected[start..end] { + match attribute { + A::FontSize(value) => { + satisfied_parts += usize::from(style.size == value); + style.size = value; + } + A::Color(_) => { + satisfied_parts += usize::from(style.color == 0x563412); + style.color = 0x563412; + } + A::Bold(value) => { + satisfied_parts += usize::from(style.bold == value); + style.bold = value; + } + _ => unreachable!(), + } + } + assert_eq!(text(&server.durable).2, original_text, "seed {seed}"); + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let space = &doc.spaces[&sid]; + let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; + let actual: Vec<_> = revision + .text_runs(object) + .unwrap() + .iter() + .flat_map(|run| { + std::iter::repeat_n( + Style { + size: run.format.font_size.unwrap(), + color: run.format.color.unwrap_or(0xff000000), + bold: run.format.bold.unwrap_or(false), + italic: run.format.italic.unwrap_or(false), + }, + run.text.chars().count(), + ) + }) + .collect(); + assert_eq!(actual, expected, "seed {seed}"); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.status(id).unwrap(), Some(result.1)); + let snapshot = cache.snapshot().unwrap(); + assert_eq!(snapshot.len(), server.durable.len(), "seed {seed}"); + assert!( + snapshot + .iter() + .zip(&server.durable) + .enumerate() + .all(|(offset, (a, b))| a == b || (212..252).contains(&offset)), + "seed {seed}: only confirmation version metadata may change" + ); + assert_eq!(cache.sync_once(&mut server).unwrap(), None); + assert!( + cache.snapshot().unwrap() == server.durable, + "seed {seed}: refreshed snapshot" + ); + } + assert!( + conflicts >= 16 && published >= 32 && satisfied_parts >= 128, + "conflicts={conflicts}, published={published}, already desired characters={satisfied_parts}" + ); + println!( + "128 seeds: {conflicts} conflicts, {published} publications, {satisfied_parts} already desired characters" + ); +} + +#[test] +fn inserted_empty_text_retains_formatting_and_dependent_text_across_sync() { + use onestore::{Insertion, TextAttribute as A}; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("empty.sqlite"); + let source = onestore::create_section("empty.one", "Original", "Author").unwrap(); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let (sid, page) = doc.pages().unwrap()[0]; + let insertion = Insertion::outline(page, 144.0, 144.0, "", "Author").unwrap(); + let cache = Replica::create(&path, &source).unwrap(); + let first = cache.insert(&source, sid, &insertion).unwrap().unwrap(); + let second = cache + .format( + &cache.snapshot().unwrap(), + sid, + insertion.text_object(), + 0..0, + &[A::Bold(true), A::FontSize(18.0)], + ) + .unwrap() + .unwrap(); + let third = cache + .edit_text( + &cache.snapshot().unwrap(), + sid, + insertion.text_object(), + 0..0, + "Typed 🦀", + ) + .unwrap() + .unwrap(); + let pending = cache.pending().unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), pending); + let mut server = Server::new(&source); + for id in [first, second, third] { + let result = cache.sync_once(&mut server).unwrap(); + assert!( + matches!(result,Some((actual,EditStatus::Published{..}))if actual==id), + "{result:?}" + ); + } + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let space = &doc.spaces[&sid]; + let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; + let runs = revision.text_runs(insertion.text_object()).unwrap(); + assert_eq!(runs.iter().map(|r| r.text).collect::(), "Typed 🦀"); + assert!( + runs.iter() + .all(|r| r.format.bold == Some(true) && r.format.font_size == Some(18.0)) + ); + assert_eq!(cache.snapshot().unwrap(), server.durable); +} + +#[test] +fn every_visual_attribute_rebases_with_an_independent_remote_attribute() { + use onestore::TextAttribute as A; + use serde_json::json; + let mut cases = Vec::new(); + for value in [false, true] { + cases.extend([ + (A::Bold(!value), A::Bold(value), "bold", json!(value)), + (A::Italic(!value), A::Italic(value), "italic", json!(value)), + ( + A::Underline(!value), + A::Underline(value), + "underline", + json!(value), + ), + (A::Strike(!value), A::Strike(value), "strike", json!(value)), + ( + A::Superscript(!value), + A::Superscript(value), + "superscript", + json!(value), + ), + ( + A::Subscript(!value), + A::Subscript(value), + "subscript", + json!(value), + ), + ]); + } + cases.extend([ + ( + A::Font("Georgia".into()), + A::Font("Arial".into()), + "font", + json!("Arial"), + ), + ( + A::FontSize(11.0), + A::FontSize(18.0), + "font_size", + json!(18.0), + ), + ( + A::Color(None), + A::Color(Some([1, 2, 3])), + "color", + json!(0x030201), + ), + ( + A::Color(Some([1, 2, 3])), + A::Color(None), + "color", + json!(0xff000000_u32), + ), + ( + A::Highlight(None), + A::Highlight(Some([4, 5, 6])), + "highlight", + json!(0x060504), + ), + ( + A::Highlight(Some([4, 5, 6])), + A::Highlight(None), + "highlight", + json!(0xff000000_u32), + ), + ]); + for (baseline, desired, field, value) in cases { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("attribute.sqlite"); + let source = onestore::create_section("attribute.one", "abc", "Author").unwrap(); + let (sid, object, _) = text(&source); + let source = PreparedEdit::format(&source, sid, object, 0..3, &[baseline]) + .unwrap() + .as_bytes() + .to_vec(); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .format(&source, sid, object, 0..3, &[desired]) + .unwrap() + .unwrap(); + let moved = PreparedEdit::text(&source, sid, object, 0..0, "Z").unwrap(); + let (other, other_field, other_value) = if field == "font_size" { + (A::Italic(true), "italic", json!(true)) + } else { + (A::FontSize(22.0), "font_size", json!(22.0)) + }; + let remote = PreparedEdit::format(moved.as_bytes(), sid, object, 1..4, &[other]).unwrap(); + let mut server = Server::new(remote.as_bytes()); + drop(cache); + let cache = Replica::open(&path).unwrap(); + let result = cache.sync_once(&mut server).unwrap(); + assert!( + matches!(result,Some((observed,EditStatus::Published{..}))if observed==id), + "{field}={value}: {result:?}" + ); + assert_eq!(text(&server.durable).2, "Zabc"); + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let space = &doc.spaces[&sid]; + let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; + let mut position = 0; + for run in revision.text_runs(object).unwrap() { + let format = serde_json::to_value(run.format).unwrap(); + for _ in run.text.chars() { + if position > 0 { + assert_eq!(format[field], value, "{field}, character {position}"); + assert_eq!( + format[other_field], other_value, + "{field}, character {position}" + ); + } + position += 1; + } + } + assert_eq!(position, 4); + } +} + +#[test] +fn reviewed_insertion_placements_preserve_identity_and_dependent_operations() { + use notebook::Operation; + use onestore::{Insertion, TextAttribute as A}; + for outline_case in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("placement.sqlite"); + let base = onestore::create_section("placement.one", "Original", "Author").unwrap(); + let (sid, _, _) = text(&base); + let store = Store::parse(&base).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let (_, page) = doc.pages().unwrap()[0]; + let space = &doc.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let parent = *view.nodes[&page] + .children + .iter() + .find(|id| matches!(view.nodes[id].kind, Kind::Outline { .. })) + .unwrap(); + let anchor = Insertion::paragraph(parent, None, "Temporary anchor", "Author").unwrap(); + let source = PreparedEdit::insert(&base, sid, &anchor) + .unwrap() + .as_bytes() + .to_vec(); + let insertion = if outline_case { + Insertion::outline(page, 144.0, 144.0, "Offline", "Author").unwrap() + } else { + Insertion::paragraph(parent, Some(anchor.object()), "Offline", "Author").unwrap() + } + .with_formatting(0..7, &[A::Italic(true)]) + .unwrap(); + let cache = Replica::create(&path, &source).unwrap(); + let first = cache.insert(&source, sid, &insertion).unwrap().unwrap(); + let second = cache + .format( + &cache.snapshot().unwrap(), + sid, + insertion.text_object(), + 0..7, + &[A::Bold(true)], + ) + .unwrap() + .unwrap(); + let third = cache + .edit_text( + &cache.snapshot().unwrap(), + sid, + insertion.text_object(), + 7..7, + " 🦀", + ) + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + let mut server = Server::new(&base); + if outline_case { + drop(cache); + let db = rusqlite::Connection::open(&path).unwrap(); + db.execute( + "INSERT INTO conflicts VALUES (?1,2)", + [i64::try_from(first).unwrap()], + ) + .unwrap(); + db.execute("UPDATE replica SET base=?1", [&base]).unwrap(); + drop(db); + } else { + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((first, EditStatus::Conflict(ConflictKind::UnsupportedEdit))) + ); + drop(cache); + } + let cache = Replica::open(&path).unwrap(); + if outline_case { + assert!( + cache + .rebase_paragraph_conflict(first, &local, &base, parent, None) + .is_err() + ); + assert!( + cache + .rebase_outline_conflict(first, &local, &base, page, f32::NAN, 288.0) + .is_err() + ); + assert!( + matches!(cache.rebase_outline_conflict(first,&source,&base,page,288.0,360.0),Err(Error::Io(e))if e.kind()==io::ErrorKind::ResourceBusy) + ); + cache + .rebase_outline_conflict(first, &local, &base, page, 288.0, 360.0) + .unwrap(); + } else { + assert!( + cache + .rebase_outline_conflict(first, &local, &base, page, 288.0, 360.0) + .is_err() + ); + assert!( + cache + .rebase_paragraph_conflict(first, &local, &base, parent, Some(anchor.object())) + .is_err() + ); + assert!( + matches!(cache.rebase_paragraph_conflict(first,&source,&base,parent,None),Err(Error::Io(e))if e.kind()==io::ErrorKind::ResourceBusy) + ); + cache + .rebase_paragraph_conflict(first, &local, &base, parent, None) + .unwrap(); + } + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.pending().unwrap()[1..], pending[1..]); + let Operation::Insert(rebased) = cache.pending().unwrap().remove(0).operation else { + panic!() + }; + assert_eq!(rebased.object(), insertion.object()); + assert_eq!(rebased.text_object(), insertion.text_object()); + assert_eq!( + serde_json::to_value(&rebased).unwrap()["formats"], + serde_json::to_value(&insertion).unwrap()["formats"] + ); + assert_eq!(cache.status(first).unwrap(), Some(EditStatus::Pending)); + drop(cache); + let cache = Replica::open(&path).unwrap(); + for id in [first, second, third] { + let result = cache.sync_once(&mut server).unwrap(); + assert!( + matches!(result,Some((actual,EditStatus::Published{..}))if actual==id), + "{result:?}" + ); + } + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let space = &doc.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let runs = view.text_runs(insertion.text_object()).unwrap(); + assert_eq!( + runs.iter().map(|r| r.text).collect::(), + "Offline 🦀" + ); + assert!(runs.iter().all(|r| r.format.bold == Some(true))); + assert!(runs.iter().all(|r| r.format.italic == Some(true))); + if outline_case { + assert_eq!( + ( + view.nodes[&insertion.object()].layout.x, + view.nodes[&insertion.object()].layout.y + ), + (Some(288.0), Some(360.0)) + ); + } else { + assert_eq!( + view.nodes[&parent].children.last(), + Some(&insertion.object()) + ); + } + assert!(!view.nodes.contains_key(&anchor.object())); + assert_eq!(server.publications, 3); + assert!(cache.pending().unwrap().is_empty()); + assert_eq!(cache.snapshot().unwrap(), server.durable); + } +} + +#[test] +fn placement_reviews_cannot_replace_pending_or_uncertain_attempts() { + use onestore::Insertion; + for outline_case in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("attempt-placement.sqlite"); + let source = onestore::create_section("placement.one", "Original", "Author").unwrap(); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let doc = Document::parse(&index).unwrap(); + let (sid, page) = doc.pages().unwrap()[0]; + let space = &doc.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let parent = *view.nodes[&page] + .children + .iter() + .find(|id| matches!(view.nodes[id].kind, Kind::Outline { .. })) + .unwrap(); + let insertion = if outline_case { + Insertion::outline(page, 144.0, 144.0, "Offline", "Author").unwrap() + } else { + Insertion::paragraph(parent, None, "Offline", "Author").unwrap() + }; + let cache = Replica::create(&path, &source).unwrap(); + let id = cache.insert(&source, sid, &insertion).unwrap().unwrap(); + let local = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + let mut server = Server::new(&source); + for attempted in [false, true] { + if attempted { + server.fault = Fault::UnknownBefore; + assert!( + matches!(cache.sync_once(&mut server),Err(Error::Remote(e))if e.state==CommitState::Unknown) + ); + } + let state = cache.status(id).unwrap(); + let result = if outline_case { + cache.rebase_outline_conflict(id, &local, &source, page, 288.0, 360.0) + } else { + cache.rebase_paragraph_conflict(id, &local, &source, parent, None) + }; + assert!(matches!(result,Err(Error::Io(e))if e.kind()==io::ErrorKind::InvalidInput)); + assert_eq!(cache.pending().unwrap(), pending); + assert_eq!(cache.status(id).unwrap(), state); + assert_eq!(cache.snapshot().unwrap(), local); + } + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), pending); + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::AwaitingConfirmation { .. })) + )); + assert_eq!(server.publications, 1); + } +} diff --git a/crates/notebook/tests/sync/outline.rs b/crates/notebook/tests/sync/outline.rs new file mode 100644 index 0000000000000000000000000000000000000000..7863cb221020478fff27934fe832ec49f95933c6 --- /dev/null +++ b/crates/notebook/tests/sync/outline.rs @@ -0,0 +1,719 @@ +use super::*; +use onestore::OutlineEdit as Change; + +pub(super) fn fixture() -> (Vec, ExGuid, ExGuid, ExGuid, ExGuid) { + let source = onestore::create_section("layout.one", "Original 🦀 é", "Author").unwrap(); + let (sid, text, _) = text(&source); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let space = &document.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let paragraph = *view + .nodes + .iter() + .find(|(_, node)| node.content == [text]) + .unwrap() + .0; + let outline = *view + .nodes + .iter() + .find(|(_, node)| node.children.contains(¶graph)) + .unwrap() + .0; + (source, sid, outline, paragraph, text) +} + +pub(super) fn node(source: &[u8], sid: ExGuid, object: ExGuid) -> serde_json::Value { + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + let document = Document::parse(&index).unwrap(); + let space = &document.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + serde_json::to_value(&view.nodes[&object]).unwrap() +} + +#[test] +fn layout_and_dependent_text_survive_reopen_and_independent_remote_formatting() { + let (source, sid, outline, paragraph, text_id) = fixture(); + for (object, change) in [ + (outline, Change::Position { x: 180.0, y: 216.0 }), + ( + outline, + Change::Width { + points: 144.0, + user_set: true, + }, + ), + (paragraph, Change::Collapsed(true)), + ] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("cache.sqlite"); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .outline(&source, sid, object, change) + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let expected = node(&local, sid, object); + let dependent = cache + .edit_text(&local, sid, text_id, 0..0, "Local ") + .unwrap() + .unwrap(); + let pending = cache.pending().unwrap(); + let local = cache.snapshot().unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), pending); + assert!(cache.snapshot().unwrap() == local); + let remote = PreparedEdit::format( + &source, + sid, + text_id, + 0..8, + &[onestore::TextAttribute::Bold(true)], + ) + .unwrap(); + let mut server = Server::new(remote.as_bytes()); + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n==id) + ); + let actual = node(&server.durable, sid, object); + assert_eq!(actual["layout"], expected["layout"]); + assert_eq!(actual["kind"], expected["kind"]); + assert_eq!(cache.pending().unwrap().len(), 1); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n==dependent) + ); + assert_eq!(text(&server.durable).2, "Local Original 🦀 é"); + assert!(cache.pending().unwrap().is_empty()); + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let space = &document.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + assert!(view.text_runs(text_id).unwrap()[0].format.bold.unwrap()); + assert_eq!(server.publications, 2); + drop(cache); + let cache = Replica::open(&path).unwrap(); + for id in [id, dependent] { + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::Published { .. }) + )); + } + } +} + +#[test] +fn rebased_outline_movement_uses_remote_title_text_and_confirms_after_reopen() { + let (source, sid, outline, _, text_id) = fixture(); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let (_, page) = document.pages().unwrap()[0]; + let space = &document.spaces[&sid]; + let metadata = space.revisions[&space.contexts[&ExGuid::default()]].roots[&2]; + let second = onestore::Insertion::outline(page, 144.0, 36.0, "Second", "Author").unwrap(); + let source = PreparedEdit::insert(&source, sid, &second) + .unwrap() + .as_bytes() + .to_vec(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("titles.sqlite"); + let cache = Replica::create(&path, &source).unwrap(); + let movement = cache + .outline( + &source, + sid, + outline, + Change::Position { x: 216.0, y: 72.0 }, + ) + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + assert_eq!(node(&local, sid, metadata)["kind"]["title"], "Second"); + let dependent = cache + .edit_text(&local, sid, text_id, 0..0, "Local ") + .unwrap() + .unwrap(); + drop(cache); + let remote = + PreparedEdit::text(&source, sid, second.text_object(), 0..6, "Remote second 🐈").unwrap(); + let mut server = Server::new(remote.as_bytes()); + server.fault = Fault::UnknownAfter; + let cache = Replica::open(&path).unwrap(); + assert!(cache.sync_once(&mut server).is_err()); + assert!(matches!( + cache.status(movement).unwrap(), + Some(EditStatus::AwaitingConfirmation { .. }) + )); + assert_eq!( + node(&server.durable, sid, metadata)["kind"]["title"], + "Original 🦀 é" + ); + assert_eq!( + node(&server.visible, sid, metadata)["kind"]["title"], + "Remote second 🐈" + ); + drop(cache); + for id in [movement, dependent] { + let cache = Replica::open(&path).unwrap(); + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) + ); + assert_eq!( + node(&server.durable, sid, metadata)["kind"]["title"], + "Remote second 🐈" + ); + assert_eq!( + node(&server.durable, sid, page)["kind"]["alternate_title"], + "Remote second 🐈" + ); + } + let cache = Replica::open(&path).unwrap(); + assert!(cache.pending().unwrap().is_empty()); + assert_eq!( + node(&cache.snapshot().unwrap(), sid, text_id)["kind"]["text"], + "Local Original 🦀 é" + ); + assert_eq!(server.publications, 2); +} + +#[test] +fn competing_layout_requires_current_review_and_preserves_dependent_edits() { + let (source, sid, outline, _, text_id) = fixture(); + for (local_change, remote_change) in [ + ( + Change::Position { x: 180.0, y: 216.0 }, + Change::Position { x: 288.0, y: 216.0 }, + ), + ( + Change::Width { + points: 144.0, + user_set: true, + }, + Change::Width { + points: 216.0, + user_set: false, + }, + ), + ] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("cache.sqlite"); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .outline(&source, sid, outline, local_change) + .unwrap() + .unwrap(); + let dependent = cache + .edit_text(&cache.snapshot().unwrap(), sid, text_id, 0..0, "Local ") + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let queue = cache.pending().unwrap(); + let remote = PreparedEdit::outline(&source, sid, outline, remote_change).unwrap(); + let mut server = Server::new(remote.as_bytes()); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, EditStatus::Conflict(ConflictKind::LayoutChanged))) + ); + assert_eq!(server.publications, 0); + assert_eq!(cache.pending().unwrap(), queue); + assert!(cache.snapshot().unwrap() == local); + assert!( + cache + .rebase_layout_conflict(id, &source, &server.visible) + .is_err() + ); + assert!(cache.rebase_layout_conflict(id, &local, &source).is_err()); + assert!( + cache + .rebase_conflict(id, &local, &server.visible, 0..0) + .is_err() + ); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!( + cache.status(id).unwrap(), + Some(EditStatus::Conflict(ConflictKind::LayoutChanged)) + ); + cache + .rebase_layout_conflict(id, &local, &server.visible) + .unwrap(); + let reviewed = cache.pending().unwrap(); + assert_eq!(reviewed[1], queue[1]); + assert!(cache.snapshot().unwrap() == local); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), reviewed); + for id in [id, dependent] { + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == id) + ); + } + assert_eq!( + node(&server.durable, sid, outline)["layout"], + node(&local, sid, outline)["layout"] + ); + assert_eq!(text(&server.durable).2, "Local Original 🦀 é"); + } +} + +#[test] +fn independent_axes_converge_without_overwriting_competing_values() { + let (source, sid, outline, _, _) = fixture(); + let directory = tempfile::tempdir().unwrap(); + let cache = Replica::create(directory.path().join("cache.sqlite"), &source).unwrap(); + let original = node(&source, sid, outline); + let local = Change::Position { x: 180.0, y: 216.0 }; + let id = cache + .outline(&source, sid, outline, local) + .unwrap() + .unwrap(); + let remote = PreparedEdit::outline( + &source, + sid, + outline, + Change::Position { + x: original["layout"]["x"].as_f64().unwrap() as f32, + y: 216.0, + }, + ) + .unwrap(); + let mut server = Server::new(remote.as_bytes()); + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n==id) + ); + let actual = node(&server.durable, sid, outline); + assert_eq!(actual["layout"]["x"], 180.0); + assert_eq!(actual["layout"]["y"], 216.0); +} + +#[test] +fn twelve_offline_writers_preserve_all_layout_intents_through_review_and_restarts() { + let (source, sid, outline, paragraph, _) = fixture(); + let directory = tempfile::tempdir().unwrap(); + std::thread::scope(|scope| { + let mut writers = Vec::new(); + for actor in 0..12 { + let source = &source; + let path = directory.path().join(format!("{actor}.sqlite")); + writers.push(scope.spawn(move || { + let cache = Replica::create(&path, source).unwrap(); + for round in 0..4 { + for (object, change) in [ + ( + outline, + Change::Position { + x: (actor + 2) as f32 * 36.0, + y: (round + 2) as f32 * 36.0, + }, + ), + ( + outline, + Change::Width { + points: (actor + round + 2) as f32 * 36.0, + user_set: true, + }, + ), + (paragraph, Change::Collapsed((actor + round) % 2 != 0)), + ] { + assert!( + cache + .outline(&cache.snapshot().unwrap(), sid, object, change) + .unwrap() + .is_some() + ); + } + } + assert_eq!(cache.pending().unwrap().len(), 12); + })); + } + for writer in writers { + writer.join().unwrap(); + } + }); + let mut server = Server::new(&source); + let mut expected_layout = node(&source, sid, outline)["layout"].clone(); + let mut expected_collapse = None; + let mut reviewed = 0; + for round in 0..12 { + for actor in 0..12 { + let path = directory.path().join(format!("{actor}.sqlite")); + let cache = Replica::open(&path).unwrap(); + let first = cache.pending().unwrap()[0].clone(); + let notebook::Operation::Outline(edit) = first.operation else { + panic!() + }; + let local = cache.snapshot().unwrap(); + let result = cache.sync_once(&mut server).unwrap(); + if result == Some((first.id, EditStatus::Conflict(ConflictKind::LayoutChanged))) { + let remote = cache.remote_snapshot().unwrap(); + cache + .rebase_layout_conflict(first.id, &local, &remote) + .unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((id, EditStatus::Published { .. })) if id == first.id) + ); + reviewed += 1; + } else { + assert!( + matches!(result, Some((id, EditStatus::Published { .. })) if id == first.id) + ); + } + match edit.change { + Change::Position { x, y } => { + expected_layout["x"] = x.into(); + expected_layout["y"] = y.into(); + } + Change::Width { points, user_set } => { + expected_layout["max_width"] = points.into(); + expected_layout["width_set_by_user"] = user_set.into(); + } + Change::Collapsed(value) => expected_collapse = Some(u8::from(value)), + } + assert_eq!( + node(&server.durable, sid, outline)["layout"], + expected_layout + ); + assert_eq!( + node(&server.durable, sid, paragraph)["kind"]["collapse_state"], + serde_json::json!(expected_collapse) + ); + assert_eq!(text(&server.durable).2, "Original 🦀 é"); + } + for actor in 0..12 { + let cache = Replica::open(directory.path().join(format!("{actor}.sqlite"))).unwrap(); + assert_eq!(cache.pending().unwrap().len(), 11 - round); + } + } + assert!(reviewed >= 12); + for actor in 0..12 { + let cache = Replica::open(directory.path().join(format!("{actor}.sqlite"))).unwrap(); + assert!(cache.pending().unwrap().is_empty()); + for id in 1..=12 { + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::Published { .. }) + )); + } + } +} + +#[test] +fn uncertain_layout_is_confirmed_by_revision_and_never_by_converged_values() { + let (source, sid, outline, _, text_id) = fixture(); + for fault in [Fault::UnknownBefore, Fault::UnknownAfter] { + let before = matches!(fault, Fault::UnknownBefore); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("cache.sqlite"); + let cache = Replica::create(&path, &source).unwrap(); + let change = Change::Width { + points: 144.0, + user_set: true, + }; + let id = cache + .outline(&source, sid, outline, change) + .unwrap() + .unwrap(); + cache + .edit_text(&cache.snapshot().unwrap(), sid, text_id, 0..0, "Local ") + .unwrap(); + let local = cache.snapshot().unwrap(); + let mut server = Server::new(&source); + server.fault = fault; + assert!(cache.sync_once(&mut server).is_err()); + let status = cache.status(id).unwrap().unwrap(); + assert!(matches!(status, EditStatus::AwaitingConfirmation { .. })); + drop(cache); + let cache = Replica::open(&path).unwrap(); + if before { + let converged = PreparedEdit::outline(&source, sid, outline, change).unwrap(); + server.visible = converged.as_bytes().to_vec(); + assert_eq!(cache.sync_once(&mut server).unwrap(), Some((id, status))); + assert!( + cache + .rebase_layout_conflict(id, &local, &server.visible) + .is_err() + ); + assert_eq!(cache.pending().unwrap().len(), 2); + assert!(cache.snapshot().unwrap() == local); + let archive = directory.path().join("recovery.sqlite"); + cache.export_recovery(&archive).unwrap(); + let recovery = notebook::Recovery::open(&archive).unwrap(); + assert_eq!(recovery.pending().unwrap(), cache.pending().unwrap()); + assert_eq!(recovery.status(id).unwrap(), Some(status)); + assert_eq!(server.confirmations, 0); + } else { + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n==id) + ); + assert_eq!( + node(&server.durable, sid, outline)["layout"]["max_width"], + 144.0 + ); + assert_eq!(server.confirmations, 1); + } + assert_eq!(server.publications, 1); + } +} + +#[test] +fn native_moves_deletions_and_layout_changes_merge_or_retain_explicit_conflicts() { + let source = include_bytes!("../../../../corpus/outline-edit/before/notebook/synthetic.one"); + let native = include_bytes!("../../../../corpus/outline-edit/after/notebook/synthetic.one"); + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let mut server = Server::new(native); + let mut counts = [0; 3]; + let mut records = Vec::new(); + for (sid, page) in document.pages().unwrap() { + let space = &document.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let Kind::Metadata { + title: Some(name), .. + } = &view.nodes[&view.roots[&2]].kind + else { + continue; + }; + if name == "Unicode rich text" { + continue; + } + let outlines: Vec<_> = view.nodes[&page] + .children + .iter() + .filter(|id| matches!(view.nodes[id].kind, Kind::Outline { .. })) + .copied() + .collect(); + if outlines.len() != 2 { + continue; + } + let outline = outlines[0]; + let mut pending = vec![outline]; + let mut target = None; + while let Some(id) = pending.pop() { + let node = &view.nodes[&id]; + pending.extend(&node.children); + if node.content.first().is_some_and(|text| matches!(&view.nodes[text].kind, Kind::RichText { text, .. } if text.starts_with("Target "))) { + target = Some(id); + } + } + let target = target.unwrap(); + let text_id = view.nodes[&view.nodes[&outlines[1]].children[0]].content[0]; + let (object, change, conflict) = match name.as_str() { + "Move outline" => ( + outline, + Change::Position { x: 252.0, y: 288.0 }, + Some(ConflictKind::LayoutChanged), + ), + "Resize outline" | "Automatic outline size" => ( + outline, + Change::Width { + points: 252.0, + user_set: true, + }, + Some(ConflictKind::LayoutChanged), + ), + "Delete outline" => ( + outline, + Change::Width { + points: 252.0, + user_set: true, + }, + Some(ConflictKind::TargetUnavailable), + ), + "Delete leaf" | "Delete subtree" | "Delete only paragraph" => ( + target, + Change::Collapsed(true), + Some(ConflictKind::TargetUnavailable), + ), + "Indent subtree" | "Outdent subtree" => ( + target, + Change::Collapsed(true), + Some(ConflictKind::StructureChanged), + ), + "Expand subtree" => (target, Change::Collapsed(false), None), + "Collapse subtree" | "Move leaf down" | "Move subtree down" | "Move subtree up" => { + (target, Change::Collapsed(true), None) + } + _ => panic!("Unexpected native case {name}"), + }; + let expected_layout = if conflict == Some(ConflictKind::TargetUnavailable) { + None + } else { + let mut layout = node(&server.visible, sid, object)["layout"].clone(); + match change { + Change::Position { x, y } => { + layout["x"] = x.into(); + layout["y"] = y.into(); + } + Change::Width { points, user_set } => { + layout["max_width"] = points.into(); + layout["width_set_by_user"] = user_set.into(); + } + Change::Collapsed(_) => {} + } + Some(layout) + }; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("cache.sqlite"); + let cache = Replica::create(&path, source).unwrap(); + let id = cache.outline(source, sid, object, change).unwrap().unwrap(); + let dependent = cache + .edit_text(&cache.snapshot().unwrap(), sid, text_id, 0..0, "Local ") + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let queue = cache.pending().unwrap(); + let result = cache.sync_once(&mut server).unwrap().unwrap(); + let mut retained = false; + if let Some(kind) = conflict { + assert_eq!(result, (id, EditStatus::Conflict(kind)), "{name}"); + assert_eq!(cache.pending().unwrap(), queue); + assert!(cache.snapshot().unwrap() == local); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Conflict(kind))); + if kind == ConflictKind::TargetUnavailable { + assert!( + cache + .rebase_layout_conflict(id, &local, &server.visible) + .is_err() + ); + counts[2] += 1; + retained = true; + } else { + cache + .rebase_layout_conflict(id, &local, &server.visible) + .unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == id) + ); + counts[1] += 1; + } + } else { + assert!( + matches!(result, (n, EditStatus::Published { .. }) if n == id), + "{name}" + ); + counts[0] += 1; + drop(cache); + } + let cache = Replica::open(&path).unwrap(); + if retained { + assert_eq!(cache.pending().unwrap(), queue); + assert!(cache.snapshot().unwrap() == local); + } else { + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == dependent) + ); + assert_eq!( + node(&server.durable, sid, object)["layout"], + expected_layout.unwrap(), + "{name}" + ); + if matches!(change, Change::Collapsed(_)) { + assert_eq!( + node(&server.durable, sid, object)["kind"]["collapse_state"], + node(&local, sid, object)["kind"]["collapse_state"] + ); + } + assert!( + node(&server.durable, sid, text_id)["kind"]["text"] + .as_str() + .unwrap() + .starts_with("Local ") + ); + drop(cache); + let cache = Replica::open(&path).unwrap(); + for id in [id, dependent] { + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::Published { .. }) + )); + } + } + records.push(serde_json::json!({"name": name, "space": sid, "object": object, "change": change, "retained": retained, "dependent_text": text_id})); + } + assert_eq!(counts, [5, 5, 4]); + assert_eq!(server.publications, 18); + if let Some(output) = std::env::var_os("ONESTORE_OFFLINE_OUTLINE_OUTPUT") { + let output = std::path::PathBuf::from(output); + assert!(output.is_absolute()); + std::fs::create_dir(&output).unwrap(); + std::fs::write(output.join("synthetic.one"), &server.durable).unwrap(); + std::fs::write( + output.with_extension("json"), + serde_json::to_vec_pretty(&records).unwrap(), + ) + .unwrap(); + } +} + +#[test] +fn a_new_native_wrap_reservation_requires_review_before_width_replacement() { + let source = include_bytes!("../../../../corpus/outline-edit/before/notebook/synthetic.one"); + let native = include_bytes!("../../../../corpus/outline-edit/after/notebook/synthetic.one"); + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let (sid, page) = document.pages().unwrap().into_iter().find(|(sid, _)| { + let space = &document.spaces[sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + matches!(&view.nodes[&view.roots[&2]].kind, Kind::Metadata { title: Some(name), .. } if name=="Move outline") + }).unwrap(); + let space = &document.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let outline = *view.nodes[&page] + .children + .iter() + .find(|id| matches!(view.nodes[id].kind, Kind::Outline { .. })) + .unwrap(); + let old = node(source, sid, outline); + let remote = node(native, sid, outline); + assert_eq!(old["layout"]["max_width"], remote["layout"]["max_width"]); + assert_eq!( + old["layout"]["width_set_by_user"], + remote["layout"]["width_set_by_user"] + ); + assert!(remote["layout"]["reserved_width"].is_number()); + let directory = tempfile::tempdir().unwrap(); + let cache = Replica::create(directory.path().join("cache.sqlite"), source).unwrap(); + let id = cache + .outline( + source, + sid, + outline, + Change::Width { + points: 144.0, + user_set: true, + }, + ) + .unwrap() + .unwrap(); + let mut server = Server::new(native); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, EditStatus::Conflict(ConflictKind::LayoutChanged))) + ); + cache + .rebase_layout_conflict(id, &cache.snapshot().unwrap(), &server.visible) + .unwrap(); + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n==id) + ); + let after = node(&server.durable, sid, outline); + assert_eq!(after["layout"]["x"], remote["layout"]["x"]); + assert_eq!(after["layout"]["y"], remote["layout"]["y"]); + assert_eq!(after["layout"]["max_width"], 144.0); + assert!(after["layout"]["reserved_width"].is_null()); +} diff --git a/crates/notebook/tests/sync/page.rs b/crates/notebook/tests/sync/page.rs new file mode 100644 index 0000000000000000000000000000000000000000..5cae6719919da89b9e2939df52f6ffb306cf804f --- /dev/null +++ b/crates/notebook/tests/sync/page.rs @@ -0,0 +1,350 @@ +use super::*; +use notebook::{Operation, Recovery}; +use onestore::{Insertion, PageCreation}; +use std::collections::BTreeMap; + +#[test] +fn twelve_replica_page_schedules_retain_acknowledged_pages_through_interruptions() { + for seed in 0..24_u64 { + let mut random = seed + 1956; + let mut input = Vec::new(); + for step in 0..48 { + random ^= random << 13; + random ^= random >> 7; + random ^= random << 17; + let mut action = random.to_le_bytes(); + if step < 12 { + action[0] = step; + action[1] = 1; + } + input.extend_from_slice(&action); + } + page_schedule::run(&input); + } +} + +#[test] +fn version_two_uncertain_text_migrates_without_replay_or_lost_receipts() { + let source = onestore::create_section("pages.one", "Original", "Author").unwrap(); + let (sid, oid, _) = text(&source); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("legacy.sqlite"); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .edit_text(&source, sid, oid, 0..0, "Old ") + .unwrap() + .unwrap(); + let snapshot = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + let store = Store::parse(&snapshot).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let revision = index.spaces[&sid].labels[&(ExGuid::default(), 1)]; + drop(cache); + let db = rusqlite::Connection::open(&path).unwrap(); + db.execute_batch( + "DROP TABLE edits; DROP TABLE assets; + CREATE TABLE edits ( + id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(id>0), space TEXT NOT NULL, + object TEXT NOT NULL, before_text TEXT NOT NULL, + start INTEGER NOT NULL CHECK(start BETWEEN 0 AND 4294967295), + end INTEGER NOT NULL CHECK(end BETWEEN start AND 4294967295), replacement TEXT NOT NULL + ) STRICT; + ALTER TABLE attempt RENAME COLUMN revisions TO revision; + PRAGMA user_version=2;", + ) + .unwrap(); + db.execute( + "INSERT INTO edits VALUES (?1,?2,?3,'Original',0,0,'Old ')", + rusqlite::params![i64::try_from(id).unwrap(), sid.to_string(), oid.to_string()], + ) + .unwrap(); + db.execute( + "INSERT INTO attempt VALUES (1,?1,?2)", + rusqlite::params![i64::try_from(id).unwrap(), revision.to_string()], + ) + .unwrap(); + db.execute( + "INSERT INTO receipts VALUES (1000,?1)", + [revision.to_string()], + ) + .unwrap(); + db.execute("UPDATE sqlite_sequence SET seq=1000 WHERE name='edits'", []) + .unwrap(); + drop(db); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), pending); + assert_eq!( + cache.status(id).unwrap(), + Some(EditStatus::AwaitingConfirmation { revision }) + ); + assert_eq!( + cache.status(1000).unwrap(), + Some(EditStatus::Published { revision }) + ); + assert!(cache.snapshot().unwrap() == snapshot); + let mut server = Server::new(&snapshot); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, EditStatus::Published { revision })) + ); + assert_eq!(server.publications, 0); + assert_eq!(server.confirmations, 1); + assert_eq!( + cache + .edit_text(&cache.snapshot().unwrap(), sid, oid, 0..0, "Next ") + .unwrap(), + Some(1001) + ); +} + +#[test] +fn offline_page_creation_rebases_with_dependent_edits_and_duplicate_titles() { + let source = onestore::create_section("pages.one", "Original", "Author").unwrap(); + let directory = tempfile::tempdir().unwrap(); + let mut server = Server::new(&source); + let mut expected = Vec::new(); + for actor in 0..12 { + let path = directory.path().join(format!("{actor}.sqlite")); + let cache = Replica::create(&path, &source).unwrap(); + let page = PageCreation::new(None, Some("Same 🦋 é"), "Offline author").unwrap(); + let id = cache.create_page(&source, &page).unwrap().unwrap(); + let insertion = Insertion::outline(page.object(), 36.0, 36.0, "Body", "Author").unwrap(); + cache + .insert(&cache.snapshot().unwrap(), page.space(), &insertion) + .unwrap() + .unwrap(); + cache + .edit_text( + &cache.snapshot().unwrap(), + page.space(), + insertion.text_object(), + 4..4, + &format!(" {actor}"), + ) + .unwrap() + .unwrap(); + let original = cache.pending().unwrap(); + assert!( + matches!(&original[0].operation, Operation::CreatePage(retained) if retained == &page) + ); + let local = cache.snapshot().unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), original); + assert_eq!(cache.snapshot().unwrap(), local); + for edit in original { + assert!(matches!(cache.sync_once(&mut server).unwrap(), + Some((published, EditStatus::Published { .. })) if published == edit.id)); + } + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::Published { .. }) + )); + expected.push(( + page.space(), + page.object(), + insertion.text_object(), + format!("Body {actor}"), + )); + } + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let pages = document.pages().unwrap(); + assert_eq!(pages.len(), 13); + for (actual, (sid, page, text, expected)) in pages[1..].iter().zip(&expected) { + assert_eq!(*actual, (*sid, *page)); + let space = &document.spaces[sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + assert!(matches!(&view.nodes[text].kind, Kind::RichText { text, .. } if text == expected)); + } + assert_eq!(server.publications, 36); + if let Some(output) = std::env::var_os("ONESTORE_OFFLINE_PAGE_OUTPUT") { + std::fs::create_dir(&output).unwrap(); + std::fs::write( + std::path::Path::new(&output).join("pages.one"), + &server.durable, + ) + .unwrap(); + } +} + +#[test] +fn uncertain_page_publication_retains_both_revisions_and_never_replays() { + for fault in [ + Fault::UnknownBefore, + Fault::UnknownAfter, + Fault::PanicBefore, + Fault::PanicAfter, + ] { + let source = onestore::create_section("pages.one", "Original", "Author").unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("pages.sqlite"); + let cache = Replica::create(&path, &source).unwrap(); + let page = PageCreation::new(None, Some("Created"), "Author").unwrap(); + let id = cache.create_page(&source, &page).unwrap().unwrap(); + let local = cache.snapshot().unwrap(); + let mut server = Server::new(&source); + server.fault = fault; + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + cache.sync_once(&mut server) + })); + let attempted = cache.status(id).unwrap().unwrap(); + assert!(matches!(attempted, EditStatus::AwaitingConfirmation { .. })); + let archive = directory.path().join("recovery.sqlite"); + cache.export_recovery(&archive).unwrap(); + assert_eq!( + Recovery::open(&archive).unwrap().status(id).unwrap(), + Some(attempted) + ); + drop(cache); + let db = rusqlite::Connection::open(&path).unwrap(); + let encoded: String = db + .query_row("SELECT revisions FROM attempt", [], |row| row.get(0)) + .unwrap(); + let revisions: BTreeMap = serde_json::from_str(&encoded).unwrap(); + assert_eq!(revisions.len(), 2); + assert!(revisions.contains_key(&page.space())); + drop(db); + let cache = Replica::open(&path).unwrap(); + let observed = server.visible.clone(); + let result = cache.sync_once(&mut server).unwrap().unwrap(); + let visible = matches!(fault, Fault::UnknownAfter | Fault::PanicAfter); + assert_eq!(server.publications, 1); + assert_eq!(server.confirmations, usize::from(visible)); + if visible { + assert!(matches!(result.1, EditStatus::Published { .. })); + assert!(cache.snapshot().unwrap() == observed); + assert!(observed[..212] == server.durable[..212]); + assert!(observed[252..] == server.durable[252..]); + } else { + assert_eq!(result, (id, attempted)); + assert_eq!(cache.snapshot().unwrap(), local); + } + } +} + +#[test] +fn surviving_page_revision_alone_does_not_confirm_section_publication() { + let source = onestore::create_section("pages.one", "Original", "Author").unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("pages.sqlite"); + let cache = Replica::create(&path, &source).unwrap(); + let page = PageCreation::new(None, None, "Author").unwrap(); + let id = cache.create_page(&source, &page).unwrap().unwrap(); + let mut server = Server::new(&source); + server.fault = Fault::UnknownAfter; + assert!(cache.sync_once(&mut server).is_err()); + let attempted = cache.status(id).unwrap().unwrap(); + drop(cache); + let db = rusqlite::Connection::open(&path).unwrap(); + let encoded: String = db + .query_row("SELECT revisions FROM attempt", [], |row| row.get(0)) + .unwrap(); + let revisions: BTreeMap = serde_json::from_str(&encoded).unwrap(); + drop(db); + let complete = server.visible.clone(); + let (§ion, &retired) = revisions + .iter() + .find(|(sid, _)| **sid != page.space()) + .unwrap(); + let store = Store::parse(&complete).unwrap(); + let mut positions = Vec::new(); + for list in store.lists.values() { + for node in &list.nodes { + if node.id == 0x1e + && node.payload[..16] == retired.guid + && node.payload[16..20] == retired.n.to_le_bytes() + { + positions.push(node.offset); + } + } + } + assert_eq!(positions.len(), 1); + // Replacing only the revision identity models maintenance retiring the section proof. + let at = positions[0] + 4; + server.visible[at] ^= 0x40; + let store = Store::parse(&server.visible).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + assert!(!index.spaces[§ion].revisions.contains_key(&retired)); + assert!( + index.spaces[&page.space()] + .revisions + .contains_key(&revisions[&page.space()]) + ); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.sync_once(&mut server).unwrap(), Some((id, attempted))); + assert_eq!(server.confirmations, 0); + assert_eq!(server.publications, 1); + server.visible = complete; + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Published { .. })) + )); + assert_eq!(server.publications, 1); +} + +#[test] +fn changed_page_anchor_requires_review_without_regenerating_dependent_identities() { + let source = + include_bytes!("../../../../corpus/page-lifecycle/03-renamed/notebook/Lifecycle.one"); + let remote = + include_bytes!("../../../../corpus/page-lifecycle/04-nested/notebook/Lifecycle.one"); + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let pages = Document::parse(&index).unwrap().pages().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("pages.sqlite"); + let cache = Replica::create(&path, source).unwrap(); + let page = PageCreation::new(Some(pages[4].0), Some("Created"), "Author").unwrap(); + let id = cache.create_page(source, &page).unwrap().unwrap(); + let insertion = + Insertion::outline(page.object(), 36.0, 36.0, "Retained body", "Author").unwrap(); + cache + .insert(&cache.snapshot().unwrap(), page.space(), &insertion) + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let pending = cache.pending().unwrap(); + let mut server = Server::new(remote); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, EditStatus::Conflict(ConflictKind::StructureChanged))) + ); + assert!( + cache + .rebase_page_creation_conflict(id, source, remote, None) + .is_err() + ); + assert!( + cache + .rebase_page_creation_conflict(id, &local, source, None) + .is_err() + ); + cache + .rebase_page_creation_conflict(id, &local, remote, None) + .unwrap(); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.pending().unwrap()[1], pending[1]); + let Operation::CreatePage(reviewed) = &cache.pending().unwrap()[0].operation else { + panic!() + }; + assert_eq!(*reviewed, page.reposition(None).unwrap()); + drop(cache); + let cache = Replica::open(&path).unwrap(); + for _ in 0..2 { + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Published { .. })) + )); + } + assert_eq!(server.publications, 2); + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let space = &document.spaces[&page.space()]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + assert!(matches!(&view.nodes[&insertion.text_object()].kind, + Kind::RichText { text, .. } if text == "Retained body")); +} diff --git a/crates/notebook/tests/sync/pages.rs b/crates/notebook/tests/sync/pages.rs new file mode 100644 index 0000000000000000000000000000000000000000..a751c6740c86a288e41a106020ed21dffe2b0c4b --- /dev/null +++ b/crates/notebook/tests/sync/pages.rs @@ -0,0 +1,631 @@ +use super::*; +use notebook::{Operation, Recovery}; +use onestore::{PageEdit, PagePosition}; +use std::collections::BTreeMap; + +const SOURCE: &[u8] = + include_bytes!("../../../../corpus/page-lifecycle/04-nested/notebook/Lifecycle.one"); + +fn order(source: &[u8]) -> Vec<(ExGuid, u32)> { + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + let document = Document::parse(&index).unwrap(); + document + .pages() + .unwrap() + .iter() + .map(|(sid, _)| { + let space = &document.spaces[sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let Kind::Metadata { level, .. } = view.nodes[&view.roots[&2]].kind else { + panic!() + }; + (*sid, level.unwrap_or(1)) + }) + .collect() +} + +fn texts(source: &[u8]) -> BTreeMap<(ExGuid, ExGuid), String> { + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let mut texts = BTreeMap::new(); + for (sid, space) in &document.spaces { + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + for (oid, node) in &view.nodes { + if let Kind::RichText { text, .. } = &node.kind { + texts.insert((*sid, *oid), text.clone()); + } + } + } + texts +} + +#[test] +fn atomic_page_batches_survive_reopen_with_dependent_text() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("pages.sqlite"); + let cache = Replica::create(&path, SOURCE).unwrap(); + let before = order(SOURCE); + let edits: Vec<_> = before[3..6] + .iter() + .map(|(sid, level)| PageEdit::move_to(*sid, None, *level).unwrap()) + .collect(); + let id = cache.pages(SOURCE, &edits).unwrap().unwrap(); + assert!(cache.pages(SOURCE, &edits).is_err()); + let mut expected_text = texts(SOURCE); + let (&(sid, oid), original) = expected_text + .iter() + .find(|(_, text)| text.starts_with("Body parent")) + .unwrap(); + let changed = format!("Offline {original}"); + expected_text.insert((sid, oid), changed); + cache + .edit_text(&cache.snapshot().unwrap(), sid, oid, 0..0, "Offline ") + .unwrap() + .unwrap(); + let queue = cache.pending().unwrap(); + assert!(matches!(&queue[0].operation, Operation::Pages(batch) if batch.edits == edits)); + let local = cache.snapshot().unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), queue); + assert_eq!(cache.snapshot().unwrap(), local); + let mut server = Server::new(SOURCE); + for edit in queue { + assert!(matches!(cache.sync_once(&mut server).unwrap(), + Some((published, EditStatus::Published { .. })) if published == edit.id)); + } + let expected = [ + before[..3].to_vec(), + before[6..].to_vec(), + before[3..6].to_vec(), + ] + .concat(); + assert_eq!(order(&server.durable), expected); + assert_eq!(texts(&server.durable), expected_text); + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::Published { .. }) + )); + assert_eq!(server.publications, 2); +} + +#[test] +fn indentation_only_edits_preserve_remote_page_movement() { + let directory = tempfile::tempdir().unwrap(); + let cache = Replica::create(directory.path().join("pages.sqlite"), SOURCE).unwrap(); + let before = order(SOURCE); + let sid = before[4].0; + let id = cache + .pages(SOURCE, &[PageEdit::set_level(sid, 3).unwrap()]) + .unwrap() + .unwrap(); + let mut server = Server::new(SOURCE); + PreparedEdit::pages( + SOURCE, + &[PageEdit::move_to(sid, Some(before[7].0), 2).unwrap()], + ) + .unwrap() + .commit(&mut server) + .unwrap(); + let mut expected = order(&server.durable); + expected.iter_mut().find(|p| p.0 == sid).unwrap().1 = 3; + assert!(matches!(cache.sync_once(&mut server).unwrap(), + Some((published, EditStatus::Published { .. })) if published == id)); + assert_eq!(order(&server.durable), expected); +} + +#[test] +fn competing_page_moves_require_current_review_and_retain_intent_identities() { + let directory = tempfile::tempdir().unwrap(); + let cache = Replica::create(directory.path().join("pages.sqlite"), SOURCE).unwrap(); + let pages = order(SOURCE); + let sid = pages[6].0; + let original = PageEdit::move_to(sid, None, 1).unwrap(); + let id = cache + .pages(SOURCE, std::slice::from_ref(&original)) + .unwrap() + .unwrap(); + let mut server = Server::new(SOURCE); + PreparedEdit::pages( + SOURCE, + &[PageEdit::move_to(sid, Some(pages[0].0), 1).unwrap()], + ) + .unwrap() + .commit(&mut server) + .unwrap(); + let remote = server.visible.clone(); + let local = cache.snapshot().unwrap(); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, EditStatus::Conflict(ConflictKind::StructureChanged))) + ); + assert_eq!(server.publications, 0); + assert_eq!(cache.snapshot().unwrap(), local); + let revised = original + .reposition(PagePosition::Before(Some(pages[2].0)), 1) + .unwrap(); + let fresh = PageEdit::move_to(sid, Some(pages[2].0), 1).unwrap(); + assert!( + cache + .rebase_pages_conflict(id, &local, &remote, &[fresh]) + .is_err() + ); + assert!( + cache + .rebase_pages_conflict(id, SOURCE, &remote, std::slice::from_ref(&revised)) + .is_err() + ); + assert!( + cache + .rebase_pages_conflict(id, &local, SOURCE, std::slice::from_ref(&revised)) + .is_err() + ); + cache + .rebase_pages_conflict(id, &local, &remote, std::slice::from_ref(&revised)) + .unwrap(); + assert!(matches!(&cache.pending().unwrap()[0].operation, + Operation::Pages(batch) if batch.edits == [revised.clone()])); + let expected = PreparedEdit::pages(&remote, &[revised]).unwrap(); + assert!(matches!(cache.sync_once(&mut server).unwrap(), + Some((published, EditStatus::Published { .. })) if published == id)); + assert_eq!(order(&server.durable), order(expected.as_bytes())); +} + +#[test] +fn one_competing_level_prevents_publication_of_the_whole_batch() { + let directory = tempfile::tempdir().unwrap(); + let cache = Replica::create(directory.path().join("pages.sqlite"), SOURCE).unwrap(); + let pages = order(SOURCE); + let id = cache + .pages( + SOURCE, + &[ + PageEdit::set_level(pages[4].0, 3).unwrap(), + PageEdit::set_level(pages[5].0, 2).unwrap(), + ], + ) + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let mut server = Server::new(SOURCE); + PreparedEdit::pages(SOURCE, &[PageEdit::set_level(pages[5].0, 1).unwrap()]) + .unwrap() + .commit(&mut server) + .unwrap(); + let remote = server.durable.clone(); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, EditStatus::Conflict(ConflictKind::StructureChanged))) + ); + assert_eq!(server.publications, 0); + assert_eq!(server.durable, remote); + assert_eq!(cache.snapshot().unwrap(), local); +} + +#[test] +fn uncertain_page_batches_survive_recovery_and_do_not_replay() { + for fault in [ + Fault::UnknownBefore, + Fault::UnknownAfter, + Fault::PanicBefore, + Fault::PanicAfter, + ] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("pages.sqlite"); + let mut cache = Replica::create(&path, SOURCE).unwrap(); + let pages = order(SOURCE); + let edits = [ + PageEdit::set_level(pages[4].0, 3).unwrap(), + PageEdit::set_level(pages[5].0, 2).unwrap(), + ]; + let id = cache.pages(SOURCE, &edits).unwrap().unwrap(); + let local = cache.snapshot().unwrap(); + let queue = cache.pending().unwrap(); + let mut server = Server::new(SOURCE); + server.fault = fault; + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + cache.sync_once(&mut server) + })); + drop(cache); + cache = Replica::open(&path).unwrap(); + let attempted = cache.status(id).unwrap().unwrap(); + assert!(matches!(attempted, EditStatus::AwaitingConfirmation { .. })); + let archive = directory.path().join("recovery.sqlite"); + cache.export_recovery(&archive).unwrap(); + let recovery = Recovery::open(&archive).unwrap(); + assert_eq!(recovery.pending().unwrap(), queue); + assert_eq!(recovery.status(id).unwrap(), Some(attempted)); + assert_eq!(recovery.snapshot().unwrap(), local); + assert!( + cache + .rebase_pages_conflict(id, &local, SOURCE, &edits) + .is_err() + ); + let result = cache.sync_once(&mut server).unwrap().unwrap(); + assert_eq!(server.publications, 1); + if matches!(fault, Fault::UnknownAfter | Fault::PanicAfter) { + assert!(matches!(result.1, EditStatus::Published { .. })); + assert_eq!(server.confirmations, 1); + } else { + assert_eq!(result, (id, attempted)); + assert_eq!(server.confirmations, 0); + } + } +} + +#[test] +fn an_unchanged_section_revision_cannot_confirm_a_changed_page() { + let before = order(SOURCE); + let sid = before[4].0; + let store = Store::parse(SOURCE).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let root = index.root; + let doc = Document::parse(&index).unwrap(); + let space = &doc.spaces[&root]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let copy = *view.nodes.iter().find(|(_, node)| + matches!(&node.kind, Kind::Metadata { title: Some(title), .. } if title == "child")) + .unwrap().0; + let source = + onestore::replace_property_bytes(SOURCE, root, copy, 0x14001dff, &3_u32.to_le_bytes()) + .unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("pages.sqlite"); + let cache = Replica::create(&path, &source).unwrap(); + let id = cache + .pages(&source, &[PageEdit::set_level(sid, 3).unwrap()]) + .unwrap() + .unwrap(); + let mut server = Server::new(&source); + server.fault = Fault::UnknownAfter; + assert!(cache.sync_once(&mut server).is_err()); + let attempted = cache.status(id).unwrap().unwrap(); + drop(cache); + let db = rusqlite::Connection::open(&path).unwrap(); + let encoded: String = db + .query_row("SELECT revisions FROM attempt", [], |r| r.get(0)) + .unwrap(); + let proofs: BTreeMap = serde_json::from_str(&encoded).unwrap(); + assert_eq!(proofs.len(), 2); + let store = Store::parse(&source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + assert_eq!( + proofs[&root], + index.spaces[&root].labels[&(ExGuid::default(), 1)] + ); + assert!(!index.spaces[&sid].revisions.contains_key(&proofs[&sid])); + drop(db); + let cache = Replica::open(&path).unwrap(); + let complete = server.visible.clone(); + server.visible = source; + assert_eq!(cache.sync_once(&mut server).unwrap(), Some((id, attempted))); + assert_eq!(server.confirmations, 0); + server.visible = complete; + assert!(matches!( + cache.sync_once(&mut server).unwrap(), + Some((_, EditStatus::Published { .. })) + )); + assert_eq!(server.publications, 1); + assert_eq!(server.confirmations, 1); +} + +#[test] +fn an_explicit_anchor_is_retained_even_when_originally_in_place() { + let directory = tempfile::tempdir().unwrap(); + let cache = Replica::create(directory.path().join("pages.sqlite"), SOURCE).unwrap(); + let pages = order(SOURCE); + let id = cache + .pages( + SOURCE, + &[ + PageEdit::move_to(pages[6].0, Some(pages[7].0), 1).unwrap(), + PageEdit::set_level(pages[4].0, 3).unwrap(), + ], + ) + .unwrap() + .unwrap(); + let mut server = Server::new(SOURCE); + PreparedEdit::pages( + SOURCE, + &[PageEdit::move_to(pages[6].0, Some(pages[0].0), 1).unwrap()], + ) + .unwrap() + .commit(&mut server) + .unwrap(); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, EditStatus::Conflict(ConflictKind::StructureChanged))) + ); + assert_eq!(server.publications, 0); +} + +#[test] +fn convergent_page_indentation_requires_confirmation_without_republishing() { + let directory = tempfile::tempdir().unwrap(); + let cache = Replica::create(directory.path().join("pages.sqlite"), SOURCE).unwrap(); + let sid = order(SOURCE)[4].0; + let id = cache + .pages(SOURCE, &[PageEdit::set_level(sid, 1).unwrap()]) + .unwrap() + .unwrap(); + let mut server = Server::new(SOURCE); + PreparedEdit::pages(SOURCE, &[PageEdit::set_level(sid, 1).unwrap()]) + .unwrap() + .commit(&mut server) + .unwrap(); + server.fault = Fault::Confirm; + assert!(cache.sync_once(&mut server).is_err()); + assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); + assert_eq!(server.publications, 0); + assert!(matches!(cache.sync_once(&mut server).unwrap(), + Some((published, EditStatus::Published { .. })) if published == id)); + assert_eq!(server.publications, 0); + assert_eq!(server.confirmations, 2); +} + +#[test] +fn schema_ten_migration_preserves_multi_space_uncertainty() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("pages.sqlite"); + let cache = Replica::create(&path, SOURCE).unwrap(); + let page = onestore::PageCreation::new(None, Some("Migration"), "Author").unwrap(); + let id = cache.create_page(SOURCE, &page).unwrap().unwrap(); + let mut server = Server::new(SOURCE); + server.fault = Fault::UnknownBefore; + assert!(cache.sync_once(&mut server).is_err()); + let local = cache.snapshot().unwrap(); + let queue = cache.pending().unwrap(); + let status = cache.status(id).unwrap(); + drop(cache); + let db = rusqlite::Connection::open(&path).unwrap(); + db.pragma_update(None, "user_version", 10).unwrap(); + let proofs: String = db + .query_row("SELECT revisions FROM attempt", [], |r| r.get(0)) + .unwrap(); + drop(db); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.pending().unwrap(), queue); + assert_eq!(cache.status(id).unwrap(), status); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, status.unwrap())) + ); + assert_eq!(server.publications, 1); + drop(cache); + let db = rusqlite::Connection::open(&path).unwrap(); + assert_eq!( + db.query_row("SELECT revisions FROM attempt", [], |r| r + .get::<_, String>(0)) + .unwrap(), + proofs + ); + assert_eq!( + db.pragma_query_value(None, "user_version", |r| r.get::<_, u32>(0)) + .unwrap(), + 11 + ); +} + +#[test] +fn native_page_changes_reconcile_with_atomic_offline_batches_and_review() { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../corpus/page-lifecycle/movement"); + let provenance: serde_json::Value = + serde_json::from_slice(&std::fs::read(root.join("provenance.json")).unwrap()).unwrap(); + let phases = provenance["cold"].as_object().unwrap(); + assert_eq!(phases.len(), 9); + let original = order(SOURCE); + let original_text = texts(SOURCE); + let (&(text_space, text_object), body) = original_text + .iter() + .find(|(_, text)| text.starts_with("Body parent")) + .unwrap(); + let mut expected_text = original_text.clone(); + expected_text.insert((text_space, text_object), format!("Offline {body}")); + let mut counts = [0; 2]; + for mode in ["indent", "group"] { + for phase in phases.keys() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("pages.sqlite"); + let cache = Replica::create(&path, SOURCE).unwrap(); + let edits = if mode == "indent" { + vec![PageEdit::set_level(original[4].0, 3).unwrap()] + } else { + original[3..6] + .iter() + .map(|(sid, level)| PageEdit::move_to(*sid, None, *level).unwrap()) + .collect() + }; + let id = cache.pages(SOURCE, &edits).unwrap().unwrap(); + let text_id = cache + .edit_text( + &cache.snapshot().unwrap(), + text_space, + text_object, + 0..0, + "Offline ", + ) + .unwrap() + .unwrap(); + let native = std::fs::read(root.join(phase).join("notebook/Lifecycle.one")).unwrap(); + let mut server = Server::new(&native); + let local = cache.snapshot().unwrap(); + let expected_conflict = if mode == "indent" { + matches!( + phase.as_str(), + "04-subpage-move" | "07-promoted-root" | "08-collapsed-group-move" + ) + } else { + !matches!( + phase.as_str(), + "02-promoted-parent" | "03-selected-group-move" | "06-promoted-level-two" + ) + }; + let first = cache.sync_once(&mut server).unwrap().unwrap(); + assert_eq!(first.0, id); + let mut reviewed = edits.clone(); + if expected_conflict { + assert_eq!( + first.1, + EditStatus::Conflict(ConflictKind::StructureChanged), + "{mode}:{phase}" + ); + assert_eq!(server.publications, 0); + assert_eq!(cache.snapshot().unwrap(), local); + if mode == "indent" && phase == "08-collapsed-group-move" { + reviewed[0] = reviewed[0].reposition(PagePosition::Keep, 1).unwrap(); + } + cache + .rebase_pages_conflict(id, &local, &native, &reviewed) + .unwrap(); + assert!(matches!(cache.sync_once(&mut server).unwrap(), + Some((published, EditStatus::Published { .. })) if published == id)); + } else { + assert!( + matches!(first.1, EditStatus::Published { .. }), + "{mode}:{phase}" + ); + } + counts[usize::from(expected_conflict)] += 1; + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert!(matches!(cache.sync_once(&mut server).unwrap(), + Some((published, EditStatus::Published { .. })) if published == text_id)); + assert!(cache.pending().unwrap().is_empty()); + let mut expected = order(&native); + for edit in &reviewed { + let at = expected.iter().position(|p| p.0 == edit.space()).unwrap(); + expected[at].1 = edit.level(); + if let PagePosition::Before(before) = edit.position() { + let page = expected.remove(at); + let at = before.map_or(expected.len(), |sid| { + expected.iter().position(|p| p.0 == sid).unwrap() + }); + expected.insert(at, page); + } + } + assert_eq!(order(&server.durable), expected, "{mode}:{phase}"); + assert_eq!(texts(&server.durable), expected_text, "{mode}:{phase}"); + assert_eq!( + server.publications, + 1 + usize::from(expected != order(&native)) + ); + if let Some(output) = std::env::var_os("ONESTORE_OFFLINE_PAGE_EDIT_OUTPUT") { + let output = std::path::Path::new(&output); + assert!(output.is_absolute()); + let path = output.join(format!("{mode}-{phase}")); + std::fs::create_dir_all(&path).unwrap(); + std::fs::write(path.join("Lifecycle.one"), &server.durable).unwrap(); + std::fs::write( + path.join("manifest.json"), + serde_json::to_vec_pretty( + &serde_json::json!({"mode": mode, "phase": phase, "original": edits, + "reviewed": reviewed, "conflict": expected_conflict, + "publications": server.publications, "confirmations": server.confirmations, + "page_receipt": format!("{:?}", cache.status(id).unwrap()), + "text_receipt": format!("{:?}", cache.status(text_id).unwrap())}), + ) + .unwrap(), + ) + .unwrap(); + } + } + } + assert_eq!(counts, [9, 9]); +} + +#[test] +fn twelve_disjoint_page_batches_merge_with_dependent_bodies_without_review() { + let mut source = onestore::create_section("pages.one", "Sentinel", "Author").unwrap(); + let mut created = Vec::new(); + for _ in 0..36 { + let page = onestore::PageCreation::new(None, Some("Same title"), "Author").unwrap(); + source = PreparedEdit::create_page(&source, &page) + .unwrap() + .as_bytes() + .to_vec(); + created.push(page); + } + let mut expected = vec![order(&source)[0]]; + let mut expected_text = texts(&source); + let directory = tempfile::tempdir().unwrap(); + let mut queues = Vec::new(); + for (actor, group) in created.chunks_exact(3).enumerate() { + let path = directory.path().join(format!("{actor}.sqlite")); + let cache = Replica::create(&path, &source).unwrap(); + let edits = [ + PageEdit::move_to(group[1].space(), Some(group[0].space()), 1).unwrap(), + PageEdit::set_level(group[2].space(), 2).unwrap(), + ]; + cache.pages(&source, &edits).unwrap().unwrap(); + let body = format!("Actor {actor} 🦀 é"); + let insertion = + onestore::Insertion::outline(group[1].object(), 36.0, 36.0, &body, "Author").unwrap(); + cache + .insert(&cache.snapshot().unwrap(), group[1].space(), &insertion) + .unwrap() + .unwrap(); + expected.extend([ + (group[1].space(), 1), + (group[0].space(), 1), + (group[2].space(), 2), + ]); + expected_text.insert((group[1].space(), insertion.text_object()), body); + queues.push((path, cache.pending().unwrap())); + } + let mut server = Server::new(&source); + for (path, queue) in &queues { + let cache = Replica::open(path).unwrap(); + assert_eq!(cache.pending().unwrap(), *queue); + for edit in queue { + assert!(matches!(cache.sync_once(&mut server).unwrap(), + Some((published, EditStatus::Published { .. })) if published == edit.id)); + } + assert!(cache.pending().unwrap().is_empty()); + } + assert_eq!(server.publications, 24); + assert_eq!(order(&server.durable), expected); + assert_eq!(texts(&server.durable), expected_text); + for (path, queue) in &queues { + let cache = Replica::open(path).unwrap(); + assert_eq!(cache.sync_once(&mut server).unwrap(), None); + assert_eq!(order(&cache.snapshot().unwrap()), expected); + for edit in queue { + assert!(matches!( + cache.status(edit.id).unwrap(), + Some(EditStatus::Published { .. }) + )); + } + } + if let Some(output) = std::env::var_os("ONESTORE_OFFLINE_PAGE_CLIENT_OUTPUT") { + let output = std::path::Path::new(&output); + assert!(output.is_absolute()); + std::fs::create_dir_all(output).unwrap(); + std::fs::write(output.join("pages.one"), &server.durable).unwrap(); + let operations: Vec<_> = queues + .iter() + .map(|(_, queue)| { + queue + .iter() + .map(|edit| { + serde_json::json!({"id": edit.id, "space": edit.space, + "operation": edit.operation}) + }) + .collect::>() + }) + .collect(); + std::fs::write( + output.join("manifest.json"), + serde_json::to_vec_pretty( + &serde_json::json!({"operations": operations, "publications": server.publications, + "expected_order": expected, "pages": 37}), + ) + .unwrap(), + ) + .unwrap(); + } +} diff --git a/crates/notebook/tests/sync/tree.rs b/crates/notebook/tests/sync/tree.rs new file mode 100644 index 0000000000000000000000000000000000000000..a0681214118c92dad8f5a6e2cb6e024407a978d7 --- /dev/null +++ b/crates/notebook/tests/sync/tree.rs @@ -0,0 +1,821 @@ +use super::*; +use onestore::{Insertion, TreeEdit}; + +#[test] +fn twelve_offline_clients_reconcile_tree_text_and_interrupted_publication() { + let mut random = 1940_u64; + for _ in 0..60 { + let input: Vec<_> = (0..384) + .map(|_| { + random ^= random << 13; + random ^= random >> 7; + random ^= random << 17; + random as u8 + }) + .collect(); + tree_schedule::run(&input); + } +} + +fn fixture() -> (Vec, ExGuid, ExGuid, [ExGuid; 4], [ExGuid; 4]) { + let (mut source, sid, outline, first, text) = super::outline::fixture(); + let mut paragraphs = [first; 4]; + let mut texts = [text; 4]; + for at in 1..4 { + let insertion = + Insertion::paragraph(outline, None, &format!("Sibling {at}"), "Author").unwrap(); + source = PreparedEdit::insert(&source, sid, &insertion) + .unwrap() + .as_bytes() + .to_vec(); + paragraphs[at] = insertion.object(); + texts[at] = insertion.text_object(); + } + (source, sid, outline, paragraphs, texts) +} + +fn children(source: &[u8], sid: ExGuid, object: ExGuid) -> Vec { + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + index.validate_current().unwrap(); + let document = Document::parse(&index).unwrap(); + let space = &document.spaces[&sid]; + space.revisions[&space.contexts[&ExGuid::default()]].nodes[&object] + .children + .clone() +} + +#[test] +fn move_preserves_remote_content_and_dependent_edits_through_reopen() { + let (source, sid, outline, paragraphs, texts) = fixture(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("cache.sqlite"); + let cache = Replica::create(&path, &source).unwrap(); + let intent = TreeEdit::move_to(paragraphs[0], outline, None, "Offline").unwrap(); + let id = cache.tree(&source, sid, &intent).unwrap().unwrap(); + let dependent = cache + .edit_text(&cache.snapshot().unwrap(), sid, texts[0], 0..0, "Local ") + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let queue = cache.pending().unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), queue); + assert_eq!(cache.snapshot().unwrap(), local); + let edited = PreparedEdit::format( + &source, + sid, + texts[0], + 0..8, + &[onestore::TextAttribute::Bold(true)], + ) + .unwrap(); + let descendant = + Insertion::paragraph(paragraphs[0], None, "New remote child", "Remote").unwrap(); + let remote = PreparedEdit::insert(edited.as_bytes(), sid, &descendant).unwrap(); + let mut server = Server::new(remote.as_bytes()); + for expected in [id, dependent] { + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == expected) + ); + } + assert_eq!( + children(&server.durable, sid, outline), + [paragraphs[1], paragraphs[2], paragraphs[3], paragraphs[0]] + ); + assert_eq!( + children(&server.durable, sid, paragraphs[0]), + [descendant.object()] + ); + let node = super::outline::node(&server.durable, sid, texts[0]); + assert_eq!(node["kind"]["text"], "Local Original 🦀 é"); + let store = Store::parse(&server.durable).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let space = &document.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + assert_eq!(view.text_runs(texts[0]).unwrap()[0].format.bold, Some(true)); + assert!(cache.pending().unwrap().is_empty()); + assert_eq!(server.publications, 2); +} + +#[test] +fn queued_format_split_and_delete_reconcile_equivalent_immutable_style_identities() { + let (source, sid, _, _, texts) = fixture(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("cache.sqlite"); + let cache = Replica::create(&path, &source).unwrap(); + cache + .format( + &source, + sid, + texts[0], + 0..8, + &[onestore::TextAttribute::Bold(true)], + ) + .unwrap(); + let split = onestore::ParagraphSplit::new(texts[0], 4, "Author").unwrap(); + cache + .split(&cache.snapshot().unwrap(), sid, &split) + .unwrap(); + cache + .tree( + &cache.snapshot().unwrap(), + sid, + &TreeEdit::delete(split.object(), "Author").unwrap(), + ) + .unwrap(); + cache + .edit_text(&cache.snapshot().unwrap(), sid, texts[0], 0..0, "Local ") + .unwrap(); + let queue = cache.pending().unwrap(); + drop(cache); + let mut server = Server::new(&source); + for intent in queue { + let cache = Replica::open(&path).unwrap(); + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((id, EditStatus::Published { .. })) if id == intent.id) + ); + } + assert_eq!( + super::outline::node(&server.durable, sid, texts[0])["kind"]["text"], + "Local Orig" + ); +} + +#[test] +fn native_empty_child_list_normalization_preserves_deletion_and_its_dependents() { + let source = include_bytes!("../../../../corpus/outline-edit/empty-children/before.one"); + let remote = include_bytes!("../../../../corpus/outline-edit/empty-children/remote.one"); + let (sid, intent): (ExGuid, TreeEdit) = serde_json::from_str(include_str!( + "../../../../corpus/outline-edit/empty-children/intent.json" + )) + .unwrap(); + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let space = &document.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let tail = view.nodes[&intent.object()].content[0]; + let (outline_id, outline) = view + .nodes + .iter() + .find(|(_, node)| node.children.contains(&intent.object())) + .unwrap(); + let left = view.nodes[&outline.children[0]].content[0]; + for changed in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("cache.sqlite"); + let cache = Replica::create(&path, source).unwrap(); + let deletion = cache.tree(source, sid, &intent).unwrap().unwrap(); + let dependent = cache + .edit_text(&cache.snapshot().unwrap(), sid, left, 0..0, "Local ") + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + let mut server = if changed { + let edited = PreparedEdit::text(remote, sid, tail, 0..2, "Changed").unwrap(); + Server::new(edited.as_bytes()) + } else { + Server::new(remote) + }; + if changed { + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((deletion, EditStatus::Conflict(ConflictKind::ContentChanged))) + ); + assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(cache.pending().unwrap().len(), 2); + assert_eq!(server.publications, 0); + } else { + for expected in [deletion, dependent] { + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((id, EditStatus::Published { .. })) if id == expected) + ); + } + assert!(cache.pending().unwrap().is_empty()); + assert_eq!(server.publications, 2); + assert!(!children(&server.durable, sid, *outline_id).contains(&intent.object())); + let node = super::outline::node(&server.durable, sid, left); + assert!(node["kind"]["text"].as_str().unwrap().starts_with("Local ")); + } + } +} + +#[test] +fn deletion_requires_review_of_remote_content_and_preserves_later_work() { + let (source, sid, outline, paragraphs, texts) = fixture(); + let child = Insertion::paragraph(paragraphs[0], None, "Descendant", "Author").unwrap(); + let source = PreparedEdit::insert(&source, sid, &child) + .unwrap() + .as_bytes() + .to_vec(); + for remote in [ + PreparedEdit::text(&source, sid, texts[0], 0..0, "Remote ") + .unwrap() + .as_bytes() + .to_vec(), + PreparedEdit::text(&source, sid, child.text_object(), 0..0, "Remote ") + .unwrap() + .as_bytes() + .to_vec(), + PreparedEdit::format( + &source, + sid, + child.text_object(), + 0..10, + &[onestore::TextAttribute::Italic(true)], + ) + .unwrap() + .as_bytes() + .to_vec(), + PreparedEdit::insert( + &source, + sid, + &Insertion::paragraph(paragraphs[0], None, "New child", "Remote").unwrap(), + ) + .unwrap() + .as_bytes() + .to_vec(), + ] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("cache.sqlite"); + let cache = Replica::create(&path, &source).unwrap(); + let intent = TreeEdit::delete(paragraphs[0], "Offline").unwrap(); + let id = cache.tree(&source, sid, &intent).unwrap().unwrap(); + let dependent = cache + .edit_text(&cache.snapshot().unwrap(), sid, texts[1], 0..0, "Local ") + .unwrap() + .unwrap(); + let queue = cache.pending().unwrap(); + let local = cache.snapshot().unwrap(); + let mut server = Server::new(&remote); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))) + ); + assert_eq!(server.publications, 0); + assert_eq!(cache.pending().unwrap(), queue); + assert_eq!(cache.snapshot().unwrap(), local); + assert!(cache.rebase_tree_conflict(id, &source, &remote).is_err()); + assert!(cache.rebase_tree_conflict(id, &local, &source).is_err()); + assert!( + cache + .rebase_tree_conflict(dependent, &local, &remote) + .is_err() + ); + assert!(cache.rebase_conflict(id, &local, &remote, 0..0).is_err()); + let archive = directory.path().join("recovery.sqlite"); + cache.export_recovery(&archive).unwrap(); + let recovery = notebook::Recovery::open(&archive).unwrap(); + assert_eq!(recovery.pending().unwrap(), queue); + assert_eq!(recovery.status(id).unwrap(), cache.status(id).unwrap()); + assert!(recovery.snapshot().unwrap() == local); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!( + cache.status(id).unwrap(), + Some(EditStatus::Conflict(ConflictKind::ContentChanged)) + ); + cache.rebase_tree_conflict(id, &local, &remote).unwrap(); + assert_eq!(cache.pending().unwrap()[1], queue[1]); + assert_eq!(cache.snapshot().unwrap(), local); + for expected in [id, dependent] { + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == expected) + ); + } + assert_eq!(children(&server.durable, sid, outline), paragraphs[1..]); + assert_eq!( + super::outline::node(&server.durable, sid, texts[1])["kind"]["text"], + "Local Sibling 1" + ); + } +} + +#[test] +fn sibling_and_ancestry_changes_have_explicit_merge_or_conflict() { + let (source, sid, outline, p, texts) = fixture(); + let insertion = Insertion::paragraph(outline, Some(p[0]), "New sibling", "Remote").unwrap(); + let cases = [ + ( + PreparedEdit::insert(&source, sid, &insertion) + .unwrap() + .as_bytes() + .to_vec(), + None, + ), + ( + PreparedEdit::tree(&source, sid, &TreeEdit::delete(p[1], "Remote").unwrap()) + .unwrap() + .as_bytes() + .to_vec(), + None, + ), + ( + PreparedEdit::tree( + &source, + sid, + &TreeEdit::move_to(p[1], outline, None, "Remote").unwrap(), + ) + .unwrap() + .as_bytes() + .to_vec(), + None, + ), + ( + PreparedEdit::tree( + &source, + sid, + &TreeEdit::move_to(p[0], outline, Some(p[2]), "Remote").unwrap(), + ) + .unwrap() + .as_bytes() + .to_vec(), + Some(ConflictKind::StructureChanged), + ), + ( + PreparedEdit::tree( + &source, + sid, + &TreeEdit::move_to(p[0], p[1], None, "Remote").unwrap(), + ) + .unwrap() + .as_bytes() + .to_vec(), + Some(ConflictKind::StructureChanged), + ), + ( + PreparedEdit::tree(&source, sid, &TreeEdit::delete(p[0], "Remote").unwrap()) + .unwrap() + .as_bytes() + .to_vec(), + Some(ConflictKind::TargetUnavailable), + ), + ]; + for (remote, expected) in cases { + let directory = tempfile::tempdir().unwrap(); + let cache = Replica::create(directory.path().join("cache.sqlite"), &source).unwrap(); + let intent = TreeEdit::move_to(p[0], outline, None, "Offline").unwrap(); + let id = cache.tree(&source, sid, &intent).unwrap().unwrap(); + let mut server = Server::new(&remote); + let result = cache.sync_once(&mut server).unwrap().unwrap(); + assert_eq!(result.0, id); + if let Some(kind) = expected { + assert_eq!(result.1, EditStatus::Conflict(kind)); + assert_eq!(server.publications, 0); + } else { + assert!(matches!(result.1, EditStatus::Published { .. })); + assert_eq!(children(&server.durable, sid, outline).last(), Some(&p[0])); + assert_eq!( + super::outline::node(&server.durable, sid, texts[0])["kind"]["text"], + "Original 🦀 é" + ); + } + } +} + +#[test] +fn moved_destination_and_missing_anchor_retain_conflicts_but_unrelated_text_does_not() { + let (source, sid, outline, p, texts) = fixture(); + let child = Insertion::paragraph(p[1], None, "Anchor", "Author").unwrap(); + let source = PreparedEdit::insert(&source, sid, &child) + .unwrap() + .as_bytes() + .to_vec(); + let intent = TreeEdit::move_to(p[0], p[1], Some(child.object()), "Offline").unwrap(); + let cases = [ + ( + PreparedEdit::tree( + &source, + sid, + &TreeEdit::move_to(p[1], p[2], None, "Remote").unwrap(), + ) + .unwrap() + .as_bytes() + .to_vec(), + ConflictKind::StructureChanged, + ), + ( + PreparedEdit::tree( + &source, + sid, + &TreeEdit::move_to(child.object(), outline, None, "Remote").unwrap(), + ) + .unwrap() + .as_bytes() + .to_vec(), + ConflictKind::UnsupportedEdit, + ), + ]; + for (remote, kind) in cases { + let directory = tempfile::tempdir().unwrap(); + let cache = Replica::create(directory.path().join("cache.sqlite"), &source).unwrap(); + let id = cache.tree(&source, sid, &intent).unwrap().unwrap(); + let mut server = Server::new(&remote); + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, EditStatus::Conflict(kind))) + ); + assert_eq!(server.publications, 0); + } + let directory = tempfile::tempdir().unwrap(); + let cache = Replica::create(directory.path().join("cache.sqlite"), &source).unwrap(); + let id = cache + .tree(&source, sid, &TreeEdit::delete(p[0], "Offline").unwrap()) + .unwrap() + .unwrap(); + let remote = PreparedEdit::text(&source, sid, texts[1], 0..0, "Remote ").unwrap(); + let mut server = Server::new(remote.as_bytes()); + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == id) + ); + assert_eq!( + super::outline::node(&server.durable, sid, texts[1])["kind"]["text"], + "Remote Sibling 1" + ); +} + +#[test] +fn unknown_tree_attempts_require_revision_evidence_even_when_effect_is_visible() { + let (source, sid, outline, p, _) = fixture(); + for deletion in [false, true] { + for fault in [ + Fault::UnknownBefore, + Fault::UnknownAfter, + Fault::Before, + Fault::Committed, + ] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("cache.sqlite"); + let cache = Replica::create(&path, &source).unwrap(); + let intent = if deletion { + TreeEdit::delete(p[0], "Offline") + } else { + TreeEdit::move_to(p[0], outline, None, "Offline") + } + .unwrap(); + let id = cache.tree(&source, sid, &intent).unwrap().unwrap(); + let mut server = Server::new(&source); + server.fault = fault; + assert!(cache.sync_once(&mut server).is_err()); + let local = cache.snapshot().unwrap(); + let before = cache.status(id).unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.status(id).unwrap(), before); + if matches!(fault, Fault::UnknownBefore) { + let independent = if deletion { + TreeEdit::delete(p[0], "Other") + } else { + TreeEdit::move_to(p[0], outline, None, "Other") + } + .unwrap(); + server = Server::new( + PreparedEdit::tree(&source, sid, &independent) + .unwrap() + .as_bytes(), + ); + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::AwaitingConfirmation { .. })) if n == id) + ); + assert!( + cache + .rebase_tree_conflict(id, &local, &server.visible) + .is_err() + ); + assert_eq!(server.publications, 0); + assert_eq!(cache.snapshot().unwrap(), local); + } else if matches!(fault, Fault::Committed) { + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::Published { .. }) + )); + } else { + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == id) + ); + assert_eq!( + server.publications, + if matches!(fault, Fault::Before) { 2 } else { 1 } + ); + let saved = cache.snapshot().unwrap(); + assert!(saved[..212] == server.durable[..212]); + assert!(saved[252..] == server.durable[252..]); + assert_eq!( + children(&saved, sid, outline), + children(&server.durable, sid, outline) + ); + } + } + } +} + +#[test] +fn independently_satisfied_move_confirms_without_republication() { + let (source, sid, outline, p, _) = fixture(); + let directory = tempfile::tempdir().unwrap(); + let cache = Replica::create(directory.path().join("cache.sqlite"), &source).unwrap(); + let id = cache + .tree( + &source, + sid, + &TreeEdit::move_to(p[0], outline, None, "Offline").unwrap(), + ) + .unwrap() + .unwrap(); + let remote = PreparedEdit::tree( + &source, + sid, + &TreeEdit::move_to(p[0], outline, None, "Remote").unwrap(), + ) + .unwrap(); + let mut server = Server::new(remote.as_bytes()); + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == id) + ); + assert_eq!(server.publications, 0); + assert_eq!(server.confirmations, 1); +} + +#[test] +fn emptied_cell_replacement_is_durable_and_cannot_be_silently_omitted_on_replay() { + let source = + include_bytes!("../../../../corpus/outline-edit/tree/before/notebook/synthetic.one"); + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let (sid, page) = document.pages().unwrap().into_iter().find(|(sid, _)| { + let space = &document.spaces[sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + matches!(&view.nodes[&view.roots[&2]].kind, Kind::Metadata { title: Some(title), .. } if title == "Delete sole cell paragraph") + }).unwrap(); + let space = &document.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let mut pending = vec![page]; + let mut selected = None; + let mut other = None; + while let Some(id) = pending.pop() { + let node = &view.nodes[&id]; + pending.extend(&node.children); + pending.extend(&node.content); + if matches!(node.kind, Kind::Cell { .. }) { + let paragraph = node.children[0]; + let text = view.nodes[¶graph].content[0]; + if matches!(&view.nodes[&text].kind, Kind::RichText { text, .. } if text.starts_with("Target ")) + { + selected = Some((id, paragraph)); + } else { + other = Some((id, text)); + } + } + } + let (cell, target) = selected.unwrap(); + let (other_cell, other_text) = other.unwrap(); + for move_out in [false, true] { + for competing_child in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("cell.sqlite"); + let cache = Replica::create(&path, source).unwrap(); + let intent = if move_out { + TreeEdit::move_to(target, other_cell, None, "Offline") + } else { + TreeEdit::delete(target, "Offline") + } + .unwrap(); + let id = cache.tree(source, sid, &intent).unwrap().unwrap(); + let local = cache.snapshot().unwrap(); + let replacement = children(&local, sid, cell); + assert_eq!(replacement.len(), 1); + assert_ne!(replacement[0], target); + let replacement_text: ExGuid = + super::outline::node(&local, sid, replacement[0])["content"][0] + .as_str() + .unwrap() + .parse() + .unwrap(); + let dependent = cache + .edit_text(&local, sid, replacement_text, 0..0, "Local replacement") + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let queue = cache.pending().unwrap(); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), queue); + assert_eq!(cache.snapshot().unwrap(), local); + let remote = if competing_child { + PreparedEdit::insert( + source, + sid, + &Insertion::paragraph(cell, None, "Remote sibling", "Remote").unwrap(), + ) + .unwrap() + .as_bytes() + .to_vec() + } else { + PreparedEdit::text(source, sid, other_text, 0..0, "Remote ") + .unwrap() + .as_bytes() + .to_vec() + }; + let mut server = Server::new(&remote); + if competing_child { + assert_eq!( + cache.sync_once(&mut server).unwrap(), + Some((id, EditStatus::Conflict(ConflictKind::StructureChanged))) + ); + assert_eq!(server.publications, 0); + assert!(cache.rebase_tree_conflict(id, &local, &remote).is_err()); + assert_eq!(cache.pending().unwrap(), queue); + assert_eq!(cache.snapshot().unwrap(), local); + } else { + for expected in [id, dependent] { + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == expected) + ); + } + assert_eq!(children(&server.durable, sid, cell), replacement); + assert_eq!( + super::outline::node(&server.durable, sid, replacement_text)["kind"]["text"], + "Local replacement" + ); + assert_eq!( + super::outline::node(&server.durable, sid, other_text)["kind"]["text"], + "Remote Other cell" + ); + if let Some(output) = std::env::var_os("ONESTORE_OFFLINE_TREE_OUTPUT") { + let output = std::path::PathBuf::from(output) + .join(if move_out { "cell-move" } else { "cell-delete" }) + .join("candidate"); + std::fs::create_dir_all(&output).unwrap(); + std::fs::write(output.join("synthetic.one"), &server.durable).unwrap(); + } + } + } + } +} + +#[test] +fn native_tree_and_layout_changes_reconcile_without_discarding_unreviewed_content() { + let source = include_bytes!("../../../../corpus/outline-edit/before/notebook/synthetic.one"); + let native = include_bytes!("../../../../corpus/outline-edit/after/notebook/synthetic.one"); + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let mut server = Server::new(native); + let mut counts = [0; 3]; + let mut records = Vec::new(); + for (sid, page) in document.pages().unwrap() { + let space = &document.spaces[&sid]; + let view = &space.revisions[&space.contexts[&ExGuid::default()]]; + let Kind::Metadata { + title: Some(name), .. + } = &view.nodes[&view.roots[&2]].kind + else { + continue; + }; + if name == "Unicode rich text" { + continue; + } + let outlines: Vec<_> = view.nodes[&page] + .children + .iter() + .copied() + .filter(|id| matches!(view.nodes[id].kind, Kind::Outline { .. })) + .collect(); + if outlines.len() != 2 { + continue; + } + let outline = outlines[0]; + let other = view.nodes[&outlines[1]].children[0]; + let dependent_text = view.nodes[&other].content[0]; + let mut pending = vec![outline]; + let mut paragraphs = std::collections::BTreeMap::new(); + while let Some(id) = pending.pop() { + let node = &view.nodes[&id]; + pending.extend(&node.children); + if let Some(text) = node.content.first() + && let Kind::RichText { text, .. } = &view.nodes[text].kind + { + paragraphs.insert(text.as_str(), id); + } + } + let target = *paragraphs + .iter() + .find(|(text, _)| text.starts_with("Target ")) + .unwrap() + .1; + let (intent, conflict) = match name.as_str() { + "Move leaf down" | "Move subtree down" => { + (TreeEdit::move_to(target, outline, None, "Offline"), None) + } + "Move subtree up" => ( + TreeEdit::move_to(target, outline, Some(paragraphs["Anchor"]), "Offline"), + None, + ), + "Indent subtree" | "Outdent subtree" => ( + TreeEdit::delete(target, "Offline"), + Some(ConflictKind::StructureChanged), + ), + "Collapse subtree" | "Expand subtree" => ( + TreeEdit::delete(target, "Offline"), + Some(ConflictKind::ContentChanged), + ), + "Delete leaf" | "Delete subtree" | "Delete only paragraph" => ( + TreeEdit::delete(target, "Offline"), + Some(ConflictKind::TargetUnavailable), + ), + "Delete outline" => ( + TreeEdit::delete(outline, "Offline"), + Some(ConflictKind::TargetUnavailable), + ), + "Move outline" | "Resize outline" | "Automatic outline size" => ( + TreeEdit::delete(outline, "Offline"), + Some(ConflictKind::ContentChanged), + ), + _ => panic!("{name}"), + }; + let intent = intent.unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("native.sqlite"); + let cache = Replica::create(&path, source).unwrap(); + let id = cache.tree(source, sid, &intent).unwrap().unwrap(); + let dependent = cache + .edit_text( + &cache.snapshot().unwrap(), + sid, + dependent_text, + 0..0, + "Offline ", + ) + .unwrap() + .unwrap(); + let local = cache.snapshot().unwrap(); + let queue = cache.pending().unwrap(); + let old_publications = server.publications; + let (actual, status) = cache.sync_once(&mut server).unwrap().unwrap(); + assert_eq!(actual, id); + if let Some(kind) = conflict { + assert_eq!(status, EditStatus::Conflict(kind), "{name}"); + assert_eq!(server.publications, old_publications); + assert_eq!(cache.pending().unwrap(), queue); + assert!(cache.snapshot().unwrap() == local); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.status(id).unwrap(), Some(status)); + if kind == ConflictKind::TargetUnavailable { + counts[2] += 1; + assert!( + cache + .rebase_tree_conflict(id, &local, &server.visible) + .is_err() + ); + records.push(serde_json::json!({"page": name, "space": sid, "result": "retained"})); + continue; + } + counts[1] += 1; + cache + .rebase_tree_conflict(id, &local, &server.visible) + .unwrap(); + assert_eq!(cache.pending().unwrap()[1], queue[1]); + for expected in [id, dependent] { + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == expected) + ); + } + } else { + counts[0] += 1; + assert!(matches!(status, EditStatus::Published { .. }), "{name}"); + assert_eq!(server.publications, old_publications); + assert!( + matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == dependent) + ); + } + assert_eq!( + super::outline::node(&server.durable, sid, dependent_text)["kind"]["text"], + format!( + "Offline {}", + match &view.nodes[&dependent_text].kind { + Kind::RichText { text, .. } => text, + _ => panic!(), + } + ) + ); + records.push(serde_json::json!({"page": name, "space": sid, "result": if conflict.is_some() { "reviewed" } else { "converged" }})); + } + assert_eq!(counts, [3, 7, 4]); + if let Some(output) = std::env::var_os("ONESTORE_OFFLINE_TREE_OUTPUT") { + let output = std::path::PathBuf::from(output); + std::fs::create_dir_all(output.join("candidate")).unwrap(); + std::fs::write(output.join("candidate/synthetic.one"), &server.durable).unwrap(); + std::fs::write( + output.join("manifest.json"), + serde_json::to_vec_pretty(&records).unwrap(), + ) + .unwrap(); + } +} diff --git a/crates/onestore-diagnostic/Cargo.toml b/crates/onestore-diagnostic/Cargo.toml deleted file mode 100644 index cc4b96f0e146f2cff6bab02ca13e8d29d920c1bb..0000000000000000000000000000000000000000 --- a/crates/onestore-diagnostic/Cargo.toml +++ /dev/null @@ -1,11 +0,0 @@ -[package] -name = "onestore-diagnostic" -version = "0.1.0" -edition = "2024" -publish = false - -[dependencies] -onestore = { path = "../onestore" } -onestore-notebook = { path = "../onestore-notebook" } -serde = { version = "1.0.229", features = ["derive"] } -serde_json = "1.0.151" diff --git a/crates/onestore-diagnostic/src/main.rs b/crates/onestore-diagnostic/src/main.rs deleted file mode 100644 index 5b39a4f147a8221205d3a636b59d6d46934080bd..0000000000000000000000000000000000000000 --- a/crates/onestore-diagnostic/src/main.rs +++ /dev/null @@ -1,126 +0,0 @@ -use onestore::{ExGuid, Insertion, PreparedEdit, TextAttribute}; -use serde::Deserialize; -use serde_json::{Value, json}; -use std::{ - env, fs, - io::{self, Write}, -}; - -#[derive(Deserialize)] -#[serde(deny_unknown_fields)] -struct Edit { - space: ExGuid, - object: ExGuid, - action: Action, -} - -#[derive(Deserialize)] -#[serde(tag = "type", deny_unknown_fields)] -enum Action { - Text { - start: u32, - end: u32, - replacement: String, - }, - Format { - start: u32, - end: u32, - attributes: Vec, - }, - Paragraph { - before: Option, - text: String, - author: String, - }, - Outline { - x: f32, - y: f32, - text: String, - author: String, - }, -} - -fn run(args: &[std::ffi::OsString]) -> Result> { - if let [mode, root] = args - && mode == "catalog" - { - let catalog = onestore_notebook::discover( - &mut onestore_notebook::Local::open(root)?, - onestore_notebook::Limits { - entries: 100_000, - bytes_per_file: 256 * 1024 * 1024, - depth: 64, - }, - )?; - return Ok(json!({"ok": true, "catalog": catalog})); - } - if args.len() != 3 - || !["snapshot", "check", "commit"] - .iter() - .any(|mode| args[0] == *mode) - { - return Err("Usage: onestore-diagnostic catalog ROOT | snapshot FILE NEW_SNAPSHOT | check SNAPSHOT - | commit FILE SNAPSHOT; edits arrive as JSON on stdin".into()); - } - if args[0] == "snapshot" { - let bytes = onestore::read_file(&args[1])?; - let mut file = fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&args[2])?; - file.write_all(&bytes)?; - file.sync_all()?; - return Ok(json!({"ok": true, "bytes": bytes.len()})); - } - let check = args[0] == "check"; - if check && args[2] != "-" { - return Err("The check command requires '-' as its final argument".into()); - } - let bytes = fs::read(if check { &args[1] } else { &args[2] })?; - let edit: Edit = serde_json::from_reader(io::stdin().lock())?; - let sid = edit.space; - let oid = edit.object; - let prepared = match edit.action { - Action::Text { - start, - end, - replacement, - } => PreparedEdit::text(&bytes, sid, oid, start..end, &replacement)?, - Action::Format { - start, - end, - attributes, - } => PreparedEdit::format(&bytes, sid, oid, start..end, &attributes)?, - Action::Paragraph { - before, - text, - author, - } => PreparedEdit::insert( - &bytes, - sid, - &Insertion::paragraph(oid, before, &text, &author)?, - )?, - Action::Outline { x, y, text, author } => { - PreparedEdit::insert(&bytes, sid, &Insertion::outline(oid, x, y, &text, &author)?)? - } - }; - if check { - return Ok(json!({"ok": true})); - } - Ok(match prepared.commit_file(&args[1]) { - Ok(()) => json!({"ok": true, "state": "Committed"}), - Err(error) => json!({"ok": false, "state": format!("{:?}", error.state), - "kind": format!("{:?}", error.error.kind()), "error": error.error.to_string()}), - }) -} - -fn main() { - let args: Vec<_> = env::args_os().skip(1).collect(); - let result = run(&args).unwrap_or_else(|error| { - let mut result = json!({"ok": false, "kind": "Input", "error": error.to_string()}); - if args.first().is_some_and(|mode| mode == "commit") { - result["state"] = json!("NotCommitted"); - } - result - }); - println!("{result}"); -} diff --git a/crates/onestore-notebook/Cargo.toml b/crates/onestore-notebook/Cargo.toml deleted file mode 100644 index 476d94e25b04ebe68f127dc163221eb4a63840ed..0000000000000000000000000000000000000000 --- a/crates/onestore-notebook/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "onestore-notebook" -version = "0.1.0" -edition = "2024" -publish = false - -[features] -smb = ["dep:onestore-smb"] -protected = ["onestore/protected", "dep:zeroize"] - -[dependencies] -onestore = { path = "../onestore" } -onestore-smb = { path = "../onestore-smb", optional = true } -serde = { version = "1", features = ["derive"] } -thiserror = "2" -zeroize = { version = "1.9.0", optional = true } - -[dev-dependencies] -serde_json = "1" -tempfile = "3" diff --git a/crates/onestore-notebook/README.md b/crates/onestore-notebook/README.md deleted file mode 100644 index f60e6b49b3a66c82955ed3545fb85400a3e385d0..0000000000000000000000000000000000000000 --- a/crates/onestore-notebook/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# onestore-notebook - -Read-only notebook discovery over a caller-supplied root. This experimental -sibling crate keeps directory traversal out of the single-file storage parser. - -Discovery returns ordered sections and nested groups with file identities and -share-relative paths. Section display-name overrides remain distinct from file -names. TOC references whose identities are absent from the directory remain -inspectable; a cached filename never substitutes for an identity match. -Reserved `_onefiles` directories are excluded from section-group traversal. -Encrypted sections and valid storage with an unreadable document graph retain -their identity as `Locked` or `Unreadable`, without being presented as empty pages. -Malformed storage, failed reads and ambiguous identities reject the discovery. - -Each file read must be a consistent, bounded snapshot. The result is an -observation across multiple files, not an atomic notebook transaction or -authorization to publish an edit. Refresh rejects observed topology changes and -duplicate physical files with the same logical identity. Retain the last accepted -catalog if discovery fails; a connection failure does not mean files were deleted. - -`read_external_asset` resolves a validated UUID `.onebin` filename beneath the -selected section's sibling `_onefiles` folder and returns exact bounded bytes. -Missing files, permissions and size failures retain their I/O error kinds; an -empty payload is a successful empty buffer. Embedded payloads remain available -directly from the core document model. diff --git a/crates/onestore-notebook/examples/discover.rs b/crates/onestore-notebook/examples/discover.rs deleted file mode 100644 index fb73310d27b998ff66f3dde3a4f2dfc364e0829b..0000000000000000000000000000000000000000 --- a/crates/onestore-notebook/examples/discover.rs +++ /dev/null @@ -1,40 +0,0 @@ -use onestore_notebook::{Limits, Local, discover}; - -fn main() -> Result<(), Box> { - let args: Vec<_> = std::env::args_os().skip(1).collect(); - let limits = Limits { - entries: 100_000, - bytes_per_file: 256 * 1024 * 1024, - depth: 64, - }; - let catalog = match args.as_slice() { - [root] => discover(&mut Local::open(root)?, limits)?, - #[cfg(feature = "smb")] - [flag, address, share, root] if flag == "--smb" => { - use onestore_smb::{Client, Credentials}; - let username = std::env::var("ONESTORE_SMB_USERNAME").unwrap_or_default(); - let password = std::env::var("ONESTORE_SMB_PASSWORD").unwrap_or_default(); - let domain = std::env::var("ONESTORE_SMB_DOMAIN").unwrap_or_default(); - let client = Client::connect( - address.to_str().ok_or("Use a UTF-8 server address")?, - share.to_str().ok_or("Use a UTF-8 share name")?, - Credentials { - username: &username, - password: &password, - domain: &domain, - }, - std::time::Duration::from_secs(5), - )?; - discover( - &mut onestore_notebook::Smb::new( - &client, - root.to_str().ok_or("Use a UTF-8 notebook path")?, - )?, - limits, - )? - } - _ => return Err("Provide ROOT, or --smb ADDRESS SHARE ROOT with the smb feature".into()), - }; - serde_json::to_writer_pretty(std::io::stdout().lock(), &catalog)?; - Ok(()) -} diff --git a/crates/onestore-notebook/examples/document.rs b/crates/onestore-notebook/examples/document.rs deleted file mode 100644 index f38d1d5e6ad757f9faed31a8e013ecd12638fd9a..0000000000000000000000000000000000000000 --- a/crates/onestore-notebook/examples/document.rs +++ /dev/null @@ -1,170 +0,0 @@ -use onestore::{ - FileDataReference, RevisionIndex, Store, - document::{Document, Kind}, -}; -use std::{ - borrow::Cow, - collections::BTreeSet, - env, fs, - io::{self, BufWriter, Write}, - path::{Path, PathBuf}, -}; - -fn write_json( - path: impl AsRef, - value: &impl serde::Serialize, -) -> Result<(), Box> { - let mut output = BufWriter::new(fs::File::create(path)?); - serde_json::to_writer(&mut output, value)?; - output.flush()?; - Ok(()) -} - -fn main() -> Result<(), Box> { - let mut args = env::args_os().skip(1); - let path = args.next().ok_or("Provide a .one or .onetoc2 file.")?; - let next = args.next(); - let (destination, option) = if next.as_deref() == Some(std::ffi::OsStr::new("--password-file")) - { - (None, next) - } else { - (next.map(PathBuf::from), args.next()) - }; - let password_file = match (option, args.next(), args.next()) { - (None, None, None) => None, - (Some(flag), Some(path), None) if flag == "--password-file" => Some(PathBuf::from(path)), - _ => return Err("Provide a source file, an optional new export directory, and optionally --password-file PATH.".into()), - }; - let source_path = PathBuf::from(path); - let bytes = onestore::read_file(&source_path)?; - let store = Store::parse(&bytes)?; - let index = RevisionIndex::parse(&store)?; - #[cfg(feature = "protected")] - let unlocked = if let Some(path) = &password_file { - use std::io::Read; - let mut password = zeroize::Zeroizing::new(String::new()); - fs::File::open(path)? - .take(65537) - .read_to_string(&mut password)?; - if password.len() > 65536 { - return Err("The password file exceeds 64 KiB.".into()); - } - Some(onestore::protected::UnlockedSection::open( - &index, - &password, - Default::default(), - )?) - } else { - None - }; - #[cfg(not(feature = "protected"))] - if password_file.is_some() { - return Err( - "Build this exporter with the protected feature to open a password-protected section." - .into(), - ); - } - #[cfg(feature = "protected")] - let document = match &unlocked { - Some(section) => section.document()?, - None => Document::parse(&index)?, - }; - #[cfg(not(feature = "protected"))] - let document = Document::parse(&index)?; - if let Some(destination) = destination { - #[cfg(unix)] - { - use std::os::unix::fs::DirBuilderExt; - fs::DirBuilder::new() - .mode(if password_file.is_some() { - 0o700 - } else { - 0o777 - }) - .create(&destination)?; - } - #[cfg(not(unix))] - fs::create_dir(&destination)?; - fs::create_dir(destination.join("assets"))?; - let parent = source_path - .parent() - .filter(|parent| !parent.as_os_str().is_empty()) - .unwrap_or_else(|| Path::new(".")); - let mut source = onestore_notebook::Local::open(parent)?; - let section = source_path - .file_name() - .and_then(|name| name.to_str()) - .ok_or("Use a UTF-8 section filename")?; - let mut assets = Vec::new(); - let mut seen = BTreeSet::new(); - for node in document - .spaces - .values() - .flat_map(|s| s.revisions.values()) - .flat_map(|r| r.nodes.values()) - { - if let Kind::File { - reference, payload, .. - } = &node.kind - && seen.insert(serde_json::to_string(reference)?) - { - let data = match reference { - FileDataReference::Internal(_) => { - Cow::Borrowed(payload.ok_or("Missing embedded file data")?) - } - FileDataReference::External(filename) => { - if password_file.is_some() { - assets.push(serde_json::json!({"reference":reference,"path":null,"error":{"kind":"Unsupported","message":"Protected external payload export is not supported"}})); - continue; - } - match onestore_notebook::read_external_asset( - &mut source, - section, - filename, - 256 * 1024 * 1024, - ) { - Ok(bytes) => Cow::Owned(bytes), - Err(error) => { - let kind = match &error { - onestore_notebook::Error::Io { error, .. } => { - format!("{:?}", error.kind()) - } - _ => "InvalidData".into(), - }; - assets.push(serde_json::json!({"reference":reference,"path":null,"error":{"kind":kind,"message":error.to_string()}})); - continue; - } - } - } - FileDataReference::Invalid => { - assets.push(serde_json::json!({"reference":reference,"path":null,"error":{"kind":"InvalidData","message":"The document marks this payload as unavailable"}})); - continue; - } - }; - let path = format!("assets/{}.bin", assets.len()); - fs::write(destination.join(&path), data)?; - assets.push(serde_json::json!({"reference":reference,"path":path})); - } - } - write_json(destination.join("assets.json"), &assets)?; - let mut text = std::collections::BTreeMap::new(); - for (sid, space) in &document.spaces { - let mut revisions = std::collections::BTreeMap::new(); - for (rid, revision) in &space.revisions { - let mut objects = std::collections::BTreeMap::new(); - for (oid, node) in &revision.nodes { - if matches!(node.kind, Kind::RichText { .. }) { - objects.insert(*oid, revision.text_runs(*oid)?); - } - } - revisions.insert(*rid, objects); - } - text.insert(*sid, revisions); - } - write_json(destination.join("text.json"), &text)?; - write_json(destination.join("document.json"), &document)?; - } else { - serde_json::to_writer(io::stdout().lock(), &document)?; - } - Ok(()) -} diff --git a/crates/onestore-notebook/examples/externalize_fixture.rs b/crates/onestore-notebook/examples/externalize_fixture.rs deleted file mode 100644 index 2b82cd3798d8c710e79c14cd3f60a308588661b0..0000000000000000000000000000000000000000 --- a/crates/onestore-notebook/examples/externalize_fixture.rs +++ /dev/null @@ -1,101 +0,0 @@ -//! Builds a new external-payload test fixture from a native capture with reserved fragment space. -use onestore::{FileDataReference, RevisionIndex, Store}; -use std::{fs, path::PathBuf}; -fn main() -> Result<(), Box> { - let args: Vec<_> = std::env::args_os().skip(1).collect(); - if args.len() != 2 { - return Err("Provide a native source section and a new fixture directory".into()); - } - let source = fs::read(&args[0])?; - let destination = PathBuf::from(&args[1]); - fs::create_dir(&destination)?; - fs::create_dir(destination.join("synthetic_onefiles"))?; - let store = Store::parse(&source)?; - assert!(store.checksum_mismatches.is_empty()); - RevisionIndex::parse(&store)?.validate_current()?; - let mut output = source.clone(); - let mut changed = 0; - for list in store.lists.values() { - for chunk in &list.fragments { - let start = chunk.offset as usize; - let end = start + chunk.length as usize - 20; - let mut body = Vec::new(); - let mut touched = false; - for node in list - .nodes - .iter() - .filter(|node| node.offset >= start + 16 && node.offset < end) - { - let raw = u32::from_le_bytes(source[node.offset..node.offset + 4].try_into()?); - let size = ((raw >> 10) & 0x1fff) as usize; - let mut encoded = source[node.offset..node.offset + size].to_vec(); - if matches!(node.id, 0x72 | 0x73) { - let offset = 4 + 4 + 4 + if node.id == 0x72 { 1 } else { 4 }; - let count = - u32::from_le_bytes(encoded[offset..offset + 4].try_into()?) as usize; - let text = String::from_utf16( - &encoded[offset + 4..offset + 4 + count * 2] - .chunks_exact(2) - .map(|v| u16::from_le_bytes(v.try_into().unwrap())) - .collect::>(), - )?; - if let FileDataReference::Internal(guid) = text.parse()? { - let filename = format!( - "{}.onebin", - text.strip_prefix("{") - .unwrap() - .strip_suffix('}') - .unwrap() - ); - let payload = store.file_data(guid)?; - let path = destination.join("synthetic_onefiles").join(&filename); - if path.exists() { - assert_eq!(fs::read(&path)?, payload); - } else { - fs::write(path, payload)?; - } - let reference = format!("{filename}"); - let data: Vec<_> = reference - .encode_utf16() - .flat_map(u16::to_le_bytes) - .collect(); - encoded.splice(offset + 4..offset + 4 + count * 2, data.iter().copied()); - encoded[offset..offset + 4] - .copy_from_slice(&(data.len() as u32 / 2).to_le_bytes()); - assert!(encoded.len() <= 0x1fff); - let header = (raw & !(0x1fff << 10)) | ((encoded.len() as u32) << 10); - encoded[..4].copy_from_slice(&header.to_le_bytes()); - changed += 1; - touched = true; - } - } - body.extend(encoded); - } - if touched { - assert!( - body.len() + 4 <= end - start - 16, - "Native fragment has insufficient reserved space" - ); - body.extend_from_slice(&(0xff_u32 | (4 << 10)).to_le_bytes()); - output[start + 16..end].fill(0); - output[start + 16..start + 16 + body.len()].copy_from_slice(&body); - } - } - } - assert!(changed > 0); - let converted = Store::parse(&output)?; - assert!(converted.checksum_mismatches.is_empty()); - RevisionIndex::parse(&converted)?.validate_current()?; - fs::write(destination.join("synthetic.one"), &output)?; - fs::write( - destination.join("Open Notebook.onetoc2"), - onestore::create_table_of_contents( - "Open Notebook.onetoc2", - &[("synthetic.one", converted.header.file_id)], - )?, - )?; - println!( - "Converted {changed} file-data declarations without changing payload identities or bytes" - ); - Ok(()) -} diff --git a/crates/onestore-notebook/src/lib.rs b/crates/onestore-notebook/src/lib.rs deleted file mode 100644 index 68a31a7c239c406376e6f23a0e014af8f3f7e120..0000000000000000000000000000000000000000 --- a/crates/onestore-notebook/src/lib.rs +++ /dev/null @@ -1,413 +0,0 @@ -#![forbid(unsafe_code)] -#![doc = include_str!("../README.md")] - -use onestore::{ - FileType, RevisionIndex, Store, - document::{Document, Kind}, -}; -use serde::Serialize; -use std::{ - collections::{BTreeMap, BTreeSet}, - io, -}; - -mod source; -pub use source::Local; -#[cfg(feature = "smb")] -pub use source::Smb; - -#[derive(Debug, Serialize)] -pub struct Section { - pub path: String, - /// Header.guidFile, also used by FileIdentityGuid in a parent TOC. - pub file_id: [u8; 16], - pub state: SectionState, -} - -#[derive(Debug, Serialize)] -pub enum SectionState { - Readable { - /// An explicit SectionDisplayName; otherwise use the current filename without its extension. - name: Option, - }, - Locked, - Unreadable(onestore::Error), -} - -#[derive(Debug, Serialize)] -pub struct Folder { - pub path: String, - pub toc: Option, - pub sections: Vec
, - pub groups: Vec, -} - -#[derive(Debug, Serialize)] -pub struct Toc { - pub filename: String, - pub file_id: [u8; 16], - /// Stale or unavailable TOC references; these are not inferred active sections. - pub unresolved: Vec, -} - -#[derive(Debug, Serialize)] -pub struct TocReference { - /// Native TOCs can retain an identity after removing its cached filename. - pub filename: Option, - pub file: [u8; 16], - pub order: u32, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum EntryKind { - File, - Directory, - Other, -} - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct Entry { - pub name: String, - pub kind: EntryKind, -} - -/// Rooted file access. Paths are relative, UTF-8, and use `/` between components. -pub trait Source { - /// Return the complete immediate directory or an error, never a truncated success. - fn entries(&mut self, path: &str, limit: usize) -> io::Result>; - /// Return a consistent file snapshot, rejecting images larger than the byte limit. - fn read(&mut self, path: &str, limit: usize) -> io::Result>; - /// Reads an external payload with the same completeness and size guarantees. - fn read_asset(&mut self, path: &str, limit: usize) -> io::Result> { - self.read(path, limit) - } -} - -/// Reads an external file-data reference from the section's sibling `_onefiles` folder. -/// `NotFound` in `Error::Io` is distinct from a successfully read zero-byte payload. -pub fn read_external_asset( - source: &mut impl Source, - section: &str, - filename: &str, - limit: usize, -) -> Result, Error> { - let (stem, extension) = section.rsplit_once('.').ok_or_else(|| Error::Entry { - path: section.into(), - })?; - if !extension.eq_ignore_ascii_case("one") - || !section.split('/').all(component) - || stem.ends_with('/') - || stem.is_empty() - { - return Err(Error::Entry { - path: section.into(), - }); - } - format!("{filename}") - .parse::() - .map_err(|_| Error::Entry { - path: filename.into(), - })?; - let path = format!("{stem}_onefiles/{filename}"); - let bytes = source.read_asset(&path, limit).map_err(|error| Error::Io { - path: path.clone(), - error, - })?; - if bytes.len() > limit { - return Err(Error::Limit { path }); - } - Ok(bytes) -} - -pub struct Limits { - pub entries: usize, - pub bytes_per_file: usize, - pub depth: usize, -} - -#[derive(Debug, thiserror::Error)] -pub enum Error { - #[error("Cannot read {path}: {error}")] - Io { - path: String, - #[source] - error: io::Error, - }, - #[error("Invalid notebook file {path}: {error}")] - Document { - path: String, - #[source] - error: onestore::Error, - }, - #[error("Discovery limit exceeded at {path}")] - Limit { path: String }, - #[error("Directory changed during discovery: {path}")] - Changed { path: String }, - #[error("Invalid or unsupported directory entry: {path}")] - Entry { path: String }, - #[error("File identity occurs at both {first} and {second}")] - DuplicateIdentity { - file: [u8; 16], - first: String, - second: String, - }, -} - -/// Discovers rooted notebook topology within caller-specified work and size limits. -/// Reserved `_onefiles` directories contain payloads, not section groups. -pub fn discover(source: &mut impl Source, limits: Limits) -> Result { - let mut remaining = limits.entries; - let mut identities = BTreeMap::new(); - scan(source, "", &limits, 0, &mut remaining, &mut identities) -} - -fn scan( - source: &mut impl Source, - path: &str, - limits: &Limits, - depth: usize, - remaining: &mut usize, - identities: &mut BTreeMap<[u8; 16], String>, -) -> Result { - if depth > limits.depth { - return Err(Error::Limit { path: path.into() }); - } - let mut listing = source - .entries(path, *remaining) - .map_err(|error| Error::Io { - path: path.into(), - error, - })?; - *remaining = remaining - .checked_sub(listing.len()) - .ok_or_else(|| Error::Limit { path: path.into() })?; - listing.sort(); - let mut names = BTreeSet::new(); - for entry in &listing { - if !component(&entry.name) || !names.insert(&entry.name) { - return Err(Error::Entry { - path: join(path, &entry.name), - }); - } - } - let mut result = Folder { - path: path.into(), - toc: None, - sections: Vec::new(), - groups: Vec::new(), - }; - for entry in &listing { - let child = join(path, &entry.name); - if entry.kind == EntryKind::Directory { - if entry.name.to_ascii_lowercase().ends_with("_onefiles") { - continue; - } - result.groups.push(scan( - source, - &child, - limits, - depth + 1, - remaining, - identities, - )?); - continue; - } - let lower = entry.name.to_ascii_lowercase(); - let expected = if lower.ends_with(".one") { - FileType::Section - } else if lower.ends_with(".onetoc2") { - FileType::TableOfContents - } else { - continue; - }; - if entry.kind != EntryKind::File - || (expected == FileType::TableOfContents && result.toc.is_some()) - { - return Err(Error::Entry { path: child }); - } - let bytes = source - .read(&child, limits.bytes_per_file) - .map_err(|error| Error::Io { - path: child.clone(), - error, - })?; - if bytes.len() > limits.bytes_per_file { - return Err(Error::Limit { path: child }); - } - let store = Store::parse(&bytes).map_err(|error| Error::Document { - path: child.clone(), - error, - })?; - if store.header.file_type != expected || !store.checksum_mismatches.is_empty() { - return Err(Error::Document { - path: child, - error: onestore::Error { - offset: 0, - message: "Unexpected file type or checksum mismatch", - }, - }); - } - let file_id = store.header.file_id; - let parsed = (|| { - let index = RevisionIndex::parse(&store)?; - let document = Document::parse(&index)?; - let revision = document.active(document.root)?; - if expected == FileType::Section { - if revision - .roots - .get(&1) - .and_then(|id| revision.nodes.get(id)) - .is_some_and(|node| matches!(node.kind, Kind::Encrypted { .. })) - { - result.sections.push(Section { - path: child.clone(), - file_id, - state: SectionState::Locked, - }); - return Ok(()); - } - index.validate_current()?; - document.pages()?; - let name = revision - .roots - .get(&2) - .and_then(|id| revision.nodes.get(id)) - .and_then(|node| match &node.kind { - Kind::SectionMetadata { name, .. } => name.clone(), - _ => None, - }); - result.sections.push(Section { - path: child.clone(), - file_id, - state: SectionState::Readable { name }, - }); - } else { - index.validate_current()?; - let node = revision.roots.get(&1).and_then(|id| revision.nodes.get(id)); - let Some(Kind::Toc { entries, .. }) = node.map(|node| &node.kind) else { - return Err(onestore::Error { - offset: 0, - message: "Missing notebook TOC root", - }); - }; - let mut seen = BTreeSet::new(); - let mut unresolved = Vec::new(); - for id in entries { - let Some(Kind::Toc { - filename, - identity: Some(file), - order: Some(order), - .. - }) = revision.nodes.get(id).map(|node| &node.kind) - else { - return Err(onestore::Error { - offset: 0, - message: "Incomplete notebook TOC reference", - }); - }; - if filename.as_deref().is_some_and(|name| !component(name)) - || !seen.insert(*file) - { - return Err(onestore::Error { - offset: 0, - message: "Invalid or duplicated notebook TOC reference", - }); - } - unresolved.push(TocReference { - filename: filename.clone(), - file: *file, - order: *order, - }); - } - result.toc = Some(Toc { - filename: entry.name.clone(), - file_id, - unresolved, - }); - } - Ok(()) - })(); - if let Err(error) = parsed { - if expected == FileType::Section { - result.sections.push(Section { - path: child.clone(), - file_id, - state: SectionState::Unreadable(error), - }); - } else { - return Err(Error::Document { path: child, error }); - } - } - if let Some(first) = identities.insert(file_id, child.clone()) { - return Err(Error::DuplicateIdentity { - file: file_id, - first, - second: child, - }); - } - } - let order: BTreeMap<_, _> = result - .toc - .iter() - .flat_map(|toc| &toc.unresolved) - .map(|entry| (entry.file, entry.order)) - .collect(); - result.sections.sort_by(|a, b| { - (order.get(&a.file_id).copied().unwrap_or(u32::MAX), &a.path) - .cmp(&(order.get(&b.file_id).copied().unwrap_or(u32::MAX), &b.path)) - }); - result.groups.sort_by(|a, b| { - ( - a.toc - .as_ref() - .and_then(|toc| order.get(&toc.file_id).copied()) - .unwrap_or(u32::MAX), - &a.path, - ) - .cmp(&( - b.toc - .as_ref() - .and_then(|toc| order.get(&toc.file_id).copied()) - .unwrap_or(u32::MAX), - &b.path, - )) - }); - let present: BTreeSet<_> = result - .sections - .iter() - .map(|section| section.file_id) - .chain( - result - .groups - .iter() - .filter_map(|group| group.toc.as_ref().map(|toc| toc.file_id)), - ) - .collect(); - if let Some(toc) = &mut result.toc { - toc.unresolved - .retain(|entry| !present.contains(&entry.file)); - } - let mut observed = source - .entries(path, listing.len()) - .map_err(|error| Error::Io { - path: path.into(), - error, - })?; - observed.sort(); - if observed != listing { - return Err(Error::Changed { path: path.into() }); - } - Ok(result) -} - -fn component(name: &str) -> bool { - !name.is_empty() && name != "." && name != ".." && !name.contains(['/', '\\', '\0']) -} - -fn join(parent: &str, name: &str) -> String { - if parent.is_empty() { - name.into() - } else { - format!("{parent}/{name}") - } -} diff --git a/crates/onestore-notebook/src/source.rs b/crates/onestore-notebook/src/source.rs deleted file mode 100644 index 4bceacd8600fe84c2497dfc43faa7d8c69b2a308..0000000000000000000000000000000000000000 --- a/crates/onestore-notebook/src/source.rs +++ /dev/null @@ -1,121 +0,0 @@ -use super::{Entry, EntryKind, Source, component}; -use std::{ - io, - path::{Path, PathBuf}, -}; - -/// A canonical local root; symbolic-link entries are not traversed. -pub struct Local { - root: PathBuf, -} - -impl Local { - pub fn open(root: impl AsRef) -> io::Result { - let root = root.as_ref().canonicalize()?; - if !root.is_dir() { - return Err(io::ErrorKind::NotADirectory.into()); - } - Ok(Self { root }) - } - - fn path(&self, relative: &str) -> io::Result { - if !relative.is_empty() && !relative.split('/').all(component) { - return Err(io::ErrorKind::InvalidInput.into()); - } - let path = self.root.join(relative).canonicalize()?; - if !path.starts_with(&self.root) { - return Err(io::ErrorKind::PermissionDenied.into()); - } - Ok(path) - } -} - -impl Source for Local { - fn entries(&mut self, path: &str, limit: usize) -> io::Result> { - let mut entries = Vec::new(); - for entry in std::fs::read_dir(self.path(path)?)? { - let entry = entry?; - if entries.len() == limit { - return Err(io::ErrorKind::FileTooLarge.into()); - } - let name = entry - .file_name() - .into_string() - .map_err(|_| io::ErrorKind::InvalidData)?; - let kind = entry.file_type()?; - entries.push(Entry { - name, - kind: if kind.is_dir() { - EntryKind::Directory - } else if kind.is_file() { - EntryKind::File - } else { - EntryKind::Other - }, - }); - } - Ok(entries) - } - - fn read(&mut self, path: &str, limit: usize) -> io::Result> { - onestore::read_file_limited(self.path(path)?, limit) - } -} - -#[cfg(feature = "smb")] -/// A share-relative root on an existing blocking SMB connection. -pub struct Smb<'a> { - client: &'a onestore_smb::Client, - root: String, -} - -#[cfg(feature = "smb")] -impl<'a> Smb<'a> { - pub fn new(client: &'a onestore_smb::Client, root: &str) -> io::Result { - let root = root.replace('\\', "/"); - if !root.is_empty() && !root.split('/').all(component) { - return Err(io::ErrorKind::InvalidInput.into()); - } - Ok(Self { client, root }) - } - - fn path(&self, relative: &str) -> io::Result { - if !relative.is_empty() && !relative.split('/').all(component) { - return Err(io::ErrorKind::InvalidInput.into()); - } - Ok(if relative.is_empty() { - self.root.clone() - } else { - super::join(&self.root, relative) - }) - } -} - -#[cfg(feature = "smb")] -impl Source for Smb<'_> { - fn entries(&mut self, path: &str, limit: usize) -> io::Result> { - Ok(self - .client - .read_dir(&self.path(path)?, limit)? - .into_iter() - .map(|entry| Entry { - name: entry.name, - kind: if entry.attributes & 0x400 != 0 { - EntryKind::Other - } else if entry.attributes & 0x10 != 0 { - EntryKind::Directory - } else { - EntryKind::File - }, - }) - .collect()) - } - - fn read(&mut self, path: &str, limit: usize) -> io::Result> { - self.client.read_storage(&self.path(path)?, limit) - } - - fn read_asset(&mut self, path: &str, limit: usize) -> io::Result> { - self.client.read_asset(&self.path(path)?, limit) - } -} diff --git a/crates/onestore-notebook/tests/assets.rs b/crates/onestore-notebook/tests/assets.rs deleted file mode 100644 index 47f50d2e80a1a5aeed909bbcfc8e99b850796987..0000000000000000000000000000000000000000 --- a/crates/onestore-notebook/tests/assets.rs +++ /dev/null @@ -1,162 +0,0 @@ -use onestore_notebook::{Error, Local, read_external_asset}; -use std::{fs, io}; - -#[test] -fn nested_external_payloads_are_exact_and_empty_is_distinct_from_missing() { - let root = tempfile::tempdir().unwrap(); - let directory = root.path().join("Group 🦀/Section é_onefiles"); - fs::create_dir_all(&directory).unwrap(); - let filename = "2a83ae62-6754-4383-8e2b-4033ff3cfba1.onebin"; - let mut source = Local::open(root.path()).unwrap(); - let section = "Group 🦀/Section é.ONE"; - assert!( - matches!(read_external_asset(&mut source, section, filename, 0), - Err(Error::Io { error, .. }) if error.kind() == io::ErrorKind::NotFound) - ); - fs::write(directory.join(filename), []).unwrap(); - assert!( - read_external_asset(&mut source, section, filename, 0) - .unwrap() - .is_empty() - ); - let bytes: Vec<_> = (0..65537).map(|index| (index % 251) as u8).collect(); - fs::write(directory.join(filename), &bytes).unwrap(); - assert!( - matches!(read_external_asset(&mut source, section, filename, bytes.len()-1), - Err(Error::Io { error, .. }) if error.kind() == io::ErrorKind::FileTooLarge) - ); - assert_eq!( - read_external_asset(&mut source, section, filename, bytes.len()).unwrap(), - bytes - ); - assert_eq!(fs::read(directory.join(filename)).unwrap(), bytes); -} - -#[test] -fn invalid_asset_locations_fail_before_accessing_the_source() { - struct Unused; - impl onestore_notebook::Source for Unused { - fn entries(&mut self, _: &str, _: usize) -> io::Result> { - panic!("Unexpected enumeration") - } - fn read(&mut self, _: &str, _: usize) -> io::Result> { - panic!("Unexpected read") - } - } - let filename = "2a83ae62-6754-4383-8e2b-4033ff3cfba1.onebin"; - for section in [ - "", - ".one", - "Group/.one", - "/Section.one", - "../Section.one", - "Group/../Section.one", - "Group\\Section.one", - "Section.onetoc2", - ] { - assert!(matches!( - read_external_asset(&mut Unused, section, filename, 100), - Err(Error::Entry { .. }) - )); - } - for name in [ - "", - "attachment.png", - "../2a83ae62-6754-4383-8e2b-4033ff3cfba1.onebin", - "2a83ae62-6754-4383-8e2b-4033ff3cfba1.onebin:other", - "", - ] { - assert!(matches!( - read_external_asset(&mut Unused, "Section.one", name, 100), - Err(Error::Entry { .. }) - )); - } -} - -#[test] -fn reserved_payload_directories_are_not_notebook_groups() { - let root = tempfile::tempdir().unwrap(); - fs::write( - root.path().join("Section.ONE"), - onestore::create_section("Section.one", "Fixture", "Author").unwrap(), - ) - .unwrap(); - for name in ["section_onefiles", "orphan_onefiles", "ordinary"] { - fs::create_dir(root.path().join(name)).unwrap(); - } - fs::write( - root.path().join("section_onefiles/unfinished.onebin"), - b"payload", - ) - .unwrap(); - let catalog = onestore_notebook::discover( - &mut Local::open(root.path()).unwrap(), - onestore_notebook::Limits { - entries: 4, - bytes_per_file: 1 << 20, - depth: 1, - }, - ) - .unwrap(); - assert_eq!(catalog.sections.len(), 1); - assert_eq!( - catalog - .groups - .iter() - .map(|group| group.path.as_str()) - .collect::>(), - ["ordinary"] - ); -} - -#[test] -#[cfg(feature = "smb")] -#[ignore = "requires a disposable Samba mirror of corpus/native-external-assets/notebook at ONESTORE_SMB_NOTEBOOK"] -fn live_external_payloads() { - use onestore::{ - FileDataReference, RevisionIndex, Store, - document::{Document, Kind}, - }; - let root = std::path::Path::new("../../corpus/native-external-assets/notebook"); - let bytes = fs::read(root.join("synthetic.one")).unwrap(); - let store = Store::parse(&bytes).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let client = onestore_smb::Client::connect( - &std::env::var("ONESTORE_SMB_LAB").unwrap(), - "agent", - onestore_smb::Credentials::default(), - std::time::Duration::from_secs(5), - ) - .unwrap(); - let mut remote = - onestore_notebook::Smb::new(&client, &std::env::var("ONESTORE_SMB_NOTEBOOK").unwrap()) - .unwrap(); - let mut local = Local::open(root).unwrap(); - let mut seen = std::collections::BTreeSet::new(); - for node in document - .spaces - .values() - .flat_map(|space| space.revisions.values()) - .flat_map(|revision| revision.nodes.values()) - { - if let Kind::File { - reference: FileDataReference::External(filename), - .. - } = &node.kind - && seen.insert(filename) - { - let expected = fs::read(root.join("synthetic_onefiles").join(filename)).unwrap(); - assert_eq!( - read_external_asset(&mut local, "synthetic.one", filename, expected.len()).unwrap(), - expected - ); - assert_eq!( - read_external_asset(&mut remote, "synthetic.one", filename, expected.len()) - .unwrap(), - expected - ); - } - } - assert_eq!(seen.len(), 3); -} diff --git a/crates/onestore-notebook/tests/discovery.rs b/crates/onestore-notebook/tests/discovery.rs deleted file mode 100644 index 89a6f21debf9b218b64f59dad76fae89c6262830..0000000000000000000000000000000000000000 --- a/crates/onestore-notebook/tests/discovery.rs +++ /dev/null @@ -1,280 +0,0 @@ -use onestore_notebook::{Entry, Error, Limits, Local, Source, discover}; -use std::{fs, io, path::Path}; - -fn limits() -> Limits { - Limits { - entries: 1000, - bytes_per_file: 16 * 1024 * 1024, - depth: 16, - } -} - -fn fixture(root: &Path) -> Vec { - let section = onestore::create_section("one.one", "Retained 🦀", "Fixture").unwrap(); - fs::write(root.join("one.one"), §ion).unwrap(); - let identity = onestore::Store::parse(§ion).unwrap().header.file_id; - let toc = onestore::create_table_of_contents("Open Notebook.onetoc2", &[("one.one", identity)]) - .unwrap(); - fs::write(root.join("Open Notebook.onetoc2"), toc).unwrap(); - section -} - -#[test] -fn rename_preserves_identity_and_does_not_trust_a_stale_cached_filename() { - let root = tempfile::tempdir().unwrap(); - let section = fixture(root.path()); - let mut source = Local::open(root.path()).unwrap(); - let before = discover(&mut source, limits()).unwrap(); - fs::rename( - root.path().join("one.one"), - root.path().join("Renamed 🦀.ONE"), - ) - .unwrap(); - let after = discover(&mut source, limits()).unwrap(); - assert_eq!(before.sections[0].file_id, after.sections[0].file_id); - assert_eq!(after.sections[0].path, "Renamed 🦀.ONE"); - assert!(after.toc.as_ref().unwrap().unresolved.is_empty()); - assert_eq!( - fs::read(root.path().join("Renamed 🦀.ONE")).unwrap(), - section - ); - fs::write( - root.path().join("one.one"), - onestore::create_section("one.one", "Different", "Fixture").unwrap(), - ) - .unwrap(); - let with_replacement = discover(&mut source, limits()).unwrap(); - assert_eq!( - with_replacement.sections[0].file_id, - before.sections[0].file_id - ); - assert_eq!(with_replacement.sections[0].path, "Renamed 🦀.ONE"); - assert_ne!( - with_replacement.sections[1].file_id, - before.sections[0].file_id - ); - fs::remove_file(root.path().join("Renamed 🦀.ONE")).unwrap(); - let absent = discover(&mut source, limits()).unwrap(); - assert_eq!(absent.toc.as_ref().unwrap().unresolved.len(), 1); - assert_eq!( - absent.toc.as_ref().unwrap().unresolved[0].file, - before.sections[0].file_id - ); - assert_ne!( - absent.sections[0].file_id, - absent.toc.as_ref().unwrap().unresolved[0].file - ); -} - -#[test] -fn copied_identities_are_ambiguous_even_across_different_groups() { - let root = tempfile::tempdir().unwrap(); - fixture(root.path()); - fs::create_dir(root.path().join("group")).unwrap(); - fs::copy( - root.path().join("one.one"), - root.path().join("group/other.one"), - ) - .unwrap(); - assert!( - matches!(discover(&mut Local::open(root.path()).unwrap(), limits()), - Err(Error::DuplicateIdentity { first, second, .. }) - if [first.as_str(), second.as_str()].contains(&"one.one") - && [first.as_str(), second.as_str()].contains(&"group/other.one")) - ); -} - -#[test] -fn locked_and_unreadable_sections_retain_their_storage_identity() { - let root = tempfile::tempdir().unwrap(); - for (name, fixture) in [ - ( - "locked.one", - "native-encrypted/encrypted-01/notebook/synthetic.one", - ), - ( - "stub.one", - "native-encrypted/cold-encrypted-02/notebook/Open Notebook.one", - ), - ] { - fs::copy( - Path::new("../../corpus").join(fixture), - root.path().join(name), - ) - .unwrap(); - } - let catalog = discover(&mut Local::open(root.path()).unwrap(), limits()).unwrap(); - assert!(catalog.toc.is_none()); - assert!(matches!( - catalog.sections[0].state, - onestore_notebook::SectionState::Locked - )); - assert!(matches!( - catalog.sections[1].state, - onestore_notebook::SectionState::Unreadable(_) - )); - for section in catalog.sections { - let bytes = fs::read(root.path().join(section.path)).unwrap(); - assert_eq!( - section.file_id, - onestore::Store::parse(&bytes).unwrap().header.file_id - ); - } -} - -#[test] -fn a_group_rename_retains_toc_identity_and_parent_order() { - let root = tempfile::tempdir().unwrap(); - let native = Path::new("../../corpus/m6/native-features-01/notebook"); - for relative in [ - "Open Notebook.onetoc2", - "Group A/Open Notebook.onetoc2", - "Group A/Duplicate.one", - "Group B/Open Notebook.onetoc2", - "Group B/Nested/Open Notebook.onetoc2", - "Group B/Nested/Duplicate.one", - ] { - let target = root.path().join(relative); - fs::create_dir_all(target.parent().unwrap()).unwrap(); - fs::copy(native.join(relative), target).unwrap(); - } - let before = discover(&mut Local::open(root.path()).unwrap(), limits()).unwrap(); - assert_eq!(before.groups[0].path, "Group A"); - fs::rename(root.path().join("Group A"), root.path().join("Renamed 🦀")).unwrap(); - let catalog = discover(&mut Local::open(root.path()).unwrap(), limits()).unwrap(); - assert_eq!(catalog.groups[0].path, "Renamed 🦀"); - assert_eq!( - catalog.groups[0].toc.as_ref().unwrap().file_id, - before.groups[0].toc.as_ref().unwrap().file_id - ); - assert_eq!(catalog.groups[1].path, "Group B"); - assert_eq!( - catalog.groups[0].sections[0].path, - "Renamed 🦀/Duplicate.one" - ); - assert_eq!( - serde_json::to_value(&catalog.toc.unwrap().unresolved).unwrap(), - serde_json::to_value(&before.toc.unwrap().unresolved).unwrap() - ); -} - -#[test] -fn limits_and_incomplete_files_do_not_produce_a_catalog() { - let root = tempfile::tempdir().unwrap(); - fixture(root.path()); - let mut source = Local::open(root.path()).unwrap(); - assert!( - discover( - &mut source, - Limits { - entries: 1, - ..limits() - } - ) - .is_err() - ); - assert!( - discover( - &mut source, - Limits { - bytes_per_file: 8, - ..limits() - } - ) - .is_err() - ); - fs::create_dir(root.path().join("group")).unwrap(); - assert!(matches!( - discover( - &mut source, - Limits { - depth: 0, - ..limits() - } - ), - Err(Error::Limit { .. }) - )); - fs::write(root.path().join("one.one"), b"unfinished").unwrap(); - assert!( - matches!(discover(&mut source, limits()), Err(Error::Document { path, .. }) if path == "one.one") - ); -} - -#[test] -fn a_failed_refresh_retains_the_previous_catalog_as_an_independent_value() { - struct Changing { - source: Local, - reads: usize, - fail: bool, - } - impl Source for Changing { - fn entries(&mut self, path: &str, limit: usize) -> io::Result> { - self.reads += 1; - let mut entries = self.source.entries(path, limit)?; - if self.reads == 2 { - if self.fail { - return Err(io::ErrorKind::ConnectionAborted.into()); - } - entries[0].name.push_str(".renamed"); - } - Ok(entries) - } - fn read(&mut self, path: &str, limit: usize) -> io::Result> { - self.source.read(path, limit) - } - } - let root = tempfile::tempdir().unwrap(); - fixture(root.path()); - let mut source = Local::open(root.path()).unwrap(); - let catalog = discover(&mut source, limits()).unwrap(); - let before = serde_json::to_value(&catalog).unwrap(); - for fail in [false, true] { - let mut changing = Changing { - source: Local::open(root.path()).unwrap(), - reads: 0, - fail, - }; - let error = discover(&mut changing, limits()).unwrap_err(); - if fail { - assert!( - matches!(error, Error::Io { error, .. } if error.kind() == io::ErrorKind::ConnectionAborted) - ); - } else { - assert!(matches!(error, Error::Changed { .. })); - } - assert_eq!(serde_json::to_value(&catalog).unwrap(), before); - } -} - -#[test] -fn local_access_stays_inside_the_selected_root() { - let root = tempfile::tempdir().unwrap(); - fixture(root.path()); - let mut source = Local::open(root.path()).unwrap(); - for path in [ - "../one.one", - "/one.one", - "x/../one.one", - "one.one\0", - "x\\one.one", - ] { - assert_eq!( - source.read(path, 100).unwrap_err().kind(), - io::ErrorKind::InvalidInput - ); - } - #[cfg(unix)] - { - let other = tempfile::tempdir().unwrap(); - fs::write(other.path().join("other.one"), b"outside").unwrap(); - std::os::unix::fs::symlink(other.path().join("other.one"), root.path().join("link.one")) - .unwrap(); - assert_eq!( - source.read("link.one", 100).unwrap_err().kind(), - io::ErrorKind::PermissionDenied - ); - assert!( - matches!(discover(&mut source, limits()), Err(Error::Entry { path }) if path == "link.one") - ); - } -} diff --git a/crates/onestore-offline/Cargo.toml b/crates/onestore-offline/Cargo.toml deleted file mode 100644 index 8fe583780935ca0b1f0ba3c470361f964299b4fa..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "onestore-offline" -version = "0.1.0" -edition = "2024" -publish = false - -[features] -smb = ["dep:onestore-smb", "onestore-notebook/smb"] - -[dependencies] -onestore = { path = "../onestore" } -onestore-notebook = { path = "../onestore-notebook" } -onestore-smb = { path = "../onestore-smb", optional = true } -rusqlite = { version = "=0.40.2", features = ["bundled", "backup"] } -thiserror = "2" -serde = { version = "1", features = ["derive"] } -serde_json = "1" -tempfile = "3" -sha2 = "0.11" - -[[example]] -name = "smb_offline_client" -required-features = ["smb"] diff --git a/crates/onestore-offline/README.md b/crates/onestore-offline/README.md deleted file mode 100644 index fd4aa367554e978a9910f58f70608df58ffa3d03..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/README.md +++ /dev/null @@ -1,323 +0,0 @@ -# onestore-offline - -Durable local editing for a OneNote file, in an optional sibling crate. The current -foundation stores a complete working image and typed editing intents in a local -SQLite database. `sync_once` provides a reconciliation step and `start_sync` owns -automatic polling and reconnects. Local success does not -acknowledge publication to a shared notebook. - -```no_run -use onestore::ExGuid; -use onestore_offline::Replica; -# fn example(path: &std::path::Path, source: &[u8], space: ExGuid, text: ExGuid) -# -> Result<(), Box> { -// `space` and `text` are identities from the supplied section's document model. -let cache = Replica::create(path, source)?; -let snapshot = cache.snapshot()?; -let local_id = cache.edit_text(&snapshot, space, text, 0..0, "Offline edit ")?; -drop(cache); - -let reopened = Replica::open(path)?; -let current = reopened.snapshot()?; -let pending = reopened.pending()?; -assert_eq!(pending.last().map(|edit| edit.id), local_id); -# Ok(()) -# } -``` - -`insert` accepts the core library's `Insertion` value and durably retains its -intent identities. `Insertion::with_formatting` queues text and -its styles as one intent and one publication. Keep that value across retries; its `text_object()` identifies -the new text for subsequent offline edits. Pending entries expose -`Operation::Text(TextEdit)`, `Operation::Insert(Insertion)`, -`Operation::Format(FormatEdit)`, `Operation::Split(SplitEdit)`, -`Operation::Join(JoinEdit)`, `Operation::Outline(OutlineEdit)`, -`Operation::Tree(TreeEdit)`, `Operation::CreatePage(PageCreation)` and -`Operation::Pages(PageEdits)` through their -`operation` field. Synchronization applies these in queue order, so an inserted -outline can precede its paragraphs and their later edits. Missing anchors or -existing insertion identities preserve a conflict and the complete local image. - -`create_page` accepts the core `PageCreation` intent and queues its page space -and section entry as one publication. Subsequent outlines and title edits use -the intent's stable identities immediately after local acknowledgement. -Independent page additions rebase against the current section order; duplicate -titles remain distinct. An unavailable or no-longer-leading insertion anchor -produces `StructureChanged`. `rebase_page_creation_conflict(id, local, remote, -before)` reviews a replacement anchor against both cache images while retaining -the new page and dependent object identities. Existing page identities require -reconciliation; a matching page alone does not establish a receipt. - -`pages(snapshot, edits)` queues a slice of core `PageEdit` intents as one atomic -publication. `PagePosition::Keep` preserves remote movement during indentation-only -edits. Every requested level remains explicit: a competing remote level produces -`StructureChanged` unless it already matches the requested level. Moves compare -the selected page's position relative to surviving observed pages; an unchanged -or already-satisfied position can proceed, and new remote pages remain present. -Indistinguishable competing moves retain a conflict and the complete local image. - -`rebase_pages_conflict(id, local, remote, edits)` reviews the batch against both -cache images. Use `PageEdit::reposition(position, level)` on the retained intents -to revise anchors or indentation; replacing page or allocated series identities -is rejected. Dependent edits remain queued. Schema 11 adds the durable batch; -schema 10 migration preserves existing multi-space publication evidence unchanged. -Attempt evidence includes every changed space plus the receipt space when that -space is unchanged. An existing section revision alone cannot confirm a page edit. -The [offline page corpus](../../corpus/page-lifecycle/offline-edits/README.md) -contains native comparisons, reproduction commands and stateful sanitizer seeds. - -`split` and `join` accept the core `ParagraphSplit` and `ParagraphJoin` intents. -Splits retain allocated identities when the original UTF-16 boundary rebases; -dependent edits can address `ParagraphSplit::text_object()` immediately after -local acknowledgement. Both operations retain observed parent paths, children, -indentation, paragraph/list state and tags. Changed structure produces -`StructureChanged`. Remote text changes must leave unambiguous boundary mappings; -current remote character styles are preserved. Reconciliation does not silently -discard a newly added right tag or move an unobserved child. - -`rebase_conflict` accepts a zero-length reviewed range for a split, preserving its -allocated identities. `rebase_join_conflict(id, local, remote)` reviews the original -join against current images. It rejects a change in which text identity survives, -because dependent edits still address that identity. As with other conflict -reviews, stale images, pending operations and uncertain attempts cannot be rebased. -An uncertain structural operation requires its original attempted revision for -confirmation; matching text or structure does not establish a receipt. - -`rebase_paragraph_conflict(id, local, remote, parent, before)` and -`rebase_outline_conflict(id, local, remote, page, x, y)` accept reviewed replacement -placements for the oldest insertion conflict. They preserve creation time, text, -author and object identities, keeping later edits attached to their original targets. -The images must still match the cache; the new placement must be valid in the remote -image. Paragraph intents cannot become outlines or vice versa. Pending edits and -uncertain publication attempts cannot be repositioned through conflict review. - -```no_run -use onestore::{ExGuid, Insertion, TextAttribute}; -use onestore_offline::{EditStatus, Replica}; -# fn add_outline(cache: &Replica, space: ExGuid, page: ExGuid) -# -> Result<(), Box> { -let outline = Insertion::outline(page, 144.0, 216.0, "Offline outline", "Author")? - .with_formatting(0..7, &[TextAttribute::Bold(true)])?; -let id = cache.insert(&cache.snapshot()?, space, &outline)?; -if let Some(id) = id { - // A running worker may already have advanced this state. - match cache.status(id)? { - Some(EditStatus::Published { revision }) => println!("{revision}"), - state => println!("{state:?}"), - } -} -# Ok(()) -# } -``` - -`format` accepts the core `TextAttribute` slice and a UTF-16 range. Its durable intent -retains the observed text and selected attribute values. Remote text changes must -leave an unambiguous range; independent remote attributes merge, while competing -values preserve `FormattingChanged`. A remote value that already matches the -requested value is accepted. Enabling superscript or subscript also checks the -opposite attribute that the operation clears. If the remote image already satisfies -the whole operation, guarded confirmation still precedes a durable receipt. - -`outline` accepts a target object and the core `onestore::OutlineEdit`: outline -position/width or a paragraph's saved collapse default. It retains the target's -ancestry and the properties the operation changes. Independent content, formatting -and layout properties merge; competing selected values produce `LayoutChanged`, -and changed ancestry produces `StructureChanged`. Width changes also check the -reserved wrapping width that they clear. Missing targets preserve the local branch. -`rebase_layout_conflict(id, local, remote)` explicitly reviews the original change -against both current images, preserving dependent intents. An uncertain layout -attempt requires its original revision; converged values cannot establish its receipt. - -`tree` accepts the core `onestore::TreeEdit`. Moves preserve independent remote -text, formatting and descendants. A competing ancestor, indentation or sibling -crossing produces `StructureChanged`; unrelated sibling insertions/deletions can -merge. An anchor must remain a direct child of the requested destination. An -already satisfied move still requires guarded confirmation before acknowledgement. -Deletion compares the selected raw property graph, including referenced styles, -tags, unknown fields and internal attachments. Changed content produces -`ContentChanged`; property order, CompactID numbering and modification timestamps -do not affect that comparison. Immutable records compare by content, and empty -child lists compare equally to absent child lists. Mutable object identities -remain significant. Missing targets retain `TargetUnavailable`. -`rebase_tree_conflict(id, local, remote)` reviews the original move/deletion against -both current images. An emptied cell's replacement paragraph/text identities must -remain the same before replay or review; later queued edits keep their targets. -Uncertain tree attempts require their original revision for confirmation, even -when an independent move or deletion has the same visible effect. - -Recognized earlier caches migrate transactionally to version ten. Publication -attempts retain every changed space's revision; legacy attempts retain their -single-space evidence. Earlier recovery archives remain readable without migration. -Version-eight -deletion observations retain their original identity-sensitive preconditions; -explicit conflict review upgrades them to the immutable-content comparison. -The migration retains images, local IDs, publication attempts, conflicts, -receipts and the autoincrement sequence; it does not reuse acknowledged IDs when -the pending queue is empty. - -Share one `Replica` between application threads. Each edit compares its supplied -snapshot under the cache transaction; stale snapshots return `Io(ResourceBusy)`. -The intent and its resulting image commit together. No-op edits return `None`. -Keep the cache on a local filesystem: the connection holds exclusive ownership -between transactions, and a second open fails busy. No network wait occurs in a -local edit. After a database error, reopen and inspect the durable state before -retrying. - -`sync_once(&mut remote)` processes the oldest pending edit through a `Remote` -implementation, returning its ID and `EditStatus`. A durable `Published` receipt -survives reopening. Publication attempts are recorded before network I/O; a retained -attempt requires every recorded revision to remain present, followed by comparison, -flushing and refreshed header version -metadata before acknowledgement. If a formatting attempt's revision is missing, -the complete requested effect can instead be confirmed on a uniquely aligned -range; its receipt identifies that confirmed current revision. Otherwise the -missing attempt remains `AwaitingConfirmation`. Neither path replays an uncertain -publication. Overlapping or ambiguous edits retain `Conflict` status, their complete -local image and the last observed remote image returned by `remote_snapshot`. -Transport errors return `Error::Remote` or `Error::RemoteIo`; inspect `status(id)` -after the error to distinguish a retained attempt from a pending edit or receipt. -The error return does not roll back a locally acknowledged intent. - -Rebasing accepts only character mappings shared by every minimum insertion/deletion -alignment. UTF-16 ranges must preserve Unicode scalar boundaries. A bounded -alignment search also leaves a conflict when it cannot establish a unique mapping. -The current operation processes one queue head; while edits remain, `snapshot` -preserves the complete local working image. An empty queue can refresh from the -remote image. Synchronization holds a separate owner lock, so local edits can -continue during network waits; competing synchronization calls return `WouldBlock`. - -`rebase_conflict(id, local, remote, range)` lets a caller explicitly place the -oldest conflicting text or formatting intent at a reviewed UTF-16 range in the remote image. -The supplied images must still match `snapshot()` and `remote_snapshot()`. -It preserves the original replacement or requested attributes, intent ID, complete local working image -and every later intent; the selected range and remote paragraph become the -intent's new comparison base in one local transaction. Formatting also captures -the reviewed attribute values as its new precondition. This operation performs -no network I/O, clears the conflict to `Pending`, and wakes the worker. -Publication still reads the latest remote image and uses exact guarded comparison; -another overlapping remote edit can produce a new conflict. An uncertain attempt -cannot be rebased, and selecting text that already equals the replacement does -not create a publication acknowledgement. - -With the optional `smb` feature, `SmbRemote::new(client, path, limit)` binds an -`onestore_smb::Client` to one share-relative file and snapshot limit. Remote identity uses the logical root -object space, which survives the tested native compaction that replaces the file ID. - -An `Arc` can own one background worker. Supply a connection factory, poll -interval and observer; successful publications drain immediately, durable local -edits wake the worker, and `wake()` requests an immediate reachability retry. -Transport failures discard the old connection and retry through the factory; -Read contention and `NotCommitted` operations with `WouldBlock` or `ResourceBusy` -reuse the connection. Contended `NotCommitted` operations use randomized backoff, -capped at one second, to separate competing retry cycles. Cancellation interrupts this delay; local wake notifications -remain coalesced until its end. -`RemoteIo` distinguishes connection/read failures from -local `Io` errors. Cache/document errors stop the worker. Observers receive every -attempt's result on the worker thread, including unchanged conflict/uncertain -statuses; durable edit state remains available through `status`. - -```no_run -# #[cfg(feature = "smb")] -# fn example(cache: std::sync::Arc, username: String, password: String) -# -> Result<(), Box> { -use onestore_offline::SmbRemote; -use onestore_smb::{Client, Credentials}; -use std::time::Duration; - -let worker = cache.start_sync( - Duration::from_secs(2), - move || { - Client::connect( - "server:445", "notes", - Credentials { username: &username, password: &password, domain: "" }, - Duration::from_secs(5), - ).map(|client| SmbRemote::new(client, "Personal/Video.one", 64 * 1024 * 1024)) - }, - |result| { - if let Err(error) = result { eprintln!("{error}"); } - }, -)?; -// Retain `worker` while synchronization should run; local edits wake it automatically. -worker.stop()?; -# Ok(()) -# } -``` - -`stop()` cancels future steps and joins the worker, returning a fatal cache error -or worker panic. Dropping it requests cancellation without waiting. An in-flight -step completes before releasing ownership; a replacement worker cannot start -while the old one still owns the replica. Remote operations and callbacks must -have bounded execution times if shutdown latency matters. A dropped worker can -briefly retain the cache; use `stop()` before requiring an immediate reopen. -Cancellation and application restart retain pending edits and publication attempts. -Credentials belong to the factory, not the cache database. - -Creation refuses existing paths. Opening recognizes the application identity and -schema version, validates database integrity and both notebook images, and rejects -unsupported journal modes without converting them. Failed initialization preserves -the file for inspection. SQLite uses DELETE journaling, EXTRA synchronization and -fullfsync; each required setting is queried back. - -The cache and its pending intents contain notebook content. The core `onestore` -crate stays independent of SQLite and network runtimes. Device and simulator -examples compile and link for iOS; recorded process-interruption tests do not -establish physical power-loss durability. Evidence is tracked in [Milestone 9](../../evidence/MILESTONE9.md). - -The SMB-enabled `smb_offline_client` example is an owned-lab workload for -`tools/native_collaboration.py --offline --embedded-smb`. It separates local -acknowledgements, remote publication attempts, persisted receipts and cache reopen -checks. Its append-specific conflict review policy lives in the test client; -the library continues to preserve conflicts requiring an explicit decision. - -## Recovery archives - -`export_recovery(new_path)` captures both complete notebook images, the typed -queue, uncertain attempts, conflicts, receipts, downloaded media and the edit-ID sequence in one -SQLite snapshot. It refuses existing destinations and leaves the live queue -unchanged. Export to a local directory from a background thread: copying holds -the cache mutex while capturing the database. Failure after the final rename can -leave a complete archive at the requested path; it never acknowledges a remote -edit. Archives contain notebook content and use a separate database identity, so -`Replica::open` rejects them as writable caches. - -```no_run -use onestore_offline::{Recovery, Replica}; -# fn example(cache: &Replica) -> Result<(), Box> { -cache.export_recovery("review.sqlite")?; -let review = Recovery::open("review.sqlite")?; -let counts = review.summary()?; -let local = review.snapshot()?; -let remote = review.remote_snapshot()?; -let pending = review.pending()?; -let receipts = review.receipts()?; -# Ok(()) -# } -``` - -`Recovery` provides read-only inspection and no synchronization or restore method. -`status(id)` preserves the same state interpretation as the live replica. -`recovery_summary()` on the live replica and `summary()` on an archive return -counts and byte sizes without notebook text, paths, authors or credentials. -Opening an archive validates its schema and images without migration. A recovery -archive is evidence for a reviewed recovery decision, not a second active queue. - -## Downloaded media - -`fetch_asset(source, section, filename, limit)` resolves a declared external -payload through `onestore-notebook::Source` and durably caches its exact bytes. -The section path is relative to the source root; the filename comes from a -`FileDataReference::External` in the retained working or remote image. Local and -SMB sources use the same API. Downloads release the cache mutex during network -I/O and recheck the reference before committing; edits can continue meanwhile. - -`cached_asset(filename, limit)` reads previously downloaded bytes without network -access. `None` means never downloaded; `Some(Vec::new())` is a downloaded empty -payload. Each read checks the stored SHA-256 and enforces the byte limit before -loading the payload. Cache contents describe the prior download, not current -server reachability or presence. A failed fetch returns its error without -silently substituting cached bytes. Different bytes for an already cached file -identity return `AssetChanged` and preserve the previous download. - -Downloads do not change pending edits, publication attempts or receipts. Recovery -archives include cached media and expose the same bounded `cached_asset` lookup. -Schema-4 archives remain readable without migration and contain no media cache. diff --git a/crates/onestore-offline/examples/cache_probe.rs b/crates/onestore-offline/examples/cache_probe.rs deleted file mode 100644 index c27e00cf46fcc8da856d280dd9874cea30f89cca..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/examples/cache_probe.rs +++ /dev/null @@ -1,238 +0,0 @@ -use onestore::{ - ExGuid, Insertion, RevisionIndex, Store, TextAttribute, - document::{Document, Kind}, -}; -use onestore_offline::Replica; -use std::{ - io::{self, BufRead, Write}, - path::Path, -}; - -fn content(bytes: &[u8]) -> (ExGuid, ExGuid, String) { - let store = Store::parse(bytes).unwrap(); - assert!(store.checksum_mismatches.is_empty()); - let index = RevisionIndex::parse(&store).unwrap(); - index.validate_current().unwrap(); - let document = Document::parse(&index).unwrap(); - document - .spaces - .iter() - .find_map(|(sid, space)| { - let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; - revision - .nodes - .iter() - .find_map(|(oid, node)| match &node.kind { - Kind::RichText { text, .. } => Some((*sid, *oid, text.clone())), - _ => None, - }) - }) - .unwrap() -} - -fn payload(operation: u64, size: usize) -> String { - format!("{operation}:🦀{}", "x".repeat(size)) -} - -fn main() -> Result<(), Box> { - let args: Vec<_> = std::env::args().collect(); - let mode = &args[1]; - let path = Path::new(&args[2]); - let operation_kind = - std::env::var("ONESTORE_CACHE_PROBE_OPERATION").unwrap_or_else(|_| "text".into()); - assert!(matches!( - operation_kind.as_str(), - "text" | "insert" | "format" - )); - let size = match std::env::var("ONESTORE_CACHE_PROBE_BYTES") { - Ok(value) => value.parse::()?, - Err(std::env::VarError::NotPresent) => 2 * 1024 * 1024, - Err(error) => return Err(error.into()), - }; - assert!(size > 0 && size <= 2 * 1024 * 1024); - let seed = std::env::var_os("ONESTORE_CACHE_PROBE_SOURCE") - .map(std::fs::read) - .transpose()?; - if mode == "init" { - let source = match seed { - Some(source) => source, - None => onestore::create_section( - "cache.one", - &if operation_kind == "format" { - payload(0, size) - } else { - "Base".into() - }, - "Fixture", - )?, - }; - Replica::create(path, &source)?; - return Ok(()); - } - let cache = Replica::open(path)?; - if mode == "read" { - let snapshot = cache.snapshot()?; - let base = cache.remote_snapshot()?; - let (sid, target, mut expected) = content(&base); - let store = Store::parse(&snapshot)?; - assert!(store.checksum_mismatches.is_empty()); - let index = RevisionIndex::parse(&store)?; - index.validate_current()?; - let document = Document::parse(&index)?; - let space = &document.spaces[&sid]; - let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; - let mut operations = Vec::new(); - let mut ids = Vec::new(); - let mut font_size = None; - for pending in cache.pending()? { - let operation = match pending.operation { - onestore_offline::Operation::Text(edit) => { - assert_eq!(operation_kind, "text"); - assert_eq!(edit.object, target); - assert_eq!(edit.before, expected); - assert_eq!( - edit.range, - 0..u32::try_from(expected.encode_utf16().count())? - ); - let operation: u64 = edit.replacement.split_once(':').unwrap().0.parse()?; - expected = payload(operation, size); - assert_eq!(edit.replacement, expected); - operation - } - onestore_offline::Operation::Insert(insertion) => { - assert_eq!(operation_kind, "insert"); - let Kind::RichText { text, .. } = - &revision.nodes[&insertion.text_object()].kind - else { - panic!("Missing inserted text") - }; - let operation: u64 = text.split_once(':').unwrap().0.parse()?; - assert_eq!(*text, payload(operation, size)); - assert_eq!(serde_json::to_value(&insertion)?["text"], *text); - let outline = &revision.nodes[&insertion.object()]; - assert!(matches!(outline.kind, Kind::Outline { .. })); - assert_eq!( - (outline.layout.x, outline.layout.y), - (Some(144.0), Some(operation as f32 * 72.0)) - ); - let (_, page) = document - .pages()? - .into_iter() - .find(|(space, _)| *space == sid) - .unwrap(); - assert!(revision.nodes[&page].children.contains(&insertion.object())); - operation - } - onestore_offline::Operation::Format(edit) => { - assert_eq!(operation_kind, "format"); - assert_eq!(edit.object, target); - assert_eq!(edit.before, expected); - assert_eq!( - edit.range, - 0..u32::try_from(expected.encode_utf16().count())? - ); - let [TextAttribute::FontSize(value)] = edit.attributes.as_slice() else { - panic!("Unexpected formatting intent") - }; - assert!((7.0..=130.0).contains(value) && value.fract() == 0.0); - font_size = Some(*value); - *value as u64 - 6 - } - onestore_offline::Operation::Split(_) - | onestore_offline::Operation::CreatePage(_) - | onestore_offline::Operation::Pages(_) - | onestore_offline::Operation::Join(_) - | onestore_offline::Operation::Outline(_) - | onestore_offline::Operation::Tree(_) => { - panic!("Unexpected operation for this fixture") - } - }; - assert!(operations.last().is_none_or(|last| *last < operation)); - assert!(ids.last().is_none_or(|last| *last < pending.id)); - operations.push(operation); - ids.push(pending.id); - } - assert!(matches!(&revision.nodes[&target].kind,Kind::RichText{text,..} if *text==expected)); - if let Some(font_size) = font_size { - assert!( - revision - .text_runs(target)? - .iter() - .all(|run| run.format.font_size == Some(font_size)) - ); - } - if operations.is_empty() { - assert!( - snapshot == base, - "An empty local queue changed its working image" - ); - } - if let Some(output) = args.get(3) { - std::fs::write(output, &snapshot)?; - } - println!( - "{}", - serde_json::json!({"operations": operations, "ids": ids, "section_bytes": snapshot.len(), "complete_payloads": true}) - ); - return Ok(()); - } - assert_eq!(mode, "edit"); - assert!( - matches!(Replica::open(path), Err(onestore_offline::Error::Database(error)) if error.sqlite_error_code() == Some(rusqlite::ErrorCode::DatabaseBusy)) - ); - println!("ready"); - io::stdout().flush()?; - let input = io::stdin(); - let mut lines = input.lock().lines(); - let instruction = lines.next().unwrap()?; - let (operation, acknowledgement) = instruction.split_once(' ').unwrap(); - let operation: u64 = operation.parse()?; - let source = cache.snapshot()?; - let (sid, oid, text) = content(&source); - let replacement = payload(operation, size); - println!("editing {operation}"); - io::stdout().flush()?; - let id = match operation_kind.as_str() { - "text" => cache.edit_text( - &source, - sid, - oid, - 0..text.encode_utf16().count().try_into()?, - &replacement, - )?, - "insert" => { - let store = Store::parse(&source)?; - let index = RevisionIndex::parse(&store)?; - let document = Document::parse(&index)?; - let (sid, page) = document.pages()?[0]; - let insertion = Insertion::outline( - page, - 144.0, - operation as f32 * 72.0, - &replacement, - "Fixture", - )?; - cache.insert(&source, sid, &insertion)? - } - "format" => { - assert!((1..=124).contains(&operation)); - cache.format( - &source, - sid, - oid, - 0..text.encode_utf16().count().try_into()?, - &[TextAttribute::FontSize(6.0 + operation as f32)], - )? - } - _ => unreachable!(), - } - .unwrap(); - if acknowledgement == "unack" { - println!("durable {operation} {id}"); - } else { - println!("ack {operation} {id}"); - } - io::stdout().flush()?; - lines.next().transpose()?; - Ok(()) -} diff --git a/crates/onestore-offline/examples/recovery_probe.rs b/crates/onestore-offline/examples/recovery_probe.rs deleted file mode 100644 index 60fc68a028e5a3a72bf6b2cf9ac861960a861556..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/examples/recovery_probe.rs +++ /dev/null @@ -1,160 +0,0 @@ -mod support { - pub mod view; -} -use support::view::view; - -use onestore::{CommitError, CommitIo, PreparedEdit}; -use onestore_offline::{EditStatus, Remote, Replica}; -use serde_json::json; -use std::{ - env, - fs::{self, File, OpenOptions}, - io::{self, Write}, - os::unix::fs::FileExt, - path::Path, -}; - -fn phase(name: &str) { - println!("{}", json!({"event":"phase", "name":name})); - io::stdout().flush().unwrap(); - if env::var("ONESTORE_RECOVERY_PAUSE").ok().as_deref() == Some(name) { - let mut line = String::new(); - assert!( - io::stdin().read_line(&mut line).unwrap() > 0, - "Controller closed a paused operation" - ); - } -} - -struct Disk { - file: File, - writes: usize, - flushes: usize, -} - -impl CommitIo for Disk { - fn read_at(&mut self, offset: u64, output: &mut [u8]) -> io::Result { - self.file.read_at(output, offset) - } - fn write_at(&mut self, offset: u64, data: &[u8]) -> io::Result { - self.writes += 1; - println!( - "{}", - json!({"event":"write", "number":self.writes, "offset":offset, "bytes":data.len()}) - ); - phase(&format!("write-{}-before", self.writes)); - let result = self.file.write_at(data, offset); - phase(&format!("write-{}-after", self.writes)); - result - } - fn flush(&mut self) -> io::Result<()> { - self.flushes += 1; - phase(&format!("flush-{}-before", self.flushes)); - let result = self.file.sync_all(); - phase(&format!("flush-{}-after", self.flushes)); - result - } -} - -impl Remote for Disk { - fn read(&mut self) -> io::Result> { - phase("read-before"); - let result = onestore::read_snapshot( - |offset, output| self.file.read_at(output, offset), - 256 * 1024 * 1024, - ) - .and_then(|snapshot| snapshot.ok_or_else(|| io::ErrorKind::WouldBlock.into())); - phase("read-after"); - result - } - fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { - phase("publish-before"); - let result = edit.commit(self); - phase("publish-after"); - result - } - fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { - phase("confirm-before"); - let result = onestore::confirm_snapshot(self, snapshot); - phase("confirm-after"); - result - } -} - -fn report(cache: &Replica, root: &Path) -> Result<(), Box> { - let local = view(&cache.snapshot()?)?; - let remote = view(&fs::read(root.join("remote.one"))?)?; - let (status, revision) = match cache.status(1)? { - Some(EditStatus::Pending) => ("pending", None), - Some(EditStatus::AwaitingConfirmation { revision }) => { - ("uncertain", Some(revision.to_string())) - } - Some(EditStatus::Published { revision }) => ("published", Some(revision.to_string())), - Some(EditStatus::Conflict(_)) => ("conflict", None), - None => ("missing", None), - }; - println!( - "{}", - json!({"event":"state", "status":status, "revision":revision, - "local_text":local.text, "remote_text":remote.text, "remote_revision":remote.revision.to_string(), - "pending":cache.pending()?.iter().map(|pending|match &pending.operation { - onestore_offline::Operation::Text(edit) => json!({"id":pending.id,"before":edit.before,"replacement":edit.replacement,"range":[edit.range.start,edit.range.end]}), - operation => json!({"id":pending.id,"operation":operation}), - }).collect::>() }) - ); - Ok(()) -} - -fn main() -> Result<(), Box> { - let args: Vec<_> = env::args().skip(1).collect(); - let mode = args - .first() - .ok_or("Expected init|inspect|sync DIRECTORY [SOURCE]")?; - let root = Path::new(args.get(1).ok_or("Missing owned directory")?); - if mode == "init" && args.len() == 3 { - let source = fs::read(&args[2])?; - let target = view(&source)?; - fs::create_dir(root)?; - let mut remote = OpenOptions::new() - .write(true) - .create_new(true) - .open(root.join("remote.one"))?; - remote.write_all(&source)?; - remote.sync_all()?; - drop(remote); - let cache = Replica::create(root.join("cache.sqlite"), &source)?; - let at = u32::try_from(target.text.encode_utf16().count())?; - assert_eq!( - cache.edit_text( - &source, - target.space, - target.object, - at..at, - " [offline-recovery]" - )?, - Some(1) - ); - phase("local-after"); - report(&cache, root)?; - } else if args.len() == 2 && ["inspect", "sync"].contains(&mode.as_str()) { - let cache = Replica::open(root.join("cache.sqlite"))?; - if mode == "sync" { - let mut disk = Disk { - file: OpenOptions::new() - .read(true) - .write(true) - .open(root.join("remote.one"))?, - writes: 0, - flushes: 0, - }; - phase("sync-before"); - let result = cache.sync_once(&mut disk); - phase("sync-after"); - result?; - } - report(&cache, root)?; - } else { - return Err("Expected init|inspect|sync DIRECTORY [SOURCE]".into()); - } - Ok(()) -} diff --git a/crates/onestore-offline/examples/smb_offline_client.rs b/crates/onestore-offline/examples/smb_offline_client.rs deleted file mode 100644 index 4440ebb379af1b2f9c7c252b0f098919ff6edb41..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/examples/smb_offline_client.rs +++ /dev/null @@ -1,863 +0,0 @@ -#[path = "../../onestore/examples/support/concurrent.rs"] -mod concurrent; - -use onestore::{ - CommitError, CommitState, ExGuid, Insertion, ParagraphJoin, ParagraphSplit, PreparedEdit, - RevisionIndex, Store, TextAttribute, TreeEdit, document::Document, -}; -use onestore_offline::{EditStatus, Error, Remote, Replica, SmbRemote}; -use onestore_smb::{Client, Credentials}; -use serde_json::json; -use std::{ - env, - io::{self, Write}, - path::{Path, PathBuf}, - sync::Arc, - thread, - time::{Duration, Instant, SystemTime, UNIX_EPOCH}, -}; - -mod support { - pub mod view; -} -use concurrent::{DocumentView, document_view}; -use support::view::view; - -impl DocumentView { - fn changes(&self, before: &Self) -> Self { - let difference = - |old: &serde_json::Map, - new: &serde_json::Map| { - old.keys() - .chain(new.keys()) - .filter(|id| old.get(*id) != new.get(*id)) - .map(|id| (id.clone(), new.get(id).cloned().unwrap_or_default())) - .collect() - }; - Self { - texts: difference(&before.texts, &self.texts), - graph: difference(&before.graph, &self.graph), - } - } -} - -const DOCUMENT_OPERATIONS: [&str; 10] = [ - "insert", - "format", - "text", - "split", - "right_text", - "nest", - "unnest", - "join", - "tail_split", - "delete", -]; - -fn now() -> u128 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_micros() -} - -#[derive(Clone)] -enum Pause { - Outage(PathBuf), - FormatReply(PathBuf), -} - -struct Traced { - remote: R, - before: Option<(String, Option)>, - pause: Option, - documents: bool, -} - -impl Remote for Traced { - fn read(&mut self) -> io::Result> { - let started = now(); - let bytes = self.remote.read()?; - let observed = view(&bytes).map_err(|error| io::Error::other(error.to_string()))?; - let documents = if self.documents { - Some(document_view(&bytes).map_err(io::Error::other)?) - } else { - None - }; - println!( - "{}", - json!({"event":"read", "started_us":started, "finished_us":now(), "text":observed.text, "documents":documents.as_ref().map(|view| &view.texts), "document_graph":documents.as_ref().map(|view| &view.graph)}) - ); - self.before = Some((observed.text, documents)); - Ok(bytes) - } - fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { - let after = view(edit.as_bytes()).map_err(|error| CommitError { - state: CommitState::NotCommitted, - error: io::Error::other(error.to_string()), - })?; - let documents = if self.documents { - Some(document_view(edit.as_bytes()).map_err(|error| CommitError { - state: CommitState::NotCommitted, - error: io::Error::other(error), - })?) - } else { - None - }; - let changes = if let Some(after) = &documents { - let before = self - .before - .as_ref() - .and_then(|(_, documents)| documents.as_ref()) - .ok_or_else(|| CommitError { - state: CommitState::NotCommitted, - error: io::Error::other("Document publication has no observed source"), - })?; - Some(after.changes(before)) - } else { - None - }; - let pause = match &self.pause { - Some(Pause::Outage(marker)) => Some(( - marker, - marker.parent().unwrap().join("offline-outage-resumed"), - "outage", - )), - Some(Pause::FormatReply(marker)) - if changes.as_ref().is_some_and(|changes| { - changes.texts.len() == 1 - && self - .before - .as_ref() - .and_then(|(_, before)| before.as_ref()) - .is_some_and(|before| { - changes.texts.keys().all(|id| before.texts.contains_key(id)) - }) - }) => - { - Some((marker, marker.with_extension("resume"), "format")) - } - _ => None, - }; - if let Some((marker, resumed, kind)) = pause - && !resumed.exists() - { - std::fs::write(marker, after.revision.to_string()).map_err(|error| CommitError { - state: CommitState::NotCommitted, - error, - })?; - println!( - "{}", - json!({"event":"publication_paused", "revision":after.revision.to_string(), "kind":kind, "at_us":now()}) - ); - let deadline = Instant::now() + Duration::from_secs(120); - while !resumed.exists() { - if Instant::now() >= deadline { - return Err(CommitError { - state: CommitState::NotCommitted, - error: io::Error::new( - io::ErrorKind::TimedOut, - "Offline publication barrier timed out", - ), - }); - } - thread::sleep(Duration::from_millis(10)); - } - } - let started = now(); - let result = self.remote.publish(edit); - let finished = now(); - println!( - "{}", - json!({"event":"remote_attempt", "started_us":started, "finished_us":finished, - "revision":after.revision.to_string(), "space":after.space.to_string(), "object":after.object.to_string(), "before":self.before.as_ref().map(|(text,_)|text), "after":after.text, - "state":format!("{:?}", result.as_ref().map_or_else(|error| error.state, |_| CommitState::Committed)), - "documents":documents.as_ref().map(|view| &view.texts), "document_graph":documents.as_ref().map(|view| &view.graph), - "document_changes":changes.as_ref().map(|view| &view.texts), "document_graph_changes":changes.as_ref().map(|view| &view.graph)}) - ); - if result - .as_ref() - .is_err_and(|error| error.state == CommitState::Unknown) - && let Some(Pause::FormatReply(marker)) = &self.pause - && marker.with_extension("isolate").exists() - { - std::fs::write(marker.with_extension("isolate"), serde_json::to_vec(&json!({"space":after.space.to_string(), "revision":after.revision.to_string(), "after_us":finished})).unwrap()) - .map_err(|error| CommitError { state:CommitState::Unknown, error })?; - println!( - "{}", - json!({"event":"confirmation_paused", "revision":after.revision.to_string(), "at_us":now()}) - ); - let deadline = Instant::now() + Duration::from_secs(120); - while !marker.with_extension("confirmation-resume").exists() { - if Instant::now() >= deadline { - return Err(CommitError { - state: CommitState::Unknown, - error: io::Error::new( - io::ErrorKind::TimedOut, - "Offline confirmation barrier timed out", - ), - }); - } - thread::sleep(Duration::from_millis(10)); - } - } - result - } - fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { - let captured = (|| -> Result<_, Box> { - let store = Store::parse(snapshot)?; - let index = RevisionIndex::parse(&store)?; - let revisions = index - .spaces - .iter() - .map(|(id, space)| { - ( - id.to_string(), - space - .revisions - .keys() - .map(ToString::to_string) - .collect::>(), - ) - }) - .collect::>(); - let current = index - .spaces - .iter() - .filter_map(|(id, space)| { - space - .labels - .get(&(ExGuid::default(), 1)) - .map(|revision| (id.to_string(), revision.to_string())) - }) - .collect::>(); - let path = env::var_os("ONESTORE_OFFLINE_CONFIRM_DIR").map(|directory| { - PathBuf::from(directory).join(format!("{}-{}.one", std::process::id(), now())) - }); - if let Some(path) = &path { - std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(path)? - .write_all(snapshot)?; - } - Ok((revisions, current, path)) - })(); - let (revisions, current, capture) = captured.map_err(|error| CommitError { - state: CommitState::NotCommitted, - error: io::Error::other(error.to_string()), - })?; - let started = now(); - let result = self.remote.confirm(snapshot); - println!( - "{}", - json!({"event":"remote_confirm", "started_us":started, "finished_us":now(), "revisions":revisions, "current_revisions":current, "capture":capture.as_ref().and_then(|path| path.file_name()).map(|name| name.to_string_lossy()), "text":self.before.as_ref().map(|(text,_)|text), - "state":format!("{:?}", result.as_ref().map_or_else(|error| error.state, |_| CommitState::Committed)), "error":result.as_ref().err().map(|error|error.error.to_string())}) - ); - result - } -} - -fn tokens(text: &str) -> Option> { - let mut remaining = text.strip_prefix("Concurrent edits:")?; - let mut tokens = Vec::new(); - while !remaining.is_empty() { - let body = remaining.strip_prefix(" [w")?; - let (token, tail) = body.split_once(']')?; - let (actor, operation) = token.split_once(':')?; - if actor.is_empty() - || operation.is_empty() - || !actor - .bytes() - .chain(operation.bytes()) - .all(|b| b.is_ascii_digit()) - || tokens.contains(&token) - { - return None; - } - tokens.push(token); - remaining = tail; - } - Some(tokens) -} - -// This owned workload explicitly resolves append conflicts after all retained tokens. -fn append_position(before: &str, current: &str, token: &str) -> Option { - let original = tokens(before)?; - let present = tokens(current)?; - let mut retained = present.iter(); - for token in original { - retained.find(|&&candidate| candidate == token)?; - } - if current.contains(token) { - return None; - } - u32::try_from(current.encode_utf16().count()).ok() -} - -fn queue_document( - cache: &Replica, - actor: &str, - operation: usize, - parent: Option, - deadline: Instant, -) -> Result<(ExGuid, [u64; DOCUMENT_OPERATIONS.len()]), Box> { - let source = cache.snapshot()?; - let store = Store::parse(&source)?; - let index = RevisionIndex::parse(&store)?; - let document = Document::parse(&index)?; - let (space, page) = document.pages()?[0]; - let text = format!("Document {actor}:{operation} 🦀"); - let insertion = if operation.is_multiple_of(2) { - let column: u32 = actor - .strip_prefix('w') - .ok_or("Missing writer number")? - .parse()?; - Insertion::outline( - page, - 144.0 + column as f32 * 240.0, - 144.0 + operation as f32 * 72.0, - &text, - "Offline document writer", - )? - } else { - Insertion::paragraph( - parent.ok_or("Missing prior outline")?, - None, - &text, - "Offline document writer", - )? - } - .with_formatting( - 0..u32::try_from(text.encode_utf16().count())?, - &[ - TextAttribute::FontSize(13.5), - TextAttribute::Color(Some([0x44, 0x55, 0x66])), - ], - )?; - let parent = if operation.is_multiple_of(2) { - insertion.object() - } else { - parent.unwrap() - }; - let end = u32::try_from(text.encode_utf16().count())?; - let split = ParagraphSplit::new(insertion.text_object(), end - 2, "Offline document writer")?; - let tail_split = - ParagraphSplit::new(insertion.text_object(), end + 1, "Offline document writer")?; - let join = ParagraphJoin::new( - insertion.text_object(), - split.text_object(), - "Offline document writer", - )?; - let attributes = [ - TextAttribute::Bold(true), - TextAttribute::FontSize(18.0 + (operation % 9) as f32), - TextAttribute::Color(Some([0x12, 0x34, 0x56])), - ]; - let mut ids = [0; DOCUMENT_OPERATIONS.len()]; - for (step, id) in ids.iter_mut().enumerate() { - let kind = DOCUMENT_OPERATIONS[step]; - let tree = match kind { - "nest" => { - let source = cache.snapshot()?; - let store = Store::parse(&source)?; - let index = RevisionIndex::parse(&store)?; - let document = Document::parse(&index)?; - let section = &document.spaces[&space]; - let view = §ion.revisions[§ion.contexts[&ExGuid::default()]]; - let paragraph = view - .nodes - .iter() - .find(|(_, node)| node.content == [insertion.text_object()]) - .map(|(id, _)| *id) - .ok_or("Missing inserted paragraph")?; - Some(TreeEdit::move_to( - split.object(), - paragraph, - None, - "Offline document writer", - )?) - } - "unnest" => Some(TreeEdit::move_to( - split.object(), - parent, - None, - "Offline document writer", - )?), - "delete" => Some(TreeEdit::delete( - tail_split.object(), - "Offline document writer", - )?), - _ => None, - }; - let range = match kind { - "text" => end - 3..end, - "split" => end - 2..end - 2, - "tail_split" => end + 1..end + 1, - "right_text" => 0..2, - _ => 1..end - 2, - }; - let target = if kind == "right_text" { - split.text_object() - } else if kind == "delete" { - tail_split.text_object() - } else { - insertion.text_object() - }; - let replacement = match kind { - "text" => Some(" e\u{301}🐈"), - "right_text" => Some("B🦋"), - _ => None, - }; - loop { - if Instant::now() >= deadline { - return Err("Document queue timed out; cache retained".into()); - } - let source = cache.snapshot()?; - let started = now(); - let result = match kind { - "insert" => cache.insert(&source, space, &insertion), - "format" => cache.format(&source, space, target, range.clone(), &attributes), - "text" | "right_text" => { - cache.edit_text(&source, space, target, range.clone(), replacement.unwrap()) - } - "split" => cache.split(&source, space, &split), - "tail_split" => cache.split(&source, space, &tail_split), - "join" => cache.join(&source, space, &join), - "nest" | "unnest" | "delete" => cache.tree(&source, space, tree.as_ref().unwrap()), - _ => unreachable!(), - }; - match result { - Ok(Some(acknowledged)) => { - *id = acknowledged; - println!( - "{}", - json!({"event":"local_document_commit","id":acknowledged,"operation":operation,"kind":kind, - "space":space.to_string(),"object":target.to_string(),"document":insertion.text_object().to_string(), - "text":text,"insertion":if kind=="insert" {Some(&insertion)} else {None}, - "split":match kind { "split" => Some(&split), "tail_split" => Some(&tail_split), _ => None }, - "tree":tree, - "joined":if kind=="join" {Some(join.texts().map(|id| id.to_string()))} else {None}, - "range":[range.start,range.end],"attributes":attributes,"replacement":replacement, - "started_us":started,"finished_us":now()}) - ); - break; - } - Err(Error::Io(error)) if error.kind() == io::ErrorKind::ResourceBusy => {} - other => return Err(format!("Unexpected document queue result: {other:?}").into()), - } - } - } - Ok((parent, ids)) -} - -fn main() -> Result<(), Box> { - let args: Vec<_> = env::args().skip(1).collect(); - if args.len() != 7 - || !["read", "write"].contains(&args[0].as_str()) - || !args[2].bytes().all(|b| b.is_ascii_alphanumeric()) - { - return Err("Expected read|write FILE ACTOR OPERATIONS START_FILE STOP_FILE SEED".into()); - } - let documents = env::var_os("ONESTORE_OFFLINE_DOCUMENTS").is_some(); - let address = env::var("ONESTORE_SMB_LAB")?; - let share = env::var("ONESTORE_SMB_SHARE")?; - let initial = Client::connect( - &address, - &share, - Credentials::default(), - Duration::from_secs(5), - )?; - if args[0] == "read" { - return concurrent::run( - &args, - |path| initial.read(path, 256 * 1024 * 1024), - |_, _, _, _, _, _| unreachable!(), - ); - } - let timeout: u64 = env::var("ONESTORE_CLIENT_TIMEOUT_MS") - .unwrap_or_else(|_| "600000".into()) - .parse()?; - let deadline = Instant::now() - .checked_add(Duration::from_millis(timeout)) - .ok_or("Invalid timeout")?; - let operations: usize = args[3].parse()?; - let mut seed: u64 = args[6].parse()?; - if operations == 0 { - return Err("Expected positive operations".into()); - } - let source = initial.read(&args[1], 256 * 1024 * 1024)?; - drop(initial); - let cache_path = Path::new(&args[4]) - .parent() - .ok_or("Missing workload directory")? - .join(format!("{}.sqlite", args[2])); - let cache = Arc::new(Replica::create(&cache_path, &source)?); - println!( - "{}", - json!({"event":"ready", "pid":std::process::id(), "actor":args[2], "offline":true, "document_operations":documents,"document_graph":documents,"document_kinds":if documents {DOCUMENT_OPERATIONS.as_slice()} else {&[]}}) - ); - while !Path::new(&args[4]).exists() { - if Instant::now() >= deadline { - return Err("Start barrier timed out".into()); - } - thread::sleep(Duration::from_millis(5)); - } - let path = args[1].clone(); - let outage = env::var_os("ONESTORE_OFFLINE_OUTAGE_DIR").map(PathBuf::from); - let pause = outage - .as_ref() - .map(|directory| Pause::Outage(directory.join(format!("offline-paused-{}", args[2])))) - .or_else(|| { - env::var_os("ONESTORE_OFFLINE_FORMAT_REPLY_DIR").map(|directory| { - Pause::FormatReply( - PathBuf::from(directory).join(format!("offline-paused-{}", args[2])), - ) - }) - }); - let (fatal_tx, fatal_rx) = std::sync::mpsc::channel(); - let worker = cache.start_sync(Duration::from_millis(50), move || { - let client = Client::connect(&address, &share, Credentials::default(), Duration::from_secs(5))?; - println!("{}", json!({"event":"transport_connected", "at_us":now()})); - Ok(Traced { remote: SmbRemote::new(client, &path, 256 * 1024 * 1024), before:None, pause:pause.clone(), documents }) - }, move |result| { - if let Err(error) = result { - println!("{}", json!({"event":"sync_error", "error":error.to_string(), "at_us":now()})); - if !matches!(error, Error::RemoteIo(_) | Error::Remote(_)) && !matches!(error, Error::Io(error) if error.kind() == io::ErrorKind::WouldBlock) { - let _ = fatal_tx.send(error.to_string()); - } - } - })?; - let mut ids = Vec::new(); - let mut document_ids = std::collections::BTreeSet::new(); - let mut document_parent = None; - let result = (|| -> Result<(), Box> { - let mut generated = 0; - let mut received = 0; - loop { - match fatal_rx.try_recv() { - Ok(error) => return Err(error.into()), - Err(std::sync::mpsc::TryRecvError::Disconnected) => { - return Err("Worker exited before completion".into()); - } - Err(std::sync::mpsc::TryRecvError::Empty) => {} - } - while received < ids.len() { - match cache.status(ids[received])? { - Some(EditStatus::Published { revision }) => { - println!( - "{}", - json!({"event":if document_ids.contains(&ids[received]) {"document_receipt"} else {"remote_receipt"}, "id":ids[received], "revision":revision.to_string(), "at_us":now()}) - ); - received += 1; - } - Some(_) => break, - None => return Err("Local intent disappeared".into()), - } - } - if Instant::now() >= deadline { - return Err("Offline workload timed out; cache retained".into()); - } - let pending = cache.pending()?; - let capacity = if outage - .as_ref() - .is_some_and(|directory| !directory.join("offline-outage-down").exists()) - { - 1 - } else if documents && outage.is_some() { - 8 * (1 + DOCUMENT_OPERATIONS.len()) - } else { - 8 - }; - if generated < operations && pending.len() < capacity { - let source = cache.snapshot()?; - let target = view(&source)?; - let at = u32::try_from(target.text.encode_utf16().count())?; - let token = format!(" [{}:{}]", args[2], generated); - let started = now(); - match cache.edit_text(&source, target.space, target.object, at..at, &token) { - Ok(Some(id)) => { - println!( - "{}", - json!({"event":"local_commit", "id":id, "operation":generated, "space":target.space.to_string(), "object":target.object.to_string(), "before":target.text, "token":token, "started_us":started, "finished_us":now()}) - ); - ids.push(id); - if documents { - let (parent, added) = queue_document( - &cache, - &args[2], - generated, - document_parent, - deadline, - )?; - document_parent = Some(parent); - document_ids.extend(added); - ids.extend(added); - } - generated += 1; - } - Err(Error::Io(error)) if error.kind() == io::ErrorKind::ResourceBusy => {} - other => return Err(format!("Unexpected local result: {other:?}").into()), - } - } - if let Some(intent) = pending.first() - && matches!(cache.status(intent.id)?, Some(EditStatus::Conflict(_))) - { - let local = cache.snapshot()?; - let remote = cache.remote_snapshot()?; - let current = view(&remote)?; - let onestore_offline::Operation::Text(edit) = &intent.operation else { - return Err(format!( - "Document intent {} requires review: {:?}", - intent.id, - cache.status(intent.id)? - ) - .into()); - }; - let at = append_position(&edit.before, ¤t.text, &edit.replacement) - .ok_or("Append model disagrees with retained history")?; - match cache.rebase_conflict(intent.id, &local, &remote, at..at) { - Ok(()) => println!( - "{}", - json!({"event":"reviewed_append", "id":intent.id, "before":edit.before, "remote":current.text, "token":edit.replacement, "at_us":now()}) - ), - Err(Error::Io(error)) - if [ - io::ErrorKind::ResourceBusy, - io::ErrorKind::WouldBlock, - io::ErrorKind::InvalidInput, - ] - .contains(&error.kind()) => {} - Err(error) => return Err(error.into()), - } - } - if generated == operations && received == ids.len() { - if !cache.pending()?.is_empty() { - return Err("Acknowledged queue did not drain".into()); - } - break; - } - seed = seed - .wrapping_mul(6364136223846793005) - .wrapping_add(1442695040888963407); - thread::sleep(Duration::from_millis(1 + (seed >> 32) % 7)); - } - Ok(()) - })(); - let stopped = worker.stop(); - result?; - stopped?; - drop(cache); - let reopened = Replica::open(&cache_path)?; - if !reopened.pending()?.is_empty() { - return Err("Pending edits reappeared after reopen".into()); - } - for id in ids { - let Some(EditStatus::Published { revision }) = reopened.status(id)? else { - return Err("Receipt did not survive reopen".into()); - }; - println!( - "{}", - json!({"event":if document_ids.contains(&id) {"reopened_document_receipt"} else {"reopened_receipt"}, "id":id, "revision":revision.to_string()}) - ); - } - println!( - "{}", - json!({"event":"done", "operations":operations, "at_us":now()}) - ); - Ok(()) -} - -#[test] -fn append_review_requires_a_unique_ordered_history_and_an_absent_new_token() { - assert_eq!( - append_position( - "Concurrent edits: [w0:0]", - "Concurrent edits: [w1:0] [w0:0]", - " [w0:1]" - ), - Some(31) - ); - for current in [ - "Concurrent edits:", - "Concurrent edits: [w0:0] [w0:0]", - "Concurrent edits: [w0:1] [w0:0]", - "Concurrent edits: changed [w0:0]", - ] { - assert_eq!( - append_position("Concurrent edits: [w0:0]", current, " [w0:1]"), - None - ); - } - assert_eq!( - append_position( - "Concurrent edits: [w0:0] [w1:0]", - "Concurrent edits: [w1:0] [w0:0]", - " [w0:1]" - ), - None - ); -} - -#[cfg(test)] -#[path = "../../onestore/tests/support/disk.rs"] -mod disk; - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn document_workload_retains_dependencies_and_receipts_across_reopen() { - struct Server { - disk: disk::Disk, - lost_reply: bool, - } - impl Remote for Server { - fn read(&mut self) -> io::Result> { - Ok(self.disk.visible.clone()) - } - fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { - edit.commit(&mut self.disk)?; - if std::mem::take(&mut self.lost_reply) { - Err(CommitError { - state: CommitState::Unknown, - error: io::Error::from(io::ErrorKind::ConnectionAborted), - }) - } else { - Ok(()) - } - } - fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { - onestore::confirm_snapshot(&mut self.disk, snapshot) - } - } - let source = - onestore::create_section("workload.one", "Concurrent edits:", "Author").unwrap(); - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("cache.sqlite"); - let mut cache = Replica::create(&path, &source).unwrap(); - let mut remote = Traced { - remote: Server { - disk: disk::Disk { - visible: source.clone(), - durable: source.clone(), - operation: 0, - fail_at: None, - write_limit: 97, - random: 1, - }, - lost_reply: false, - }, - before: None, - pause: None, - documents: true, - }; - println!( - "{}", - json!({"event":"ready", "actor":"w0", "offline":true, - "document_operations":true, "document_graph":true, "document_kinds":DOCUMENT_OPERATIONS}) - ); - let deadline = Instant::now() + Duration::from_secs(30); - let mut parent = None; - let mut ids = Vec::new(); - for operation in 0..2 { - let (outline, added) = - queue_document(&cache, "w0", operation, parent, deadline).unwrap(); - parent = Some(outline); - ids.extend(added); - drop(cache); - cache = Replica::open(&path).unwrap(); - } - let local = cache.snapshot().unwrap(); - assert_eq!( - cache.pending().unwrap().len(), - 2 * DOCUMENT_OPERATIONS.len() - ); - for (step, id) in ids.iter().enumerate() { - remote.remote.lost_reply = matches!( - DOCUMENT_OPERATIONS[step % DOCUMENT_OPERATIONS.len()], - "split" | "nest" | "unnest" | "join" | "tail_split" | "delete" - ); - let result = cache.sync_once(&mut remote); - let state = if result.is_err() { - assert!(matches!( - result, - Err(Error::Remote(CommitError { - state: CommitState::Unknown, - .. - })) - )); - drop(cache); - cache = Replica::open(&path).unwrap(); - cache.sync_once(&mut remote).unwrap().unwrap() - } else { - result.unwrap().unwrap() - }; - assert_eq!(state.0, *id); - let EditStatus::Published { revision } = state.1 else { - panic!("{state:?}") - }; - println!( - "{}", - json!({"event":"document_receipt", "id":id, "revision":revision.to_string(), "at_us":now()}) - ); - drop(cache); - cache = Replica::open(&path).unwrap(); - assert_eq!( - cache.status(*id).unwrap(), - Some(EditStatus::Published { revision }) - ); - if step + 1 < ids.len() { - assert_eq!(cache.snapshot().unwrap(), local); - } - } - assert!(cache.pending().unwrap().is_empty()); - for id in ids { - let Some(EditStatus::Published { revision }) = cache.status(id).unwrap() else { - panic!() - }; - println!( - "{}", - json!({"event":"reopened_document_receipt", "id":id, "revision":revision.to_string()}) - ); - } - remote.read().unwrap(); - println!("{}", json!({"event":"done"})); - } - - #[test] - fn publication_differences_retain_removed_text_and_graph_identities() { - let before = DocumentView { - texts: serde_json::from_value(json!({"left":{"text":"ab"}, "right":{"text":"cd"}, "untouched":{"text":"ef"}})).unwrap(), - graph: serde_json::from_value(json!({"outline":{"children":["a","b"]}, "a":{"content":["left"]}, "b":{"content":["right"]}})).unwrap(), - }; - let after = DocumentView { - texts: serde_json::from_value( - json!({"left":{"text":"abcd"}, "untouched":{"text":"ef"}}), - ) - .unwrap(), - graph: serde_json::from_value( - json!({"outline":{"children":["a"]}, "a":{"content":["left"]}}), - ) - .unwrap(), - }; - let changes = after.changes(&before); - assert_eq!( - changes.texts, - serde_json::from_value::>( - json!({"left":{"text":"abcd"}, "right":null}) - ) - .unwrap() - ); - assert_eq!( - changes.graph, - serde_json::from_value::>( - json!({"outline":{"children":["a"]}, "b":null}) - ) - .unwrap() - ); - assert_eq!(before.changes(&after).texts["right"], before.texts["right"]); - assert_eq!(before.changes(&before), DocumentView::default()); - } -} diff --git a/crates/onestore-offline/examples/support/view.rs b/crates/onestore-offline/examples/support/view.rs deleted file mode 100644 index 280be0443d90acca9f8db8dd3d7f0dc63d1b5ecc..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/examples/support/view.rs +++ /dev/null @@ -1,57 +0,0 @@ -use onestore::{ - ExGuid, RevisionIndex, Store, - document::{Document, Kind}, -}; - -pub struct View { - pub space: ExGuid, - pub object: ExGuid, - pub revision: ExGuid, - pub text: String, -} - -pub fn view(bytes: &[u8]) -> Result> { - let store = Store::parse(bytes)?; - if !store.checksum_mismatches.is_empty() { - return Err("Transaction checksum damage".into()); - } - let index = RevisionIndex::parse(&store)?; - index.validate_current()?; - let document = Document::parse(&index)?; - let mut found = Vec::new(); - for (sid, page) in document.pages()? { - let space = &document.spaces[&sid]; - let rid = space.contexts[&ExGuid::default()]; - let revision = &space.revisions[&rid]; - let mut pending = vec![page]; - let mut seen = std::collections::BTreeSet::new(); - while let Some(oid) = pending.pop() { - if !seen.insert(oid) { - continue; - } - let node = &revision.nodes[&oid]; - pending.extend( - node.children - .iter() - .chain(&node.content) - .chain(&node.structure) - .copied(), - ); - if let Kind::RichText { text, .. } = &node.kind - && text.starts_with("Concurrent edits:") - { - revision.text_runs(oid)?; - found.push(View { - space: sid, - object: oid, - revision: rid, - text: text.clone(), - }); - } - } - } - if found.len() != 1 { - return Err("Expected one concurrent-edit paragraph".into()); - } - Ok(found.pop().unwrap()) -} diff --git a/crates/onestore-offline/src/assets.rs b/crates/onestore-offline/src/assets.rs deleted file mode 100644 index 9f61197f3a188cfb90d3bc60dc65502595b8a7a9..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/src/assets.rs +++ /dev/null @@ -1,134 +0,0 @@ -use super::*; -use onestore::FileDataReference; -use rusqlite::OptionalExtension; -use sha2::{Digest, Sha256}; - -impl Replica { - /// Reads a previously downloaded external payload without network access. - /// Absence is distinct from an empty payload; every returned buffer passes its stored checksum. - pub fn cached_asset(&self, filename: &str, limit: usize) -> Result>> { - let key = key(filename)?; - let connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - cached(&connection, &key, limit) - } - - /// Fetches a declared external payload and durably retains it without changing the edit queue. - /// Different bytes for an already cached identity return `AssetChanged`, preserving the cache. - /// Network I/O does not hold the cache mutex; a stale reference fails before local publication. - pub fn fetch_asset( - &self, - source: &mut impl onestore_notebook::Source, - section: &str, - filename: &str, - limit: usize, - ) -> Result> { - let key = key(filename)?; - { - let connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - if !referenced(&connection, &key)? { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "The retained document images do not reference this external payload", - ) - .into()); - } - } - let bytes = onestore_notebook::read_external_asset(source, section, filename, limit)?; - let mut connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; - if !referenced(&transaction, &key)? { - return Err(io::Error::new( - io::ErrorKind::ResourceBusy, - "The external payload reference changed during download", - ) - .into()); - } - let previous = match cached(&transaction, &key, bytes.len()) { - Err(Error::Io(error)) if error.kind() == io::ErrorKind::FileTooLarge => { - return Err(Error::AssetChanged); - } - other => other?, - }; - if let Some(previous) = previous { - if previous != bytes { - return Err(Error::AssetChanged); - } - } else { - transaction.execute( - "INSERT INTO assets(name,data,sha256) VALUES (?1,?2,?3)", - params![key, &bytes, &Sha256::digest(&bytes)[..]], - )?; - } - transaction.commit()?; - Ok(bytes) - } -} - -pub(crate) fn key(filename: &str) -> Result { - format!("{filename}").parse::()?; - Ok(filename.to_ascii_lowercase()) -} - -fn referenced(connection: &Connection, key: &str) -> Result { - for column in ["working", "base"] { - let image: Vec = connection.query_row( - &format!("SELECT {column} FROM replica WHERE id=1"), - [], - |row| row.get(0), - )?; - let store = Store::parse(&image)?; - let index = RevisionIndex::parse(&store)?; - let document = Document::parse(&index)?; - if document.spaces.values().flat_map(|space| space.revisions.values()) - .flat_map(|revision| revision.nodes.values()).any(|node| { - matches!(&node.kind, Kind::File { reference: FileDataReference::External(name), .. } if name.eq_ignore_ascii_case(key)) - }) { - return Ok(true); - } - } - Ok(false) -} - -pub(crate) fn cached(connection: &Connection, key: &str, limit: usize) -> Result>> { - let version: u32 = connection.pragma_query_value(None, "user_version", |row| row.get(0))?; - if version < 5 { - return Ok(None); - } - let length: Option = connection - .query_row( - "SELECT length(data) FROM assets WHERE name=?1", - [key], - |row| row.get(0), - ) - .optional()?; - let Some(length) = length else { - return Ok(None); - }; - let length = - usize::try_from(length).map_err(|_| io::Error::from(io::ErrorKind::InvalidData))?; - if length > limit { - return Err(io::Error::from(io::ErrorKind::FileTooLarge).into()); - } - let (bytes, expected): (Vec, Vec) = connection.query_row( - "SELECT data,sha256 FROM assets WHERE name=?1", - [key], - |row| Ok((row.get(0)?, row.get(1)?)), - )?; - if bytes.len() != length || Sha256::digest(&bytes)[..] != expected { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Cached external payload checksum mismatch", - ) - .into()); - } - Ok(Some(bytes)) -} diff --git a/crates/onestore-offline/src/formatting.rs b/crates/onestore-offline/src/formatting.rs deleted file mode 100644 index c229f53032d87d21e919e5ec87d8b665608fd62f..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/src/formatting.rs +++ /dev/null @@ -1,203 +0,0 @@ -use super::*; -use onestore::TextAttribute; -use serde::{Deserialize, Serialize}; -use serde_json::{Value, json}; -use std::collections::BTreeMap; - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct Span { - end: u32, - values: Vec, -} - -/// A formatting intent and the text/attribute values observed before local publication. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct FormatEdit { - pub object: ExGuid, - pub before: String, - pub range: Range, - pub attributes: Vec, - observed: Vec, -} - -fn fields(attributes: &[TextAttribute]) -> BTreeMap<&'static str, Value> { - let mut fields = BTreeMap::new(); - for attribute in attributes { - let (name, value) = match attribute { - TextAttribute::Bold(v) => ("bold", json!(v)), - TextAttribute::Italic(v) => ("italic", json!(v)), - TextAttribute::Underline(v) => ("underline", json!(v)), - TextAttribute::Strike(v) => ("strike", json!(v)), - TextAttribute::Superscript(v) => { - if *v { - fields.insert("subscript", json!(false)); - } - ("superscript", json!(v)) - } - TextAttribute::Subscript(v) => { - if *v { - fields.insert("superscript", json!(false)); - } - ("subscript", json!(v)) - } - TextAttribute::Font(v) => ("font", json!(v)), - TextAttribute::FontSize(v) => ("font_size", json!(v)), - TextAttribute::Color(v) | TextAttribute::Highlight(v) => ( - if matches!(attribute, TextAttribute::Color(_)) { - "color" - } else { - "highlight" - }, - json!(v.map_or(0xff000000, |[r, g, b]| u32::from_le_bytes([r, g, b, 0]))), - ), - }; - fields.insert(name, value); - } - fields -} - -fn observe( - source: &[u8], - space: ExGuid, - object: ExGuid, - range: Range, - fields: &BTreeMap<&str, Value>, -) -> Result> { - let store = Store::parse(source)?; - let index = RevisionIndex::parse(&store)?; - let document = Document::parse(&index)?; - let revision = document.active(space)?; - let mut spans: Vec = Vec::new(); - let mut start = 0; - for run in revision.text_runs(object)? { - let end = - start + u32::try_from(run.text.encode_utf16().count()).map_err(io::Error::other)?; - if (start < range.end && range.start < end) || (start == 0 && end == 0 && range == (0..0)) { - let format = serde_json::to_value(run.format).map_err(io::Error::other)?; - let values = fields - .iter() - .map(|(name, desired)| { - let value = &format[*name]; - if value.is_null() && desired.is_boolean() { - json!(false) - } else if value.is_null() && matches!(*name, "color" | "highlight") { - json!(0xff000000_u32) - } else { - value.clone() - } - }) - .collect::>(); - let end = end.min(range.end) - range.start; - if let Some(last) = spans.last_mut().filter(|s| s.values == values) { - last.end = end; - } else { - spans.push(Span { end, values }); - } - } - start = end; - } - Ok(spans) -} - -impl Replica { - /// Durably records a visual-formatting change and its observed attribute values. - /// Independent remote attributes can merge; competing values preserve a conflict. - pub fn format( - &self, - source: &[u8], - space: ExGuid, - object: ExGuid, - range: Range, - attributes: &[TextAttribute], - ) -> Result> { - let (edit, prepared) = FormatEdit::capture(source, space, object, range, attributes)?; - self.record(source, space, Operation::Format(edit), &prepared) - } -} - -impl FormatEdit { - pub(crate) fn capture<'a>( - source: &'a [u8], - space: ExGuid, - object: ExGuid, - range: Range, - attributes: &[TextAttribute], - ) -> Result<(Self, PreparedEdit<'a>)> { - let prepared = PreparedEdit::format(source, space, object, range.clone(), attributes)?; - let before = paragraph(source, space, object)?.ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "Prepared formatting has no text target", - ) - })?; - let observed = observe(source, space, object, range.clone(), &fields(attributes))?; - let edit = Self { - object, - before, - range, - attributes: attributes.to_vec(), - observed, - }; - Ok((edit, prepared)) - } - - pub(crate) fn prepare<'a>( - &self, - snapshot: &'a [u8], - space: ExGuid, - ) -> Result, ConflictKind>> { - let Some(text) = paragraph(snapshot, space, self.object)? else { - return Ok(Err(ConflictKind::TargetUnavailable)); - }; - let Some(range) = rebase::rebase(&self.before, &text, self.range.clone()) else { - return Ok(Err(ConflictKind::TextChanged)); - }; - let prepared = match PreparedEdit::format( - snapshot, - space, - self.object, - range.clone(), - &self.attributes, - ) { - Ok(prepared) => prepared, - Err(_) => return Ok(Err(ConflictKind::UnsupportedEdit)), - }; - let desired = fields(&self.attributes); - let observed = observe(snapshot, space, self.object, range.clone(), &desired)?; - let wanted: Vec<_> = desired.values().collect(); - for spans in [&self.observed, &observed] { - if spans.is_empty() - || (!range.is_empty() && spans[0].end == 0) - || spans.last().unwrap().end != range.end - range.start - || spans.windows(2).any(|s| s[0].end >= s[1].end) - || spans.iter().any(|s| s.values.len() != wanted.len()) - { - return Ok(Err(ConflictKind::UnsupportedEdit)); - } - } - let (mut before, mut after) = (0, 0); - while before < self.observed.len() && after < observed.len() { - let old = &self.observed[before]; - let current = &observed[after]; - if old - .values - .iter() - .zip(¤t.values) - .zip(&wanted) - .any(|((old, new), wanted)| new != old && new != *wanted) - { - return Ok(Err(ConflictKind::FormattingChanged)); - } - let end = old.end.min(current.end); - if old.end == end { - before += 1; - } - if current.end == end { - after += 1; - } - } - Ok(Ok(prepared)) - } -} diff --git a/crates/onestore-offline/src/lib.rs b/crates/onestore-offline/src/lib.rs deleted file mode 100644 index 58a57badb609e2b9223be5f3d70d81c645050297..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/src/lib.rs +++ /dev/null @@ -1,432 +0,0 @@ -#![forbid(unsafe_code)] -#![doc = include_str!("../README.md")] - -use onestore::{ - ExGuid, Insertion, PageCreation, PreparedEdit, RevisionIndex, Store, - document::{Document, Kind}, -}; -use rusqlite::{Connection, OpenFlags, TransactionBehavior, params}; -use std::{fs::OpenOptions, io, ops::Range, path::Path, sync::Mutex, time::Duration}; - -mod assets; -mod formatting; -mod outline; -mod pages; -mod paragraph; -mod rebase; -mod recovery; -mod schema; -mod tree; -pub use formatting::FormatEdit; -pub use outline::OutlineEdit; -pub use pages::PageEdits; -pub use paragraph::{JoinEdit, SplitEdit}; -pub use recovery::{Recovery, RecoverySummary}; -pub use tree::TreeEdit; -mod sync; -pub use sync::{ConflictKind, EditStatus, Remote}; -mod worker; -pub use worker::SyncWorker; -#[cfg(feature = "smb")] -mod smb; -#[cfg(feature = "smb")] -pub use smb::SmbRemote; - -#[derive(Debug, thiserror::Error)] -pub enum Error { - #[error(transparent)] - Database(#[from] rusqlite::Error), - #[error(transparent)] - Io(#[from] io::Error), - #[error(transparent)] - Document(#[from] onestore::Error), - #[error(transparent)] - Remote(#[from] onestore::CommitError), - #[error(transparent)] - RemoteIo(io::Error), - #[error(transparent)] - Notebook(#[from] onestore_notebook::Error), - #[error("External payload identity now refers to different bytes")] - AssetChanged, -} - -type Result = std::result::Result; - -const APPLICATION_ID: u32 = 0x4f4e454f; -const SCHEMA_VERSION: u32 = 11; - -/// Text and its observed precondition, retained across cache reopen and rebasing. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct TextEdit { - pub object: ExGuid, - pub before: String, - pub range: Range, - pub replacement: String, -} - -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub enum Operation { - CreatePage(PageCreation), - Pages(PageEdits), - Text(TextEdit), - Insert(Insertion), - Format(FormatEdit), - Split(SplitEdit), - Join(JoinEdit), - Outline(OutlineEdit), - Tree(TreeEdit), -} - -/// A locally acknowledged intent; its ID remains stable across cache reopen. -#[derive(Debug, Clone, PartialEq)] -pub struct PendingEdit { - pub id: u64, - pub space: ExGuid, - pub operation: Operation, -} - -/// Owns one local cache. Share this handle between threads; a second open fails busy. -/// SQLite's exclusive connection retains ownership between local transactions. -pub struct Replica { - connection: Mutex, - synchronization: Mutex<()>, - worker: Mutex>, -} - -impl Replica { - /// Seeds a new cache from a validated notebook image, refusing any existing path. - /// An initialization error preserves the created file for inspection. - pub fn create(path: impl AsRef, source: &[u8]) -> Result { - validate(source)?; - let mut options = OpenOptions::new(); - options.read(true).write(true).create_new(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.mode(0o600); - } - drop(options.open(path.as_ref())?); - Self::connect(path.as_ref(), Some(source)) - } - - /// Reopens an existing cache and its durable pending edits without network access. - /// Unrecognized databases and unsupported journal modes are rejected without conversion. - pub fn open(path: impl AsRef) -> Result { - Self::connect(path.as_ref(), None) - } - - fn connect(path: &Path, source: Option<&[u8]>) -> Result { - let mut connection = cache_connection(path)?; - let transaction = connection.transaction_with_behavior(TransactionBehavior::Exclusive)?; - let application: u32 = - transaction.pragma_query_value(None, "application_id", |row| row.get(0))?; - let version: u32 = - transaction.pragma_query_value(None, "user_version", |row| row.get(0))?; - if let Some(source) = source { - let tables: i64 = - transaction - .query_row("SELECT count(*) FROM sqlite_schema", [], |row| row.get(0))?; - if application != 0 || version != 0 || tables != 0 { - return Err(io::Error::new( - io::ErrorKind::AlreadyExists, - "Cache initialization found an existing database", - ) - .into()); - } - transaction.pragma_update(None, "application_id", APPLICATION_ID)?; - transaction.pragma_update(None, "user_version", SCHEMA_VERSION)?; - transaction.execute_batch( - " - CREATE TABLE replica ( - id INTEGER PRIMARY KEY CHECK(id=1), - base BLOB NOT NULL, - working BLOB NOT NULL - ) STRICT; - ", - )?; - schema::create(&transaction)?; - transaction.execute("INSERT INTO replica VALUES (1, ?1, ?1)", [source])?; - } else { - if application != APPLICATION_ID || !(1..=SCHEMA_VERSION).contains(&version) { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Unrecognized cache or unsupported schema version", - ) - .into()); - } - validate_images(&transaction)?; - if version < SCHEMA_VERSION { - schema::migrate(&transaction, version)?; - transaction.pragma_update(None, "user_version", SCHEMA_VERSION)?; - } - pending(&transaction)?; - } - transaction.commit()?; - Ok(Self { - connection: Mutex::new(connection), - synchronization: Mutex::new(()), - worker: Mutex::new(std::sync::Weak::new()), - }) - } - - /// Returns the latest complete locally committed image, including pending edits. - pub fn snapshot(&self) -> Result> { - let connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - Ok( - connection.query_row("SELECT working FROM replica WHERE id=1", [], |row| { - row.get(0) - })?, - ) - } - - pub fn pending(&self) -> Result> { - let connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - pending(&connection) - } - - /// Atomically records an intent and its resulting local image; returns its durable ID. - /// Unchanged text returns `None`. A stale image returns `Io(ResourceBusy)`. - /// On synchronization, replacement text inherits the remote style at the rebased start. - /// After a database error, reopen and inspect the cache before retrying the edit. - pub fn edit_text( - &self, - source: &[u8], - space: ExGuid, - object: ExGuid, - range: Range, - replacement: &str, - ) -> Result> { - let edit = PreparedEdit::text(source, space, object, range.clone(), replacement)?; - let before = paragraph(source, space, object)?.ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "Prepared edit has no text target", - ) - })?; - self.record( - source, - space, - Operation::Text(TextEdit { - object, - before, - range, - replacement: replacement.to_owned(), - }), - &edit, - ) - } - - /// Durably queues a validated insertion with its stable object identities. - /// Uses the same snapshot and local-acknowledgement contract as `edit_text`. - pub fn insert( - &self, - source: &[u8], - space: ExGuid, - insertion: &Insertion, - ) -> Result> { - let edit = PreparedEdit::insert(source, space, insertion)?; - self.record(source, space, Operation::Insert(insertion.clone()), &edit) - } - - /// Queues a new page and its section entry with stable identities for dependent edits. - pub fn create_page(&self, source: &[u8], page: &PageCreation) -> Result> { - let edit = PreparedEdit::create_page(source, page)?; - self.record( - source, - page.space(), - Operation::CreatePage(page.clone()), - &edit, - ) - } - - fn record( - &self, - source: &[u8], - space: ExGuid, - operation: Operation, - edit: &PreparedEdit<'_>, - ) -> Result> { - let mut connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; - let current: Vec = - transaction.query_row("SELECT working FROM replica WHERE id=1", [], |row| { - row.get(0) - })?; - if current != source { - return Err(io::Error::new( - io::ErrorKind::ResourceBusy, - "The local snapshot changed before this edit", - ) - .into()); - } - if edit.as_bytes() == source { - return Ok(None); - } - transaction.execute( - "INSERT INTO edits(space, operation) VALUES (?1, ?2)", - params![ - space.to_string(), - serde_json::to_string(&operation).map_err(io::Error::other)? - ], - )?; - let id = u64::try_from(transaction.last_insert_rowid()).map_err(io::Error::other)?; - transaction.execute( - "UPDATE replica SET working=?1 WHERE id=1", - [edit.as_bytes()], - )?; - transaction.commit()?; - drop(connection); - self.wake_sync(); - Ok(Some(id)) - } - - fn wake_sync(&self) { - if let Ok(worker) = self.worker.lock() - && let Some(worker) = worker.upgrade() - { - worker.wake(); - } - } -} - -fn active_paths( - view: &onestore::document::Revision<'_>, - pages: &[ExGuid], - objects: &[ExGuid], -) -> Option>> { - let parents = view.parents(pages).ok()?; - objects - .iter() - .map(|object| { - if !parents.contains_key(object) && !pages.contains(object) { - return None; - } - let mut path = Vec::new(); - let mut at = *object; - while !pages.contains(&at) { - let [parent] = parents.get(&at)?.as_slice() else { - return None; - }; - if path.len() >= view.nodes.len() { - return None; - } - path.push(*parent); - at = *parent; - } - Some(path) - }) - .collect() -} - -fn paragraph(source: &[u8], space: ExGuid, object: ExGuid) -> Result> { - let store = Store::parse(source)?; - let index = RevisionIndex::parse(&store)?; - let document = Document::parse(&index)?; - let node = document - .active(space) - .ok() - .and_then(|revision| revision.nodes.get(&object)); - Ok(match node.map(|node| &node.kind) { - Some(Kind::RichText { text, .. }) => Some(text.clone()), - _ => None, - }) -} - -fn validate(source: &[u8]) -> Result { - let store = Store::parse(source)?; - if !store.checksum_mismatches.is_empty() { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Notebook transaction checksum damage", - ) - .into()); - } - let index = RevisionIndex::parse(&store)?; - index.validate_current()?; - Document::parse(&index)?; - Ok(index.root) -} - -fn pending(connection: &Connection) -> Result> { - let mut query = connection.prepare("SELECT id, space, operation FROM edits ORDER BY id")?; - let mut rows = query.query([])?; - let mut edits = Vec::new(); - while let Some(row) = rows.next()? { - let edit = PendingEdit { - id: u64::try_from(row.get::<_, i64>(0)?).map_err(io::Error::other)?, - space: row.get::<_, String>(1)?.parse()?, - operation: serde_json::from_str(&row.get::<_, String>(2)?) - .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?, - }; - if matches!(&edit.operation, Operation::CreatePage(page) if page.space() != edit.space) { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Cached page identity differs from its creation intent", - ) - .into()); - } - edits.push(edit); - } - Ok(edits) -} - -fn cache_connection(path: &Path) -> Result { - let connection = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_WRITE)?; - connection.busy_timeout(Duration::ZERO)?; - connection.execute_batch( - "PRAGMA locking_mode=EXCLUSIVE; PRAGMA synchronous=EXTRA; PRAGMA fullfsync=ON; PRAGMA foreign_keys=ON;", - )?; - for (name, expected) in [("locking_mode", "exclusive"), ("journal_mode", "delete")] { - let actual: String = connection.pragma_query_value(None, name, |row| row.get(0))?; - if actual != expected { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Unsupported cache locking or journal mode", - ) - .into()); - } - } - for (name, expected) in [("synchronous", 3), ("fullfsync", 1), ("foreign_keys", 1)] { - let actual: i64 = connection.pragma_query_value(None, name, |row| row.get(0))?; - if actual != expected { - return Err(io::Error::new( - io::ErrorKind::Unsupported, - "Required cache synchronization is unavailable", - ) - .into()); - } - } - Ok(connection) -} - -fn validate_images(connection: &Connection) -> Result<()> { - let integrity: String = connection.query_row("PRAGMA quick_check", [], |row| row.get(0))?; - if integrity != "ok" { - return Err( - io::Error::new(io::ErrorKind::InvalidData, "Cache integrity check failed").into(), - ); - } - let (base, working): (Vec, Vec) = - connection.query_row("SELECT base, working FROM replica WHERE id=1", [], |row| { - Ok((row.get(0)?, row.get(1)?)) - })?; - if validate(&base)? != validate(&working)? { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Cache images belong to different documents", - ) - .into()); - } - Ok(()) -} diff --git a/crates/onestore-offline/src/outline.rs b/crates/onestore-offline/src/outline.rs deleted file mode 100644 index 8651735ac02a49b0cb619a15a99b8648b63312d7..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/src/outline.rs +++ /dev/null @@ -1,138 +0,0 @@ -use super::*; -use serde::{Deserialize, Serialize}; -use serde_json::{Value, json}; - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct Observed { - path: Vec, - values: Vec, -} - -/// A layout or saved expansion intent with its original ancestry and property values. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct OutlineEdit { - pub object: ExGuid, - pub change: onestore::OutlineEdit, - observed: Observed, -} - -fn observe( - source: &[u8], - space: ExGuid, - object: ExGuid, - change: onestore::OutlineEdit, -) -> Result> { - let store = Store::parse(source)?; - let index = RevisionIndex::parse(&store)?; - let document = Document::parse(&index)?; - let Ok(view) = document.active(space) else { - return Ok(None); - }; - let pages = document.pages_in(space)?; - if pages.len() != 1 { - return Ok(None); - } - let Some(mut paths) = active_paths(view, &pages, &[object]) else { - return Ok(None); - }; - let path = paths.pop().unwrap(); - let node = &view.nodes[&object]; - let values = match (change, &node.kind) { - (onestore::OutlineEdit::Position { .. }, Kind::Outline { .. }) => { - vec![json!(node.layout.x), json!(node.layout.y)] - } - (onestore::OutlineEdit::Width { .. }, Kind::Outline { .. }) => { - vec![ - json!(node.layout.max_width), - json!(node.layout.width_set_by_user.unwrap_or(false)), - json!(node.layout.reserved_width), - ] - } - (onestore::OutlineEdit::Collapsed(_), Kind::Paragraph { collapse_state, .. }) => { - vec![json!(collapse_state.unwrap_or(0))] - } - _ => return Ok(None), - }; - Ok(Some(Observed { path, values })) -} - -impl Replica { - /// Durably records layout or saved expansion changes, preserving independent remote edits. - pub fn outline( - &self, - source: &[u8], - space: ExGuid, - object: ExGuid, - change: onestore::OutlineEdit, - ) -> Result> { - let (edit, prepared) = OutlineEdit::capture(source, space, object, change)?; - self.record(source, space, Operation::Outline(edit), &prepared) - } -} - -impl OutlineEdit { - pub(crate) fn capture<'a>( - source: &'a [u8], - space: ExGuid, - object: ExGuid, - change: onestore::OutlineEdit, - ) -> Result<(Self, PreparedEdit<'a>)> { - let prepared = PreparedEdit::outline(source, space, object, change)?; - let observed = observe(source, space, object, change)?.ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "Prepared outline edit has no active target", - ) - })?; - Ok(( - Self { - object, - change, - observed, - }, - prepared, - )) - } - - pub(crate) fn prepare<'a>( - &self, - snapshot: &'a [u8], - space: ExGuid, - ) -> Result, ConflictKind>> { - let Some(current) = observe(snapshot, space, self.object, self.change)? else { - return Ok(Err(ConflictKind::TargetUnavailable)); - }; - if current.path != self.observed.path { - return Ok(Err(ConflictKind::StructureChanged)); - } - let prepared = match PreparedEdit::outline(snapshot, space, self.object, self.change) { - Ok(prepared) => prepared, - Err(_) => return Ok(Err(ConflictKind::UnsupportedEdit)), - }; - let wanted = - observe(prepared.as_bytes(), space, self.object, self.change)?.ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "Prepared outline edit lost its target", - ) - })?; - if self.observed.values.len() != current.values.len() - || wanted.values.len() != current.values.len() - { - return Ok(Err(ConflictKind::UnsupportedEdit)); - } - if self - .observed - .values - .iter() - .zip(¤t.values) - .zip(&wanted.values) - .any(|((before, current), wanted)| current != before && current != wanted) - { - return Ok(Err(ConflictKind::LayoutChanged)); - } - Ok(Ok(prepared)) - } -} diff --git a/crates/onestore-offline/src/pages.rs b/crates/onestore-offline/src/pages.rs deleted file mode 100644 index f77536addde20c68f8e8f0c76a2c315d7f883cbb..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/src/pages.rs +++ /dev/null @@ -1,168 +0,0 @@ -use super::*; -use onestore::{PageEdit, PagePosition}; -use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; - -type PageOrder = Vec<(ExGuid, u32)>; - -/// An atomic page batch with its observed order and indentation. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PageEdits { - pub edits: Vec, - observed: PageOrder, -} - -fn observe(source: &[u8]) -> Result> { - let store = Store::parse(source)?; - let index = RevisionIndex::parse(&store)?; - let document = Document::parse(&index)?; - let mut pages = Vec::new(); - for (sid, _) in document.pages()? { - let view = document.active(sid)?; - let Some(metadata) = view.roots.get(&2).and_then(|id| view.nodes.get(id)) else { - return Ok(None); - }; - let Kind::Metadata { level, .. } = metadata.kind else { - return Ok(None); - }; - pages.push((sid, level.unwrap_or(1))); - } - Ok(Some((document.root, pages))) -} - -fn positions(pages: &[(ExGuid, u32)]) -> Result> { - let mut positions = BTreeMap::new(); - for (at, (sid, level)) in pages.iter().enumerate() { - if sid.guid == [0; 16] - || !(1..=3).contains(level) - || positions.insert(*sid, (at, *level)).is_some() - { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Page observations need unique identities and valid indentation", - ) - .into()); - } - } - Ok(positions) -} - -impl Replica { - /// Durably queues the complete page batch using the supplied local snapshot. - pub fn pages(&self, source: &[u8], edits: &[PageEdit]) -> Result> { - let (space, batch, prepared) = PageEdits::capture(source, edits)?; - self.record(source, space, Operation::Pages(batch), &prepared) - } -} - -impl PageEdits { - fn capture<'a>( - source: &'a [u8], - edits: &[PageEdit], - ) -> Result<(ExGuid, Self, PreparedEdit<'a>)> { - let prepared = PreparedEdit::pages(source, edits)?; - let (space, observed) = observe(source)?.ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "Prepared page edits have no page metadata", - ) - })?; - positions(&observed)?; - Ok(( - space, - Self { - edits: edits.to_vec(), - observed, - }, - prepared, - )) - } - - pub(crate) fn prepare<'a>( - &self, - source: &'a [u8], - space: ExGuid, - ) -> Result, ConflictKind>> { - let Some((root, current)) = observe(source)? else { - return Ok(Err(ConflictKind::TargetUnavailable)); - }; - if root != space { - return Ok(Err(ConflictKind::TargetUnavailable)); - } - let before = positions(&self.observed)?; - let current = positions(¤t)?; - for edit in &self.edits { - let Some((_, original_level)) = before.get(&edit.space()) else { - return Ok(Err(ConflictKind::TargetUnavailable)); - }; - let Some((_, current_level)) = current.get(&edit.space()) else { - return Ok(Err(ConflictKind::TargetUnavailable)); - }; - if current_level != original_level && *current_level != edit.level() { - return Ok(Err(ConflictKind::StructureChanged)); - } - } - let prepared = match PreparedEdit::pages(source, &self.edits) { - Ok(prepared) => prepared, - Err(_) => return Ok(Err(ConflictKind::StructureChanged)), - }; - let (_, wanted) = observe(prepared.as_bytes())?.ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "Prepared page edits lost page metadata", - ) - })?; - let wanted = positions(&wanted)?; - for edit in &self.edits { - if edit.position() == PagePosition::Keep { - continue; - } - let sid = edit.space(); - let peers = before - .keys() - .filter(|id| **id != sid && current.contains_key(id)); - let mut unchanged = true; - let mut satisfied = true; - for id in peers { - let observed = current[id].0 < current[&sid].0; - unchanged &= observed == (before[id].0 < before[&sid].0); - satisfied &= observed == (wanted[id].0 < wanted[&sid].0); - } - if !unchanged && !satisfied { - return Ok(Err(ConflictKind::StructureChanged)); - } - } - Ok(Ok(prepared)) - } - - pub(crate) fn review(&self, source: &[u8], space: ExGuid, edits: &[PageEdit]) -> Result { - let identities_match = edits.len() == self.edits.len() - && edits.iter().all(|edit| { - self.edits - .iter() - .find(|original| original.space() == edit.space()) - .is_some_and(|original| { - original - .reposition(edit.position(), edit.level()) - .is_ok_and(|revised| revised == *edit) - }) - }); - if !identities_match { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Retain the original page and series identities when reviewing a page batch", - ) - .into()); - } - let (root, reviewed, _) = Self::capture(source, edits)?; - if root != space { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Review the original section for page edits", - ) - .into()); - } - Ok(reviewed) - } -} diff --git a/crates/onestore-offline/src/paragraph.rs b/crates/onestore-offline/src/paragraph.rs deleted file mode 100644 index f97975e1da365fc07b4d823454389c00e2ce9810..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/src/paragraph.rs +++ /dev/null @@ -1,200 +0,0 @@ -use super::*; -use onestore::{ParagraphJoin, ParagraphSplit}; -use serde::{Deserialize, Serialize}; -use serde_json::{Value, json}; - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct Observed { - text: String, - structure: Value, -} - -/// A stable split intent and the paragraph state observed before local publication. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct SplitEdit { - pub intent: ParagraphSplit, - observed: Observed, -} - -/// A join intent and both paragraphs' observed text, placement and tag ownership. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct JoinEdit { - pub intent: ParagraphJoin, - observed: [Observed; 2], -} - -fn observe(source: &[u8], space: ExGuid, texts: &[ExGuid]) -> Result>> { - let store = Store::parse(source)?; - let index = RevisionIndex::parse(&store)?; - let document = Document::parse(&index)?; - let Ok(view) = document.active(space) else { - return Ok(None); - }; - let pages = document.pages_in(space)?; - let Some(paths) = active_paths(view, &pages, texts) else { - return Ok(None); - }; - let mut observed = Vec::new(); - for (text, path) in texts.iter().zip(paths) { - let node = &view.nodes[text]; - let Kind::RichText { text: content, .. } = &node.kind else { - return Ok(None); - }; - let Some(paragraph) = path.first().and_then(|id| view.nodes.get(id)) else { - return Ok(None); - }; - let Kind::Paragraph { lists, .. } = ¶graph.kind else { - return Ok(None); - }; - if paragraph.content != [*text] { - return Ok(None); - } - let lists: Vec<_> = lists.iter().map(|id| (*id, &view.nodes[id].kind)).collect(); - observed.push(Observed { - text: content.clone(), - structure: json!({ - "path": path, - "children": paragraph.children, - "child_level": paragraph.child_level, - "paragraph": paragraph.kind, - "lists": lists, - "tags": node.tags, - }), - }); - } - Ok(Some(observed)) -} - -impl Replica { - /// Durably queues a split with stable new identities and observed structural preconditions. - pub fn split( - &self, - source: &[u8], - space: ExGuid, - intent: &ParagraphSplit, - ) -> Result> { - let (operation, prepared) = SplitEdit::capture(source, space, intent)?; - self.record(source, space, Operation::Split(operation), &prepared) - } - - /// Durably queues a join; changed placement, children or tags require conflict review. - pub fn join( - &self, - source: &[u8], - space: ExGuid, - intent: &ParagraphJoin, - ) -> Result> { - let (operation, prepared) = JoinEdit::capture(source, space, intent)?; - self.record(source, space, Operation::Join(operation), &prepared) - } -} - -impl SplitEdit { - pub(crate) fn capture<'a>( - source: &'a [u8], - space: ExGuid, - intent: &ParagraphSplit, - ) -> Result<(Self, PreparedEdit<'a>)> { - let prepared = PreparedEdit::split(source, space, intent)?; - let Some(mut observed) = observe(source, space, &[intent.position().0])? else { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Select text in an active paragraph", - ) - .into()); - }; - Ok(( - Self { - intent: intent.clone(), - observed: observed.remove(0), - }, - prepared, - )) - } - - pub(crate) fn prepare<'a>( - &self, - source: &'a [u8], - space: ExGuid, - ) -> Result, ConflictKind>> { - let (text, offset) = self.intent.position(); - let Some(observed) = observe(source, space, &[text])? else { - return Ok(Err(ConflictKind::TargetUnavailable)); - }; - if observed[0].structure != self.observed.structure { - return Ok(Err(ConflictKind::StructureChanged)); - } - let Some(range) = rebase::rebase(&self.observed.text, &observed[0].text, offset..offset) - else { - return Ok(Err(ConflictKind::TextChanged)); - }; - Ok( - PreparedEdit::split(source, space, &self.intent.reposition(range.start)) - .map_err(|_| ConflictKind::UnsupportedEdit), - ) - } -} - -impl JoinEdit { - pub(crate) fn review(&self, source: &[u8], space: ExGuid) -> Result { - let (reviewed, _) = Self::capture(source, space, &self.intent)?; - if reviewed.observed[0].text.is_empty() != self.observed[0].text.is_empty() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "The join would change its surviving text identity", - ) - .into()); - } - Ok(reviewed) - } - - pub(crate) fn capture<'a>( - source: &'a [u8], - space: ExGuid, - intent: &ParagraphJoin, - ) -> Result<(Self, PreparedEdit<'a>)> { - let prepared = PreparedEdit::join(source, space, intent)?; - let Some(observed) = observe(source, space, &intent.texts())? else { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Select text in two active paragraphs", - ) - .into()); - }; - Ok(( - Self { - intent: intent.clone(), - observed: observed.try_into().unwrap(), - }, - prepared, - )) - } - - pub(crate) fn prepare<'a>( - &self, - source: &'a [u8], - space: ExGuid, - ) -> Result, ConflictKind>> { - let Some(observed) = observe(source, space, &self.intent.texts())? else { - return Ok(Err(ConflictKind::TargetUnavailable)); - }; - for (i, (old, new)) in self.observed.iter().zip(&observed).enumerate() { - if old.structure != new.structure { - return Ok(Err(ConflictKind::StructureChanged)); - } - let at = if i == 0 { - u32::try_from(old.text.encode_utf16().count()).map_err(io::Error::other)? - } else { - 0 - }; - if rebase::rebase(&old.text, &new.text, at..at).is_none() { - return Ok(Err(ConflictKind::TextChanged)); - } - } - Ok(PreparedEdit::join(source, space, &self.intent) - .map_err(|_| ConflictKind::UnsupportedEdit)) - } -} diff --git a/crates/onestore-offline/src/rebase.rs b/crates/onestore-offline/src/rebase.rs deleted file mode 100644 index 9a6dce0a8b16201774a6482925ad04659b6ab1b6..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/src/rebase.rs +++ /dev/null @@ -1,297 +0,0 @@ -use std::ops::Range; - -const INFINITY: u32 = 1 << 30; -const MAX_CELLS: usize = 4_000_000; - -struct Matrix { - band: usize, - values: Vec, -} - -impl Matrix { - fn get(&self, row: usize, column: usize) -> u32 { - let Some(column) = column - .checked_add(self.band) - .and_then(|at| at.checked_sub(row)) - else { - return INFINITY; - }; - let width = self.band * 2 + 1; - if column >= width { - return INFINITY; - } - self.values - .get(row * width + column) - .copied() - .unwrap_or(INFINITY) - } - - fn build(before: &[char], after: &[char], band: usize) -> Self { - let width = band * 2 + 1; - let mut matrix = Self { - band, - values: vec![INFINITY; (before.len() + 1) * width], - }; - for row in 0..=before.len() { - for column in row.saturating_sub(band)..=after.len().min(row + band) { - let mut cost = if row == 0 && column == 0 { 0 } else { INFINITY }; - if row > 0 { - cost = cost.min(matrix.get(row - 1, column) + 1); - } - if column > 0 { - cost = cost.min(matrix.get(row, column - 1) + 1); - } - if row > 0 && column > 0 && before[row - 1] == after[column - 1] { - cost = cost.min(matrix.get(row - 1, column - 1)); - } - matrix.values[row * width + column + band - row] = cost; - } - } - matrix - } -} - -/// Requires the same mapping in every minimum insertion/deletion alignment. -pub(crate) fn rebase(before: &str, after: &str, range: Range) -> Option> { - if range.start > range.end { - return None; - } - let mut a: Vec<_> = before.chars().collect(); - let offsets: Vec<_> = std::iter::once(0) - .chain(a.iter().scan(0_u32, |at, character| { - *at = at.checked_add(character.len_utf16() as u32)?; - Some(*at) - })) - .collect(); - let local = - offsets.binary_search(&range.start).ok()?..offsets.binary_search(&range.end).ok()?; - if before == after { - return Some(range); - } - let mut b: Vec<_> = after.chars().collect(); - let (n, m) = (a.len(), b.len()); - let mut band = n.abs_diff(m).max(1); - let forward = loop { - let width = band.checked_mul(2)?.checked_add(1)?; - if (n + 1).checked_mul(width)? > MAX_CELLS { - return None; - } - let matrix = Matrix::build(&a, &b, band); - if matrix.get(n, m) <= band as u32 { - break matrix; - } - band *= 2; - }; - let distance = forward.get(n, m); - a.reverse(); - b.reverse(); - let backward = Matrix::build(&a, &b, band); - a.reverse(); - b.reverse(); - let position = |index: usize| { - let mut matched = None; - for column in index.saturating_sub(band)..=m.min(index + band) { - let cost = forward.get(index, column); - if cost + 1 + backward.get(n - index - 1, m - column) == distance { - return None; - } - if b.get(column) == Some(&a[index]) - && cost + backward.get(n - index - 1, m - column - 1) == distance - { - if matched.is_some() { - return None; - } - matched = Some(column); - } - } - matched - }; - let mapped = if local.is_empty() { - if local.start > 0 { - position(local.start - 1)?; - } - if local.start < n { - position(local.start)?; - } - let mut boundary = None; - for column in local.start.saturating_sub(band)..=m.min(local.start + band) { - if forward.get(local.start, column) + backward.get(n - local.start, m - column) - == distance - { - if boundary.is_some() { - return None; - } - boundary = Some(column); - } - } - let at = boundary?; - at..at - } else { - let start = position(local.start)?; - for index in local.start + 1..local.end { - if position(index)? != start + index - local.start { - return None; - } - } - start..start + local.len() - }; - let start = b[..mapped.start] - .iter() - .map(|character| character.len_utf16()) - .sum::(); - let end = start - + b[mapped] - .iter() - .map(|character| character.len_utf16()) - .sum::(); - Some(u32::try_from(start).ok()?..u32::try_from(end).ok()?) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn optimal_paths(a: &[char], b: &[char]) -> Vec> { - fn visit( - a: &[char], - b: &[char], - path: &mut Vec<(usize, usize)>, - cost: usize, - best: &mut usize, - found: &mut Vec>, - ) { - if cost > *best { - return; - } - let (i, j) = *path.last().unwrap(); - if (i, j) == (a.len(), b.len()) { - if cost < *best { - found.clear(); - *best = cost; - } - found.push(path.clone()); - return; - } - for (next_i, next_j, charge) in [(i + 1, j + 1, 0), (i + 1, j, 1), (i, j + 1, 1)] { - if next_i > a.len() || next_j > b.len() || (charge == 0 && a[i] != b[j]) { - continue; - } - path.push((next_i, next_j)); - visit(a, b, path, cost + charge, best, found); - path.pop(); - } - } - let mut found = Vec::new(); - let mut best = usize::MAX; - visit(a, b, &mut vec![(0, 0)], 0, &mut best, &mut found); - found - } - - #[test] - fn bounded_alignment_agrees_with_exhaustive_paths_for_every_small_unicode_edit() { - let mut words = vec![String::new()]; - for length in 1..=4 { - for bits in 0..1 << length { - words.push( - (0..length) - .map(|bit| if bits & (1 << bit) == 0 { 'a' } else { '🦀' }) - .collect(), - ); - } - } - let mut cases = 0; - for before in &words { - let a: Vec<_> = before.chars().collect(); - for after in &words { - let b: Vec<_> = after.chars().collect(); - let paths = optimal_paths(&a, &b); - for start in 0..=a.len() { - for end in start..=a.len() { - let mut expected = None; - let mut valid = true; - for path in &paths { - let mapping: Vec<_> = (0..a.len()) - .map(|i| { - path.windows(2).find_map(|edge| { - (edge[0].0 == i && edge[1] == (i + 1, edge[0].1 + 1)) - .then_some(edge[0].1) - }) - }) - .collect(); - let candidate = if start == end { - let vertices: Vec<_> = path - .iter() - .filter(|(i, _)| *i == start) - .map(|(_, j)| *j) - .collect(); - if (start > 0 && mapping[start - 1].is_none()) - || (start < a.len() && mapping[start].is_none()) - || vertices.len() != 1 - { - None - } else { - Some(vertices[0]..vertices[0]) - } - } else if let Some(first) = mapping[start] { - (start..end) - .all(|i| mapping[i] == Some(first + i - start)) - .then_some(first..first + end - start) - } else { - None - }; - let Some(candidate) = candidate else { - valid = false; - break; - }; - if expected.as_ref().is_some_and(|old| *old != candidate) { - valid = false; - break; - } - expected = Some(candidate); - } - let expected = if valid { - expected.map(|range| { - b[..range.start].iter().map(|c| c.len_utf16() as u32).sum() - ..b[..range.end].iter().map(|c| c.len_utf16() as u32).sum() - }) - } else { - None - }; - let range = a[..start].iter().map(|c| c.len_utf16() as u32).sum() - ..a[..end].iter().map(|c| c.len_utf16() as u32).sum(); - assert_eq!( - rebase(before, after, range), - expected, - "{before:?} -> {after:?}, characters {start}..{end}" - ); - cases += 1; - } - } - } - } - assert_eq!(cases, 10881); - } - - #[test] - fn maps_disjoint_unicode_edits_and_rejects_ambiguous_or_overlapping_changes() { - assert_eq!(rebase("ab🦀cd", "Xab🦀cYd", 2..4), Some(3..5)); - assert_eq!(rebase("abc", "XabcY", 1..2), Some(2..3)); - assert_eq!(rebase("abc", "XabcY", 1..1), Some(2..2)); - assert_eq!(rebase("abc", "abcX", 3..3), None); - assert_eq!(rebase("abc", "ac", 1..2), None); - assert_eq!(rebase("abc", "", 1..1), None); - assert_eq!(rebase("aaaa", "aaaaa", 1..2), None); - assert_eq!(rebase("ab🦀cd", "ab🦀cd", 3..4), None); - assert_eq!(rebase("abc", "abc", 4..4), None); - } - - #[test] - fn long_paragraphs_with_small_remote_changes_use_a_narrow_band() { - let text = format!("{}🦀{}", "a".repeat(8192), "b".repeat(8192)); - assert_eq!( - rebase(&text, &format!("X{text}Y"), 8192..8194), - Some(8193..8195) - ); - assert_eq!(rebase(&"a".repeat(8192), &"b".repeat(8192), 1..2), None); - } -} diff --git a/crates/onestore-offline/src/recovery.rs b/crates/onestore-offline/src/recovery.rs deleted file mode 100644 index 5b87b1c733d53acaba966f24c4fe60ba5df548e7..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/src/recovery.rs +++ /dev/null @@ -1,190 +0,0 @@ -use super::*; -use rusqlite::backup::{Backup, StepResult}; -use std::{collections::BTreeMap, fs::File}; - -const RECOVERY_ID: u32 = 0x4f4e4552; - -/// Counts and image sizes without notebook text, paths, authors or credentials. -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] -pub struct RecoverySummary { - pub queued_edits: u64, - pub conflicts: u64, - pub uncertain_edits: u64, - pub published_receipts: u64, - pub working_bytes: u64, - pub remote_bytes: u64, - pub cached_assets: u64, - pub cached_asset_bytes: u64, -} - -/// Read-only recovery evidence; it cannot publish or acknowledge an edit. -pub struct Recovery { - connection: Connection, -} - -impl Recovery { - /// Opens an exported archive without migration or conversion into a writable replica. - pub fn open(path: impl AsRef) -> Result { - let connection = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; - connection.busy_timeout(Duration::ZERO)?; - connection.execute_batch("BEGIN")?; - let application: u32 = - connection.pragma_query_value(None, "application_id", |row| row.get(0))?; - let version: u32 = connection.pragma_query_value(None, "user_version", |row| row.get(0))?; - if application != RECOVERY_ID || !(4..=SCHEMA_VERSION).contains(&version) { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Unrecognized recovery archive or unsupported schema version", - ) - .into()); - } - validate_images(&connection)?; - for edit in pending(&connection)? { - sync::status(&connection, edit.id)?; - } - receipts(&connection)?; - Ok(Self { connection }) - } - - pub fn summary(&self) -> Result { - summary(&self.connection) - } - - pub fn snapshot(&self) -> Result> { - Ok(self - .connection - .query_row("SELECT working FROM replica WHERE id=1", [], |row| { - row.get(0) - })?) - } - - pub fn remote_snapshot(&self) -> Result> { - Ok(self - .connection - .query_row("SELECT base FROM replica WHERE id=1", [], |row| row.get(0))?) - } - - pub fn pending(&self) -> Result> { - pending(&self.connection) - } - - pub fn status(&self, id: u64) -> Result> { - sync::status(&self.connection, id) - } - - pub fn receipts(&self) -> Result> { - receipts(&self.connection) - } - - /// Reads a previously downloaded external payload without accessing its former server. - pub fn cached_asset(&self, filename: &str, limit: usize) -> Result>> { - assets::cached(&self.connection, &assets::key(filename)?, limit) - } -} - -impl Replica { - /// Summarizes durable state without exposing notebook content. - pub fn recovery_summary(&self) -> Result { - let connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - summary(&connection) - } - - /// Exports a consistent archive to a new local path without changing the live queue. - /// Archives contain notebook content and cannot be opened as writable replicas. - /// An error after publication can leave a complete archive at the destination. - pub fn export_recovery(&self, path: impl AsRef) -> Result<()> { - let path = path.as_ref(); - if path.file_name().is_none() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Provide a new recovery archive filename", - ) - .into()); - } - let parent = path - .parent() - .filter(|parent| !parent.as_os_str().is_empty()) - .unwrap_or_else(|| Path::new(".")); - let temporary = tempfile::Builder::new() - .prefix(".onestore-recovery-") - .tempfile_in(parent)?; - let mut destination = cache_connection(temporary.path())?; - { - let source = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - let backup = Backup::new(&source, &mut destination)?; - if backup.step(-1)? != StepResult::Done { - return Err(io::Error::new( - io::ErrorKind::WouldBlock, - "Recovery export could not acquire the database snapshot", - ) - .into()); - } - } - destination.pragma_update(None, "application_id", RECOVERY_ID)?; - destination.close().map_err(|(_, error)| error)?; - temporary.as_file().sync_all()?; - temporary - .persist_noclobber(path) - .map_err(|error| error.error)?; - File::open(parent)?.sync_all()?; - Ok(()) - } -} - -fn summary(connection: &Connection) -> Result { - let version: u32 = connection.pragma_query_value(None, "user_version", |row| row.get(0))?; - let (cached_assets, cached_asset_bytes) = if version < 5 { - (0, 0) - } else { - connection.query_row( - "SELECT count(*),coalesce(sum(length(data)),0) FROM assets", - [], - |row| Ok((unsigned(row, 0)?, unsigned(row, 1)?)), - )? - }; - Ok(connection.query_row( - "SELECT (SELECT count(*) FROM edits), (SELECT count(*) FROM conflicts), - (SELECT count(*) FROM attempt), (SELECT count(*) FROM receipts), - length(working), length(base) FROM replica WHERE id=1", - [], - |row| { - Ok(RecoverySummary { - queued_edits: unsigned(row, 0)?, - conflicts: unsigned(row, 1)?, - uncertain_edits: unsigned(row, 2)?, - published_receipts: unsigned(row, 3)?, - working_bytes: unsigned(row, 4)?, - remote_bytes: unsigned(row, 5)?, - cached_assets, - cached_asset_bytes, - }) - }, - )?) -} - -fn receipts(connection: &Connection) -> Result> { - let mut statement = - connection.prepare("SELECT edit_id, revision FROM receipts ORDER BY edit_id")?; - let mut rows = statement.query([])?; - let mut receipts = BTreeMap::new(); - while let Some(row) = rows.next()? { - receipts.insert(unsigned(row, 0)?, row.get::<_, String>(1)?.parse()?); - } - Ok(receipts) -} - -fn unsigned(row: &rusqlite::Row<'_>, column: usize) -> rusqlite::Result { - u64::try_from(row.get::<_, i64>(column)?).map_err(|error| { - rusqlite::Error::FromSqlConversionFailure( - column, - rusqlite::types::Type::Integer, - Box::new(error), - ) - }) -} diff --git a/crates/onestore-offline/src/schema.rs b/crates/onestore-offline/src/schema.rs deleted file mode 100644 index b392c200c66b01dc09c0a76dc128b06f03f2766b..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/src/schema.rs +++ /dev/null @@ -1,143 +0,0 @@ -use super::*; -use rusqlite::{OptionalExtension, Transaction}; - -const CONFLICTS: &str = "CREATE TABLE conflicts ( - edit_id INTEGER PRIMARY KEY REFERENCES edits(id) ON DELETE CASCADE, - kind INTEGER NOT NULL CHECK(kind BETWEEN 0 AND 6) -) STRICT;"; - -const ASSETS: &str = "CREATE TABLE assets ( - name TEXT PRIMARY KEY NOT NULL, - data BLOB NOT NULL, - sha256 BLOB NOT NULL CHECK(length(sha256)=32) -) STRICT;"; - -pub(crate) fn create(transaction: &Transaction<'_>) -> Result<()> { - transaction.execute_batch( - "CREATE TABLE edits ( - id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(id>0), - space TEXT NOT NULL, - operation TEXT NOT NULL - ) STRICT; - CREATE TABLE attempt ( - id INTEGER PRIMARY KEY CHECK(id=1), - edit_id INTEGER NOT NULL UNIQUE REFERENCES edits(id) ON DELETE CASCADE, - revisions TEXT NOT NULL - ) STRICT; - CREATE TABLE receipts ( - edit_id INTEGER PRIMARY KEY CHECK(edit_id>0), - revision TEXT NOT NULL - ) STRICT;", - )?; - transaction.execute_batch(CONFLICTS)?; - transaction.execute_batch(ASSETS)?; - Ok(()) -} - -pub(crate) fn migrate(transaction: &Transaction<'_>, version: u32) -> Result<()> { - if version >= 10 { - return Ok(()); - } - if version >= 3 { - transaction.execute_batch("ALTER TABLE conflicts RENAME TO old_conflicts;")?; - transaction.execute_batch(CONFLICTS)?; - transaction.execute_batch( - "INSERT INTO conflicts SELECT * FROM old_conflicts; DROP TABLE old_conflicts;", - )?; - if version < 5 { - transaction.execute_batch(ASSETS)?; - } - transaction.execute_batch("ALTER TABLE attempt RENAME COLUMN revision TO revisions;")?; - migrate_attempts(transaction)?; - return Ok(()); - } - - let mut query = transaction.prepare( - "SELECT id, space, object, before_text, start, end, replacement FROM edits ORDER BY id", - )?; - let mut rows = query.query([])?; - let mut edits = Vec::new(); - while let Some(row) = rows.next()? { - edits.push(PendingEdit { - id: u64::try_from(row.get::<_, i64>(0)?).map_err(io::Error::other)?, - space: row.get::<_, String>(1)?.parse()?, - operation: Operation::Text(TextEdit { - object: row.get::<_, String>(2)?.parse()?, - before: row.get(3)?, - range: row.get(4)?..row.get(5)?, - replacement: row.get(6)?, - }), - }); - } - drop(rows); - drop(query); - let sequence: i64 = transaction - .query_row( - "SELECT seq FROM sqlite_sequence WHERE name='edits'", - [], - |row| row.get(0), - ) - .optional()? - .unwrap_or(0); - let (mut attempts, mut conflicts, mut receipts) = (Vec::new(), Vec::new(), Vec::new()); - if version == 2 { - let mut query = transaction.prepare("SELECT edit_id, revision FROM attempt")?; - attempts = query - .query_map([], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) - })? - .collect::>()?; - let mut query = transaction.prepare("SELECT edit_id, kind FROM conflicts")?; - conflicts = query - .query_map([], |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)))? - .collect::>()?; - let mut query = transaction.prepare("SELECT edit_id, revision FROM receipts")?; - receipts = query - .query_map([], |row| { - Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) - })? - .collect::>()?; - transaction - .execute_batch("DROP TABLE attempt; DROP TABLE conflicts; DROP TABLE receipts;")?; - } - transaction.execute_batch("DROP TABLE edits;")?; - create(transaction)?; - for edit in edits { - transaction.execute( - "INSERT INTO edits(id,space,operation) VALUES (?1,?2,?3)", - params![ - i64::try_from(edit.id).map_err(io::Error::other)?, - edit.space.to_string(), - serde_json::to_string(&edit.operation).map_err(io::Error::other)? - ], - )?; - } - // A drained queue must not reuse IDs belonging to existing durable receipts. - transaction.execute("DELETE FROM sqlite_sequence WHERE name='edits'", [])?; - transaction.execute( - "INSERT INTO sqlite_sequence(name,seq) VALUES ('edits',?1)", - [sequence], - )?; - for (id, revision) in attempts { - transaction.execute( - "INSERT INTO attempt VALUES (1,?1,?2)", - params![id, revision], - )?; - } - for (id, kind) in conflicts { - transaction.execute("INSERT INTO conflicts VALUES (?1,?2)", params![id, kind])?; - } - for (id, revision) in receipts { - transaction.execute("INSERT INTO receipts VALUES (?1,?2)", params![id, revision])?; - } - migrate_attempts(transaction)?; - Ok(()) -} - -fn migrate_attempts(transaction: &Transaction<'_>) -> Result<()> { - transaction.execute_batch( - "UPDATE attempt SET revisions=json_object( - (SELECT space FROM edits WHERE edits.id=attempt.edit_id), revisions);", - )?; - Ok(()) -} diff --git a/crates/onestore-offline/src/smb.rs b/crates/onestore-offline/src/smb.rs deleted file mode 100644 index 6076ee03937af7bce342e751daa9e65cd447dad7..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/src/smb.rs +++ /dev/null @@ -1,36 +0,0 @@ -use crate::Remote; -use onestore::{CommitError, PreparedEdit}; -use onestore_smb::Client; -use std::io; - -/// Binds every reconciliation operation to one share-relative file and read limit. -/// Connection loss retires the client; reconnect before subsequent sync attempts. -pub struct SmbRemote { - client: Client, - path: String, - limit: usize, -} - -impl SmbRemote { - pub fn new(client: Client, path: impl Into, limit: usize) -> Self { - Self { - client, - path: path.into(), - limit, - } - } -} - -impl Remote for SmbRemote { - fn read(&mut self) -> io::Result> { - self.client.read(&self.path, self.limit) - } - - fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { - self.client.commit_prepared(&self.path, edit) - } - - fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { - self.client.confirm_snapshot(&self.path, snapshot) - } -} diff --git a/crates/onestore-offline/src/sync.rs b/crates/onestore-offline/src/sync.rs deleted file mode 100644 index 199845ff7184d5d3ca9d7b468fcde94d8d599468..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/src/sync.rs +++ /dev/null @@ -1,671 +0,0 @@ -use super::*; -use onestore::{CommitError, CommitState}; -use rusqlite::OptionalExtension; -use std::{ - collections::BTreeMap, - sync::{MutexGuard, TryLockError}, -}; - -/// A single remote file with fresh reads and native-compatible guarded publication. -/// Errors retain publication state; confirmation compares, flushes, and notifies cached readers. -pub trait Remote { - fn read(&mut self) -> io::Result>; - fn publish(&mut self, edit: &PreparedEdit<'_>) -> std::result::Result<(), CommitError>; - fn confirm(&mut self, snapshot: &[u8]) -> std::result::Result<(), CommitError>; -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[repr(i64)] -pub enum ConflictKind { - TextChanged = 0, - TargetUnavailable = 1, - UnsupportedEdit = 2, - FormattingChanged = 3, - StructureChanged = 4, - LayoutChanged = 5, - ContentChanged = 6, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum EditStatus { - Pending, - /// Retained publication attempt; this revision alone may be insufficient to confirm it. - AwaitingConfirmation { - revision: ExGuid, - }, - Conflict(ConflictKind), - /// Revision of the intent's space when its complete effect was confirmed. - Published { - revision: ExGuid, - }, -} - -impl Replica { - /// Returns a durable receipt or the persisted state of a locally acknowledged edit. - pub fn status(&self, id: u64) -> Result> { - let connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - status(&connection, id) - } - - /// The last observed remote image, retained alongside the complete local working image. - /// Observation alone does not acknowledge any pending edit's remote durability. - pub fn remote_snapshot(&self) -> Result> { - let connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - Ok(connection.query_row("SELECT base FROM replica WHERE id=1", [], |row| row.get(0))?) - } - - /// Reconciles one pending edit, or refreshes the working image when the queue is empty. - /// Network I/O holds synchronization ownership without holding the cache mutex. - /// Uncertain edits are never replayed; retired formatting may confirm its complete observed effect. - pub fn sync_once(&self, remote: &mut impl Remote) -> Result> { - let _owner = self.sync_owner()?; - let snapshot = remote.read().map_err(Error::RemoteIo)?; - let identity = validate(&snapshot)?; - let (intent, attempted) = { - let mut connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - let transaction = - connection.transaction_with_behavior(TransactionBehavior::Immediate)?; - let base: Vec = - transaction - .query_row("SELECT base FROM replica WHERE id=1", [], |row| row.get(0))?; - let base_store = Store::parse(&base)?; - if RevisionIndex::parse(&base_store)?.root != identity { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Remote snapshot belongs to another document", - ) - .into()); - } - let Some(intent) = pending(&transaction)?.into_iter().next() else { - transaction.execute( - "UPDATE replica SET base=?1, working=?1 WHERE id=1", - [&snapshot], - )?; - transaction.commit()?; - return Ok(None); - }; - let attempted = transaction - .query_row( - "SELECT revisions FROM attempt WHERE edit_id=?1", - [i64::try_from(intent.id).map_err(io::Error::other)?], - |row| row.get::<_, String>(0), - ) - .optional()?; - transaction.execute("UPDATE replica SET base=?1 WHERE id=1", [&snapshot])?; - transaction.commit()?; - (intent, attempted) - }; - if let Some(encoded) = attempted { - let revisions = attempted_revisions(&encoded, intent.space)?; - let mut revision = revisions[&intent.space]; - let store = Store::parse(&snapshot)?; - let index = RevisionIndex::parse(&store)?; - if !revisions.iter().all(|(space, revision)| { - index - .spaces - .get(space) - .is_some_and(|space| space.revisions.contains_key(revision)) - }) { - let satisfied = match &intent.operation { - Operation::Format(edit) if revisions.len() == 1 => edit - .prepare(&snapshot, intent.space)? - .is_ok_and(|prepared| prepared.as_bytes() == snapshot), - _ => false, - }; - if !satisfied { - return Ok(Some(( - intent.id, - EditStatus::AwaitingConfirmation { revision }, - ))); - } - revision = index.active(intent.space)?; - } - if let Err(error) = remote.confirm(&snapshot) { - if error.state == CommitState::Committed { - self.acknowledge(intent.id, revision, &snapshot)?; - } - return Err(error.into()); - } - self.acknowledge(intent.id, revision, &snapshot)?; - return Ok(Some((intent.id, EditStatus::Published { revision }))); - } - let candidate = match &intent.operation { - Operation::CreatePage(page) => PreparedEdit::create_page(&snapshot, page) - .map_err(|_| ConflictKind::StructureChanged), - Operation::Pages(edit) => edit.prepare(&snapshot, intent.space)?, - Operation::Format(edit) => edit.prepare(&snapshot, intent.space)?, - Operation::Split(edit) => edit.prepare(&snapshot, intent.space)?, - Operation::Join(edit) => edit.prepare(&snapshot, intent.space)?, - Operation::Outline(edit) => edit.prepare(&snapshot, intent.space)?, - Operation::Tree(edit) => edit.prepare(&snapshot, intent.space)?, - Operation::Insert(insertion) => { - PreparedEdit::insert(&snapshot, intent.space, insertion) - .map_err(|_| ConflictKind::UnsupportedEdit) - } - Operation::Text(edit) => paragraph(&snapshot, intent.space, edit.object)? - .ok_or(ConflictKind::TargetUnavailable) - .and_then(|text| { - crate::rebase::rebase(&edit.before, &text, edit.range.clone()) - .ok_or(ConflictKind::TextChanged) - }) - .and_then(|range| { - PreparedEdit::text( - &snapshot, - intent.space, - edit.object, - range, - &edit.replacement, - ) - .map_err(|_| ConflictKind::UnsupportedEdit) - }), - }; - let prepared = match candidate { - Ok(prepared) => prepared, - Err(kind) => { - let connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - connection.execute("INSERT INTO conflicts(edit_id, kind) VALUES (?1, ?2) ON CONFLICT(edit_id) DO UPDATE SET kind=excluded.kind", params![i64::try_from(intent.id).map_err(io::Error::other)?, kind as i64])?; - return Ok(Some((intent.id, EditStatus::Conflict(kind)))); - } - }; - if prepared.as_bytes() == snapshot { - let store = Store::parse(&snapshot)?; - let index = RevisionIndex::parse(&store)?; - let revision = index.active(intent.space)?; - if let Err(error) = remote.confirm(&snapshot) { - if error.state == CommitState::Committed { - self.acknowledge(intent.id, revision, &snapshot)?; - } - return Err(error.into()); - } - self.acknowledge(intent.id, revision, &snapshot)?; - return Ok(Some((intent.id, EditStatus::Published { revision }))); - } - let store = Store::parse(prepared.as_bytes())?; - let index = RevisionIndex::parse(&store)?; - let revision = index.active(intent.space)?; - let before_store = Store::parse(&snapshot)?; - let before = RevisionIndex::parse(&before_store)?; - let mut revisions: BTreeMap<_, _> = index - .spaces - .iter() - .filter_map(|(sid, space)| { - let revision = *space.labels.get(&(ExGuid::default(), 1))?; - (before - .spaces - .get(sid) - .and_then(|s| s.labels.get(&(ExGuid::default(), 1))) - != Some(&revision)) - .then_some((*sid, revision)) - }) - .collect(); - // The receipt's space can be unchanged in a multi-space edit. - revisions.insert(intent.space, revision); - { - let mut connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - let transaction = - connection.transaction_with_behavior(TransactionBehavior::Immediate)?; - transaction.execute( - "DELETE FROM conflicts WHERE edit_id=?1", - [i64::try_from(intent.id).map_err(io::Error::other)?], - )?; - transaction.execute( - "INSERT INTO attempt(id, edit_id, revisions) VALUES (1, ?1, ?2)", - params![ - i64::try_from(intent.id).map_err(io::Error::other)?, - serde_json::to_string(&revisions).map_err(io::Error::other)? - ], - )?; - transaction.commit()?; - } - match remote.publish(&prepared) { - Ok(()) => {} - Err(error) if error.state == CommitState::NotCommitted => { - let connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - connection.execute( - "DELETE FROM attempt WHERE edit_id=?1", - [i64::try_from(intent.id).map_err(io::Error::other)?], - )?; - return Err(error.into()); - } - Err(error) if error.state == CommitState::Committed => { - self.acknowledge(intent.id, revision, prepared.as_bytes())?; - return Err(error.into()); - } - Err(error) => return Err(error.into()), - } - self.acknowledge(intent.id, revision, prepared.as_bytes())?; - Ok(Some((intent.id, EditStatus::Published { revision }))) - } - - /// Places the oldest text, formatting or split conflict at a reviewed remote UTF-16 range. - /// The requested replacement/attributes, local image, intent ID and later edits are preserved. - /// Both supplied images must match `snapshot` and `remote_snapshot`; stale review - /// returns `Io(ResourceBusy)`. Uncertain publication attempts cannot be rebased. - pub fn rebase_conflict( - &self, - id: u64, - local: &[u8], - remote: &[u8], - range: Range, - ) -> Result<()> { - self.resolve_conflict(id, local, remote, |intent| { - Ok(match intent.operation { - Operation::Text(mut edit) => { - let prepared = PreparedEdit::text( - remote, - intent.space, - edit.object, - range.clone(), - &edit.replacement, - )?; - if prepared.as_bytes() == remote { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "The selected range already contains the replacement", - ) - .into()); - } - edit.before = - paragraph(remote, intent.space, edit.object)?.ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "The remote text target is unavailable", - ) - })?; - edit.range = range; - Operation::Text(edit) - } - Operation::Format(edit) => Operation::Format( - FormatEdit::capture( - remote, - intent.space, - edit.object, - range, - &edit.attributes, - )? - .0, - ), - Operation::Split(edit) => { - if !range.is_empty() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Choose a single split boundary", - ) - .into()); - } - Operation::Split( - crate::paragraph::SplitEdit::capture( - remote, - intent.space, - &edit.intent.reposition(range.start), - )? - .0, - ) - } - Operation::Join(_) => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Review the paragraph join using rebase_join_conflict", - ) - .into()); - } - Operation::Outline(_) => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Review layout changes using rebase_layout_conflict", - ) - .into()); - } - Operation::Tree(_) => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Review the subtree edit using rebase_tree_conflict", - ) - .into()); - } - Operation::Insert(_) => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Insertion conflicts require a placement, not a text range", - ) - .into()); - } - Operation::CreatePage(_) => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Review page placement using rebase_page_creation_conflict", - ) - .into()); - } - Operation::Pages(_) => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Review page edits using rebase_pages_conflict", - ) - .into()); - } - }) - }) - } - - /// Repositions an unattempted page-creation conflict, retaining dependent object identities. - pub fn rebase_page_creation_conflict( - &self, - id: u64, - local: &[u8], - remote: &[u8], - before: Option, - ) -> Result<()> { - self.resolve_conflict(id, local, remote, |intent| { - let Operation::CreatePage(page) = intent.operation else { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Select a page-creation conflict", - ) - .into()); - }; - let page = page.reposition(before)?; - PreparedEdit::create_page(remote, &page)?; - Ok(Operation::CreatePage(page)) - }) - } - - /// Reviews a page batch against both cache images, retaining its page and series identities. - pub fn rebase_pages_conflict( - &self, - id: u64, - local: &[u8], - remote: &[u8], - edits: &[onestore::PageEdit], - ) -> Result<()> { - self.resolve_conflict(id, local, remote, |intent| { - let Operation::Pages(batch) = intent.operation else { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Select a page movement or indentation conflict", - ) - .into()); - }; - Ok(Operation::Pages(batch.review( - remote, - intent.space, - edits, - )?)) - }) - } - - /// Reviews a join against both current cache images, retaining its original text identities. - /// The surviving text identity must remain the same for dependent edits. - pub fn rebase_join_conflict(&self, id: u64, local: &[u8], remote: &[u8]) -> Result<()> { - self.resolve_conflict(id, local, remote, |intent| { - let Operation::Join(edit) = intent.operation else { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Select a paragraph join conflict", - ) - .into()); - }; - Ok(Operation::Join(edit.review(remote, intent.space)?)) - }) - } - - /// Repositions the oldest paragraph insertion conflict against reviewed cache images. - /// Object identities and dependent edits are retained; uncertain attempts cannot be moved. - pub fn rebase_paragraph_conflict( - &self, - id: u64, - local: &[u8], - remote: &[u8], - parent: ExGuid, - before: Option, - ) -> Result<()> { - self.resolve_conflict(id, local, remote, |intent| { - let Operation::Insert(insertion) = intent.operation else { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Select a paragraph insertion conflict", - ) - .into()); - }; - let insertion = insertion.reposition_paragraph(parent, before)?; - PreparedEdit::insert(remote, intent.space, &insertion)?; - Ok(Operation::Insert(insertion)) - }) - } - - /// Repositions the oldest outline insertion conflict against reviewed cache images. - /// Object identities and dependent edits are retained; uncertain attempts cannot be moved. - pub fn rebase_outline_conflict( - &self, - id: u64, - local: &[u8], - remote: &[u8], - page: ExGuid, - x: f32, - y: f32, - ) -> Result<()> { - self.resolve_conflict(id, local, remote, |intent| { - let Operation::Insert(insertion) = intent.operation else { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Select an outline insertion conflict", - ) - .into()); - }; - let insertion = insertion.reposition_outline(page, x, y)?; - PreparedEdit::insert(remote, intent.space, &insertion)?; - Ok(Operation::Insert(insertion)) - }) - } - - /// Reviews the original layout intent against both current cache images. - /// Competing property values are replaced only after this explicit review. - pub fn rebase_layout_conflict(&self, id: u64, local: &[u8], remote: &[u8]) -> Result<()> { - self.resolve_conflict(id, local, remote, |intent| { - let Operation::Outline(edit) = intent.operation else { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Select an outline layout conflict", - ) - .into()); - }; - Ok(Operation::Outline( - OutlineEdit::capture(remote, intent.space, edit.object, edit.change)?.0, - )) - }) - } - - /// Reviews the original subtree intent against both current cache images. - /// Replacement paragraph identities must remain unchanged for dependent edits. - pub fn rebase_tree_conflict(&self, id: u64, local: &[u8], remote: &[u8]) -> Result<()> { - self.resolve_conflict(id, local, remote, |intent| { - let Operation::Tree(edit) = intent.operation else { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Select a subtree move or deletion conflict", - ) - .into()); - }; - Ok(Operation::Tree(edit.review(remote, intent.space)?)) - }) - } - - fn resolve_conflict( - &self, - id: u64, - local: &[u8], - remote: &[u8], - update: impl FnOnce(PendingEdit) -> Result, - ) -> Result<()> { - let owner = self.sync_owner()?; - let intent = self - .pending()? - .into_iter() - .next() - .filter(|intent| intent.id == id) - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "Only the oldest conflict can be rebased", - ) - })?; - let operation = update(intent)?; - let mut connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; - let (base, working): (Vec, Vec) = - transaction.query_row("SELECT base, working FROM replica WHERE id=1", [], |row| { - Ok((row.get(0)?, row.get(1)?)) - })?; - if working != local || base != remote { - return Err(io::Error::new( - io::ErrorKind::ResourceBusy, - "The reviewed cache images changed", - ) - .into()); - } - let id = i64::try_from(id).map_err(io::Error::other)?; - let eligible: bool = transaction.query_row( - "SELECT EXISTS(SELECT 1 FROM conflicts WHERE edit_id=?1) AND NOT EXISTS(SELECT 1 FROM attempt)", - [id], |row| row.get(0), - )?; - if !eligible { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "The edit is not an unattempted conflict", - ) - .into()); - } - transaction.execute( - "UPDATE edits SET operation=?1 WHERE id=?2", - params![ - serde_json::to_string(&operation).map_err(io::Error::other)?, - id - ], - )?; - transaction.execute("DELETE FROM conflicts WHERE edit_id=?1", [id])?; - transaction.commit()?; - drop(connection); - drop(owner); - self.wake_sync(); - Ok(()) - } - - fn sync_owner(&self) -> Result> { - Ok(self - .synchronization - .try_lock() - .map_err(|error| match error { - TryLockError::WouldBlock => io::Error::from(io::ErrorKind::WouldBlock), - TryLockError::Poisoned(_) => { - io::Error::other("Synchronization owner panicked; reopen the cache") - } - })?) - } - - fn acknowledge(&self, id: u64, revision: ExGuid, snapshot: &[u8]) -> Result<()> { - let id = i64::try_from(id).map_err(io::Error::other)?; - let mut connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; - transaction.execute( - "INSERT INTO receipts(edit_id, revision) VALUES (?1, ?2)", - params![id, revision.to_string()], - )?; - transaction.execute("DELETE FROM edits WHERE id=?1", [id])?; - transaction.execute("UPDATE replica SET base=?1, working=CASE WHEN EXISTS(SELECT 1 FROM edits) THEN working ELSE ?1 END WHERE id=1", [snapshot])?; - transaction.commit()?; - Ok(()) - } -} - -pub(crate) fn status(connection: &Connection, id: u64) -> Result> { - let id = i64::try_from(id).map_err(io::Error::other)?; - if let Some(revision) = connection - .query_row( - "SELECT revision FROM receipts WHERE edit_id=?1", - [id], - |row| row.get::<_, String>(0), - ) - .optional()? - { - return Ok(Some(EditStatus::Published { - revision: revision.parse()?, - })); - } - let version: u32 = connection.pragma_query_value(None, "user_version", |row| row.get(0))?; - let column = if version < 10 { - "revision" - } else { - "revisions" - }; - let record: Option<(Option, Option, String)> = connection.query_row( - &format!("SELECT attempt.{column}, conflicts.kind, edits.space FROM edits LEFT JOIN attempt ON attempt.edit_id=edits.id LEFT JOIN conflicts ON conflicts.edit_id=edits.id WHERE edits.id=?1"), [id], |row| Ok((row.get(0)?,row.get(1)?, row.get(2)?))).optional()?; - Ok(match record { - None => None, - Some((Some(revision), _, space)) => Some(EditStatus::AwaitingConfirmation { - revision: if version < 10 { - revision.parse()? - } else { - let space = space.parse()?; - attempted_revisions(&revision, space)?[&space] - }, - }), - Some((None, Some(kind), _)) => Some(EditStatus::Conflict(match kind { - 0 => ConflictKind::TextChanged, - 1 => ConflictKind::TargetUnavailable, - 2 => ConflictKind::UnsupportedEdit, - 3 => ConflictKind::FormattingChanged, - 4 => ConflictKind::StructureChanged, - 5 => ConflictKind::LayoutChanged, - 6 => ConflictKind::ContentChanged, - _ => { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Unknown cached conflict kind", - ) - .into()); - } - })), - Some((None, None, _)) => Some(EditStatus::Pending), - }) -} - -fn attempted_revisions(encoded: &str, space: ExGuid) -> Result> { - let revisions: BTreeMap = serde_json::from_str(encoded) - .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; - if !revisions.contains_key(&space) - || revisions - .iter() - .any(|(sid, rid)| sid.guid == [0; 16] || rid.guid == [0; 16]) - { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Cached publication evidence is incomplete", - ) - .into()); - } - Ok(revisions) -} diff --git a/crates/onestore-offline/src/tree.rs b/crates/onestore-offline/src/tree.rs deleted file mode 100644 index 83282746bbf20c1b3fb92b862b8ed02b3b4c35a5..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/src/tree.rs +++ /dev/null @@ -1,541 +0,0 @@ -use super::*; -use onestore::{FileDataReference, IdStream, ObjectData, PropertySets, ResolvedRevision, Value}; -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; -use std::collections::{BTreeMap, BTreeSet}; - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(untagged, deny_unknown_fields)] -enum Content { - Legacy([u8; 32]), - Semantic { sha256: [u8; 32] }, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct Observed { - path: Vec<(ExGuid, u8)>, - siblings: BTreeMap, - destination: Vec<(ExGuid, u8)>, - content: Option, -} - -/// A subtree intent with observed placement, deletion content and replacement identities. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct TreeEdit { - pub intent: onestore::TreeEdit, - observed: Observed, - created: BTreeSet, -} - -fn fingerprint( - store: &Store<'_>, - raw: &ResolvedRevision<'_>, - root: ExGuid, - legacy: bool, -) -> Result<[u8; 32]> { - let mut objects = BTreeMap::new(); - let mut visiting = BTreeSet::new(); - let mut pending = vec![(root, false)]; - while let Some((id, complete)) = pending.pop() { - if objects.contains_key(&id) { - continue; - } - let object = &raw.objects[&id]; - let references = object.references()?; - if !complete { - if !visiting.insert(id) { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Subtree property references form a cycle", - ) - .into()); - } - pending.push((id, true)); - pending.extend(references.objects.iter().map(|id| (*id, false))); - continue; - } - visiting.remove(&id); - let mut hash = Sha256::new(); - hash.update(object.jcid.to_le_bytes()); - match object.data { - ObjectData::Properties(bytes) => { - hash.update([0]); - let properties = PropertySets::parse(bytes)?; - let mut object_ids = references.objects.into_iter(); - let mut spaces = references.object_spaces.into_iter(); - let mut contexts = references.contexts.into_iter(); - let mut fields = Vec::new(); - for set in &properties.sets { - let mut values = BTreeMap::new(); - for property in set { - let mut value = Sha256::new(); - match &property.value { - Value::NoData => {} - Value::Bytes(bytes) => value.update(bytes), - Value::References { - stream, - compact_ids, - } => { - let references = match stream { - IdStream::Objects => &mut object_ids, - IdStream::ObjectSpaces => &mut spaces, - IdStream::Contexts => &mut contexts, - }; - for reference in references.take(compact_ids.len() / 4) { - if !legacy && *stream == IdStream::Objects { - let immutable = - raw.objects[&reference].jcid & 0x100000 != 0; - value.update([u8::from(immutable)]); - if !immutable { - value.update(reference.guid); - value.update(reference.n.to_le_bytes()); - } - value.update(objects[&reference]); - } else { - value.update(reference.guid); - value.update(reference.n.to_le_bytes()); - } - } - } - Value::Sets(_) => {} - } - if property.id == 0x14001d7a - || (!legacy - && property.id == 0x24001c20 - && matches!(&property.value, Value::References { compact_ids, .. } if compact_ids.is_empty())) - { - continue; - } - values.insert(property.id, value); - } - fields.push(values); - } - // Arena indices and CompactIDs can change without changing property content. - let mut sets = vec![[0; 32]; properties.sets.len()]; - for at in (0..properties.sets.len()).rev() { - for property in &properties.sets[at] { - if let Value::Sets(children) = &property.value { - let value = fields[at].get_mut(&property.id).unwrap(); - for child in children.clone() { - value.update(sets[child]); - } - } - } - let mut set = Sha256::new(); - for (id, value) in &fields[at] { - set.update(id.to_le_bytes()); - set.update(value.clone().finalize()); - } - sets[at] = set.finalize().into(); - } - hash.update(sets[0]); - } - ObjectData::File { - reference, - extension, - } => { - hash.update([1]); - hash.update(Sha256::digest(reference)); - hash.update(Sha256::digest(extension)); - if let Some(FileDataReference::Internal(guid)) = object.file_reference()? { - hash.update(Sha256::digest(store.file_data(guid)?)); - } - } - ObjectData::Encrypted(_) => unreachable!("references rejected encrypted content"), - } - objects.insert(id, <[u8; 32]>::from(hash.finalize())); - } - if !legacy { - return Ok(objects[&root]); - } - let mut hash = Sha256::new(); - for (id, value) in objects { - hash.update(id.guid); - hash.update(id.n.to_le_bytes()); - hash.update(value); - } - Ok(hash.finalize().into()) -} - -fn observe( - source: &[u8], - space: ExGuid, - intent: &onestore::TreeEdit, - legacy: bool, -) -> Result> { - let store = Store::parse(source)?; - let index = RevisionIndex::parse(&store)?; - let document = Document::parse(&index)?; - let Ok(view) = document.active(space) else { - return Ok(None); - }; - let pages = document.pages_in(space)?; - if pages.len() != 1 { - return Ok(None); - } - let object = intent.object(); - let mut targets = vec![object]; - if let Some((parent, _)) = intent.destination() { - targets.push(parent); - } - let Some(paths) = active_paths(view, &pages, &targets) else { - return Ok(None); - }; - let Some(parent) = paths[0].first() else { - return Ok(None); - }; - let children = &view.nodes[parent].children; - let Some(position) = children.iter().position(|id| *id == object) else { - return Ok(None); - }; - let levels = |path: &[ExGuid]| { - path.iter() - .map(|id| (*id, view.nodes[id].child_level.unwrap_or(1))) - .collect::>() - }; - let destination = if let Some((parent, _)) = intent.destination() { - levels(&[&[parent], paths[1].as_slice()].concat()) - } else { - Vec::new() - }; - Ok(Some(Observed { - path: levels(&paths[0]), - siblings: children - .iter() - .enumerate() - .filter_map(|(at, id)| (*id != object).then_some((*id, at < position))) - .collect(), - destination, - content: if intent.destination().is_none() { - let sha256 = fingerprint(&store, &index.resolve_active(space)?, object, legacy)?; - Some(if legacy { - Content::Legacy(sha256) - } else { - Content::Semantic { sha256 } - }) - } else { - None - }, - })) -} - -fn mutable_objects(source: &[u8], space: ExGuid) -> Result> { - let store = Store::parse(source)?; - let index = RevisionIndex::parse(&store)?; - let raw = index.resolve_active(space)?; - Ok(raw - .reachable()? - .into_iter() - .filter(|id| raw.objects[id].jcid & 0x100000 == 0) - .collect()) -} - -impl Replica { - /// Queues a move or deletion, retaining content changes for explicit deletion review. - pub fn tree( - &self, - source: &[u8], - space: ExGuid, - intent: &onestore::TreeEdit, - ) -> Result> { - let (edit, prepared) = TreeEdit::capture(source, space, intent)?; - self.record(source, space, Operation::Tree(edit), &prepared) - } -} - -impl TreeEdit { - pub(crate) fn capture<'a>( - source: &'a [u8], - space: ExGuid, - intent: &onestore::TreeEdit, - ) -> Result<(Self, PreparedEdit<'a>)> { - let prepared = PreparedEdit::tree(source, space, intent)?; - let observed = observe(source, space, intent, false)?.ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "Prepared subtree edit has no active target", - ) - })?; - let before = mutable_objects(source, space)?; - let after = mutable_objects(prepared.as_bytes(), space)?; - Ok(( - Self { - intent: intent.clone(), - observed, - created: &after - &before, - }, - prepared, - )) - } - - pub(crate) fn prepare<'a>( - &self, - source: &'a [u8], - space: ExGuid, - ) -> Result, ConflictKind>> { - let Some(current) = observe( - source, - space, - &self.intent, - matches!(self.observed.content, Some(Content::Legacy(_))), - )? - else { - return Ok(Err(ConflictKind::TargetUnavailable)); - }; - let prepared = match PreparedEdit::tree(source, space, &self.intent) { - Ok(prepared) => prepared, - Err(_) => return Ok(Err(ConflictKind::UnsupportedEdit)), - }; - let after = mutable_objects(prepared.as_bytes(), space)?; - if prepared.as_bytes() == source { - return Ok(if self.created.is_subset(&after) { - Ok(prepared) - } else { - Err(ConflictKind::StructureChanged) - }); - } - if current.path != self.observed.path - || current.destination != self.observed.destination - || current.siblings.iter().any(|(id, before)| { - self.observed - .siblings - .get(id) - .is_some_and(|was_before| before != was_before) - }) - || &after - &mutable_objects(source, space)? != self.created - { - return Ok(Err(ConflictKind::StructureChanged)); - } - if current.content != self.observed.content { - return Ok(Err(ConflictKind::ContentChanged)); - } - Ok(Ok(prepared)) - } - - pub(crate) fn review(&self, source: &[u8], space: ExGuid) -> Result { - let (reviewed, _) = Self::capture(source, space, &self.intent)?; - if reviewed.created != self.created { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "The subtree edit would change its replacement paragraph identities", - ) - .into()); - } - Ok(reviewed) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use onestore::Object; - use std::sync::Arc; - - fn set(fields: &[(u32, Vec)]) -> Vec { - let mut bytes = u16::try_from(fields.len()).unwrap().to_le_bytes().to_vec(); - for (id, _) in fields { - bytes.extend(id.to_le_bytes()); - } - for (_, value) in fields { - bytes.extend(value); - } - bytes - } - - #[test] - fn deletion_fingerprint_resolves_references_and_nested_sets_without_storage_order() { - let source = onestore::create_section("fingerprint.one", "Original", "Author").unwrap(); - let store = Store::parse(&source).unwrap(); - let root = ExGuid { - guid: [1; 16], - n: 1, - }; - let child = ExGuid { - guid: [2; 16], - n: 7, - }; - let hash = |reverse: bool, global: u32, timestamp: u32, change: usize, immutable: bool| { - let mut fields = vec![ - (0x14001d7a, timestamp.to_le_bytes().to_vec()), - (0x20000001, vec![]), - ( - 0x44000002, - set(&[(0x14000003, (u32::from(change == 1)).to_le_bytes().to_vec())]), - ), - ( - 0x44000004, - set(&[(if change == 2 { 0x08000005 } else { 0x88000005 }, vec![])]), - ), - ]; - if change != 5 { - fields.push((0x24001c20, 0_u32.to_le_bytes().to_vec())); - } - if change == 6 { - fields.push((0x24000007, 0_u32.to_le_bytes().to_vec())); - } - if reverse { - fields.reverse(); - } - let mut bytes = 0x80000001_u32.to_le_bytes().to_vec(); - bytes.extend(((global << 8) | child.n).to_le_bytes()); - bytes.extend(set(&fields)); - let mut child_bytes = 0x80000000_u32.to_le_bytes().to_vec(); - child_bytes.extend(set(&[( - 0x14000006, - (u32::from(change == 3)).to_le_bytes().to_vec(), - )])); - let ids = Arc::new(BTreeMap::from([( - global, - if change == 4 { [3; 16] } else { child.guid }, - )])); - let target = ExGuid { - guid: ids[&global], - ..child - }; - let raw = ResolvedRevision { - roots: BTreeMap::from([(1, root)]), - objects: BTreeMap::from([ - ( - root, - Object { - jcid: 0x6000d, - reference_count: 1, - data: ObjectData::Properties(&bytes), - global_ids: ids.clone(), - }, - ), - ( - target, - Object { - jcid: if immutable { 0x12004d } else { 0x6000e }, - reference_count: 1, - data: ObjectData::Properties(&child_bytes), - global_ids: ids, - }, - ), - ]), - }; - fingerprint(&store, &raw, root, false).unwrap() - }; - let original = hash(false, 0, 1, 0, false); - assert_eq!(original, hash(true, 137, 2, 0, false)); - assert_eq!(original, hash(true, 137, 2, 5, false)); - assert_ne!(original, hash(true, 137, 2, 6, false)); - for change in 1..=4 { - assert_ne!(original, hash(true, 137, 2, change, false)); - } - let immutable = hash(false, 0, 1, 0, true); - assert_eq!(immutable, hash(true, 137, 2, 4, true)); - for change in 1..=3 { - assert_ne!(immutable, hash(true, 137, 2, change, true)); - } - } - - #[test] - fn version_eight_deletion_observations_survive_migration_and_upgrade_only_after_review() { - struct Server(Vec); - impl Remote for Server { - fn read(&mut self) -> io::Result> { - Ok(self.0.clone()) - } - fn publish( - &mut self, - edit: &PreparedEdit<'_>, - ) -> std::result::Result<(), onestore::CommitError> { - self.0 = edit.as_bytes().to_vec(); - Ok(()) - } - fn confirm( - &mut self, - snapshot: &[u8], - ) -> std::result::Result<(), onestore::CommitError> { - assert!(self.0 == snapshot); - Ok(()) - } - } - let source = onestore::create_section("legacy.one", "AlphaOmega", "Author").unwrap(); - let store = Store::parse(&source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let (sid, _) = document.pages().unwrap()[0]; - let view = document.active(sid).unwrap(); - let text = *view.nodes.iter().find(|(_, node)| matches!(&node.kind, Kind::RichText { text, .. } if text == "AlphaOmega")).unwrap().0; - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("legacy.sqlite"); - let cache = Replica::create(&path, &source).unwrap(); - cache - .format( - &source, - sid, - text, - 0..10, - &[onestore::TextAttribute::Bold(true)], - ) - .unwrap(); - let split = onestore::ParagraphSplit::new(text, 5, "Author").unwrap(); - cache - .split(&cache.snapshot().unwrap(), sid, &split) - .unwrap(); - let before = cache.snapshot().unwrap(); - let intent = onestore::TreeEdit::delete(split.object(), "Author").unwrap(); - let deletion = cache.tree(&before, sid, &intent).unwrap().unwrap(); - let dependent = cache - .edit_text(&cache.snapshot().unwrap(), sid, text, 0..0, "Local ") - .unwrap() - .unwrap(); - let mut queue = cache.pending().unwrap(); - let Operation::Tree(edit) = &mut queue[2].operation else { - panic!() - }; - edit.observed = observe(&before, sid, &intent, true).unwrap().unwrap(); - let serialized = serde_json::to_string(&queue[2].operation).unwrap(); - let local = cache.snapshot().unwrap(); - drop(cache); - let db = Connection::open(&path).unwrap(); - db.execute( - "UPDATE edits SET operation=?1 WHERE id=?2", - params![serialized, i64::try_from(deletion).unwrap()], - ) - .unwrap(); - db.pragma_update(None, "user_version", 8).unwrap(); - db.execute_batch("ALTER TABLE attempt RENAME COLUMN revisions TO revision;") - .unwrap(); - drop(db); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.pending().unwrap(), queue); - assert!(cache.snapshot().unwrap() == local); - let mut server = Server(source); - for _ in 0..2 { - assert!(matches!( - cache.sync_once(&mut server).unwrap(), - Some((_, EditStatus::Published { .. })) - )); - } - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((deletion, EditStatus::Conflict(ConflictKind::ContentChanged))) - ); - cache - .rebase_tree_conflict(deletion, &local, &server.0) - .unwrap(); - let queue = cache.pending().unwrap(); - let Operation::Tree(edit) = &queue[0].operation else { - panic!() - }; - assert!(matches!( - edit.observed.content, - Some(Content::Semantic { .. }) - )); - for id in [deletion, dependent] { - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) - ); - } - assert_eq!( - paragraph(&server.0, sid, text).unwrap().as_deref(), - Some("Local Alpha") - ); - } -} diff --git a/crates/onestore-offline/src/worker.rs b/crates/onestore-offline/src/worker.rs deleted file mode 100644 index d506a60361e4adc11ead3d22c4b73a052ea07268..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/src/worker.rs +++ /dev/null @@ -1,167 +0,0 @@ -use super::*; -use std::{ - collections::hash_map::RandomState, - hash::BuildHasher, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - mpsc::{self, SyncSender}, - }, - thread::{self, JoinHandle}, - time::Instant, -}; - -pub(super) struct Signal { - stopped: AtomicBool, - sender: SyncSender<()>, -} - -impl Signal { - pub(super) fn wake(&self) { - // One retained notification covers edits that arrive during network I/O. - let _ = self.sender.try_send(()); - } -} - -/// Owns automatic reconciliation. Dropping requests cancellation without blocking. -/// The in-flight sync step finishes before ownership is released; `stop` waits for it. -pub struct SyncWorker { - signal: Arc, - thread: Option>>, -} - -impl SyncWorker { - /// Requests a retry, for example after a network reachability change. - /// A pending contention backoff finishes before processing the notification. - pub fn wake(&self) { - self.signal.wake(); - } - - /// Cancels future steps and waits for the current step and callback to finish. - /// A stopped worker leaves pending edits and uncertain attempts in the cache. - /// Call outside the worker's own callback, which cannot join its calling thread. - pub fn stop(mut self) -> Result<()> { - self.signal.stopped.store(true, Ordering::Release); - self.signal.wake(); - self.thread - .take() - .expect("Worker owns its thread") - .join() - .map_err(|_| io::Error::other("Synchronization worker panicked"))? - } -} - -impl Drop for SyncWorker { - fn drop(&mut self) { - self.signal.stopped.store(true, Ordering::Release); - self.signal.wake(); - } -} - -impl Replica { - /// Starts one worker, reconnecting through `connect` after transport failures. - /// Local edits wake it; `interval` controls idle polling and transport retries. - /// Contended operations returning `NotCommitted` also back off by up to one second. - /// `observe` runs on the worker after each attempt, including connection errors. - /// Cache/document errors stop the worker; inspect them through `observe` or `stop`. - /// Remote calls and callbacks must be bounded for `stop` to have bounded latency. - pub fn start_sync( - self: &Arc, - interval: Duration, - mut connect: F, - mut observe: O, - ) -> io::Result - where - R: Remote + 'static, - F: FnMut() -> io::Result + Send + 'static, - O: FnMut(&Result>) + Send + 'static, - { - if interval.is_zero() || Instant::now().checked_add(interval).is_none() { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Synchronization interval must be positive and representable", - )); - } - let mut owner = self - .worker - .lock() - .map_err(|_| io::Error::other("Synchronization worker registration panicked"))?; - if owner.upgrade().is_some() { - return Err(io::ErrorKind::WouldBlock.into()); - } - let (sender, receiver) = mpsc::sync_channel(1); - let signal = Arc::new(Signal { - stopped: AtomicBool::new(false), - sender, - }); - let replica = Arc::clone(self); - let worker_signal = Arc::clone(&signal); - let thread = thread::Builder::new() - .name("onestore-sync".into()) - .spawn(move || { - let mut remote = None; - let jitter = RandomState::new(); - let mut contention = 0_u32; - while !worker_signal.stopped.load(Ordering::Acquire) { - let result = match remote.as_mut() { - Some(remote) => replica.sync_once(remote), - None => match connect() { - Ok(connected) => { - remote = Some(connected); - continue; - } - Err(error) => Err(Error::RemoteIo(error)), - }, - }; - observe(&result); - match result { - Ok(Some((_, EditStatus::Published { .. }))) => { - contention = 0; - continue; - } - Ok(None) => contention = 0, - Ok(_) => {} - Err(Error::Remote(onestore::CommitError { - state: onestore::CommitState::NotCommitted, - ref error, - })) if matches!( - error.kind(), - io::ErrorKind::WouldBlock | io::ErrorKind::ResourceBusy - ) => - { - contention = contention.saturating_add(1); - let ceiling = (50_u64 << contention.min(5)).min(1000); - let until = Instant::now() - + Duration::from_millis(jitter.hash_one(contention) % ceiling); - // Local wakes must not keep competing writers in the same retry phase. - while !worker_signal.stopped.load(Ordering::Acquire) { - let Some(remaining) = until.checked_duration_since(Instant::now()) - else { - break; - }; - let _ = receiver.recv_timeout(remaining); - } - continue; - } - Err(Error::RemoteIo(ref error)) - if matches!( - error.kind(), - io::ErrorKind::WouldBlock | io::ErrorKind::ResourceBusy - ) => {} - Err(Error::RemoteIo(_) | Error::Remote(_)) => remote = None, - Err(Error::Io(ref error)) if error.kind() == io::ErrorKind::WouldBlock => {} - Err(error) => return Err(error), - } - if !worker_signal.stopped.load(Ordering::Acquire) { - let _ = receiver.recv_timeout(interval); - } - } - Ok(()) - })?; - *owner = Arc::downgrade(&signal); - Ok(SyncWorker { - signal, - thread: Some(thread), - }) - } -} diff --git a/crates/onestore-offline/tests/assets.rs b/crates/onestore-offline/tests/assets.rs deleted file mode 100644 index 3186efe6def0171e62db0640966f925ca9ec0f11..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/tests/assets.rs +++ /dev/null @@ -1,449 +0,0 @@ -use onestore_offline::{EditStatus, Error, Operation, Recovery, Replica}; -use std::{ - fs, io, - path::Path, - sync::{Barrier, mpsc}, - time::Duration, -}; - -const FIXTURE: &str = "../../corpus/native-external-assets/notebook"; -const LEGACY: &str = "../../corpus/offline-v4/live.sqlite"; - -fn copied_cache(root: &Path) -> Replica { - let path = root.join("cache.sqlite"); - assert!(!path.exists()); - fs::copy(LEGACY, &path).unwrap(); - Replica::open(path).unwrap() -} - -fn payload(size: usize) -> (String, Vec) { - fs::read_dir(Path::new(FIXTURE).join("synthetic_onefiles")) - .unwrap() - .map(|entry| entry.unwrap().path()) - .find_map(|path| { - let bytes = fs::read(&path).unwrap(); - (bytes.len() == size) - .then(|| (path.file_name().unwrap().to_str().unwrap().into(), bytes)) - }) - .unwrap() -} - -struct Payload(Option>>); -impl onestore_notebook::Source for Payload { - fn entries(&mut self, _: &str, _: usize) -> io::Result> { - panic!("Unexpected enumeration") - } - fn read(&mut self, _: &str, _: usize) -> io::Result> { - self.0.take().expect("Unexpected repeated payload read") - } -} - -#[test] -fn downloaded_media_survives_migration_reopen_and_recovery_with_the_queue_intact() { - let directory = tempfile::tempdir().unwrap(); - let original = fs::read(LEGACY).unwrap(); - let cache = copied_cache(directory.path()); - let working = cache.snapshot().unwrap(); - let remote = cache.remote_snapshot().unwrap(); - let pending = cache.pending().unwrap(); - assert_eq!( - pending.iter().map(|edit| edit.id).collect::>(), - [1, 2] - ); - let uncertain = cache.status(1).unwrap(); - assert!(matches!( - uncertain, - Some(EditStatus::AwaitingConfirmation { .. }) - )); - let mut source = onestore_notebook::Local::open(FIXTURE).unwrap(); - for size in [0, 1024] { - let (name, bytes) = payload(size); - assert!(cache.cached_asset(&name, size).unwrap().is_none()); - assert_eq!( - cache - .fetch_asset(&mut source, "synthetic.one", &name, size) - .unwrap(), - bytes - ); - assert_eq!( - cache - .cached_asset(&name.to_ascii_lowercase(), size) - .unwrap(), - Some(bytes) - ); - } - let summary = cache.recovery_summary().unwrap(); - assert_eq!( - (summary.cached_assets, summary.cached_asset_bytes), - (2, 1024) - ); - assert_eq!(cache.snapshot().unwrap(), working); - assert_eq!(cache.remote_snapshot().unwrap(), remote); - assert_eq!(cache.pending().unwrap(), pending); - assert_eq!(cache.status(1).unwrap(), uncertain); - cache - .export_recovery(directory.path().join("recovery.sqlite")) - .unwrap(); - drop(cache); - let cache = Replica::open(directory.path().join("cache.sqlite")).unwrap(); - let recovery = Recovery::open(directory.path().join("recovery.sqlite")).unwrap(); - assert_eq!(recovery.summary().unwrap(), summary); - assert_eq!(recovery.pending().unwrap(), pending); - assert_eq!(recovery.status(1).unwrap(), uncertain); - assert!(recovery.receipts().unwrap().is_empty()); - for size in [0, 1024] { - let (name, bytes) = payload(size); - assert_eq!( - cache.cached_asset(&name, size).unwrap(), - Some(bytes.clone()) - ); - assert_eq!(recovery.cached_asset(&name, size).unwrap(), Some(bytes)); - } - assert_eq!(fs::read(LEGACY).unwrap(), original); -} - -#[test] -fn an_original_version_four_archive_remains_read_only_and_has_no_cached_media() { - let path = "../../corpus/offline-v4/recovery.sqlite"; - let before = fs::read(path).unwrap(); - let archive = Recovery::open(path).unwrap(); - assert_eq!(archive.pending().unwrap().len(), 2); - assert!(matches!( - archive.status(1).unwrap(), - Some(EditStatus::AwaitingConfirmation { .. }) - )); - assert_eq!(archive.summary().unwrap().cached_assets, 0); - assert_eq!(archive.summary().unwrap().cached_asset_bytes, 0); - assert!(archive.cached_asset(&payload(0).0, 0).unwrap().is_none()); - drop(archive); - assert_eq!(fs::read(path).unwrap(), before); -} - -#[test] -fn unsuccessful_refreshes_and_changed_identity_data_preserve_the_downloaded_payload() { - let directory = tempfile::tempdir().unwrap(); - let cache = copied_cache(directory.path()); - let (name, bytes) = payload(1024); - let mut source = Payload(Some(Ok(bytes.clone()))); - cache - .fetch_asset(&mut source, "synthetic.one", &name, 1024) - .unwrap(); - let before = fs::read(directory.path().join("cache.sqlite")).unwrap(); - let mut unavailable = Payload(Some(Err(io::ErrorKind::ConnectionReset.into()))); - assert!( - matches!(cache.fetch_asset(&mut unavailable, "synthetic.one", &name, 1024), - Err(Error::Notebook(onestore_notebook::Error::Io { error, .. })) if error.kind() == io::ErrorKind::ConnectionReset) - ); - for size in [0, 1024, 2048] { - let mut changed = Payload(Some(Ok(vec![9; size]))); - assert!(matches!( - cache.fetch_asset(&mut changed, "synthetic.one", &name, 2048), - Err(Error::AssetChanged) - )); - } - assert!( - matches!(cache.cached_asset(&name, 1023), Err(Error::Io(error)) if error.kind() == io::ErrorKind::FileTooLarge) - ); - assert_eq!(cache.cached_asset(&name, 1024).unwrap(), Some(bytes)); - assert_eq!( - fs::read(directory.path().join("cache.sqlite")).unwrap(), - before - ); -} - -#[test] -fn unreferenced_payloads_are_rejected_before_io_or_local_changes() { - let directory = tempfile::tempdir().unwrap(); - let cache = copied_cache(directory.path()); - let mut unused = Payload(None); - assert!( - matches!(cache.fetch_asset(&mut unused, "synthetic.one", "00000000-0000-0000-0000-000000000001.onebin", 100), - Err(Error::Io(error)) if error.kind() == io::ErrorKind::InvalidInput) - ); - assert!( - cache - .fetch_asset(&mut unused, "synthetic.one", "../payload.onebin", 100) - .is_err() - ); - assert_eq!(cache.recovery_summary().unwrap().cached_assets, 0); -} - -#[test] -fn download_network_wait_does_not_block_local_edits() { - struct Waiting { - entered: mpsc::Sender<()>, - released: mpsc::Receiver<()>, - bytes: Vec, - } - impl onestore_notebook::Source for Waiting { - fn entries(&mut self, _: &str, _: usize) -> io::Result> { - panic!("Unexpected enumeration") - } - fn read(&mut self, _: &str, _: usize) -> io::Result> { - self.entered.send(()).unwrap(); - self.released.recv_timeout(Duration::from_secs(5)).unwrap(); - Ok(self.bytes.clone()) - } - } - let directory = tempfile::tempdir().unwrap(); - let cache = copied_cache(directory.path()); - let (name, bytes) = payload(1024); - let (entered, waiting) = mpsc::channel(); - let (release, released) = mpsc::channel(); - let mut source = Waiting { - entered, - released, - bytes: bytes.clone(), - }; - std::thread::scope(|scope| { - let download = scope.spawn(|| { - cache - .fetch_asset(&mut source, "synthetic.one", &name, 1024) - .unwrap() - }); - waiting.recv_timeout(Duration::from_secs(5)).unwrap(); - let pending = cache.pending().unwrap(); - let Operation::Text(edit) = &pending[0].operation else { - panic!() - }; - let id = cache - .edit_text( - &cache.snapshot().unwrap(), - pending[0].space, - edit.object, - 0..0, - "during download ", - ) - .unwrap() - .unwrap(); - release.send(()).unwrap(); - assert_eq!(download.join().unwrap(), bytes); - assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); - }); -} - -#[test] -fn concurrent_downloads_publish_one_immutable_cache_entry() { - let directory = tempfile::tempdir().unwrap(); - let cache = copied_cache(directory.path()); - let (name, bytes) = payload(1024); - let ready = Barrier::new(8); - std::thread::scope(|scope| { - for _ in 0..8 { - scope.spawn(|| { - let mut source = onestore_notebook::Local::open(FIXTURE).unwrap(); - ready.wait(); - assert_eq!( - cache - .fetch_asset(&mut source, "synthetic.one", &name, 1024) - .unwrap(), - bytes - ); - }); - } - }); - assert_eq!(cache.recovery_summary().unwrap().cached_assets, 1); -} - -#[test] -fn a_native_refresh_removing_the_reference_rejects_an_inflight_download() { - struct Native; - impl onestore_offline::Remote for Native { - fn read(&mut self) -> io::Result> { - fs::read("../../corpus/native-external-assets/native/synthetic.one") - } - fn publish(&mut self, _: &onestore::PreparedEdit<'_>) -> Result<(), onestore::CommitError> { - panic!("Unexpected publication") - } - fn confirm(&mut self, _: &[u8]) -> Result<(), onestore::CommitError> { - panic!("Unexpected confirmation") - } - } - struct Refresh<'a>(&'a Replica); - impl onestore_notebook::Source for Refresh<'_> { - fn entries(&mut self, _: &str, _: usize) -> io::Result> { - panic!("Unexpected enumeration") - } - fn read(&mut self, _: &str, _: usize) -> io::Result> { - assert_eq!(self.0.sync_once(&mut Native).unwrap(), None); - Ok(payload(1024).1) - } - } - let root = tempfile::tempdir().unwrap(); - let source = fs::read(Path::new(FIXTURE).join("synthetic.one")).unwrap(); - let cache = Replica::create(root.path().join("cache.sqlite"), &source).unwrap(); - let (name, _) = payload(1024); - assert!( - matches!(cache.fetch_asset(&mut Refresh(&cache), "synthetic.one", &name, 1024), - Err(Error::Io(error)) if error.kind() == io::ErrorKind::ResourceBusy) - ); - assert!(cache.cached_asset(&name, 1024).unwrap().is_none()); - assert_eq!( - cache.snapshot().unwrap(), - fs::read("../../corpus/native-external-assets/native/synthetic.one").unwrap() - ); -} - -#[test] -fn local_failure_and_bad_cached_bytes_never_become_successful_downloads() { - let directory = tempfile::tempdir().unwrap(); - drop(copied_cache(directory.path())); - let path = directory.path().join("cache.sqlite"); - let connection = rusqlite::Connection::open(&path).unwrap(); - connection.execute_batch("CREATE TRIGGER fail_asset BEFORE INSERT ON assets BEGIN SELECT RAISE(ABORT,'Test asset failure'); END").unwrap(); - drop(connection); - let (name, bytes) = payload(1024); - let cache = Replica::open(&path).unwrap(); - let mut source = onestore_notebook::Local::open(FIXTURE).unwrap(); - assert!(matches!( - cache.fetch_asset(&mut source, "synthetic.one", &name, 1024), - Err(Error::Database(_)) - )); - assert!(cache.cached_asset(&name, 1024).unwrap().is_none()); - assert_eq!(cache.pending().unwrap().len(), 2); - drop(cache); - let connection = rusqlite::Connection::open(&path).unwrap(); - connection.execute_batch("DROP TRIGGER fail_asset").unwrap(); - drop(connection); - let cache = Replica::open(&path).unwrap(); - cache - .fetch_asset(&mut source, "synthetic.one", &name, 1024) - .unwrap(); - drop(cache); - let connection = rusqlite::Connection::open(&path).unwrap(); - let mut damaged = bytes; - damaged[0] ^= 1; - connection - .execute("UPDATE assets SET data=?1", [damaged]) - .unwrap(); - drop(connection); - let cache = Replica::open(&path).unwrap(); - assert!( - matches!(cache.cached_asset(&name, 1024), Err(Error::Io(error)) if error.kind() == io::ErrorKind::InvalidData) - ); - assert!( - matches!(cache.fetch_asset(&mut source, "synthetic.one", &name, 1024), Err(Error::Io(error)) if error.kind() == io::ErrorKind::InvalidData) - ); - cache - .export_recovery(directory.path().join("damaged.sqlite")) - .unwrap(); - let archive = Recovery::open(directory.path().join("damaged.sqlite")).unwrap(); - assert!(archive.cached_asset(&name, 1024).is_err()); - assert_eq!(archive.pending().unwrap().len(), 2); -} - -#[test] -fn abrupt_process_exit_retains_only_completed_downloads_and_archives() { - const CHILD: &str = "ONESTORE_ASSET_EXIT_CASE"; - if let Ok(phase) = std::env::var(CHILD) { - struct ExitDuringRead; - impl onestore_notebook::Source for ExitDuringRead { - fn entries(&mut self, _: &str, _: usize) -> io::Result> { - panic!("Unexpected enumeration") - } - fn read(&mut self, _: &str, _: usize) -> io::Result> { - std::process::exit(83) - } - } - let root = std::env::var("ONESTORE_ASSET_EXIT_ROOT").unwrap(); - let cache = copied_cache(Path::new(&root)); - let (name, bytes) = payload(1024); - if phase == "download" { - cache - .fetch_asset(&mut ExitDuringRead, "synthetic.one", &name, bytes.len()) - .unwrap(); - panic!("Read returned after process exit"); - } - cache - .fetch_asset( - &mut onestore_notebook::Local::open(FIXTURE).unwrap(), - "synthetic.one", - &name, - bytes.len(), - ) - .unwrap(); - if phase == "archive" { - cache - .export_recovery(Path::new(&root).join("recovery.sqlite")) - .unwrap(); - } - std::process::exit(83); - } - for phase in ["download", "cached", "archive"] { - let root = tempfile::tempdir().unwrap(); - let output = std::process::Command::new(std::env::current_exe().unwrap()) - .args([ - "--exact", - "abrupt_process_exit_retains_only_completed_downloads_and_archives", - ]) - .env(CHILD, phase) - .env("ONESTORE_ASSET_EXIT_ROOT", root.path()) - .output() - .unwrap(); - assert_eq!( - output.status.code(), - Some(83), - "{}", - String::from_utf8_lossy(&output.stderr) - ); - let cache = Replica::open(root.path().join("cache.sqlite")).unwrap(); - let (name, bytes) = payload(1024); - let expected = (phase != "download").then_some(bytes); - assert_eq!(cache.cached_asset(&name, 1024).unwrap(), expected); - assert_eq!(cache.pending().unwrap().len(), 2); - assert!(matches!( - cache.status(1).unwrap(), - Some(EditStatus::AwaitingConfirmation { .. }) - )); - if phase == "archive" { - let recovery = Recovery::open(root.path().join("recovery.sqlite")).unwrap(); - assert_eq!(recovery.cached_asset(&name, 1024).unwrap(), expected); - assert_eq!(recovery.pending().unwrap(), cache.pending().unwrap()); - assert_eq!(recovery.status(1).unwrap(), cache.status(1).unwrap()); - } - } -} - -#[test] -#[cfg(feature = "smb")] -#[ignore = "requires a disposable Samba mirror at ONESTORE_SMB_NOTEBOOK"] -fn live_smb_downloads_survive_disconnect_and_cache_reopen() { - let root = tempfile::tempdir().unwrap(); - let cache = copied_cache(root.path()); - let client = onestore_smb::Client::connect( - &std::env::var("ONESTORE_SMB_LAB").unwrap(), - "agent", - onestore_smb::Credentials::default(), - Duration::from_secs(5), - ) - .unwrap(); - let notebook = std::env::var("ONESTORE_SMB_NOTEBOOK").unwrap(); - let mut source = onestore_notebook::Smb::new(&client, ¬ebook).unwrap(); - for size in [0, 771, 1024] { - let (name, bytes) = payload(size); - assert_eq!( - cache - .fetch_asset(&mut source, "synthetic.one", &name, size) - .unwrap(), - bytes - ); - } - drop(client); - cache - .export_recovery(root.path().join("recovery.sqlite")) - .unwrap(); - drop(cache); - let cache = Replica::open(root.path().join("cache.sqlite")).unwrap(); - let recovery = Recovery::open(root.path().join("recovery.sqlite")).unwrap(); - for size in [0, 771, 1024] { - let (name, bytes) = payload(size); - assert_eq!( - cache.cached_asset(&name, size).unwrap(), - Some(bytes.clone()) - ); - assert_eq!(recovery.cached_asset(&name, size).unwrap(), Some(bytes)); - } - assert_eq!(cache.pending().unwrap(), recovery.pending().unwrap()); - assert_eq!(cache.status(1).unwrap(), recovery.status(1).unwrap()); - assert_eq!(cache.recovery_summary().unwrap().cached_assets, 3); -} diff --git a/crates/onestore-offline/tests/cache.rs b/crates/onestore-offline/tests/cache.rs deleted file mode 100644 index 554e50de35068d76ea96c781fb4eb53785136459..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/tests/cache.rs +++ /dev/null @@ -1,932 +0,0 @@ -use onestore::{ - ExGuid, RevisionIndex, Store, - document::{Document, Kind}, -}; -use onestore_offline::{Error, Replica}; -use std::{ - collections::BTreeSet, - fs, - io::ErrorKind, - sync::Barrier, - time::{Duration, Instant}, -}; - -fn target(source: &[u8]) -> (ExGuid, ExGuid, String) { - let store = Store::parse(source).unwrap(); - assert!(store.checksum_mismatches.is_empty()); - let index = RevisionIndex::parse(&store).unwrap(); - index.validate_current().unwrap(); - let doc = Document::parse(&index).unwrap(); - doc.spaces - .iter() - .find_map(|(sid, space)| { - let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; - revision - .nodes - .iter() - .find_map(|(oid, node)| match &node.kind { - Kind::RichText { text, .. } => Some((*sid, *oid, text.clone())), - _ => None, - }) - }) - .unwrap() -} - -#[test] -fn cache_reopen_preserves_exact_images_and_intents() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("section.sqlite"); - let source = onestore::create_section("section.one", "café 🦀", "Fixture").unwrap(); - let replica = Replica::create(&path, &source).unwrap(); - assert_eq!(replica.snapshot().unwrap(), source); - assert!(replica.pending().unwrap().is_empty()); - let (sid, oid, _) = target(&source); - assert_eq!( - replica.edit_text(&source, sid, oid, 0..0, "").unwrap(), - None - ); - assert_eq!( - replica.edit_text(&source, sid, oid, 0..4, "café").unwrap(), - None - ); - let first = replica - .edit_text(&source, sid, oid, 5..7, "🐈 日本語") - .unwrap() - .unwrap(); - let edited = replica.snapshot().unwrap(); - assert_eq!(target(&edited).2, "café 🐈 日本語"); - let intents = replica.pending().unwrap(); - assert_eq!(intents.len(), 1); - assert_eq!(intents[0].id, first); - let onestore_offline::Operation::Text(first_edit) = &intents[0].operation else { - panic!() - }; - assert_eq!(first_edit.before, "café 🦀"); - assert_eq!(first_edit.range, 5..7); - assert_eq!(first_edit.replacement, "🐈 日本語"); - assert_eq!((intents[0].space, first_edit.object), (sid, oid)); - drop(replica); - let replica = Replica::open(&path).unwrap(); - assert_eq!(replica.snapshot().unwrap(), edited); - assert_eq!(replica.pending().unwrap(), intents); - let second = replica - .edit_text(&edited, sid, oid, 0..0, "Recovered ") - .unwrap() - .unwrap(); - assert!(second > first); - let onestore_offline::Operation::Text(second_edit) = &replica.pending().unwrap()[1].operation - else { - panic!() - }; - assert_eq!(second_edit.before, "café 🐈 日本語"); -} - -#[test] -fn failed_edits_preserve_both_intent_queue_and_working_image() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("section.sqlite"); - let source = onestore::create_section("section.one", "café 🦀", "Fixture").unwrap(); - let replica = Replica::create(&path, &source).unwrap(); - let (sid, oid, _) = target(&source); - for (range, replacement) in [(6..7, "X"), (0..u32::MAX, "X"), (0..1, "\n")] { - assert!( - replica - .edit_text(&source, sid, oid, range, replacement) - .is_err() - ); - assert_eq!(replica.snapshot().unwrap(), source); - assert!(replica.pending().unwrap().is_empty()); - } - drop(replica); - let connection = rusqlite::Connection::open(&path).unwrap(); - connection.execute_batch("CREATE TRIGGER fail_image BEFORE UPDATE ON replica BEGIN SELECT RAISE(ABORT, 'Injected image update failure'); END;").unwrap(); - drop(connection); - let replica = Replica::open(&path).unwrap(); - assert!(matches!( - replica.edit_text(&source, sid, oid, 0..0, "lost? "), - Err(Error::Database(_)) - )); - drop(replica); - let replica = Replica::open(&path).unwrap(); - assert_eq!(replica.snapshot().unwrap(), source); - assert!(replica.pending().unwrap().is_empty()); -} - -#[test] -fn concurrent_recovery_exports_capture_one_complete_acknowledged_queue() { - use onestore_offline::{Operation, Recovery}; - - let directory = tempfile::tempdir().unwrap(); - let source = onestore::create_section("recovery.one", "base", "Fixture").unwrap(); - let (space, object, _) = target(&source); - let replica = Replica::create(directory.path().join("live.sqlite"), &source).unwrap(); - let start = Barrier::new(4); - std::thread::scope(|scope| { - for writer in 0..3 { - let (replica, start) = (&replica, &start); - scope.spawn(move || { - start.wait(); - for edit in 0..20 { - loop { - let source = replica.snapshot().unwrap(); - match replica.edit_text( - &source, - space, - object, - 0..0, - &format!("[{writer}:{edit}] "), - ) { - Ok(Some(_)) => break, - Err(Error::Io(error)) if error.kind() == ErrorKind::ResourceBusy => { - continue; - } - other => panic!("Unexpected local edit: {other:?}"), - } - } - } - }); - } - start.wait(); - for n in 0..12 { - let path = directory.path().join(format!("recovery-{n}.sqlite")); - replica.export_recovery(&path).unwrap(); - let archive = Recovery::open(path).unwrap(); - let pending = archive.pending().unwrap(); - let mut expected = String::new(); - for edit in pending.iter().rev() { - let Operation::Text(edit) = &edit.operation else { - panic!() - }; - assert_eq!(edit.range, 0..0); - expected.push_str(&edit.replacement); - } - expected.push_str("base"); - assert_eq!(target(&archive.snapshot().unwrap()).2, expected); - assert_eq!(archive.remote_snapshot().unwrap(), source); - assert_eq!( - archive.summary().unwrap().queued_edits, - pending.len() as u64 - ); - assert!(archive.receipts().unwrap().is_empty()); - } - }); - assert_eq!(replica.pending().unwrap().len(), 60); -} - -#[test] -fn ownership_and_foreign_file_rejection_preserve_existing_data() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("section.sqlite"); - assert!(Replica::open(&path).is_err()); - assert!(!path.exists()); - assert!(Replica::create(&path, b"invalid").is_err()); - assert!(!path.exists()); - let source = onestore::create_section("section.one", "Owned", "Fixture").unwrap(); - let replica = Replica::create(&path, &source).unwrap(); - for _ in 0..3 { - assert!( - matches!(Replica::open(&path), Err(Error::Database(error)) if error.sqlite_error_code() == Some(rusqlite::ErrorCode::DatabaseBusy)) - ); - assert!( - matches!(Replica::create(&path, &source), Err(Error::Io(error)) if error.kind() == ErrorKind::AlreadyExists) - ); - assert_eq!(replica.snapshot().unwrap(), source); - } - drop(replica); - for sql in [ - "PRAGMA application_id=0", - "PRAGMA application_id=1330529615; PRAGMA user_version=99", - ] { - let connection = rusqlite::Connection::open(&path).unwrap(); - connection.execute_batch(sql).unwrap(); - drop(connection); - let before = fs::read(&path).unwrap(); - assert!(Replica::open(&path).is_err()); - assert_eq!(fs::read(&path).unwrap(), before); - } - let foreign = dir.path().join("foreign.sqlite"); - let connection = rusqlite::Connection::open(&foreign).unwrap(); - connection - .execute_batch( - "CREATE TABLE unrelated (value TEXT); INSERT INTO unrelated VALUES ('preserve');", - ) - .unwrap(); - drop(connection); - let before = fs::read(&foreign).unwrap(); - assert!(Replica::open(&foreign).is_err()); - assert_eq!(fs::read(&foreign).unwrap(), before); - let incomplete = dir.path().join("incomplete.sqlite"); - fs::write(&incomplete, []).unwrap(); - assert!(Replica::open(&incomplete).is_err()); - assert_eq!(fs::read(&incomplete).unwrap(), b""); -} - -#[test] -fn twelve_local_editors_reject_stale_ranges_and_preserve_every_acknowledgement() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("section.sqlite"); - let source = onestore::create_section("section.one", "Shared café 🦀", "Fixture").unwrap(); - let replica = Replica::create(&path, &source).unwrap(); - let (sid, oid, _) = target(&source); - let barrier = Barrier::new(12); - let deadline = Instant::now() + Duration::from_secs(60); - let outcomes = std::thread::scope(|scope| { - let handles: Vec<_> = (0..12) - .map(|writer| { - let (replica, source, barrier) = (&replica, &source, &barrier); - scope.spawn(move || { - barrier.wait(); - let first = - replica.edit_text(source, sid, oid, 0..0, &format!("[initial-{writer}] ")); - let mut ids = Vec::new(); - let first_won = match first { - Ok(Some(id)) => { - ids.push(id); - true - } - Err(Error::Io(error)) if error.kind() == ErrorKind::ResourceBusy => false, - other => panic!("Unexpected first edit: {other:?}"), - }; - for edit in 0..20 { - loop { - assert!( - Instant::now() < deadline, - "Writer {writer} stopped progressing at {edit}" - ); - let source = replica.snapshot().unwrap(); - match replica.edit_text( - &source, - sid, - oid, - 0..0, - &format!("[{writer}-{edit}] "), - ) { - Ok(Some(id)) => { - ids.push(id); - break; - } - Err(Error::Io(error)) - if error.kind() == ErrorKind::ResourceBusy => {} - other => panic!("Unexpected edit: {other:?}"), - } - } - } - (first_won, ids) - }) - }) - .collect(); - handles - .into_iter() - .map(|handle| handle.join().unwrap()) - .collect::>() - }); - assert_eq!(outcomes.iter().filter(|(won, _)| *won).count(), 1); - let ids: BTreeSet<_> = outcomes.into_iter().flat_map(|(_, ids)| ids).collect(); - assert_eq!(ids.len(), 241); - let final_bytes = replica.snapshot().unwrap(); - let content = target(&final_bytes).2; - let mut expected = "Shared café 🦀".to_owned(); - for pending in replica.pending().unwrap() { - let onestore_offline::Operation::Text(edit) = pending.operation else { - panic!() - }; - assert_eq!(edit.before, expected); - assert_eq!(edit.range, 0..0); - expected.insert_str(0, &edit.replacement); - } - assert_eq!(content, expected); - for writer in 0..12 { - for edit in 0..20 { - assert_eq!(content.matches(&format!("[{writer}-{edit}] ")).count(), 1); - } - } - assert_eq!( - replica - .pending() - .unwrap() - .iter() - .map(|edit| edit.id) - .collect::>(), - ids - ); - assert!( - matches!(replica.edit_text(&source, sid, oid, 0..0, ""), Err(Error::Io(error)) if error.kind() == ErrorKind::ResourceBusy) - ); - drop(replica); - let reopened = Replica::open(&path).unwrap(); - assert_eq!(reopened.snapshot().unwrap(), final_bytes); - assert_eq!(reopened.pending().unwrap().len(), 241); -} - -#[test] -fn seeded_unicode_edits_and_restarts_match_an_independent_text_model() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("model.sqlite"); - let mut text = "ab🚀ab🦀 é repeated repeated".to_owned(); - let mut source = onestore::create_section("model.one", &text, "Fixture").unwrap(); - let mut replica = Replica::create(&path, &source).unwrap(); - let (space, object, _) = target(&source); - let mut random = 911_u64; - let mut next = || { - random ^= random << 13; - random ^= random >> 7; - random ^= random << 17; - random - }; - let mut intents = Vec::new(); - for step in 0..1024 { - let boundaries: Vec<_> = text - .char_indices() - .map(|(at, _)| at) - .chain([text.len()]) - .collect(); - let first = boundaries[next() as usize % boundaries.len()]; - let last = boundaries[next() as usize % boundaries.len()]; - let bytes = first.min(last)..first.max(last); - let range = u32::try_from(text[..bytes.start].encode_utf16().count()).unwrap() - ..u32::try_from(text[..bytes.end].encode_utf16().count()).unwrap(); - let replacement = ["", "🐈", "日本語", "repeated", "é", "ab🦀ab"][next() as usize % 6]; - let mut expected = text.clone(); - expected.replace_range(bytes, replacement); - let acknowledgement = replica - .edit_text(&source, space, object, range.clone(), replacement) - .unwrap(); - if let Some(id) = acknowledgement { - assert_ne!(text, expected); - assert!( - intents - .last() - .is_none_or(|edit: &onestore_offline::PendingEdit| edit.id < id) - ); - intents.push(onestore_offline::PendingEdit { - id, - space, - operation: onestore_offline::Operation::Text(onestore_offline::TextEdit { - object, - before: text, - range, - replacement: replacement.into(), - }), - }); - assert!( - matches!(replica.edit_text(&source, space, object, 0..0, "stale"), Err(Error::Io(error)) if error.kind() == ErrorKind::ResourceBusy) - ); - } else { - assert_eq!(text, expected); - } - text = expected; - source = replica.snapshot().unwrap(); - assert_eq!(target(&source).2, text, "seed 911, step {step}"); - if step % 37 == 0 { - drop(replica); - replica = Replica::open(&path).unwrap(); - assert_eq!(replica.snapshot().unwrap(), source); - assert_eq!(replica.pending().unwrap(), intents); - } - } - assert!( - intents.len() > 512, - "The history checkpoint boundary was not exercised" - ); - drop(replica); - let replica = Replica::open(&path).unwrap(); - assert_eq!(replica.pending().unwrap(), intents); - assert_eq!(replica.snapshot().unwrap(), source); -} - -#[test] -#[ignore = "migrates a fresh copy of a retained version-two or version-three cache"] -fn migrate_retained_cache_copy() { - use onestore_offline::{EditStatus, Operation, PendingEdit, TextEdit}; - let source = std::path::PathBuf::from(std::env::var_os("ONESTORE_MIGRATION_SOURCE").unwrap()); - let output = std::path::PathBuf::from(std::env::var_os("ONESTORE_MIGRATION_OUTPUT").unwrap()); - assert!(source.is_absolute() && output.is_absolute()); - let mut original = fs::File::open(&source).unwrap(); - let mut destination = fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&output) - .unwrap(); - std::io::copy(&mut original, &mut destination).unwrap(); - destination.sync_all().unwrap(); - drop(destination); - let db = rusqlite::Connection::open(&output).unwrap(); - let version = db - .pragma_query_value(None, "user_version", |r| r.get::<_, u32>(0)) - .unwrap(); - assert!(matches!(version, 2 | 3)); - if std::env::var_os("ONESTORE_MIGRATION_ROLLBACK").is_some() { - assert_eq!(version, 2); - db.pragma_update(None, "foreign_keys", false).unwrap(); - db.execute( - "INSERT INTO attempt VALUES (1,999999,'{00000001-0000-0000-0000-000000000000},1')", - [], - ) - .unwrap(); - drop(db); - let before = fs::read(&output).unwrap(); - assert!( - matches!(Replica::open(&output), Err(Error::Database(rusqlite::Error::SqliteFailure(error, _))) if error.extended_code == 787) - ); - assert_eq!(fs::read(&output).unwrap(), before); - let db = rusqlite::Connection::open(&output).unwrap(); - assert_eq!( - db.pragma_query_value(None, "user_version", |r| r.get::<_, u32>(0)) - .unwrap(), - 2 - ); - println!("migration: rollback preserved {} bytes", before.len()); - return; - } - if std::env::var_os("ONESTORE_MIGRATION_CONFLICT").is_some() { - db.execute("INSERT INTO conflicts SELECT min(id),0 FROM edits", []) - .unwrap(); - } - let (base, working): (Vec, Vec) = db - .query_row("SELECT base,working FROM replica", [], |r| { - Ok((r.get(0)?, r.get(1)?)) - }) - .unwrap(); - let sequence: i64 = db - .query_row( - "SELECT seq FROM sqlite_sequence WHERE name='edits'", - [], - |r| r.get(0), - ) - .unwrap(); - let pending: Vec = - if version == 2 { - let mut query = db - .prepare("SELECT id,space,object,before_text,start,end,replacement FROM edits ORDER BY id") - .unwrap(); - let pending: Vec = query - .query_map([], |r| { - Ok(PendingEdit { - id: u64::try_from(r.get::<_, i64>(0)?).unwrap(), - space: r.get::<_, String>(1)?.parse().unwrap(), - operation: Operation::Text(TextEdit { - object: r.get::<_, String>(2)?.parse().unwrap(), - before: r.get(3)?, - range: r.get(4)?..r.get(5)?, - replacement: r.get(6)?, - }), - }) - }) - .unwrap() - .collect::>() - .unwrap(); - drop(query); - pending - } else { - let mut query = db - .prepare("SELECT id,space,operation FROM edits ORDER BY id") - .unwrap(); - query - .query_map([], |r| { - Ok(PendingEdit { - id: u64::try_from(r.get::<_, i64>(0)?).unwrap(), - space: r.get::<_, String>(1)?.parse().unwrap(), - operation: serde_json::from_str(&r.get::<_, String>(2)?).unwrap(), - }) - }) - .unwrap() - .collect::>() - .unwrap() - }; - let mut states = std::collections::BTreeMap::new(); - for edit in &pending { - states.insert(edit.id, EditStatus::Pending); - } - for table in ["receipts", "attempt"] { - let mut query = db - .prepare(&format!("SELECT edit_id,revision FROM {table}")) - .unwrap(); - for row in query - .query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?))) - .unwrap() - { - let (id, rid) = row.unwrap(); - let revision = rid.parse().unwrap(); - states.insert( - u64::try_from(id).unwrap(), - if table == "receipts" { - EditStatus::Published { revision } - } else { - EditStatus::AwaitingConfirmation { revision } - }, - ); - } - } - let mut query = db.prepare("SELECT edit_id,kind FROM conflicts").unwrap(); - for row in query - .query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, u32>(1)?))) - .unwrap() - { - let (id, kind) = row.unwrap(); - assert_eq!(kind, 0); - states.insert( - u64::try_from(id).unwrap(), - EditStatus::Conflict(onestore_offline::ConflictKind::TextChanged), - ); - } - drop(query); - drop(db); - let cache = Replica::open(&output).unwrap(); - assert_eq!(cache.snapshot().unwrap(), working); - assert_eq!(cache.remote_snapshot().unwrap(), base); - assert_eq!(cache.pending().unwrap(), pending); - for (id, status) in &states { - assert_eq!(cache.status(*id).unwrap(), Some(*status)); - } - drop(cache); - let cache = Replica::open(&output).unwrap(); - assert_eq!(cache.pending().unwrap(), pending); - let store = Store::parse(&working).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let (sid, page) = doc.pages().unwrap()[0]; - let insertion = onestore::Insertion::outline( - page, - 144.0, - 720.0, - "After cache migration", - "Migration author", - ) - .unwrap(); - let next = cache.insert(&working, sid, &insertion).unwrap().unwrap(); - assert_eq!(next, u64::try_from(sequence + 1).unwrap()); - let updated = cache.snapshot().unwrap(); - drop(cache); - let cache = Replica::open(&output).unwrap(); - assert_eq!(cache.snapshot().unwrap(), updated); - assert_eq!( - cache.pending().unwrap().last().unwrap().operation, - Operation::Insert(insertion) - ); - println!( - "migration: {}", - serde_json::json!({"pending":pending.len(),"retained_statuses":states.len(),"next_id":next,"base_bytes":base.len(),"working_bytes":working.len()}) - ); -} - -#[test] -fn twelve_local_clients_preserve_inserted_identities_and_dependent_edits() { - use onestore::Insertion; - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("parallel.sqlite"); - let source = onestore::create_section("parallel.one", "Original", "Author").unwrap(); - let store = Store::parse(&source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let (sid, page) = doc.pages().unwrap()[0]; - let cache = Replica::create(&path, &source).unwrap(); - let barrier = Barrier::new(12); - let deadline = Instant::now() + Duration::from_secs(90); - let all = std::thread::scope(|scope| { - let handles: Vec<_> = (0..12) - .map(|client| { - let (cache, barrier) = (&cache, &barrier); - scope.spawn(move || { - let outline = Insertion::outline( - page, - 72.0, - 144.0 + client as f32 * 72.0, - &format!("Client {client}"), - "Author", - ) - .unwrap(); - let mut ids = Vec::new(); - let mut objects = Vec::new(); - barrier.wait(); - for sequence in 0..4 { - let insertion = if sequence == 0 { - outline.clone() - } else { - Insertion::paragraph( - outline.object(), - None, - &format!("Paragraph {client}:{sequence}"), - "Author", - ) - .unwrap() - } - .with_formatting(0..6, &[onestore::TextAttribute::Italic(true)]) - .unwrap(); - loop { - assert!( - Instant::now() < deadline, - "Client {client} stopped at insertion {sequence}" - ); - let snapshot = cache.snapshot().unwrap(); - match cache.insert(&snapshot, sid, &insertion) { - Ok(Some(id)) => { - ids.push(id); - objects.push(insertion.text_object()); - break; - } - Err(Error::Io(e)) if e.kind() == ErrorKind::ResourceBusy => {} - other => panic!("{other:?}"), - } - } - if sequence == 0 { - continue; - } - loop { - assert!( - Instant::now() < deadline, - "Client {client} stopped at text {sequence}" - ); - let snapshot = cache.snapshot().unwrap(); - match cache.edit_text( - &snapshot, - sid, - insertion.text_object(), - 0..0, - "Edited ", - ) { - Ok(Some(id)) => { - ids.push(id); - break; - } - Err(Error::Io(e)) if e.kind() == ErrorKind::ResourceBusy => {} - other => panic!("{other:?}"), - } - } - } - (ids, objects) - }) - }) - .collect(); - handles - .into_iter() - .map(|h| h.join().unwrap()) - .collect::>() - }); - let ids: BTreeSet<_> = all - .iter() - .flat_map(|(ids, _)| ids.iter().copied()) - .collect(); - assert_eq!(ids.len(), 84); - let snapshot = cache.snapshot().unwrap(); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.snapshot().unwrap(), snapshot); - assert_eq!( - cache - .pending() - .unwrap() - .iter() - .map(|e| e.id) - .collect::>(), - ids - ); - let store = Store::parse(&snapshot).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let s = &doc.spaces[&sid]; - let v = &s.revisions[&s.contexts[&ExGuid::default()]]; - for (client, (_, objects)) in all.iter().enumerate() { - for (sequence, id) in objects.iter().enumerate() { - let wanted = if sequence == 0 { - format!("Client {client}") - } else { - format!("Edited Paragraph {client}:{sequence}") - }; - assert!(matches!(&v.nodes[id].kind,Kind::RichText{text,..} if *text==wanted)); - let runs = v.text_runs(*id).unwrap(); - assert_eq!(runs[0].format.italic, Some(true)); - assert_eq!( - runs[0].text, - if sequence == 0 { - "Client" - } else { - "Edited Paragr" - } - ); - assert!(runs[1..].iter().all(|run| run.format.italic != Some(true))); - } - } -} - -#[test] -fn unrecognized_persisted_operations_are_rejected_without_dropping_fields() { - for operation in ["Text", "Insert", "Format", "Split", "Join", "Outline"] { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("unknown.sqlite"); - let mut source = onestore::create_section("unknown.one", "Original", "Author").unwrap(); - let (sid, oid, _) = target(&source); - let split = onestore::ParagraphSplit::new(oid, 3, "Author").unwrap(); - if operation == "Join" { - source = onestore::PreparedEdit::split(&source, sid, &split) - .unwrap() - .as_bytes() - .to_vec(); - } - let cache = Replica::create(&path, &source).unwrap(); - if operation == "Insert" { - let store = Store::parse(&source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let (_, page) = doc.pages().unwrap()[0]; - let insertion = - onestore::Insertion::outline(page, 144.0, 144.0, "Inserted", "Author").unwrap(); - cache.insert(&source, sid, &insertion).unwrap(); - } else if operation == "Outline" { - let store = Store::parse(&source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let space = &doc.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let paragraph = *view - .nodes - .iter() - .find(|(_, node)| node.content == [oid]) - .unwrap() - .0; - cache - .outline( - &source, - sid, - paragraph, - onestore::OutlineEdit::Collapsed(true), - ) - .unwrap(); - } else if operation == "Text" { - cache.edit_text(&source, sid, oid, 0..0, "New ").unwrap(); - } else if operation == "Split" { - cache.split(&source, sid, &split).unwrap(); - } else if operation == "Join" { - cache - .join( - &source, - sid, - &onestore::ParagraphJoin::new(oid, split.text_object(), "Author").unwrap(), - ) - .unwrap(); - } else { - cache - .format( - &source, - sid, - oid, - 0..4, - &[onestore::TextAttribute::Bold(true)], - ) - .unwrap(); - } - drop(cache); - let db = rusqlite::Connection::open(&path).unwrap(); - let encoded: String = db - .query_row("SELECT operation FROM edits", [], |r| r.get(0)) - .unwrap(); - let mut value: serde_json::Value = serde_json::from_str(&encoded).unwrap(); - value[operation]["future_option"] = true.into(); - db.execute("UPDATE edits SET operation=?1", [value.to_string()]) - .unwrap(); - if matches!(operation, "Text" | "Insert") { - db.execute_batch("ALTER TABLE attempt RENAME COLUMN revisions TO revision; DROP TABLE assets; DROP TABLE conflicts; CREATE TABLE conflicts (edit_id INTEGER PRIMARY KEY REFERENCES edits(id) ON DELETE CASCADE, kind INTEGER NOT NULL CHECK(kind BETWEEN 0 AND 2)) STRICT; PRAGMA user_version=3;").unwrap(); - } - drop(db); - let before = fs::read(&path).unwrap(); - assert!( - matches!(Replica::open(&path),Err(Error::Io(error))if error.kind()==ErrorKind::InvalidData) - ); - assert_eq!(fs::read(&path).unwrap(), before); - } -} - -#[test] -fn prior_schema_migrations_retain_queue_evidence_assets_and_enable_content_conflicts() { - for (version, ceiling) in [(5, 3), (6, 4), (7, 5), (8, 6), (9, 6)] { - use onestore_offline::{ConflictKind, EditStatus, Recovery}; - use sha2::{Digest, Sha256}; - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("previous.sqlite"); - let source = onestore::create_section("migration.one", "Original", "Author").unwrap(); - let (sid, oid, _) = target(&source); - let cache = Replica::create(&path, &source).unwrap(); - let first = cache - .edit_text(&source, sid, oid, 0..0, "New ") - .unwrap() - .unwrap(); - let second = cache - .format( - &cache.snapshot().unwrap(), - sid, - oid, - 0..3, - &[onestore::TextAttribute::Bold(true)], - ) - .unwrap() - .unwrap(); - let queue = cache.pending().unwrap(); - let local = cache.snapshot().unwrap(); - let store = Store::parse(&local).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let revision = index.spaces[&sid].labels[&(ExGuid::default(), 1)]; - drop(cache); - let db = rusqlite::Connection::open(&path).unwrap(); - let current_version: u32 = db - .pragma_query_value(None, "user_version", |row| row.get(0)) - .unwrap(); - let asset = "00112233-4455-6677-8899-aabbccddeeff.onebin"; - let data = b"retained media"; - db.execute( - "INSERT INTO assets VALUES (?1,?2,?3)", - rusqlite::params![asset, data.as_slice(), Sha256::digest(data).as_slice()], - ) - .unwrap(); - db.execute_batch(&format!( - "ALTER TABLE attempt RENAME COLUMN revisions TO revision; - DROP TABLE conflicts; CREATE TABLE conflicts ( - edit_id INTEGER PRIMARY KEY REFERENCES edits(id) ON DELETE CASCADE, - kind INTEGER NOT NULL CHECK(kind BETWEEN 0 AND {ceiling})) STRICT; - PRAGMA user_version={version}; - UPDATE sqlite_sequence SET seq=1000 WHERE name='edits';" - )) - .unwrap(); - db.execute( - "INSERT INTO conflicts VALUES (?1,3)", - [i64::try_from(second).unwrap()], - ) - .unwrap(); - db.execute( - "INSERT INTO attempt VALUES (1,?1,?2)", - rusqlite::params![i64::try_from(first).unwrap(), revision.to_string()], - ) - .unwrap(); - db.execute( - "INSERT INTO receipts VALUES (999,?1)", - [revision.to_string()], - ) - .unwrap(); - drop(db); - let archive = directory.path().join("legacy-recovery.sqlite"); - std::fs::copy(&path, &archive).unwrap(); - let archive_db = rusqlite::Connection::open(&archive).unwrap(); - archive_db - .pragma_update(None, "application_id", 0x4f4e4552) - .unwrap(); - drop(archive_db); - let original_archive = std::fs::read(&archive).unwrap(); - let recovery = Recovery::open(&archive).unwrap(); - assert_eq!( - recovery.status(first).unwrap(), - Some(EditStatus::AwaitingConfirmation { revision }) - ); - assert_eq!(recovery.pending().unwrap(), queue); - drop(recovery); - assert!(std::fs::read(&archive).unwrap() == original_archive); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.pending().unwrap(), queue); - assert_eq!( - cache.cached_asset(asset, 1024).unwrap(), - Some(data.to_vec()) - ); - assert!(cache.snapshot().unwrap() == local); - assert!(cache.remote_snapshot().unwrap() == source); - assert_eq!( - cache.status(first).unwrap(), - Some(EditStatus::AwaitingConfirmation { revision }) - ); - assert_eq!( - cache.status(second).unwrap(), - Some(EditStatus::Conflict(ConflictKind::FormattingChanged)) - ); - assert_eq!( - cache.status(999).unwrap(), - Some(EditStatus::Published { revision }) - ); - let split = onestore::ParagraphSplit::new(oid, 3, "Author").unwrap(); - assert_eq!(cache.split(&local, sid, &split).unwrap(), Some(1001)); - let pending = cache.pending().unwrap(); - let archive = directory.path().join("recovery.sqlite"); - cache.export_recovery(&archive).unwrap(); - let recovery = Recovery::open(&archive).unwrap(); - assert_eq!(recovery.pending().unwrap(), pending); - assert_eq!( - recovery.cached_asset(asset, 1024).unwrap(), - Some(data.to_vec()) - ); - assert_eq!( - recovery.status(first).unwrap(), - cache.status(first).unwrap() - ); - assert_eq!(recovery.receipts().unwrap().get(&999), Some(&revision)); - drop(cache); - let db = rusqlite::Connection::open(&path).unwrap(); - assert_eq!( - db.pragma_query_value(None, "user_version", |row| row.get::<_, u32>(0)) - .unwrap(), - current_version - ); - db.execute("INSERT INTO conflicts VALUES (1001,6)", []) - .unwrap(); - drop(db); - let cache = Replica::open(&path).unwrap(); - assert_eq!( - cache.status(1001).unwrap(), - Some(EditStatus::Conflict(ConflictKind::ContentChanged)) - ); - } -} diff --git a/crates/onestore-offline/tests/support/page_schedule.rs b/crates/onestore-offline/tests/support/page_schedule.rs deleted file mode 100644 index fe02c8a42adcd80da6c9446132ecb762ecf97261..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/tests/support/page_schedule.rs +++ /dev/null @@ -1,254 +0,0 @@ -use crate::disk; -use onestore::{ - CommitError, ExGuid, Insertion, PageEdit, PagePosition, PreparedEdit, RevisionIndex, Store, - document::{Document, Kind}, -}; -use onestore_offline::{EditStatus, Operation, Remote, Replica}; -use std::{io, sync::LazyLock}; - -#[path = "../../../onestore/tests/support/current.rs"] -mod current; - -static SOURCE: LazyLock> = LazyLock::new(|| { - let mut source = onestore::create_section("page-schedule.one", "Original", "Author").unwrap(); - for _ in 0..3 { - let page = onestore::PageCreation::new(None, Some("Same title"), "Author").unwrap(); - source = PreparedEdit::create_page(&source, &page) - .unwrap() - .as_bytes() - .to_vec(); - } - source -}); - -fn pages(source: &[u8]) -> Vec<(ExGuid, ExGuid, u32)> { - let store = Store::parse(source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - index.validate_current().unwrap(); - let document = Document::parse(&index).unwrap(); - document - .pages() - .unwrap() - .into_iter() - .map(|(sid, page)| { - let space = &document.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let Kind::Metadata { level, .. } = view.nodes[&view.roots[&2]].kind else { - panic!() - }; - (sid, page, level.unwrap_or(1)) - }) - .collect() -} - -struct Session<'a> { - disk: &'a mut disk::Disk, - operation: Option<&'a Operation>, -} - -impl Remote for Session<'_> { - fn read(&mut self) -> io::Result> { - Ok(self.disk.visible.clone()) - } - - fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { - let mut expected = pages(&self.disk.visible); - match self.operation.unwrap() { - Operation::CreatePage(page) => expected.push((page.space(), page.object(), 1)), - Operation::Pages(batch) => { - for change in &batch.edits { - let at = expected.iter().position(|p| p.0 == change.space()).unwrap(); - expected[at].2 = change.level(); - if let PagePosition::Before(before) = change.position() { - let page = expected.remove(at); - let at = before.map_or(expected.len(), |before| { - expected.iter().position(|p| p.0 == before).unwrap() - }); - expected.insert(at, page); - } - } - } - Operation::Insert(insertion) => { - let store = Store::parse(edit.as_bytes()).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let matching: Vec<_> = document - .spaces - .values() - .filter_map(|space| { - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - view.nodes.get(&insertion.text_object()) - }) - .collect(); - assert_eq!(matching.len(), 1); - assert!( - matches!(&matching[0].kind, Kind::RichText { text, .. } if text == "Body 🦋 é") - ); - } - _ => panic!(), - } - assert_eq!(pages(edit.as_bytes()), expected); - let old = current::current(&self.disk.durable); - let new = current::current(edit.as_bytes()); - let result = edit.commit(self.disk); - let observed = current::current(&self.disk.durable); - assert!(observed == old || observed == new); - if result.is_ok() { - assert_eq!(observed, new); - } - result - } - - fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { - onestore::confirm_snapshot(self.disk, snapshot) - } -} - -pub fn run(input: &[u8]) { - let directory = tempfile::tempdir().unwrap(); - let mut replicas: [Option; 12] = std::array::from_fn(|_| None); - let mut owned: [Vec<(ExGuid, ExGuid)>; 12] = std::array::from_fn(|_| Vec::new()); - let mut disk = disk::Disk { - visible: SOURCE.clone(), - durable: SOURCE.clone(), - operation: 0, - fail_at: None, - write_limit: 4096, - random: 1, - }; - for step in input.chunks_exact(8).take(48) { - let actor = usize::from(step[0]) % replicas.len(); - let path = directory.path().join(format!("{actor}.sqlite")); - let cache = replicas[actor].get_or_insert_with(|| Replica::create(&path, &SOURCE).unwrap()); - let snapshot = cache.snapshot().unwrap(); - let pending = cache.pending().unwrap(); - match step[1] % 8 { - 0 | 1 => { - let title = (step[2] & 1 != 0).then_some("Same 🦋 é"); - let page = onestore::PageCreation::new(None, title, "Author").unwrap(); - cache.create_page(&snapshot, &page).unwrap().unwrap(); - owned[actor].push((page.space(), page.object())); - if step[1] % 8 == 1 { - let insertion = - Insertion::outline(page.object(), 36.0, 36.0, "Body 🦋 é", "Author") - .unwrap(); - cache - .insert(&cache.snapshot().unwrap(), page.space(), &insertion) - .unwrap() - .unwrap(); - } - } - 2 => { - let statuses: Vec<_> = pending - .iter() - .map(|edit| cache.status(edit.id).unwrap()) - .collect(); - replicas[actor] = None; - let cache = Replica::open(&path).unwrap(); - assert!(cache.snapshot().unwrap() == snapshot); - assert_eq!(cache.pending().unwrap(), pending); - assert_eq!( - pending - .iter() - .map(|edit| cache.status(edit.id).unwrap()) - .collect::>(), - statuses - ); - replicas[actor] = Some(cache); - } - 3 | 4 => { - disk.operation = 0; - disk.write_limit = if step[3] & 1 == 0 { 17 } else { 4096 }; - disk.fail_at = - (step[2] != 0).then_some(usize::from(u16::from_le_bytes([step[2], step[3]]))); - disk.random = u64::from(step[4]) + 1; - let prior = pending - .first() - .and_then(|edit| cache.status(edit.id).unwrap()); - let mut session = Session { - disk: &mut disk, - operation: pending.first().map(|edit| &edit.operation), - }; - let result = cache.sync_once(&mut session); - if matches!(prior, Some(EditStatus::AwaitingConfirmation { .. })) { - assert!( - result.is_err() - || !matches!(result.unwrap(), Some((_, EditStatus::Conflict(_)))) - ); - } - } - 5 => { - disk.visible.clone_from(&disk.durable); - disk.fail_at = None; - } - 6 => { - let order = pages(&snapshot); - let count = 1 + usize::from(step[3] & 1 != 0); - let selected: Vec<_> = (0..count) - .map(|i| order[(usize::from(step[2]) + i) % order.len()].0) - .collect(); - let anchor = (step[4] & 1 != 0) - .then_some(order[usize::from(step[5]) % order.len()].0) - .filter(|sid| !selected.contains(sid)); - let edits: Vec<_> = selected - .iter() - .enumerate() - .map(|(i, sid)| { - let level = u32::from(step[6 + i]) % 3 + 1; - if step[4] & 2 == 0 { - PageEdit::set_level(*sid, level).unwrap() - } else { - PageEdit::move_to(*sid, anchor, level).unwrap() - } - }) - .collect(); - let expected = PreparedEdit::pages(&snapshot, &edits); - match cache.pages(&snapshot, &edits) { - Ok(id) => { - let expected = expected.unwrap(); - assert_eq!( - current::current(&cache.snapshot().unwrap()), - current::current(expected.as_bytes()) - ); - assert_eq!(id.is_some(), expected.as_bytes() != snapshot); - } - Err(_) => { - assert!(expected.is_err()); - assert_eq!(cache.snapshot().unwrap(), snapshot); - assert_eq!(cache.pending().unwrap(), pending); - } - } - } - 7 => { - if let Some(edit) = pending.first() - && matches!( - cache.status(edit.id).unwrap(), - Some(EditStatus::Conflict(_)) - ) - && let Operation::Pages(batch) = &edit.operation - { - let remote = cache.remote_snapshot().unwrap(); - let order = pages(&remote); - let anchor = (step[2] & 1 != 0) - .then_some(order[usize::from(step[3]) % order.len()].0) - .filter(|sid| batch.edits.iter().all(|e| e.space() != *sid)); - let edits: Vec<_> = batch - .edits - .iter() - .map(|edit| edit.reposition(PagePosition::Before(anchor), 1).unwrap()) - .collect(); - let result = cache.rebase_pages_conflict(edit.id, &snapshot, &remote, &edits); - assert_eq!(cache.snapshot().unwrap(), snapshot); - if result.is_err() { - assert_eq!(cache.pending().unwrap(), pending); - } - } - } - _ => unreachable!(), - } - let local = pages(&replicas[actor].as_ref().unwrap().snapshot().unwrap()); - for page in &owned[actor] { - assert!(local.iter().any(|(sid, oid, _)| (*sid, *oid) == *page)); - } - } -} diff --git a/crates/onestore-offline/tests/support/tree_schedule.rs b/crates/onestore-offline/tests/support/tree_schedule.rs deleted file mode 100644 index 8032dba58d40dfd020373396b0c058f667615bbb..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/tests/support/tree_schedule.rs +++ /dev/null @@ -1,271 +0,0 @@ -use crate::disk; -use onestore::{ - CommitError, ExGuid, Insertion, PreparedEdit, RevisionIndex, Store, TreeEdit, - document::{Document, Kind}, -}; -use onestore_offline::{EditStatus, Operation, Remote, Replica}; -use std::{io, sync::LazyLock}; - -static SOURCE: LazyLock<(Vec, ExGuid, ExGuid)> = LazyLock::new(|| { - let mut source = - onestore::create_section("tree-schedule.one", "Original 🦀 é 0", "Author").unwrap(); - let store = Store::parse(&source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let (sid, page) = document.pages().unwrap()[0]; - let space = &document.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let outline = *view.nodes[&page] - .children - .iter() - .find(|id| matches!(view.nodes[id].kind, Kind::Outline { .. })) - .unwrap(); - for at in 1..12 { - let insert = - Insertion::paragraph(outline, None, &format!("Original 🦀 é {at}"), "Author").unwrap(); - source = PreparedEdit::insert(&source, sid, &insert) - .unwrap() - .as_bytes() - .to_vec(); - } - (source, sid, outline) -}); - -fn rows(source: &[u8]) -> Vec<(ExGuid, ExGuid, String)> { - let (_, sid, outline) = &*SOURCE; - let store = Store::parse(source).unwrap(); - assert!(store.checksum_mismatches.is_empty()); - let index = RevisionIndex::parse(&store).unwrap(); - index.validate_current().unwrap(); - let document = Document::parse(&index).unwrap(); - let space = &document.spaces[sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let (_, page) = document - .pages() - .unwrap() - .into_iter() - .find(|(space, _)| space == sid) - .unwrap(); - if !view.nodes[&page].children.contains(outline) { - return Vec::new(); - } - let node = &view.nodes[outline]; - let mut rows = Vec::new(); - for paragraph in &node.children { - let node = &view.nodes[paragraph]; - assert!(node.children.is_empty()); - let [text] = node.content.as_slice() else { - panic!() - }; - let Kind::RichText { text: value, .. } = &view.nodes[text].kind else { - panic!() - }; - rows.push((*paragraph, *text, value.clone())); - } - rows -} - -struct Session<'a> { - disk: &'a mut disk::Disk, - operation: Option<&'a Operation>, -} - -impl Remote for Session<'_> { - fn read(&mut self) -> io::Result> { - Ok(self.disk.visible.clone()) - } - fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { - let before = rows(&self.disk.visible); - let mut expected = before.clone(); - match self.operation.unwrap() { - Operation::Tree(edit) => { - let at = expected - .iter() - .position(|(id, _, _)| *id == edit.intent.object()) - .unwrap(); - let row = expected.remove(at); - if let Some((parent, anchor)) = edit.intent.destination() { - assert_eq!(parent, SOURCE.2); - let at = anchor - .map(|anchor| { - expected - .iter() - .position(|(id, _, _)| *id == anchor) - .unwrap() - }) - .unwrap_or(expected.len()); - expected.insert(at, row); - } - } - Operation::Text(edit) => { - assert_eq!(edit.range, 0..1); - let (_, _, text) = expected - .iter_mut() - .find(|(_, id, _)| *id == edit.object) - .unwrap(); - text.replace_range(0..1, &edit.replacement); - } - Operation::Format(format) => { - let [onestore::TextAttribute::FontSize(size)] = format.attributes.as_slice() else { - panic!() - }; - let store = Store::parse(edit.as_bytes()).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let space = &document.spaces[&SOURCE.1]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - assert_eq!( - view.text_runs(format.object).unwrap()[0].format.font_size, - Some(*size) - ); - } - _ => panic!(), - } - assert_eq!(rows(edit.as_bytes()), expected); - let result = edit.commit(self.disk); - let durable = rows(&self.disk.durable); - assert!(durable == before || durable == expected); - if result.is_ok() { - assert_eq!(durable, expected); - } - result - } - fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { - onestore::confirm_snapshot(self.disk, snapshot) - } -} - -pub fn run(input: &[u8]) { - let (source, sid, outline) = &*SOURCE; - let directory = tempfile::tempdir().unwrap(); - let mut replicas: [Option; 12] = std::array::from_fn(|_| None); - let mut disk = disk::Disk { - visible: source.clone(), - durable: source.clone(), - operation: 0, - fail_at: None, - write_limit: 4096, - random: 1, - }; - for step in input.chunks_exact(8).take(48) { - let actor = usize::from(step[0]) % replicas.len(); - let path = directory.path().join(format!("{actor}.sqlite")); - let cache = replicas[actor].get_or_insert_with(|| Replica::create(&path, source).unwrap()); - let snapshot = cache.snapshot().unwrap(); - let pending = cache.pending().unwrap(); - let local = rows(&snapshot); - match step[1] % 8 { - 0..=2 if !local.is_empty() => { - let target = usize::from(step[2]) % local.len(); - let id = match step[1] % 8 { - 0 => { - let before = - (step[3] & 1 == 0).then(|| local[usize::from(step[4]) % local.len()].0); - cache - .tree( - &snapshot, - *sid, - &TreeEdit::move_to(local[target].0, *outline, before, "Offline") - .unwrap(), - ) - .unwrap() - } - 1 => cache - .tree( - &snapshot, - *sid, - &TreeEdit::delete(local[target].0, "Offline").unwrap(), - ) - .unwrap(), - _ if step[3] & 1 == 0 => cache - .edit_text( - &snapshot, - *sid, - local[target].1, - 0..1, - &char::from(b'A' + step[3] % 26).to_string(), - ) - .unwrap(), - _ => cache - .format( - &snapshot, - *sid, - local[target].1, - 0..1, - &[onestore::TextAttribute::FontSize( - 12.0 + f32::from(step[3] % 20), - )], - ) - .unwrap(), - }; - let next = cache.pending().unwrap(); - assert_eq!(next[..pending.len()], pending); - assert_eq!(next.len(), pending.len() + usize::from(id.is_some())); - } - 3 | 4 => { - disk.operation = 0; - disk.fail_at = (step[4] != 0) - .then_some(usize::from(u16::from_le_bytes([step[4], step[5]])) % 2048 + 1); - disk.write_limit = if step[6] & 1 == 0 { 17 } else { 4096 }; - disk.random = u64::from(step[7]) + 1; - let _ = cache.sync_once(&mut Session { - disk: &mut disk, - operation: pending.first().map(|p| &p.operation), - }); - let next = cache.pending().unwrap(); - if next.len() == pending.len() { - assert_eq!(next, pending); - } else { - assert_eq!(next, pending[1..]); - let Some(EditStatus::Published { revision }) = - cache.status(pending[0].id).unwrap() - else { - panic!() - }; - let store = Store::parse(&disk.durable).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - assert!(index.spaces[sid].revisions.contains_key(&revision)); - } - if !next.is_empty() { - assert!(cache.snapshot().unwrap() == snapshot); - } - disk.visible.clone_from(&disk.durable); - } - 5 => { - let mut expected = rows(&disk.visible); - if !expected.is_empty() { - let target = usize::from(step[2]) % expected.len(); - let (_, text, content) = &mut expected[target]; - let at = u32::try_from(content.encode_utf16().count()).unwrap(); - content.push('R'); - let snapshot = disk.visible.clone(); - let edit = PreparedEdit::text(&snapshot, *sid, *text, at..at, "R").unwrap(); - disk.fail_at = None; - edit.commit(&mut disk).unwrap(); - assert_eq!(rows(&disk.durable), expected); - } - } - _ => { - if step[1] % 8 == 6 - && let Some(first) = pending.first() - && matches!(first.operation, Operation::Tree(_)) - && matches!( - cache.status(first.id).unwrap(), - Some(EditStatus::Conflict(_)) - ) - { - let remote = cache.remote_snapshot().unwrap(); - let _ = cache.rebase_tree_conflict(first.id, &snapshot, &remote); - } - let pending = cache.pending().unwrap(); - drop(replicas[actor].take()); - let cache = Replica::open(&path).unwrap(); - assert!(cache.snapshot().unwrap() == snapshot); - assert_eq!(cache.pending().unwrap(), pending); - replicas[actor] = Some(cache); - } - } - rows(&disk.durable); - rows(&replicas[actor].as_ref().unwrap().snapshot().unwrap()); - } -} diff --git a/crates/onestore-offline/tests/sync.rs b/crates/onestore-offline/tests/sync.rs deleted file mode 100644 index a41fcf03c1353f7cf1fde00c90fd083650d72876..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/tests/sync.rs +++ /dev/null @@ -1,3932 +0,0 @@ -use onestore::{ - CommitError, CommitIo, CommitState, ExGuid, PreparedEdit, RevisionIndex, Store, - document::{Document, Kind}, -}; -use onestore_offline::{ConflictKind, EditStatus, Error, Remote, Replica}; -use std::io; - -#[path = "../../onestore/tests/support/disk.rs"] -mod disk; -#[path = "sync/outline.rs"] -mod outline; -#[path = "sync/page.rs"] -mod page; -#[path = "support/page_schedule.rs"] -mod page_schedule; -#[path = "sync/pages.rs"] -mod pages; -#[path = "sync/tree.rs"] -mod tree; -#[path = "support/tree_schedule.rs"] -mod tree_schedule; - -mod paragraph { - use super::*; - use onestore::{Insertion, ParagraphJoin, ParagraphSplit, TextAttribute}; - - fn fixture() -> (Vec, ExGuid, ExGuid, ExGuid) { - let source = onestore::create_section("paragraph.one", "ab🦀cd", "Fixture").unwrap(); - let (sid, left, _) = text(&source); - let store = Store::parse(&source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let space = &doc.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let parent = *view - .nodes - .iter() - .find(|(_, node)| node.content == [left]) - .unwrap() - .0; - (source, sid, left, parent) - } - - fn native_reconciliation(keyboard: bool) -> (Vec, Vec) { - let source = include_bytes!( - "../../../corpus/paragraph-edit/reconciliation/before/notebook/synthetic.one" - ); - let native: &[u8] = if keyboard { - include_bytes!( - "../../../corpus/paragraph-edit/reconciliation/keyboard/notebook/synthetic.one" - ) - } else { - include_bytes!( - "../../../corpus/paragraph-edit/reconciliation/remote/notebook/synthetic.one" - ) - }; - let store = Store::parse(source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let mut server = Server::new(native); - let mut recorded = Vec::new(); - let mut counts = [0; 3]; - for (sid, page) in document.pages().unwrap() { - let space = &document.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let Kind::Metadata { - title: Some(name), .. - } = &view.nodes[&view.roots[&2]].kind - else { - continue; - }; - if !name.starts_with("Split ") && !name.starts_with("Join ") { - continue; - } - let outline = view.nodes[&page] - .children - .iter() - .find(|id| matches!(view.nodes[id].kind, Kind::Outline { .. })) - .unwrap(); - let children = &view.nodes[outline].children; - let left = view.nodes[&children[0]].content[0]; - let split = name.starts_with("Split "); - let adoption = name == "Join empty adoption"; - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("native-reconciliation.sqlite"); - let mut cache = Replica::create(&path, source).unwrap(); - let (id, target, intent) = if split { - let intent = ParagraphSplit::new(left, 2, "Offline author").unwrap(); - ( - cache.split(source, sid, &intent).unwrap().unwrap(), - intent.text_object(), - serde_json::to_value(intent).unwrap(), - ) - } else { - let right = view.nodes[&children[1]].content[0]; - let intent = ParagraphJoin::new(left, right, "Offline author").unwrap(); - ( - cache.join(source, sid, &intent).unwrap().unwrap(), - if adoption { right } else { left }, - serde_json::to_value(intent).unwrap(), - ) - }; - drop(cache); - cache = Replica::open(&path).unwrap(); - let at = if split { - 2 - } else if adoption { - 1 - } else { - 4 - }; - let dependent = cache - .edit_text( - &cache.snapshot().unwrap(), - sid, - target, - at..at + 1, - if adoption { "I" } else { "C" }, - ) - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let pending = cache.pending().unwrap(); - drop(cache); - cache = Replica::open(&path).unwrap(); - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(cache.pending().unwrap(), pending); - let previous = server.publications; - let remote = server.visible.clone(); - let (actual, status) = cache.sync_once(&mut server).unwrap().unwrap(); - assert_eq!(actual, id); - assert_eq!(cache.snapshot().unwrap(), local); - if !keyboard { - counts[2] += 1; - assert_eq!( - status, - EditStatus::Conflict(ConflictKind::TargetUnavailable), - "{name}" - ); - assert_eq!(server.publications, previous); - assert_eq!(server.visible, remote); - drop(cache); - cache = Replica::open(&path).unwrap(); - assert_eq!(cache.status(id).unwrap(), Some(status)); - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(cache.pending().unwrap(), pending); - continue; - } - let automatic = ["prefix", "format", "sibling"] - .iter() - .any(|suffix| name.ends_with(suffix)); - if automatic { - counts[0] += 1; - assert!( - matches!(status, EditStatus::Published { .. }), - "{name}: {status:?}" - ); - } else { - let expected = if name.ends_with("boundary") || adoption { - ConflictKind::TextChanged - } else { - ConflictKind::StructureChanged - }; - assert_eq!(status, EditStatus::Conflict(expected), "{name}"); - assert_eq!(server.publications, previous); - assert_eq!(server.visible, remote); - assert_eq!(cache.pending().unwrap(), pending); - drop(cache); - cache = Replica::open(&path).unwrap(); - assert_eq!(cache.status(id).unwrap(), Some(status)); - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(cache.pending().unwrap(), pending); - if adoption || name == "Join remote list" { - counts[2] += 1; - assert!(cache.rebase_join_conflict(id, &local, &remote).is_err()); - assert_eq!(cache.pending().unwrap(), pending); - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(cache.status(id).unwrap(), Some(status)); - recorded.push(serde_json::json!({"case": name, "space": sid, "intent": intent, "outcome": "retained_conflict"})); - continue; - } - counts[1] += 1; - if split { - let offset = if name.ends_with("boundary") { 3 } else { 2 }; - cache - .rebase_conflict(id, &local, &remote, offset..offset) - .unwrap(); - let onestore_offline::Operation::Split(edit) = - &cache.pending().unwrap()[0].operation - else { - panic!() - }; - assert_eq!( - edit.intent, - serde_json::from_value::(intent.clone()) - .unwrap() - .reposition(offset) - ); - } else { - cache.rebase_join_conflict(id, &local, &remote).unwrap(); - } - drop(cache); - cache = Replica::open(&path).unwrap(); - let (actual, status) = cache.sync_once(&mut server).unwrap().unwrap(); - assert_eq!(actual, id); - assert!( - matches!(status, EditStatus::Published { .. }), - "{name}: {status:?}" - ); - } - drop(cache); - cache = Replica::open(&path).unwrap(); - let first_receipt = cache.status(id).unwrap().unwrap(); - let (actual, status) = cache.sync_once(&mut server).unwrap().unwrap(); - assert_eq!(actual, dependent); - assert!( - matches!(status, EditStatus::Published { .. }), - "{name} dependent: {status:?}" - ); - assert_eq!(server.publications, previous + 2); - drop(cache); - cache = Replica::open(&path).unwrap(); - assert_eq!(cache.status(id).unwrap(), Some(first_receipt)); - assert_eq!(cache.status(dependent).unwrap(), Some(status)); - assert!(cache.pending().unwrap().is_empty()); - assert_eq!(cache.snapshot().unwrap(), server.durable); - let EditStatus::Published { revision } = first_receipt else { - panic!() - }; - let EditStatus::Published { - revision: dependent_revision, - } = status - else { - panic!() - }; - recorded.push(serde_json::json!({"case": name, "space": sid, "intent": intent, - "outcome": if automatic { "automatic" } else { "reviewed" }, "receipt": revision, "dependent_receipt": dependent_revision})); - } - assert_eq!(counts, if keyboard { [6, 8, 2] } else { [0, 0, 16] }); - assert_eq!(server.publications, if keyboard { 28 } else { 0 }); - (server.durable, recorded) - } - - #[test] - fn native_com_replacement_preserves_every_local_paragraph_branch() { - native_reconciliation(false); - } - - #[test] - fn native_changes_commute_or_retain_reviewable_paragraph_dependencies() { - native_reconciliation(true); - } - - #[test] - #[ignore = "exports native-controlled reconciliation for cold OneNote validation"] - fn export_native_reconciliation() { - let output = std::path::PathBuf::from( - std::env::var_os("ONESTORE_NATIVE_RECONCILIATION_OUTPUT").unwrap(), - ); - assert!(output.is_absolute()); - std::fs::create_dir(&output).unwrap(); - let (bytes, cases) = native_reconciliation(true); - std::fs::create_dir(output.join("candidate")).unwrap(); - std::fs::write(output.join("candidate/synthetic.one"), bytes).unwrap(); - std::fs::write( - output.join("manifest.json"), - serde_json::to_vec_pretty(&cases).unwrap(), - ) - .unwrap(); - } - - #[test] - fn native_splits_retain_independent_local_identities_and_changed_boundaries() { - let source = include_bytes!("../../../corpus/paragraph-edit/before/notebook/synthetic.one"); - let native = include_bytes!("../../../corpus/paragraph-edit/split/notebook/synthetic.one"); - let manifest: serde_json::Value = - serde_json::from_str(include_str!("../../../corpus/paragraph-edit/manifest.json")) - .unwrap(); - let store = Store::parse(source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let mut accepted = 0; - let mut conflicts = 0; - for case in manifest["cases"].as_array().unwrap() { - if case["case"] == "Split before hyperlink" { - continue; - } - let left: ExGuid = serde_json::from_value(case["original_text"].clone()).unwrap(); - let native_right: ExGuid = serde_json::from_value(case["new_text"].clone()).unwrap(); - let (sid, _) = document - .pages() - .unwrap() - .into_iter() - .find(|(sid, _)| { - let space = &document.spaces[sid]; - space.revisions[&space.contexts[&ExGuid::default()]] - .nodes - .contains_key(&left) - }) - .unwrap(); - let split = ParagraphSplit::new( - left, - u32::try_from(case["offset_utf16"].as_u64().unwrap()).unwrap(), - "Offline author", - ) - .unwrap(); - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("native-split.sqlite"); - let mut cache = Replica::create(&path, source).unwrap(); - let id = cache.split(source, sid, &split).unwrap().unwrap(); - let current = cache.snapshot().unwrap(); - let dependent = cache - .edit_text( - ¤t, - sid, - split.text_object(), - 0..0, - "Local dependent edit: ", - ) - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let pending = cache.pending().unwrap(); - let mut server = Server::new(native); - let (published, status) = cache.sync_once(&mut server).unwrap().unwrap(); - assert_eq!(published, id); - assert_eq!(cache.snapshot().unwrap(), local); - if matches!(case["case"].as_str().unwrap(), "Split end" | "Split empty") { - accepted += 1; - assert!( - matches!(status, EditStatus::Published { .. }), - "{}: {status:?}", - case["case"] - ); - drop(cache); - cache = Replica::open(&path).unwrap(); - assert_eq!(cache.sync_once(&mut server).unwrap().unwrap().0, dependent); - assert_eq!(server.publications, 2); - let store = Store::parse(&server.visible).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let space = &document.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let original: ExGuid = - serde_json::from_value(case["original_paragraph"].clone()).unwrap(); - let native_paragraph: ExGuid = - serde_json::from_value(case["new_paragraph"].clone()).unwrap(); - let parent = view - .nodes - .values() - .find(|node| node.children.contains(&original)) - .unwrap(); - let position = parent - .children - .iter() - .position(|id| *id == original) - .unwrap(); - assert_eq!( - &parent.children[position..position + 3], - &[original, split.object(), native_paragraph] - ); - assert!( - matches!(&view.nodes[&native_right].kind, Kind::RichText {text,..} if text.is_empty()) - ); - assert!( - matches!(&view.nodes[&split.text_object()].kind, Kind::RichText {text,..} if text == "Local dependent edit: ") - ); - } else { - conflicts += 1; - assert!( - matches!( - status, - EditStatus::Conflict( - ConflictKind::TextChanged | ConflictKind::StructureChanged - ) - ), - "{}: {status:?}", - case["case"] - ); - assert_eq!(server.publications, 0); - assert_eq!(cache.pending().unwrap(), pending); - drop(cache); - cache = Replica::open(&path).unwrap(); - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(cache.pending().unwrap(), pending); - assert_eq!(cache.status(id).unwrap(), Some(status)); - } - } - assert_eq!((accepted, conflicts), (2, 10)); - } - - #[test] - fn native_joins_do_not_acknowledge_or_discard_an_independent_local_branch() { - let source = include_bytes!( - "../../../corpus/paragraph-edit/join-tags/before/notebook/synthetic.one" - ); - let native = include_bytes!( - "../../../corpus/paragraph-edit/join-tags/joined/notebook/synthetic.one" - ); - let store = Store::parse(source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let mut cases = 0; - for (sid, page) in document.pages().unwrap() { - let space = &document.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let Kind::Metadata { - title: Some(name), .. - } = &view.nodes[&view.roots[&2]].kind - else { - continue; - }; - if !name.starts_with("Join ") { - continue; - } - cases += 1; - let outline = view.nodes[&page] - .children - .iter() - .find(|id| matches!(view.nodes[id].kind, Kind::Outline { .. })) - .unwrap(); - let children = &view.nodes[outline].children; - let left = view.nodes[&children[0]].content[0]; - let right = view.nodes[&children[1]].content[0]; - let Kind::RichText { - text: left_text, .. - } = &view.nodes[&left].kind - else { - panic!() - }; - let survivor = if left_text.is_empty() { right } else { left }; - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("native-join.sqlite"); - let mut cache = Replica::create(&path, source).unwrap(); - let join = ParagraphJoin::new(left, right, "Offline author").unwrap(); - let id = cache.join(source, sid, &join).unwrap().unwrap(); - let current = cache.snapshot().unwrap(); - cache - .edit_text(¤t, sid, survivor, 0..0, "Local dependent edit: ") - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let pending = cache.pending().unwrap(); - let mut server = Server::new(native); - let (conflicted, status) = cache.sync_once(&mut server).unwrap().unwrap(); - assert_eq!(conflicted, id); - assert!( - matches!( - status, - EditStatus::Conflict(ConflictKind::TargetUnavailable) - ), - "{name}: {status:?}" - ); - assert_eq!(server.publications, 0); - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(cache.pending().unwrap(), pending); - drop(cache); - cache = Replica::open(&path).unwrap(); - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(cache.pending().unwrap(), pending); - assert_eq!(cache.status(id).unwrap(), Some(status)); - } - assert_eq!(cases, 3); - } - - #[test] - fn split_join_dependencies_reopen_and_rebase_with_remote_text_and_styles() { - let (source, sid, left, parent) = fixture(); - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("paragraph.sqlite"); - let mut cache = Replica::create(&path, &source).unwrap(); - let split = ParagraphSplit::new(left, 2, "Local").unwrap(); - let child = Insertion::paragraph(split.object(), None, "Child", "Local").unwrap(); - let join = ParagraphJoin::new(left, split.text_object(), "Local").unwrap(); - let mut ids = Vec::new(); - for step in 0..6 { - let current = cache.snapshot().unwrap(); - let id = match step { - 0 => cache.split(¤t, sid, &split), - 1 => cache.edit_text(¤t, sid, split.text_object(), 0..2, "🦋"), - 2 => cache.format( - ¤t, - sid, - split.text_object(), - 2..3, - &[TextAttribute::Bold(true)], - ), - 3 => cache.insert(¤t, sid, &child), - 4 => cache.join(¤t, sid, &join), - 5 => cache.edit_text(¤t, sid, left, 5..6, "D"), - _ => unreachable!(), - } - .unwrap() - .unwrap(); - ids.push(id); - let saved = cache.snapshot().unwrap(); - let pending = cache.pending().unwrap(); - drop(cache); - cache = Replica::open(&path).unwrap(); - assert_eq!(cache.snapshot().unwrap(), saved); - assert_eq!(cache.pending().unwrap(), pending); - } - let remote = onestore::replace_text(&source, sid, left, 0..0, "Z").unwrap(); - let remote = - PreparedEdit::format(&remote, sid, left, 1..2, &[TextAttribute::Italic(true)]).unwrap(); - let mut server = Server::new(remote.as_bytes()); - let local = cache.snapshot().unwrap(); - for (i, id) in ids.iter().enumerate() { - let (published, state) = cache.sync_once(&mut server).unwrap().unwrap(); - assert_eq!(published, *id); - assert!(matches!(state, EditStatus::Published { .. }), "{state:?}"); - drop(cache); - cache = Replica::open(&path).unwrap(); - assert_eq!(cache.status(*id).unwrap(), Some(state)); - if i != ids.len() - 1 { - assert_eq!(cache.snapshot().unwrap(), local); - } - } - assert_eq!(server.publications, ids.len()); - assert_eq!(server.visible, server.durable); - assert_eq!(cache.snapshot().unwrap(), server.durable); - let store = Store::parse(&server.durable).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - index.validate_current().unwrap(); - let doc = Document::parse(&index).unwrap(); - let space = &doc.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - assert!( - matches!(&view.nodes[&left].kind, Kind::RichText { text, .. } if text == "Zab🦋cD") - ); - assert_eq!(view.nodes[&parent].children, [child.object()]); - assert_eq!(view.nodes[&parent].content, [left]); - let runs = view.text_runs(left).unwrap(); - let chars: Vec<_> = runs - .iter() - .flat_map(|run| { - run.text - .chars() - .map(|c| (c, run.format.bold, run.format.italic)) - }) - .collect(); - assert_eq!(chars.iter().find(|v| v.0 == 'a').unwrap().2, Some(true)); - assert_eq!(chars.iter().find(|v| v.0 == 'c').unwrap().1, Some(true)); - } - - #[test] - fn changed_split_children_require_fresh_review_without_reallocating_dependents() { - let (source, sid, left, parent) = fixture(); - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("paragraph.sqlite"); - let cache = Replica::create(&path, &source).unwrap(); - let split = ParagraphSplit::new(left, 2, "Local").unwrap(); - let id = cache.split(&source, sid, &split).unwrap().unwrap(); - let dependent = cache - .edit_text( - &cache.snapshot().unwrap(), - sid, - split.text_object(), - 0..2, - "🦋", - ) - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let child = Insertion::paragraph(parent, None, "Remote child", "Remote").unwrap(); - let remote = PreparedEdit::insert(&source, sid, &child).unwrap(); - let mut server = Server::new(remote.as_bytes()); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((id, EditStatus::Conflict(ConflictKind::StructureChanged))) - ); - assert_eq!(server.publications, 0); - assert_eq!(cache.snapshot().unwrap(), local); - assert!( - cache - .rebase_conflict(id, &source, &server.visible, 2..2) - .is_err() - ); - assert!( - cache - .rebase_conflict(id, &local, &server.visible, 2..3) - .is_err() - ); - cache - .rebase_conflict(id, &local, &server.visible, 2..2) - .unwrap(); - let pending = cache.pending().unwrap(); - let onestore_offline::Operation::Split(edit) = &pending[0].operation else { - panic!() - }; - assert_eq!(edit.intent, split); - assert_eq!(pending[1].id, dependent); - drop(cache); - let cache = Replica::open(&path).unwrap(); - for expected in [id, dependent] { - let (actual, state) = cache.sync_once(&mut server).unwrap().unwrap(); - assert_eq!(actual, expected); - assert!(matches!(state, EditStatus::Published { .. })); - } - let store = Store::parse(&server.durable).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let space = &doc.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - assert_eq!(view.nodes[&split.object()].children, [child.object()]); - assert!( - matches!(&view.nodes[&split.text_object()].kind, Kind::RichText { text, .. } if text == "🦋cd") - ); - } - - #[test] - fn uncertain_paragraph_operations_keep_the_original_attempt_across_reopen() { - for join in [false, true] { - for fault in [ - Fault::Before, - Fault::UnknownBefore, - Fault::UnknownAfter, - Fault::Committed, - ] { - let (source, sid, left, _) = fixture(); - let split = ParagraphSplit::new(left, 2, "Local").unwrap(); - let source = if join { - PreparedEdit::split(&source, sid, &split) - .unwrap() - .as_bytes() - .to_vec() - } else { - source - }; - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("paragraph.sqlite"); - let cache = Replica::create(&path, &source).unwrap(); - let id = if join { - cache.join( - &source, - sid, - &ParagraphJoin::new(left, split.text_object(), "Local").unwrap(), - ) - } else { - cache.split(&source, sid, &split) - } - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let intent = cache.pending().unwrap(); - let mut server = Server::new(&source); - server.fault = fault; - assert!(cache.sync_once(&mut server).is_err()); - let state = cache.status(id).unwrap().unwrap(); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.status(id).unwrap(), Some(state)); - if matches!(fault, Fault::UnknownBefore | Fault::UnknownAfter) { - assert!(matches!(state, EditStatus::AwaitingConfirmation { .. })); - assert_eq!(cache.pending().unwrap(), intent); - assert_eq!(cache.snapshot().unwrap(), local); - if matches!(fault, Fault::UnknownBefore) { - assert_eq!(cache.sync_once(&mut server).unwrap(), Some((id, state))); - assert_eq!(server.publications, 1); - continue; - } - server.fault = Fault::Confirm; - assert!(cache.sync_once(&mut server).is_err()); - assert_eq!(cache.status(id).unwrap(), Some(state)); - let (_, state) = cache.sync_once(&mut server).unwrap().unwrap(); - assert!(matches!(state, EditStatus::Published { .. })); - assert_eq!(server.publications, 1); - } else if matches!(fault, Fault::Before) { - assert_eq!(state, EditStatus::Pending); - assert!(matches!( - cache.sync_once(&mut server).unwrap().unwrap().1, - EditStatus::Published { .. } - )); - assert_eq!(server.publications, 2); - } else { - assert!(matches!(state, EditStatus::Published { .. })); - assert_eq!(server.publications, 1); - } - assert!(cache.pending().unwrap().is_empty()); - let saved = cache.snapshot().unwrap(); - if matches!(fault, Fault::UnknownAfter) { - // Confirmation refreshes version metadata after reading the acknowledged snapshot. - assert!(saved[..212] == server.durable[..212]); - assert!(saved[252..] == server.durable[252..]); - assert_eq!( - Store::parse(&server.durable).unwrap().header.generation, - Store::parse(&saved).unwrap().header.generation + 1 - ); - } else { - assert!(saved == server.durable); - } - assert_eq!(cache.sync_once(&mut server).unwrap(), None); - assert!(cache.snapshot().unwrap() == server.durable); - } - } - } - - #[test] - fn join_reviews_preserve_survivors_and_require_current_child_placement() { - for empty in [false, true] { - let (source, sid, left, _) = fixture(); - let split = ParagraphSplit::new(left, if empty { 0 } else { 2 }, "Local").unwrap(); - let source = PreparedEdit::split(&source, sid, &split) - .unwrap() - .as_bytes() - .to_vec(); - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("join.sqlite"); - let cache = Replica::create(&path, &source).unwrap(); - let intent = ParagraphJoin::new(left, split.text_object(), "Local").unwrap(); - let id = cache.join(&source, sid, &intent).unwrap().unwrap(); - let survivor = if empty { split.text_object() } else { left }; - let dependent = cache - .edit_text(&cache.snapshot().unwrap(), sid, survivor, 0..0, "Local ") - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let child = - Insertion::paragraph(split.object(), None, "Remote child", "Remote").unwrap(); - let remote = PreparedEdit::insert(&source, sid, &child) - .unwrap() - .as_bytes() - .to_vec(); - let mut server = Server::new(&remote); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((id, EditStatus::Conflict(ConflictKind::StructureChanged))) - ); - assert!(cache.rebase_join_conflict(id, &source, &remote).is_err()); - assert_eq!(server.publications, 0); - cache.rebase_join_conflict(id, &local, &remote).unwrap(); - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) - ); - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == dependent) - ); - let store = Store::parse(&server.durable).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let space = &doc.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let parent = view - .nodes - .values() - .find(|node| node.content == [survivor]) - .unwrap(); - assert_eq!(parent.children, [child.object()]); - assert!( - matches!(&view.nodes[&survivor].kind, Kind::RichText { text, .. } if text == "Local ab🦀cd") - ); - } - - let (source, sid, left, _) = fixture(); - let split = ParagraphSplit::new(left, 0, "Local").unwrap(); - let source = PreparedEdit::split(&source, sid, &split) - .unwrap() - .as_bytes() - .to_vec(); - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("identity.sqlite"); - let cache = Replica::create(&path, &source).unwrap(); - let intent = ParagraphJoin::new(left, split.text_object(), "Local").unwrap(); - let id = cache.join(&source, sid, &intent).unwrap().unwrap(); - let local = cache.snapshot().unwrap(); - let remote = onestore::replace_text(&source, sid, left, 0..0, "Remote").unwrap(); - let mut server = Server::new(&remote); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((id, EditStatus::Conflict(ConflictKind::TextChanged))) - ); - assert!(cache.rebase_join_conflict(id, &local, &remote).is_err()); - assert!(cache.snapshot().unwrap() == local); - assert_eq!(server.publications, 0); - assert_eq!(cache.pending().unwrap().len(), 1); - } - - #[test] - fn overlapping_clients_retain_each_local_branch_without_replaying_removed_targets() { - let (source, sid, left, _) = fixture(); - let split = ParagraphSplit::new(left, 2, "Seed").unwrap(); - let source = PreparedEdit::split(&source, sid, &split) - .unwrap() - .as_bytes() - .to_vec(); - let directory = tempfile::tempdir().unwrap(); - let first = Replica::create(directory.path().join("first.sqlite"), &source).unwrap(); - let second_path = directory.path().join("second.sqlite"); - let second = Replica::create(&second_path, &source).unwrap(); - let join = ParagraphJoin::new(left, split.text_object(), "First").unwrap(); - first.join(&source, sid, &join).unwrap(); - let right_split = ParagraphSplit::new(split.text_object(), 2, "Second").unwrap(); - let id = second.split(&source, sid, &right_split).unwrap().unwrap(); - second - .edit_text( - &second.snapshot().unwrap(), - sid, - right_split.text_object(), - 0..0, - "Second ", - ) - .unwrap(); - let local = second.snapshot().unwrap(); - let pending = second.pending().unwrap(); - let mut server = Server::new(&source); - assert!(matches!( - first.sync_once(&mut server).unwrap().unwrap().1, - EditStatus::Published { .. } - )); - assert_eq!( - second.sync_once(&mut server).unwrap(), - Some((id, EditStatus::Conflict(ConflictKind::TargetUnavailable))) - ); - drop(second); - let second = Replica::open(&second_path).unwrap(); - assert_eq!(second.pending().unwrap(), pending); - assert!(second.snapshot().unwrap() == local); - assert_eq!(server.publications, 1); - assert_eq!( - second.status(id).unwrap(), - Some(EditStatus::Conflict(ConflictKind::TargetUnavailable)) - ); - } - - #[test] - #[ignore = "exports reconciled paragraph edits for independent native validation"] - fn export_native_offline_paragraphs() { - use std::{fs, path::PathBuf}; - let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../corpus/paragraph-edit"); - let output = PathBuf::from(std::env::var_os("ONESTORE_OFFLINE_PARAGRAPH_OUTPUT").unwrap()); - assert!(output.is_absolute()); - fs::create_dir(&output).unwrap(); - for (name, source, manifest, split) in [ - ( - "splits", - "before/notebook/synthetic.one", - "rust-split/manifest.json", - true, - ), - ( - "joins", - "split/notebook/synthetic.one", - "rust-join/split/manifest.json", - false, - ), - ( - "inheritance", - "join-edges/before/notebook/synthetic.one", - "rust-join/inheritance/manifest.json", - false, - ), - ( - "tags", - "join-tags/before/notebook/synthetic.one", - "rust-join/tags/manifest.json", - false, - ), - ] { - let source = fs::read(root.join(source)).unwrap(); - let folder = output.join(name); - fs::create_dir(&folder).unwrap(); - let path = folder.join("cache.sqlite"); - let mut cache = Replica::create(&path, &source).unwrap(); - let mut server = Server::new(&source); - let manifest: serde_json::Value = - serde_json::from_slice(&fs::read(root.join(manifest)).unwrap()).unwrap(); - let cases = if split { &manifest["cases"] } else { &manifest }; - let mut recorded = Vec::new(); - for case in cases - .as_array() - .unwrap() - .iter() - .filter(|case| case.get("intent").is_some()) - { - let local = cache.snapshot().unwrap(); - let (text, offset) = if split { - serde_json::from_value::(case["intent"].clone()) - .unwrap() - .position() - } else { - ( - serde_json::from_value::(case["intent"].clone()) - .unwrap() - .texts()[0], - 1, - ) - }; - let store = Store::parse(&local).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let (sid, space) = doc - .spaces - .iter() - .find(|(_, space)| { - space.revisions[&space.contexts[&ExGuid::default()]] - .nodes - .contains_key(&text) - }) - .unwrap(); - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let Kind::RichText { text: content, .. } = &view.nodes[&text].kind else { - panic!() - }; - let remote_prefix = offset > 0 && !content.is_empty(); - assert!(!content.contains('☂')); - if remote_prefix { - server.visible = - onestore::replace_text(&server.visible, *sid, text, 0..0, "☂").unwrap(); - server.durable.clone_from(&server.visible); - } - let id = if split { - cache.split( - &local, - *sid, - &serde_json::from_value(case["intent"].clone()).unwrap(), - ) - } else { - cache.join( - &local, - *sid, - &serde_json::from_value(case["intent"].clone()).unwrap(), - ) - } - .unwrap() - .unwrap(); - recorded.push(serde_json::json!({"case": case["case"], "id": id, - "space": sid, "intent": case["intent"], "remote_prefix": remote_prefix})); - drop(cache); - cache = Replica::open(&path).unwrap(); - } - for case in &mut recorded { - let id = case["id"].as_u64().unwrap(); - if id % 2 == 0 { - server.fault = Fault::UnknownAfter; - assert!(cache.sync_once(&mut server).is_err()); - assert!(matches!( - cache.status(id).unwrap(), - Some(EditStatus::AwaitingConfirmation { .. }) - )); - drop(cache); - cache = Replica::open(&path).unwrap(); - } - let (actual, status) = cache.sync_once(&mut server).unwrap().unwrap(); - assert_eq!(actual, id); - let EditStatus::Published { revision } = status else { - panic!("{name}: {status:?}") - }; - case["revision"] = serde_json::to_value(revision).unwrap(); - } - assert_eq!(server.publications, recorded.len()); - assert_eq!(cache.sync_once(&mut server).unwrap(), None); - assert!(cache.snapshot().unwrap() == server.durable); - cache - .export_recovery(folder.join("recovery.sqlite")) - .unwrap(); - fs::create_dir(folder.join("candidate")).unwrap(); - fs::write(folder.join("candidate/synthetic.one"), &server.durable).unwrap(); - fs::write( - folder.join("manifest.json"), - serde_json::to_vec_pretty(&recorded).unwrap(), - ) - .unwrap(); - } - } -} - -#[derive(Clone, Copy, Default)] -enum Fault { - #[default] - None, - Before, - UnknownBefore, - UnknownAfter, - Committed, - PanicBefore, - PanicAfter, - Confirm, - ConfirmCommitted, -} - -struct Server { - visible: Vec, - durable: Vec, - fault: Fault, - publications: usize, - confirmations: usize, -} - -impl Server { - fn new(source: &[u8]) -> Self { - Self { - visible: source.to_vec(), - durable: source.to_vec(), - fault: Fault::None, - publications: 0, - confirmations: 0, - } - } -} - -fn failure(state: CommitState) -> CommitError { - CommitError { - state, - error: io::Error::from(io::ErrorKind::ConnectionAborted), - } -} - -impl CommitIo for Server { - fn read_at(&mut self, offset: u64, output: &mut [u8]) -> io::Result { - let offset = usize::try_from(offset).unwrap(); - let size = output.len().min(self.visible.len().saturating_sub(offset)); - if size > 0 { - output[..size].copy_from_slice(&self.visible[offset..offset + size]); - } - Ok(size) - } - fn write_at(&mut self, offset: u64, bytes: &[u8]) -> io::Result { - let offset = usize::try_from(offset).unwrap(); - self.visible - .resize(self.visible.len().max(offset + bytes.len()), 0); - self.visible[offset..offset + bytes.len()].copy_from_slice(bytes); - Ok(bytes.len()) - } - fn flush(&mut self) -> io::Result<()> { - self.durable.clone_from(&self.visible); - Ok(()) - } -} - -impl Remote for Server { - fn read(&mut self) -> io::Result> { - Ok(self.visible.clone()) - } - fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { - self.publications += 1; - let fault = std::mem::take(&mut self.fault); - match fault { - Fault::Before => return Err(failure(CommitState::NotCommitted)), - Fault::UnknownBefore => return Err(failure(CommitState::Unknown)), - Fault::PanicBefore => panic!("Terminated before remote I/O"), - _ => {} - } - let old = self.durable.clone(); - edit.commit(self)?; - match fault { - Fault::UnknownAfter => { - self.durable = old; - Err(failure(CommitState::Unknown)) - } - Fault::Committed => Err(failure(CommitState::Committed)), - Fault::PanicAfter => panic!("Terminated after remote publication"), - _ => Ok(()), - } - } - fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { - self.confirmations += 1; - if matches!(self.fault, Fault::Confirm) { - self.fault = Fault::None; - return Err(failure(CommitState::Unknown)); - } - onestore::confirm_snapshot(self, snapshot)?; - if matches!(self.fault, Fault::ConfirmCommitted) { - self.fault = Fault::None; - return Err(failure(CommitState::Committed)); - } - Ok(()) - } -} - -fn text(source: &[u8]) -> (ExGuid, ExGuid, String) { - let store = Store::parse(source).unwrap(); - assert!(store.checksum_mismatches.is_empty()); - let index = RevisionIndex::parse(&store).unwrap(); - index.validate_current().unwrap(); - let doc = Document::parse(&index).unwrap(); - doc.spaces - .iter() - .find_map(|(sid, space)| { - space.revisions[&space.contexts[&ExGuid::default()]] - .nodes - .iter() - .find_map(|(oid, node)| match &node.kind { - Kind::RichText { text, .. } => Some((*sid, *oid, text.clone())), - _ => None, - }) - }) - .unwrap() -} - -#[test] -fn recovery_archive_preserves_typed_queue_uncertainty_and_receipts_without_becoming_a_writer() { - use onestore::{Insertion, TextAttribute}; - use onestore_offline::{Recovery, RecoverySummary}; - - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("live.sqlite"); - let archive_path = directory.path().join("recovery.sqlite"); - let source = onestore::create_section("recovery.one", "Original", "Fixture").unwrap(); - let (space, object, _) = text(&source); - let cache = Replica::create(&path, &source).unwrap(); - let published = cache - .edit_text(&source, space, object, 0..0, "Published ") - .unwrap() - .unwrap(); - let mut server = Server::new(&source); - let (_, receipt) = cache.sync_once(&mut server).unwrap().unwrap(); - let source = cache.snapshot().unwrap(); - let store = Store::parse(&source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let (page_space, page) = document.pages().unwrap()[0]; - let outline = Insertion::outline(page, 100.0, 200.0, "Outline", "Fixture").unwrap(); - let inserted = cache - .insert(&source, page_space, &outline) - .unwrap() - .unwrap(); - let paragraph = Insertion::paragraph(outline.object(), None, "Recovery 🦀", "Fixture").unwrap(); - cache - .insert(&cache.snapshot().unwrap(), page_space, ¶graph) - .unwrap(); - cache - .format( - &cache.snapshot().unwrap(), - page_space, - paragraph.text_object(), - 0..3, - &[TextAttribute::Bold(true)], - ) - .unwrap(); - cache - .edit_text( - &cache.snapshot().unwrap(), - page_space, - paragraph.text_object(), - 0..0, - "Pending ", - ) - .unwrap(); - server.fault = Fault::UnknownBefore; - assert!(matches!( - cache.sync_once(&mut server), - Err(Error::Remote(CommitError { - state: CommitState::Unknown, - .. - })) - )); - let working = cache.snapshot().unwrap(); - let remote = cache.remote_snapshot().unwrap(); - let pending = cache.pending().unwrap(); - let uncertain = cache.status(inserted).unwrap(); - let source_file = std::fs::read(&path).unwrap(); - let summary = RecoverySummary { - queued_edits: 4, - conflicts: 0, - uncertain_edits: 1, - published_receipts: 1, - working_bytes: working.len() as u64, - remote_bytes: remote.len() as u64, - cached_assets: 0, - cached_asset_bytes: 0, - }; - assert_eq!(cache.recovery_summary().unwrap(), summary); - cache.export_recovery(&archive_path).unwrap(); - assert_eq!(std::fs::read(&path).unwrap(), source_file); - let archive_bytes = std::fs::read(&archive_path).unwrap(); - assert!(Replica::open(&archive_path).is_err()); - assert_eq!(std::fs::read(&archive_path).unwrap(), archive_bytes); - assert!(Recovery::open(&path).is_err()); - let archive = Recovery::open(&archive_path).unwrap(); - assert_eq!(archive.summary().unwrap(), summary); - assert_eq!(archive.snapshot().unwrap(), working); - assert_eq!(archive.remote_snapshot().unwrap(), remote); - assert_eq!(archive.pending().unwrap(), pending); - assert_eq!(archive.status(published).unwrap(), Some(receipt)); - assert_eq!(archive.status(inserted).unwrap(), uncertain); - for edit in &pending { - assert_eq!( - archive.status(edit.id).unwrap(), - cache.status(edit.id).unwrap() - ); - } - let EditStatus::Published { revision } = receipt else { - panic!() - }; - assert_eq!(archive.receipts().unwrap(), [(published, revision)].into()); - assert_eq!( - archive.status(u64::MAX - 1).unwrap_err().to_string(), - cache.status(u64::MAX - 1).unwrap_err().to_string() - ); - for existing in [&path, &archive_path] { - assert!( - matches!(cache.export_recovery(existing), Err(Error::Io(error)) if error.kind() == io::ErrorKind::AlreadyExists) - ); - } - assert_eq!(std::fs::read(&path).unwrap(), source_file); - assert_eq!(std::fs::read(&archive_path).unwrap(), archive_bytes); - cache - .edit_text(&working, space, object, 0..0, "Later ") - .unwrap(); - assert_eq!(archive.snapshot().unwrap(), working); - assert_eq!(archive.pending().unwrap(), pending); - drop(archive); - assert_eq!( - Recovery::open(&archive_path).unwrap().summary().unwrap(), - summary - ); - assert_eq!(std::fs::read(&archive_path).unwrap(), archive_bytes); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - assert_eq!( - std::fs::metadata(&archive_path) - .unwrap() - .permissions() - .mode() - & 0o777, - 0o600 - ); - } - let remaining: Vec<_> = std::fs::read_dir(directory.path()) - .unwrap() - .map(|entry| entry.unwrap().file_name()) - .collect(); - assert!( - remaining - .iter() - .all(|name| !name.to_string_lossy().starts_with(".onestore-recovery-")), - "Temporary recovery files remain: {remaining:?}" - ); -} - -#[test] -fn recovery_archive_retains_conflict_images_and_rejects_foreign_or_future_archives() { - use onestore_offline::Recovery; - - let directory = tempfile::tempdir().unwrap(); - let source = onestore::create_section("recovery.one", "Original", "Fixture").unwrap(); - let (space, object, _) = text(&source); - let cache = Replica::create(directory.path().join("live.sqlite"), &source).unwrap(); - let id = cache - .edit_text(&source, space, object, 0..8, "Local") - .unwrap() - .unwrap(); - let changed = onestore::replace_text(&source, space, object, 0..8, "Remote").unwrap(); - let mut server = Server::new(&changed); - let outcome = cache.sync_once(&mut server).unwrap().unwrap(); - assert_eq!( - outcome, - (id, EditStatus::Conflict(ConflictKind::TextChanged)) - ); - let path = directory.path().join("conflict.sqlite"); - cache.export_recovery(&path).unwrap(); - let archive = Recovery::open(&path).unwrap(); - assert_eq!(text(&archive.snapshot().unwrap()).2, "Local"); - assert_eq!(archive.remote_snapshot().unwrap(), changed); - assert_eq!(archive.status(id).unwrap(), Some(outcome.1)); - assert_eq!(archive.summary().unwrap().conflicts, 1); - assert_eq!(archive.summary().unwrap().uncertain_edits, 0); - drop(archive); - for sql in [ - "PRAGMA user_version=99", - "PRAGMA user_version=4; PRAGMA application_id=0", - ] { - let connection = rusqlite::Connection::open(&path).unwrap(); - connection.execute_batch(sql).unwrap(); - drop(connection); - let before = std::fs::read(&path).unwrap(); - assert!(Recovery::open(&path).is_err()); - assert_eq!(std::fs::read(&path).unwrap(), before); - } - assert_eq!(cache.status(id).unwrap(), Some(outcome.1)); - assert_eq!(text(&cache.snapshot().unwrap()).2, "Local"); -} - -#[test] -fn rebases_multiple_disjoint_remote_changes_and_persists_the_remote_receipt() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("cache.sqlite"); - let source = onestore::create_section("sync.one", "ab🦀cd", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Replica::create(&path, &source).unwrap(); - let id = cache - .edit_text(&source, sid, oid, 2..4, "🐈") - .unwrap() - .unwrap(); - let remote = onestore::replace_text(&source, sid, oid, 0..6, "Xab🦀cYd").unwrap(); - let mut server = Server::new(&remote); - let outcome = cache.sync_once(&mut server).unwrap().unwrap(); - assert_eq!(outcome.0, id); - assert!(matches!(outcome.1, EditStatus::Published { .. })); - assert_eq!(text(&server.durable).2, "Xab🐈cYd"); - assert_eq!(cache.snapshot().unwrap(), server.durable); - assert!(cache.pending().unwrap().is_empty()); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.status(id).unwrap(), Some(outcome.1)); - assert_eq!(cache.sync_once(&mut server).unwrap(), None); - assert_eq!(server.publications, 1); - assert_eq!(cache.status(id + 1).unwrap(), None); - let source = cache.snapshot().unwrap(); - let next = cache - .edit_text(&source, sid, oid, 0..0, "Later ") - .unwrap() - .unwrap(); - assert!(next > id); -} - -#[test] -fn overlapping_changes_preserve_both_images_and_survive_restart() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("cache.sqlite"); - let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Replica::create(&path, &source).unwrap(); - let id = cache - .edit_text(&source, sid, oid, 1..2, "L") - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let remote = onestore::replace_text(&source, sid, oid, 1..2, "R").unwrap(); - let mut server = Server::new(&remote); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((id, EditStatus::Conflict(ConflictKind::TextChanged))) - ); - assert_eq!(server.publications, 0); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(cache.remote_snapshot().unwrap(), remote); - assert_eq!(cache.pending().unwrap().len(), 1); - assert_eq!( - cache.status(id).unwrap(), - Some(EditStatus::Conflict(ConflictKind::TextChanged)) - ); -} - -#[test] -fn lost_replies_and_process_termination_never_blindly_replay_an_attempt() { - for fault in [ - Fault::UnknownBefore, - Fault::UnknownAfter, - Fault::PanicBefore, - Fault::PanicAfter, - ] { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("cache.sqlite"); - let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Replica::create(&path, &source).unwrap(); - let id = cache - .edit_text(&source, sid, oid, 0..0, "Once ") - .unwrap() - .unwrap(); - let mut server = Server::new(&source); - server.fault = fault; - let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - cache.sync_once(&mut server) - })); - let attempted = cache.status(id).unwrap().unwrap(); - assert!(matches!(attempted, EditStatus::AwaitingConfirmation { .. })); - drop(cache); - let cache = Replica::open(&path).unwrap(); - let result = cache.sync_once(&mut server).unwrap().unwrap(); - if matches!(fault, Fault::UnknownAfter | Fault::PanicAfter) { - assert!(matches!(result.1, EditStatus::Published { .. })); - assert_eq!(text(&server.durable).2, "Once abc"); - assert_eq!(server.confirmations, 1); - assert!(cache.pending().unwrap().is_empty()); - } else { - assert_eq!(result.1, attempted); - assert_eq!(cache.pending().unwrap().len(), 1); - assert_eq!(server.confirmations, 0); - assert_eq!(server.durable, source); - } - assert_eq!(server.publications, 1); - } -} - -#[test] -fn failed_confirmation_does_not_promote_visible_bytes_to_a_receipt() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("cache.sqlite"); - let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Replica::create(&path, &source).unwrap(); - let id = cache - .edit_text(&source, sid, oid, 0..0, "Once ") - .unwrap() - .unwrap(); - let mut server = Server::new(&source); - server.fault = Fault::UnknownAfter; - assert!(cache.sync_once(&mut server).is_err()); - server.fault = Fault::Confirm; - assert!(cache.sync_once(&mut server).is_err()); - assert_eq!(server.durable, source); - assert!(matches!( - cache.status(id).unwrap(), - Some(EditStatus::AwaitingConfirmation { .. }) - )); - assert!(matches!( - cache.sync_once(&mut server).unwrap(), - Some((_, EditStatus::Published { .. })) - )); - assert_eq!(server.publications, 1); - assert_eq!(server.confirmations, 2); - assert_eq!(server.visible, server.durable); -} - -#[test] -fn confirmation_cleanup_failure_still_records_a_durable_receipt() { - let dir = tempfile::tempdir().unwrap(); - let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Replica::create(dir.path().join("cache.sqlite"), &source).unwrap(); - let id = cache - .edit_text(&source, sid, oid, 0..0, "Once ") - .unwrap() - .unwrap(); - let mut server = Server::new(&source); - server.fault = Fault::UnknownAfter; - assert!(cache.sync_once(&mut server).is_err()); - server.fault = Fault::ConfirmCommitted; - assert!( - matches!(cache.sync_once(&mut server), Err(Error::Remote(error)) if error.state == CommitState::Committed) - ); - assert!(matches!( - cache.status(id).unwrap(), - Some(EditStatus::Published { .. }) - )); - assert!(cache.pending().unwrap().is_empty()); - assert_eq!(server.visible, server.durable); - assert_eq!(server.publications, 1); - assert_eq!(server.confirmations, 1); -} - -#[test] -fn proven_unpublished_attempts_retry_and_committed_cleanup_errors_keep_receipts() { - for fault in [Fault::Before, Fault::Committed] { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("cache.sqlite"); - let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Replica::create(&path, &source).unwrap(); - let id = cache - .edit_text(&source, sid, oid, 0..0, "Once ") - .unwrap() - .unwrap(); - let mut server = Server::new(&source); - server.fault = fault; - assert!(matches!( - cache.sync_once(&mut server), - Err(Error::Remote(_)) - )); - if matches!(fault, Fault::Before) { - assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); - assert!(matches!( - cache.sync_once(&mut server).unwrap(), - Some((_, EditStatus::Published { .. })) - )); - assert_eq!(server.publications, 2); - } else { - assert!(matches!( - cache.status(id).unwrap(), - Some(EditStatus::Published { .. }) - )); - assert_eq!(cache.sync_once(&mut server).unwrap(), None); - assert_eq!(server.publications, 1); - } - assert_eq!(text(&server.durable).2, "Once abc"); - } -} - -#[test] -fn database_failures_before_and_after_publication_preserve_recovery_state() { - for table in ["attempt", "receipts"] { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("cache.sqlite"); - let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Replica::create(&path, &source).unwrap(); - let id = cache - .edit_text(&source, sid, oid, 0..0, "Once ") - .unwrap() - .unwrap(); - drop(cache); - let db = rusqlite::Connection::open(&path).unwrap(); - db.execute_batch(&format!("CREATE TRIGGER interrupted BEFORE INSERT ON {table} BEGIN SELECT RAISE(ABORT,'Injected cache failure'); END;")).unwrap(); - drop(db); - let cache = Replica::open(&path).unwrap(); - let mut server = Server::new(&source); - assert!(matches!( - cache.sync_once(&mut server), - Err(Error::Database(_)) - )); - assert_eq!(server.publications, usize::from(table == "receipts")); - assert_eq!(cache.pending().unwrap().len(), 1); - drop(cache); - let db = rusqlite::Connection::open(&path).unwrap(); - db.execute_batch("DROP TRIGGER interrupted").unwrap(); - drop(db); - let cache = Replica::open(&path).unwrap(); - assert!(matches!( - cache.sync_once(&mut server).unwrap(), - Some((_, EditStatus::Published { .. })) - )); - assert!(matches!( - cache.status(id).unwrap(), - Some(EditStatus::Published { .. }) - )); - assert_eq!(server.publications, 1); - assert_eq!(server.confirmations, usize::from(table == "receipts")); - assert_eq!(text(&server.durable).2, "Once abc"); - } -} - -#[test] -fn twelve_local_editors_progress_during_remote_reads_publication_and_confirmation() { - struct Paused { - server: Server, - phase: &'static str, - entered: std::sync::mpsc::Sender<()>, - resume: std::sync::mpsc::Receiver<()>, - } - impl Paused { - fn wait(&self, phase: &str) { - if self.phase == phase { - self.entered.send(()).unwrap(); - self.resume - .recv_timeout(std::time::Duration::from_secs(5)) - .unwrap(); - } - } - } - impl Remote for Paused { - fn read(&mut self) -> io::Result> { - self.wait("read"); - self.server.read() - } - fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { - self.wait("publish"); - self.server.publish(edit) - } - fn confirm(&mut self, source: &[u8]) -> Result<(), CommitError> { - self.wait("confirm"); - self.server.confirm(source) - } - } - for phase in ["read", "publish", "confirm"] { - let dir = tempfile::tempdir().unwrap(); - let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Replica::create(dir.path().join("cache.sqlite"), &source).unwrap(); - let first = cache - .edit_text(&source, sid, oid, 0..0, "First ") - .unwrap() - .unwrap(); - let mut server = Server::new(&source); - if phase == "confirm" { - server.fault = Fault::UnknownAfter; - assert!(cache.sync_once(&mut server).is_err()); - } - let (entered_tx, entered_rx) = std::sync::mpsc::channel(); - let (resume_tx, resume_rx) = std::sync::mpsc::channel(); - let mut paused = Paused { - server, - phase, - entered: entered_tx, - resume: resume_rx, - }; - let mut server = std::thread::scope(|scope| { - let running = scope.spawn(|| { - let result = cache.sync_once(&mut paused); - assert!( - matches!(result, Ok(Some((id, EditStatus::Published { .. }))) if id == first) - ); - paused.server - }); - entered_rx - .recv_timeout(std::time::Duration::from_secs(5)) - .unwrap(); - assert!( - matches!(cache.sync_once(&mut Server::new(&source)),Err(Error::Io(error)) if error.kind()==io::ErrorKind::WouldBlock) - ); - assert!( - matches!(cache.rebase_conflict(first, &[], &[], 0..0), Err(Error::Io(error)) if error.kind() == io::ErrorKind::WouldBlock) - ); - let started = std::time::Instant::now(); - let handles: Vec<_> = (0..12) - .map(|writer| { - let cache = &cache; - scope.spawn(move || { - loop { - let snapshot = cache.snapshot().unwrap(); - match cache.edit_text( - &snapshot, - sid, - oid, - 0..0, - &format!("[{writer}] "), - ) { - Ok(Some(id)) => break id, - Err(Error::Io(error)) - if error.kind() == io::ErrorKind::ResourceBusy => {} - other => panic!("Unexpected local outcome {other:?}"), - } - } - }) - }) - .collect(); - let ids: std::collections::BTreeSet<_> = handles - .into_iter() - .map(|handle| handle.join().unwrap()) - .collect(); - assert_eq!(ids.len(), 12); - assert!( - started.elapsed() < std::time::Duration::from_secs(2), - "Local edits waited for remote {phase}" - ); - resume_tx.send(()).unwrap(); - running.join().unwrap() - }); - assert_eq!(cache.pending().unwrap().len(), 12); - let expected = text(&cache.snapshot().unwrap()).2; - for _ in 0..12 { - assert!(matches!( - cache.sync_once(&mut server).unwrap(), - Some((_, EditStatus::Published { .. })) - )); - } - assert!(cache.pending().unwrap().is_empty()); - assert_eq!(server.publications, 13); - assert_eq!(text(&server.durable).2, expected); - assert_eq!(cache.snapshot().unwrap(), server.durable); - } -} - -#[test] -fn unrelated_remote_files_never_replace_a_local_cache() { - let dir = tempfile::tempdir().unwrap(); - let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Replica::create(dir.path().join("cache.sqlite"), &source).unwrap(); - let other = onestore::create_section("other.one", "abc", "Fixture").unwrap(); - let mut server = Server::new(&other); - for pending in [false, true] { - if pending { - cache.edit_text(&source, sid, oid, 0..0, "Local ").unwrap(); - } - let before = cache.snapshot().unwrap(); - assert!( - matches!(cache.sync_once(&mut server),Err(Error::Io(error)) if error.kind()==io::ErrorKind::InvalidInput) - ); - assert_eq!(cache.snapshot().unwrap(), before); - assert_eq!(cache.remote_snapshot().unwrap(), source); - assert_eq!(server.publications, 0); - } -} - -#[test] -fn version_one_cache_migration_preserves_images_intents_and_local_ids() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("cache.sqlite"); - let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Replica::create(&path, &source).unwrap(); - let id = cache - .edit_text(&source, sid, oid, 0..0, "Local ") - .unwrap() - .unwrap(); - let snapshot = cache.snapshot().unwrap(); - let pending = cache.pending().unwrap(); - drop(cache); - let db = rusqlite::Connection::open(&path).unwrap(); - let current_version: u32 = db - .pragma_query_value(None, "user_version", |row| row.get(0)) - .unwrap(); - db.execute_batch( - "DROP TABLE attempt; DROP TABLE conflicts; DROP TABLE receipts; DROP TABLE edits; DROP TABLE assets; - CREATE TABLE edits ( - id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(id>0), space TEXT NOT NULL, - object TEXT NOT NULL, before_text TEXT NOT NULL, - start INTEGER NOT NULL CHECK(start BETWEEN 0 AND 4294967295), - end INTEGER NOT NULL CHECK(end BETWEEN start AND 4294967295), replacement TEXT NOT NULL - ) STRICT; PRAGMA user_version=1;", - ) - .unwrap(); - db.execute("INSERT INTO edits(id,space,object,before_text,start,end,replacement) VALUES (?1,?2,?3,'abc',0,0,'Local ')",rusqlite::params![i64::try_from(id).unwrap(),sid.to_string(),oid.to_string()]).unwrap(); - drop(db); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.snapshot().unwrap(), snapshot); - assert_eq!(cache.pending().unwrap(), pending); - assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); - let mut server = Server::new(&source); - assert!(matches!( - cache.sync_once(&mut server).unwrap(), - Some((_, EditStatus::Published { .. })) - )); - drop(cache); - let db = rusqlite::Connection::open(&path).unwrap(); - assert_eq!( - db.pragma_query_value(None, "user_version", |row| row.get::<_, u32>(0)) - .unwrap(), - current_version - ); -} - -#[test] -fn reviewed_conflict_rebase_preserves_twelve_dependent_edits_and_survives_reopen() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("cache.sqlite"); - let source = onestore::create_section("resolve.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Replica::create(&path, &source).unwrap(); - let first = cache - .edit_text(&source, sid, oid, 1..2, "L") - .unwrap() - .unwrap(); - for n in 0..12 { - cache - .edit_text( - &cache.snapshot().unwrap(), - sid, - oid, - 0..0, - &format!("[{n}] "), - ) - .unwrap(); - } - let local = cache.snapshot().unwrap(); - let pending = cache.pending().unwrap(); - let remote = onestore::replace_text(&source, sid, oid, 0..3, "aRcZ").unwrap(); - let mut server = Server::new(&remote); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((first, EditStatus::Conflict(ConflictKind::TextChanged))) - ); - cache.rebase_conflict(first, &local, &remote, 1..2).unwrap(); - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(cache.remote_snapshot().unwrap(), remote); - let rebased = cache.pending().unwrap(); - assert_eq!(&rebased[1..], &pending[1..]); - assert_eq!(rebased[0].id, first); - let onestore_offline::Operation::Text(updated) = &rebased[0].operation else { - panic!() - }; - let onestore_offline::Operation::Text(previous) = &pending[0].operation else { - panic!() - }; - assert_eq!(updated.replacement, previous.replacement); - assert_eq!(updated.before, "aRcZ"); - assert_eq!(cache.status(first).unwrap(), Some(EditStatus::Pending)); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.pending().unwrap(), rebased); - assert_eq!(cache.snapshot().unwrap(), local); - for intent in &pending { - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((id, EditStatus::Published { .. })) if id == intent.id) - ); - } - assert_eq!(server.publications, 13); - assert_eq!(text(&server.durable).2, text(&local).2 + "Z"); - assert_eq!(cache.snapshot().unwrap(), server.durable); - assert!(cache.pending().unwrap().is_empty()); -} - -#[test] -fn conflict_review_rejects_stale_images_invalid_ranges_and_nonconflicting_states() { - let dir = tempfile::tempdir().unwrap(); - let source = onestore::create_section("resolve.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Replica::create(dir.path().join("cache.sqlite"), &source).unwrap(); - let id = cache - .edit_text(&source, sid, oid, 1..2, "L") - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - assert!( - matches!(cache.rebase_conflict(id, &local, &source, 1..2), Err(Error::Io(error)) if error.kind() == io::ErrorKind::InvalidInput) - ); - let remote = onestore::replace_text(&source, sid, oid, 0..3, "🦀Rc").unwrap(); - let mut server = Server::new(&remote); - assert!(matches!( - cache.sync_once(&mut server).unwrap(), - Some((_, EditStatus::Conflict(_))) - )); - let pending = cache.pending().unwrap(); - for range in [1..2, 100..101] { - assert!(matches!( - cache.rebase_conflict(id, &local, &remote, range), - Err(Error::Document(_)) - )); - assert_eq!(cache.pending().unwrap(), pending); - } - assert!( - matches!(cache.rebase_conflict(id + 1, &local, &remote, 2..3), Err(Error::Io(error)) if error.kind() == io::ErrorKind::InvalidInput) - ); - cache.edit_text(&local, sid, oid, 0..0, "Later ").unwrap(); - let changed = cache.snapshot().unwrap(); - assert!( - matches!(cache.rebase_conflict(id, &local, &remote, 2..3), Err(Error::Io(error)) if error.kind() == io::ErrorKind::ResourceBusy) - ); - let new_remote = onestore::replace_text(&remote, sid, oid, 2..3, "Q").unwrap(); - server.visible = new_remote.clone(); - server.durable = new_remote; - assert!(matches!( - cache.sync_once(&mut server).unwrap(), - Some((_, EditStatus::Conflict(_))) - )); - assert!( - matches!(cache.rebase_conflict(id, &changed, &remote, 2..3), Err(Error::Io(error)) if error.kind() == io::ErrorKind::ResourceBusy) - ); - assert_eq!(cache.snapshot().unwrap(), changed); - assert_eq!(cache.pending().unwrap()[0], pending[0]); - assert_eq!(server.publications, 0); - - let current = cache.remote_snapshot().unwrap(); - cache.rebase_conflict(id, &changed, ¤t, 2..3).unwrap(); - server.fault = Fault::UnknownBefore; - assert!( - matches!(cache.sync_once(&mut server), Err(Error::Remote(error)) if error.state == CommitState::Unknown) - ); - let attempted = cache.status(id).unwrap(); - let pending = cache.pending().unwrap(); - assert!( - matches!(cache.rebase_conflict(id, &changed, ¤t, 2..3), Err(Error::Io(error)) if error.kind() == io::ErrorKind::InvalidInput) - ); - assert_eq!(cache.status(id).unwrap(), attempted); - assert_eq!(cache.pending().unwrap(), pending); - assert_eq!(server.publications, 1); -} - -#[test] -fn failure_between_rebase_and_conflict_clear_rolls_back_the_entire_resolution() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("cache.sqlite"); - let source = onestore::create_section("resolve.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Replica::create(&path, &source).unwrap(); - let id = cache - .edit_text(&source, sid, oid, 1..2, "L") - .unwrap() - .unwrap(); - let remote = onestore::replace_text(&source, sid, oid, 0..3, "XaRc").unwrap(); - let mut server = Server::new(&remote); - cache.sync_once(&mut server).unwrap(); - let local = cache.snapshot().unwrap(); - let pending = cache.pending().unwrap(); - let status = cache.status(id).unwrap(); - drop(cache); - let connection = rusqlite::Connection::open(&path).unwrap(); - connection.execute_batch("CREATE TRIGGER fail_clear BEFORE DELETE ON conflicts BEGIN SELECT RAISE(ABORT, 'test conflict clear failure'); END;").unwrap(); - drop(connection); - let cache = Replica::open(&path).unwrap(); - assert!(matches!( - cache.rebase_conflict(id, &local, &remote, 2..3), - Err(Error::Database(_)) - )); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.pending().unwrap(), pending); - assert_eq!(cache.status(id).unwrap(), status); - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(cache.remote_snapshot().unwrap(), remote); - assert_eq!(server.publications, 0); -} - -#[test] -fn seeded_reviewed_ranges_preserve_unicode_and_edits_inside_the_original_replacement() { - let dir = tempfile::tempdir().unwrap(); - let original: Vec<_> = "abcdefghij🦀klmnop".chars().collect(); - let mut seed = 911_u64; - for case in 0..64 { - seed ^= seed << 13; - seed ^= seed >> 7; - seed ^= seed << 17; - let at = seed as usize % original.len(); - let prefix = "[".repeat((seed >> 8) as usize % 5); - let suffix = "]".repeat((seed >> 16) as usize % 5); - let start: u32 = original[..at].iter().map(|ch| ch.len_utf16() as u32).sum(); - let end = start + original[at].len_utf16() as u32; - let source = onestore::create_section( - "resolve.one", - &original.iter().collect::(), - "Fixture", - ) - .unwrap(); - let (sid, oid, _) = text(&source); - let cache = Replica::create(dir.path().join(format!("{case}.sqlite")), &source).unwrap(); - let id = cache - .edit_text(&source, sid, oid, start..end, "λ🦊μ") - .unwrap() - .unwrap(); - let dependent = cache - .edit_text( - &cache.snapshot().unwrap(), - sid, - oid, - start + 1..start + 3, - "🐕", - ) - .unwrap() - .unwrap(); - let mut remote_text = original.clone(); - remote_text.splice(at..at + 1, "Ω🐈π".chars()); - let remote_text = prefix.clone() + &remote_text.iter().collect::() + &suffix; - let remote = onestore::replace_text( - &source, - sid, - oid, - 0..original.iter().map(|ch| ch.len_utf16() as u32).sum(), - &remote_text, - ) - .unwrap(); - let mut server = Server::new(&remote); - assert!( - matches!( - cache.sync_once(&mut server).unwrap(), - Some((_, EditStatus::Conflict(_))) - ), - "case {case}, seed {seed}" - ); - let at_remote = start + prefix.len() as u32; - cache - .rebase_conflict( - id, - &cache.snapshot().unwrap(), - &remote, - at_remote..at_remote + 4, - ) - .unwrap(); - for expected in [id, dependent] { - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == expected), - "case {case}, seed {seed}" - ); - } - let mut expected = original.clone(); - expected.splice(at..at + 1, "λ🐕μ".chars()); - let expected = prefix + &expected.iter().collect::() + &suffix; - assert_eq!( - text(&server.durable).2, - expected, - "case {case}, seed {seed}" - ); - assert_eq!(server.publications, 2); - assert!(cache.pending().unwrap().is_empty()); - } -} - -#[test] -fn remote_changes_after_review_cannot_be_overwritten_by_the_reviewed_placement() { - struct ChangedAfterRead { - server: Server, - change: Option>, - } - impl Remote for ChangedAfterRead { - fn read(&mut self) -> io::Result> { - let snapshot = self.server.read()?; - if let Some(changed) = self.change.take() { - self.server.visible = changed.clone(); - self.server.durable = changed; - } - Ok(snapshot) - } - fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { - self.server.publish(edit) - } - fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { - self.server.confirm(snapshot) - } - } - for after_read in [false, true] { - let dir = tempfile::tempdir().unwrap(); - let source = onestore::create_section("resolve.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Replica::create(dir.path().join("cache.sqlite"), &source).unwrap(); - let id = cache - .edit_text(&source, sid, oid, 1..2, "L") - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let remote = onestore::replace_text(&source, sid, oid, 1..2, "R").unwrap(); - let mut remote = ChangedAfterRead { - server: Server::new(&remote), - change: None, - }; - assert!(matches!( - cache.sync_once(&mut remote).unwrap(), - Some((_, EditStatus::Conflict(_))) - )); - cache - .rebase_conflict(id, &local, &cache.remote_snapshot().unwrap(), 1..2) - .unwrap(); - let changed = onestore::replace_text(&remote.server.visible, sid, oid, 1..2, "Q").unwrap(); - if after_read { - remote.change = Some(changed.clone()); - } else { - remote.server.visible = changed.clone(); - remote.server.durable = changed.clone(); - } - if after_read { - assert!( - matches!(cache.sync_once(&mut remote), Err(Error::Remote(error)) if error.state == CommitState::NotCommitted) - ); - assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); - } - assert_eq!( - cache.sync_once(&mut remote).unwrap(), - Some((id, EditStatus::Conflict(ConflictKind::TextChanged))) - ); - assert_eq!(remote.server.durable, changed); - assert_eq!(remote.server.visible, changed); - assert_eq!(remote.server.publications, usize::from(after_read)); - assert_eq!(cache.snapshot().unwrap(), local); - let onestore_offline::Operation::Text(edit) = &cache.pending().unwrap()[0].operation else { - panic!() - }; - assert_eq!(edit.replacement, "L"); - } -} - -mod worker { - use super::*; - use std::{ - sync::{Arc, Mutex, mpsc}, - time::{Duration, Instant}, - }; - - #[derive(Clone)] - struct Shared(Arc>); - - impl Remote for Shared { - fn read(&mut self) -> io::Result> { - self.0.lock().unwrap().read() - } - fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { - self.0.lock().unwrap().publish(edit) - } - fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { - self.0.lock().unwrap().confirm(snapshot) - } - } - - #[test] - fn reconnects_after_connect_read_and_uncertain_publish_without_replaying() { - struct Session { - shared: Shared, - fail_read: bool, - } - impl Remote for Session { - fn read(&mut self) -> io::Result> { - if self.fail_read { - return Err(io::ErrorKind::ConnectionReset.into()); - } - self.shared.read() - } - fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { - self.shared.publish(edit) - } - fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { - self.shared.confirm(snapshot) - } - } - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("cache.sqlite"); - let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Arc::new(Replica::create(&path, &source).unwrap()); - let id = cache - .edit_text(&source, sid, oid, 1..2, "🦀") - .unwrap() - .unwrap(); - let mut server = Server::new(&source); - server.fault = Fault::UnknownAfter; - let server = Arc::new(Mutex::new(server)); - let shared = Shared(Arc::clone(&server)); - let (connected_tx, connected_rx) = mpsc::channel(); - let (observed_tx, observed_rx) = mpsc::channel(); - let mut connections = 0; - let worker = cache - .start_sync( - Duration::from_millis(10), - move || { - connections += 1; - connected_tx.send(connections).unwrap(); - if connections <= 2 { - return Err(io::ErrorKind::ConnectionRefused.into()); - } - Ok(Session { - shared: shared.clone(), - fail_read: connections == 3, - }) - }, - move |result| { - observed_tx - .send(result.as_ref().copied().map_err(|error| error.to_string())) - .unwrap(); - }, - ) - .unwrap(); - let mut errors = 0; - let published = loop { - match observed_rx.recv_timeout(Duration::from_secs(5)).unwrap() { - Err(_) => errors += 1, - Ok(Some((actual, status @ EditStatus::Published { .. }))) => { - assert_eq!(actual, id); - break status; - } - other => panic!("Unexpected result: {other:?}"), - } - }; - worker.stop().unwrap(); - assert_eq!(errors, 4); - assert_eq!(connected_rx.try_iter().collect::>(), [1, 2, 3, 4, 5]); - let server = server.lock().unwrap(); - assert_eq!(server.publications, 1); - assert_eq!(server.confirmations, 1); - assert_eq!(text(&server.durable).2, "a🦀c"); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.status(id).unwrap(), Some(published)); - let cached = cache.snapshot().unwrap(); - // Confirmation changes reader-notification fields without changing the committed graph. - assert_eq!(cached[..212], server.durable[..212]); - assert_eq!(cached[252..], server.durable[252..]); - let cached_generation = Store::parse(&cached).unwrap().header.generation; - let remote_generation = Store::parse(&server.durable).unwrap().header.generation; - if cached_generation == remote_generation { - assert_eq!(cached, server.durable); - } else { - assert_eq!(cached_generation.checked_add(1), Some(remote_generation)); - } - } - - #[test] - fn local_edits_wake_an_idle_worker_and_coalesced_notifications_drain_twelve_writers() { - let dir = tempfile::tempdir().unwrap(); - let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); - let server = Arc::new(Mutex::new(Server::new(&source))); - let shared = Shared(Arc::clone(&server)); - let (observed_tx, observed_rx) = mpsc::channel(); - let (resume_tx, resume_rx) = mpsc::channel(); - let mut first = true; - let worker = cache - .start_sync( - Duration::from_secs(3600), - move || Ok(shared.clone()), - move |result| { - observed_tx.send(*result.as_ref().unwrap()).unwrap(); - if first { - first = false; - resume_rx.recv_timeout(Duration::from_secs(5)).unwrap(); - } - }, - ) - .unwrap(); - assert_eq!( - observed_rx.recv_timeout(Duration::from_secs(5)).unwrap(), - None - ); - let started = Instant::now(); - let edits = std::thread::scope(|scope| { - (0..12) - .map(|writer| { - let cache = &cache; - scope.spawn(move || { - let replacement = format!("[{writer}] "); - loop { - let snapshot = cache.snapshot().unwrap(); - match cache.edit_text(&snapshot, sid, oid, 0..0, &replacement) { - Ok(Some(id)) => break (id, replacement), - Err(Error::Io(error)) - if error.kind() == io::ErrorKind::ResourceBusy => {} - other => panic!("Unexpected local outcome: {other:?}"), - } - } - }) - }) - .collect::>() - .into_iter() - .map(|join| join.join().unwrap()) - .collect::>() - }); - resume_tx.send(()).unwrap(); - let mut published = std::collections::BTreeSet::new(); - while published.len() < 12 { - if let Some((id, status)) = observed_rx.recv_timeout(Duration::from_secs(5)).unwrap() { - assert!(matches!(status, EditStatus::Published { .. }), "{status:?}"); - assert!(published.insert(id)); - } - } - assert!( - started.elapsed() < Duration::from_secs(5), - "Edits waited for the hourly poll" - ); - worker.stop().unwrap(); - assert_eq!(published, edits.keys().copied().collect()); - let expected = edits.values().rev().cloned().collect::() + "abc"; - let server = server.lock().unwrap(); - assert_eq!(text(&server.durable).2, expected); - assert_eq!(server.publications, 12); - assert_eq!(cache.snapshot().unwrap(), server.durable); - assert!(cache.pending().unwrap().is_empty()); - } - - #[test] - fn dropping_during_publication_is_nonblocking_and_retains_ownership_until_recovery_is_recorded() - { - struct Paused { - shared: Shared, - entered: mpsc::Sender<()>, - resume: mpsc::Receiver<()>, - } - impl Remote for Paused { - fn read(&mut self) -> io::Result> { - self.shared.read() - } - fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { - self.entered.send(()).unwrap(); - self.resume.recv_timeout(Duration::from_secs(5)).unwrap(); - self.shared.publish(edit) - } - fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { - self.shared.confirm(snapshot) - } - } - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("cache.sqlite"); - let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Arc::new(Replica::create(&path, &source).unwrap()); - let id = cache - .edit_text(&source, sid, oid, 0..0, "L ") - .unwrap() - .unwrap(); - let mut server = Server::new(&source); - server.fault = Fault::UnknownAfter; - let server = Arc::new(Mutex::new(server)); - let shared = Shared(Arc::clone(&server)); - let (entered_tx, entered_rx) = mpsc::channel(); - let (resume_tx, resume_rx) = mpsc::channel(); - let mut session = Some(Paused { - shared, - entered: entered_tx, - resume: resume_rx, - }); - let worker = cache - .start_sync( - Duration::from_secs(3600), - move || Ok(session.take().unwrap()), - |_| {}, - ) - .unwrap(); - entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); - let stopped = Instant::now(); - drop(worker); - assert!(stopped.elapsed() < Duration::from_millis(500)); - let shared = Shared(Arc::clone(&server)); - let start = cache.start_sync( - Duration::from_secs(3600), - move || Ok(shared.clone()), - |_| {}, - ); - assert!(matches!(start, Err(error) if error.kind() == io::ErrorKind::WouldBlock)); - let weak = Arc::downgrade(&cache); - drop(cache); - resume_tx.send(()).unwrap(); - while weak.upgrade().is_some() { - assert!(stopped.elapsed() < Duration::from_secs(5)); - std::thread::sleep(Duration::from_millis(1)); - } - let cache = Arc::new(Replica::open(&path).unwrap()); - assert!(matches!( - cache.status(id).unwrap(), - Some(EditStatus::AwaitingConfirmation { .. }) - )); - assert_eq!(server.lock().unwrap().publications, 1); - let shared = Shared(Arc::clone(&server)); - let (tx, rx) = mpsc::channel(); - let worker = cache - .start_sync( - Duration::from_secs(3600), - move || Ok(shared.clone()), - move |result| { - tx.send(*result.as_ref().unwrap()).unwrap(); - }, - ) - .unwrap(); - assert!( - matches!(rx.recv_timeout(Duration::from_secs(5)).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) - ); - worker.stop().unwrap(); - let server = server.lock().unwrap(); - assert_eq!(server.publications, 1); - assert_eq!(server.confirmations, 1); - assert_eq!(text(&server.durable).2, "L abc"); - } - - #[test] - fn cache_failures_stop_retries_and_return_the_error_without_remote_publication() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("cache.sqlite"); - let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Replica::create(&path, &source).unwrap(); - let id = cache - .edit_text(&source, sid, oid, 0..0, "L ") - .unwrap() - .unwrap(); - drop(cache); - let connection = rusqlite::Connection::open(&path).unwrap(); - connection.execute_batch("CREATE TRIGGER fail_attempt BEFORE INSERT ON attempt BEGIN SELECT RAISE(ABORT, 'test cache write failure'); END;").unwrap(); - drop(connection); - let cache = Arc::new(Replica::open(&path).unwrap()); - let server = Arc::new(Mutex::new(Server::new(&source))); - let shared = Shared(Arc::clone(&server)); - let (tx, rx) = mpsc::channel(); - let worker = cache - .start_sync( - Duration::from_millis(1), - move || Ok(shared.clone()), - move |result| { - tx.send(matches!(result, Err(Error::Database(_)))).unwrap(); - }, - ) - .unwrap(); - assert!(rx.recv_timeout(Duration::from_secs(5)).unwrap()); - assert!(matches!(worker.stop(), Err(Error::Database(_)))); - assert!(rx.try_iter().next().is_none()); - assert_eq!(server.lock().unwrap().publications, 0); - assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); - assert_eq!(text(&cache.snapshot().unwrap()).2, "L abc"); - } - - #[test] - fn polling_preserves_conflicts_and_absent_uncertain_revisions_without_replay() { - for uncertain in [false, true] { - let dir = tempfile::tempdir().unwrap(); - let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = - Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); - let id = cache - .edit_text(&source, sid, oid, 1..2, "L") - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let mut server = if uncertain { - Server::new(&source) - } else { - Server::new(&onestore::replace_text(&source, sid, oid, 1..2, "R").unwrap()) - }; - if uncertain { - server.fault = Fault::UnknownBefore; - } - let server = Arc::new(Mutex::new(server)); - let shared = Shared(Arc::clone(&server)); - let (tx, rx) = mpsc::channel(); - let worker = cache - .start_sync( - Duration::from_millis(10), - move || Ok(shared.clone()), - move |result| { - tx.send(result.as_ref().copied().map_err(|_| ())).unwrap(); - }, - ) - .unwrap(); - if uncertain { - assert!(rx.recv_timeout(Duration::from_secs(5)).unwrap().is_err()); - } - let mut previous = None; - let started = Instant::now(); - for _ in 0..5 { - let (actual, status) = rx - .recv_timeout(Duration::from_secs(5)) - .unwrap() - .unwrap() - .unwrap(); - assert_eq!(actual, id); - if uncertain { - assert!(matches!(status, EditStatus::AwaitingConfirmation { .. })); - } else { - assert_eq!(status, EditStatus::Conflict(ConflictKind::TextChanged)); - } - if let Some(previous) = previous { - assert_eq!(previous, status); - } - previous = Some(status); - } - assert!( - started.elapsed() >= Duration::from_millis(30), - "Worker spun instead of waiting between retries" - ); - worker.stop().unwrap(); - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(cache.pending().unwrap().len(), 1); - let server = server.lock().unwrap(); - assert_eq!(server.publications, usize::from(uncertain)); - assert_eq!(server.confirmations, 0); - } - } - - #[test] - fn reachability_notification_retries_without_waiting_for_the_poll() { - let dir = tempfile::tempdir().unwrap(); - let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); - let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); - let (tx, rx) = mpsc::channel(); - let worker = cache - .start_sync( - Duration::from_secs(3600), - || -> io::Result { Err(io::ErrorKind::NotConnected.into()) }, - move |result| { - tx.send(matches!(result, Err(Error::RemoteIo(_)))).unwrap(); - }, - ) - .unwrap(); - assert!(rx.recv_timeout(Duration::from_secs(5)).unwrap()); - worker.wake(); - assert!(rx.recv_timeout(Duration::from_secs(5)).unwrap()); - worker.stop().unwrap(); - assert!(rx.try_iter().next().is_none()); - } - - #[test] - fn cancellation_during_connect_does_not_read_or_report_a_false_refresh() { - let dir = tempfile::tempdir().unwrap(); - let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); - let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); - let (entered_tx, entered_rx) = mpsc::channel(); - let (resume_tx, resume_rx) = mpsc::channel(); - let (tx, rx) = mpsc::channel(); - // An invalid image makes any unexpected read observable as an error callback. - let shared = Shared(Arc::new(Mutex::new(Server::new(&[])))); - let worker = cache - .start_sync( - Duration::from_secs(3600), - move || { - entered_tx.send(()).unwrap(); - resume_rx.recv_timeout(Duration::from_secs(5)).unwrap(); - Ok(shared.clone()) - }, - move |_| { - tx.send(()).unwrap(); - }, - ) - .unwrap(); - entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); - drop(worker); - let weak = Arc::downgrade(&cache); - drop(cache); - resume_tx.send(()).unwrap(); - let started = Instant::now(); - while weak.upgrade().is_some() { - assert!(started.elapsed() < Duration::from_secs(5)); - std::thread::sleep(Duration::from_millis(1)); - } - assert_eq!( - rx.recv_timeout(Duration::from_secs(5)), - Err(mpsc::RecvTimeoutError::Disconnected) - ); - } - - #[test] - fn invalid_intervals_and_callback_panics_leave_worker_ownership_recoverable() { - let dir = tempfile::tempdir().unwrap(); - let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); - let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); - for interval in [Duration::ZERO, Duration::MAX] { - assert!( - matches!(cache.start_sync(interval, || -> io::Result { panic!("Unexpected connection") }, |_| {}), Err(error) if error.kind() == io::ErrorKind::InvalidInput) - ); - } - let shared = Shared(Arc::new(Mutex::new(Server::new(&source)))); - let first = shared.clone(); - let (tx, rx) = mpsc::channel(); - let worker = cache - .start_sync( - Duration::from_secs(3600), - move || Ok(first.clone()), - move |_| { - tx.send(()).unwrap(); - panic!("Test observer panic"); - }, - ) - .unwrap(); - rx.recv_timeout(Duration::from_secs(5)).unwrap(); - assert!(matches!(worker.stop(), Err(Error::Io(_)))); - let (tx, rx) = mpsc::channel(); - let worker = cache - .start_sync( - Duration::from_secs(3600), - move || Ok(shared.clone()), - move |result| { - tx.send(*result.as_ref().unwrap()).unwrap(); - }, - ) - .unwrap(); - assert_eq!(rx.recv_timeout(Duration::from_secs(5)).unwrap(), None); - worker.stop().unwrap(); - assert_eq!(cache.snapshot().unwrap(), source); - } - - #[test] - fn reviewed_conflict_wakes_the_worker_and_publishes_the_original_intent_once() { - let dir = tempfile::tempdir().unwrap(); - let source = onestore::create_section("resolve.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); - let id = cache - .edit_text(&source, sid, oid, 1..2, "L") - .unwrap() - .unwrap(); - let remote = onestore::replace_text(&source, sid, oid, 1..2, "R").unwrap(); - let server = Arc::new(Mutex::new(Server::new(&remote))); - let shared = Shared(Arc::clone(&server)); - let (tx, rx) = mpsc::channel(); - let (resume_tx, resume_rx) = mpsc::channel(); - let mut first = true; - let worker = cache - .start_sync( - Duration::from_secs(3600), - move || Ok(shared.clone()), - move |result| { - tx.send(*result.as_ref().unwrap()).unwrap(); - if first { - first = false; - resume_rx.recv_timeout(Duration::from_secs(5)).unwrap(); - } - }, - ) - .unwrap(); - assert_eq!( - rx.recv_timeout(Duration::from_secs(5)).unwrap(), - Some((id, EditStatus::Conflict(ConflictKind::TextChanged))) - ); - cache - .rebase_conflict( - id, - &cache.snapshot().unwrap(), - &cache.remote_snapshot().unwrap(), - 1..2, - ) - .unwrap(); - assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); - resume_tx.send(()).unwrap(); - assert!( - matches!(rx.recv_timeout(Duration::from_secs(5)).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) - ); - worker.stop().unwrap(); - assert!(cache.pending().unwrap().is_empty()); - let server = server.lock().unwrap(); - assert_eq!(server.publications, 1); - assert_eq!(text(&server.durable).2, "aLc"); - assert_eq!(cache.snapshot().unwrap(), server.durable); - } - - #[test] - fn ordinary_read_and_unpublished_write_contention_reuse_the_connection() { - struct Busy { - server: Server, - reads: usize, - writes: usize, - } - impl Remote for Busy { - fn read(&mut self) -> io::Result> { - self.reads += 1; - match self.reads { - 1 => Err(io::ErrorKind::WouldBlock.into()), - 2 => Err(io::ErrorKind::ResourceBusy.into()), - _ => self.server.read(), - } - } - fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { - self.writes += 1; - match self.writes { - 1 | 2 => Err(CommitError { - state: CommitState::NotCommitted, - error: if self.writes == 1 { - io::ErrorKind::WouldBlock - } else { - io::ErrorKind::ResourceBusy - } - .into(), - }), - _ => self.server.publish(edit), - } - } - fn confirm(&mut self, source: &[u8]) -> Result<(), CommitError> { - self.server.confirm(source) - } - } - let dir = tempfile::tempdir().unwrap(); - let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); - let id = cache - .edit_text(&source, sid, oid, 0..0, "L ") - .unwrap() - .unwrap(); - let mut remote = Some(Busy { - server: Server::new(&source), - reads: 0, - writes: 0, - }); - let (tx, rx) = mpsc::channel(); - let worker = cache - .start_sync( - Duration::from_millis(1), - move || Ok(remote.take().expect("Contention caused a reconnect")), - move |result| { - tx.send(result.as_ref().copied().map_err(|error| error.to_string())) - .unwrap(); - }, - ) - .unwrap(); - for _ in 0..4 { - assert!(rx.recv_timeout(Duration::from_secs(5)).unwrap().is_err()); - } - assert!( - matches!(rx.recv_timeout(Duration::from_secs(5)).unwrap().unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) - ); - worker.stop().unwrap(); - assert_eq!(text(&cache.snapshot().unwrap()).2, "L abc"); - assert!(cache.pending().unwrap().is_empty()); - } - #[test] - fn publication_backoff_drains_local_wakes_without_waiting_for_the_idle_poll() { - struct BusyOnce(Server); - impl Remote for BusyOnce { - fn read(&mut self) -> io::Result> { - self.0.read() - } - fn publish(&mut self, edit: &PreparedEdit<'_>) -> Result<(), CommitError> { - let server = &mut self.0; - if server.publications == 0 { - server.publications += 1; - return Err(CommitError { - state: CommitState::NotCommitted, - error: io::ErrorKind::ResourceBusy.into(), - }); - } - server.publish(edit) - } - fn confirm(&mut self, source: &[u8]) -> Result<(), CommitError> { - self.0.confirm(source) - } - } - let dir = tempfile::tempdir().unwrap(); - let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); - let first = cache - .edit_text(&source, sid, oid, 0..0, "L ") - .unwrap() - .unwrap(); - let mut remote = Some(BusyOnce(Server::new(&source))); - let observed = Arc::clone(&cache); - let (tx, rx) = mpsc::channel(); - let mut failed = false; - let worker = cache - .start_sync( - Duration::from_secs(3600), - move || Ok(remote.take().expect("Contention must retain the session")), - move |result| match result { - Err(Error::Remote(error)) if error.state == CommitState::NotCommitted => { - assert!(!failed); - failed = true; - let source = observed.snapshot().unwrap(); - let second = observed - .edit_text(&source, sid, oid, 0..0, "Q ") - .unwrap() - .unwrap(); - tx.send((second, None)).unwrap(); - } - Ok(Some((id, status))) => tx.send((*id, Some(*status))).unwrap(), - Ok(None) => {} - other => panic!("Unexpected worker result: {other:?}"), - }, - ) - .unwrap(); - let (second, status) = rx.recv_timeout(Duration::from_secs(5)).unwrap(); - assert_eq!(status, None); - for expected in [first, second] { - let (id, status) = rx.recv_timeout(Duration::from_secs(5)).unwrap(); - assert_eq!(id, expected); - assert!(matches!(status, Some(EditStatus::Published { .. }))); - } - worker.stop().unwrap(); - assert!(cache.pending().unwrap().is_empty()); - assert_eq!(text(&cache.snapshot().unwrap()).2, "Q L abc"); - } -} - -#[test] -fn offline_insertions_survive_reopen_rebase_and_dependent_text_edits() { - use onestore::Insertion; - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("insert.sqlite"); - let source = onestore::create_section("insert.one", "Original", "Author").unwrap(); - let (sid, original, _) = text(&source); - let store = Store::parse(&source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let (_, page) = doc.pages().unwrap()[0]; - let cache = Replica::create(&path, &source).unwrap(); - let outline = Insertion::outline(page, 72.0, 144.0, "Offline outline", "Offline author") - .unwrap() - .with_formatting(0..7, &[onestore::TextAttribute::Italic(true)]) - .unwrap(); - let first = cache.insert(&source, sid, &outline).unwrap().unwrap(); - let snapshot = cache.snapshot().unwrap(); - let paragraph = Insertion::paragraph( - outline.object(), - None, - "Offline paragraph 🦀", - "Offline author", - ) - .unwrap() - .with_formatting(8..17, &[onestore::TextAttribute::Bold(true)]) - .unwrap(); - let second = cache.insert(&snapshot, sid, ¶graph).unwrap().unwrap(); - let third = cache - .edit_text( - &cache.snapshot().unwrap(), - sid, - paragraph.text_object(), - 0..0, - "Edited ", - ) - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let pending = cache.pending().unwrap(); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(cache.pending().unwrap(), pending); - let remote = onestore::replace_text(&source, sid, original, 0..0, "Remote ").unwrap(); - let mut server = Server::new(&remote); - for id in [first, second, third] { - assert!( - matches!(cache.sync_once(&mut server).unwrap(),Some((observed,EditStatus::Published{..})) if observed==id) - ); - } - assert_eq!(server.publications, 3); - assert!(cache.pending().unwrap().is_empty()); - assert_eq!(cache.snapshot().unwrap(), server.durable); - let store = Store::parse(&server.durable).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let s = &doc.spaces[&sid]; - let v = &s.revisions[&s.contexts[&ExGuid::default()]]; - for (id, wanted) in [ - (original, "Remote Original"), - (outline.text_object(), "Offline outline"), - (paragraph.text_object(), "Edited Offline paragraph 🦀"), - ] { - assert!(matches!(&v.nodes[&id].kind,Kind::RichText{text,..} if text==wanted)); - } - let outline_runs = v.text_runs(outline.text_object()).unwrap(); - assert_eq!(outline_runs[0].text, "Offline"); - assert_eq!(outline_runs[0].format.italic, Some(true)); - let runs = v.text_runs(paragraph.text_object()).unwrap(); - assert_eq!(runs[1].text, "paragraph"); - assert_eq!(runs[1].format.bold, Some(true)); - assert_eq!( - v.nodes[&outline.object()].children.last(), - Some(¶graph.object()) - ); -} - -#[test] -fn uncertain_insertions_reconcile_the_original_revision_without_duplicate_objects() { - use onestore::Insertion; - for fault in [ - Fault::Before, - Fault::UnknownBefore, - Fault::UnknownAfter, - Fault::PanicBefore, - Fault::PanicAfter, - Fault::Committed, - ] { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("uncertain-insert.sqlite"); - let source = onestore::create_section("insert.one", "Original", "Author").unwrap(); - let store = Store::parse(&source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let (sid, page) = doc.pages().unwrap()[0]; - let cache = Replica::create(&path, &source).unwrap(); - let insertion = Insertion::outline(page, 144.0, 144.0, "Uncertain insertion", "Author") - .unwrap() - .with_formatting(0..9, &[onestore::TextAttribute::Bold(true)]) - .unwrap() - .with_formatting(10..19, &[onestore::TextAttribute::Italic(true)]) - .unwrap(); - let id = cache.insert(&source, sid, &insertion).unwrap().unwrap(); - let local = cache.snapshot().unwrap(); - let mut server = Server::new(&source); - server.fault = fault; - let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - cache.sync_once(&mut server) - })); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(server.publications, 1); - if matches!(fault, Fault::UnknownBefore | Fault::PanicBefore) { - let status = cache.status(id).unwrap(); - assert!(matches!( - status, - Some(EditStatus::AwaitingConfirmation { .. }) - )); - for _ in 0..5 { - assert_eq!( - cache.sync_once(&mut server).unwrap(), - status.map(|state| (id, state)) - ); - } - assert_eq!(server.publications, 1); - assert_eq!(server.durable, source); - assert_eq!(cache.snapshot().unwrap(), local); - } else { - if !matches!(fault, Fault::Committed) { - assert!(matches!( - cache.sync_once(&mut server).unwrap(), - Some((_, EditStatus::Published { .. })) - )); - } - assert_eq!( - server.publications, - if matches!(fault, Fault::Before) { 2 } else { 1 } - ); - assert_eq!( - server.confirmations, - usize::from(matches!(fault, Fault::UnknownAfter | Fault::PanicAfter)) - ); - let store = Store::parse(&server.durable).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let s = &doc.spaces[&sid]; - let v = &s.revisions[&s.contexts[&ExGuid::default()]]; - assert_eq!(v.nodes.values().filter(|node|matches!(&node.kind,Kind::RichText{text,..} if text=="Uncertain insertion")).count(),1); - assert!(v.nodes.contains_key(&insertion.object())); - let runs = v.text_runs(insertion.text_object()).unwrap(); - assert_eq!(runs.len(), 3); - assert_eq!(runs[0].text, "Uncertain"); - assert_eq!(runs[0].format.bold, Some(true)); - assert_eq!(runs[1].text, " "); - assert_ne!(runs[1].format.bold, Some(true)); - assert_ne!(runs[1].format.italic, Some(true)); - assert_eq!(runs[2].text, "insertion"); - assert_eq!(runs[2].format.italic, Some(true)); - assert!(cache.pending().unwrap().is_empty()); - } - } -} - -#[test] -fn cross_run_text_rebases_preserve_remote_styles_and_survive_lost_replies() { - use onestore::TextAttribute as A; - for fault in [Fault::None, Fault::UnknownAfter, Fault::PanicAfter] { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("cross-run.sqlite"); - let plain = onestore::create_section("cross.one", "ab🦀cde\u{301}fg", "Author").unwrap(); - let (sid, oid, _) = text(&plain); - let bold = PreparedEdit::format(&plain, sid, oid, 1..4, &[A::Bold(true)]).unwrap(); - let source = PreparedEdit::format(bold.as_bytes(), sid, oid, 4..7, &[A::Italic(true)]) - .unwrap() - .as_bytes() - .to_vec(); - let cache = Replica::create(&path, &source).unwrap(); - let first = cache - .edit_text(&source, sid, oid, 2..7, "日本語") - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let second = cache - .edit_text(&local, sid, oid, 3..4, "🐈") - .unwrap() - .unwrap(); - let pending = cache.pending().unwrap(); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.pending().unwrap(), pending); - let prefix = PreparedEdit::text(&source, sid, oid, 0..0, "Prefix ").unwrap(); - let remote = - PreparedEdit::format(prefix.as_bytes(), sid, oid, 9..11, &[A::Underline(true)]) - .unwrap(); - let mut server = Server::new(remote.as_bytes()); - server.fault = fault; - let attempted = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - cache.sync_once(&mut server) - })); - if matches!(fault, Fault::None) { - assert!( - matches!(attempted.unwrap().unwrap(), Some((id, EditStatus::Published { .. })) if id == first) - ); - } else { - assert!(matches!( - cache.status(first).unwrap(), - Some(EditStatus::AwaitingConfirmation { .. }) - )); - } - drop(cache); - let cache = Replica::open(&path).unwrap(); - while let Some((_, status)) = cache.sync_once(&mut server).unwrap() { - assert!(matches!(status, EditStatus::Published { .. })); - } - assert_eq!(server.publications, 2); - assert!(matches!( - cache.status(second).unwrap(), - Some(EditStatus::Published { .. }) - )); - assert_eq!(text(&server.durable).2, "Prefix ab日🐈語\u{301}fg"); - assert_eq!(cache.snapshot().unwrap(), server.durable); - let store = Store::parse(&server.durable).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let space = &document.spaces[&sid]; - let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; - let runs = revision.text_runs(oid).unwrap(); - let replacement = runs.iter().find(|run| run.text == "日🐈語").unwrap(); - assert_eq!(replacement.format.bold, Some(true)); - assert_eq!(replacement.format.underline, Some(true)); - assert_ne!(replacement.format.italic, Some(true)); - let suffix = runs.last().unwrap(); - assert_eq!(suffix.text, "\u{301}fg"); - assert_ne!(suffix.format.bold, Some(true)); - assert_ne!(suffix.format.underline, Some(true)); - assert_ne!(suffix.format.italic, Some(true)); - } -} - -#[test] -fn offline_formatting_rebases_text_and_merges_independent_attributes() { - use onestore::TextAttribute as A; - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("format.sqlite"); - let source = onestore::create_section("format.one", "ab🦀cd", "Author").unwrap(); - let (sid, id, _) = text(&source); - let cache = Replica::create(&path, &source).unwrap(); - let local = cache - .format(&source, sid, id, 2..4, &[A::Bold(true)]) - .unwrap() - .unwrap(); - let pending = cache.pending().unwrap(); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.pending().unwrap(), pending); - let moved = PreparedEdit::text(&source, sid, id, 0..0, "Prefix ").unwrap(); - let styled = - PreparedEdit::format(moved.as_bytes(), sid, id, 9..11, &[A::Italic(true)]).unwrap(); - let mut server = Server::new(styled.as_bytes()); - assert!( - matches!(cache.sync_once(&mut server).unwrap(),Some((observed,EditStatus::Published{..}))if observed==local) - ); - let store = Store::parse(&server.durable).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let space = &doc.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let runs = view.text_runs(id).unwrap(); - let selected = runs.iter().find(|r| r.text == "🦀").unwrap(); - assert_eq!(selected.format.bold, Some(true)); - assert_eq!(selected.format.italic, Some(true)); - assert_eq!( - runs.iter().map(|r| r.text).collect::(), - "Prefix ab🦀cd" - ); -} - -#[test] -fn competing_font_changes_remain_preserved_conflicts() { - use onestore::TextAttribute as A; - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("conflict.sqlite"); - let source = onestore::create_section("format.one", "abcdef", "Author").unwrap(); - let (sid, id, _) = text(&source); - let cache = Replica::create(&path, &source).unwrap(); - let local_id = cache - .format(&source, sid, id, 1..5, &[A::FontSize(14.0)]) - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let pending = cache.pending().unwrap(); - let remote = PreparedEdit::format(&source, sid, id, 2..4, &[A::FontSize(18.0)]).unwrap(); - let mut server = Server::new(remote.as_bytes()); - for _ in 0..3 { - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some(( - local_id, - EditStatus::Conflict(ConflictKind::FormattingChanged) - )) - ); - } - assert_eq!(server.publications, 0); - assert_eq!(server.confirmations, 0); - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(cache.pending().unwrap(), pending); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!( - cache.status(local_id).unwrap(), - Some(EditStatus::Conflict(ConflictKind::FormattingChanged)) - ); - assert_eq!(cache.snapshot().unwrap(), local); -} - -#[test] -fn independently_satisfied_formatting_requires_confirmation_before_a_receipt() { - use onestore::TextAttribute as A; - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("satisfied.sqlite"); - let source = onestore::create_section("format.one", "abcdef", "Author").unwrap(); - let (sid, id, _) = text(&source); - let cache = Replica::create(&path, &source).unwrap(); - let local_id = cache - .format(&source, sid, id, 1..5, &[A::Bold(true)]) - .unwrap() - .unwrap(); - let remote = PreparedEdit::format(&source, sid, id, 1..5, &[A::Bold(true)]).unwrap(); - let mut server = Server::new(remote.as_bytes()); - server.durable = source.clone(); - server.fault = Fault::Confirm; - assert!( - matches!(cache.sync_once(&mut server),Err(Error::Remote(e))if e.state==CommitState::Unknown) - ); - assert_eq!(cache.status(local_id).unwrap(), Some(EditStatus::Pending)); - assert_eq!(server.publications, 0); - assert_eq!(server.durable, source); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert!( - matches!(cache.sync_once(&mut server).unwrap(),Some((id,EditStatus::Published{..}))if id==local_id) - ); - assert_eq!(server.publications, 0); - assert_eq!(server.confirmations, 2); - assert_ne!(server.durable, source); - assert!(cache.pending().unwrap().is_empty()); -} - -#[test] -fn retired_format_attempts_require_the_complete_durable_effect_without_replay() { - use onestore::TextAttribute as A; - for (complete, fault) in [ - (true, Fault::None), - (true, Fault::Confirm), - (true, Fault::ConfirmCommitted), - (false, Fault::None), - ] { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("retired-format.sqlite"); - let source = onestore::create_section("format.one", "abcdef", "Author").unwrap(); - let (sid, object, _) = text(&source); - let cache = Replica::create(&path, &source).unwrap(); - let id = cache - .format( - &source, - sid, - object, - 1..5, - &[A::Bold(true), A::FontSize(18.0)], - ) - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let mut server = Server::new(&source); - server.fault = Fault::UnknownAfter; - assert!(cache.sync_once(&mut server).is_err()); - let attempted = cache.status(id).unwrap(); - let Some(EditStatus::AwaitingConfirmation { revision: retired }) = attempted else { - panic!() - }; - drop(cache); - let prefix = PreparedEdit::text(&source, sid, object, 0..0, "prefix ").unwrap(); - let mut attributes = vec![A::Bold(true), A::Italic(true)]; - if complete { - attributes.push(A::FontSize(18.0)); - } - server.visible = PreparedEdit::format(prefix.as_bytes(), sid, object, 8..12, &attributes) - .unwrap() - .as_bytes() - .to_vec(); - let store = Store::parse(&server.visible).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let current = index.spaces[&sid].labels[&(ExGuid::default(), 1)]; - assert!(!index.spaces[&sid].revisions.contains_key(&retired)); - let cache = Replica::open(&path).unwrap(); - server.fault = fault; - let result = cache.sync_once(&mut server); - if !complete { - assert_eq!(result.unwrap(), attempted.map(|s| (id, s))); - assert_eq!(server.confirmations, 0); - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(server.durable, source); - } else { - if matches!(fault, Fault::Confirm) { - assert!(matches!(result, Err(Error::Remote(e)) if e.state == CommitState::Unknown)); - assert_eq!(cache.status(id).unwrap(), attempted); - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(server.durable, source); - let complete = server.visible.clone(); - server.visible = - PreparedEdit::format(&complete, sid, object, 8..12, &[A::Bold(false)]) - .unwrap() - .as_bytes() - .to_vec(); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - attempted.map(|s| (id, s)) - ); - assert_eq!(server.confirmations, 1); - assert_eq!(cache.snapshot().unwrap(), local); - server.visible = complete; - cache.sync_once(&mut server).unwrap(); - } else if matches!(fault, Fault::ConfirmCommitted) { - assert!( - matches!(result, Err(Error::Remote(e)) if e.state == CommitState::Committed) - ); - } else { - assert_eq!( - result.unwrap(), - Some((id, EditStatus::Published { revision: current })) - ); - } - assert_ne!(current, retired); - assert_eq!( - cache.status(id).unwrap(), - Some(EditStatus::Published { revision: current }) - ); - assert!(cache.pending().unwrap().is_empty()); - assert_eq!(text(&server.durable).2, "prefix abcdef"); - assert_ne!(server.durable, source); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!( - cache.status(id).unwrap(), - Some(EditStatus::Published { revision: current }) - ); - } - assert_eq!(server.publications, 1); - } -} - -#[test] -fn retired_text_with_equal_plaintext_but_a_different_surviving_run_remains_uncertain() { - use onestore::TextAttribute; - - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("repeated.sqlite"); - let plain = onestore::create_section("repeated.one", "aa", "Fixture").unwrap(); - let (space, object, _) = text(&plain); - let source = PreparedEdit::format(&plain, space, object, 0..1, &[TextAttribute::Bold(true)]) - .unwrap() - .as_bytes() - .to_vec(); - let cache = Replica::create(&path, &source).unwrap(); - let id = cache - .edit_text(&source, space, object, 0..1, "") - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let mut server = Server::new(&source); - server.fault = Fault::UnknownAfter; - assert!(cache.sync_once(&mut server).is_err()); - let attempt = cache.status(id).unwrap(); - server.visible = onestore::replace_text(&source, space, object, 1..2, "").unwrap(); - assert_eq!(text(&server.visible).2, text(&local).2); - let bold = |bytes: &[u8]| { - let store = Store::parse(bytes).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let space = &document.spaces[&space]; - space.revisions[&space.contexts[&ExGuid::default()]] - .text_runs(object) - .unwrap() - .into_iter() - .find(|run| !run.text.is_empty()) - .unwrap() - .format - .bold - }; - assert_eq!(bold(&server.visible), Some(true)); - assert_ne!(bold(&server.visible), bold(&local)); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - attempt.map(|state| (id, state)) - ); - assert_eq!(server.publications, 1); - assert_eq!(server.confirmations, 0); - assert_eq!(cache.snapshot().unwrap(), local); -} - -#[test] -fn uncertain_formatting_keeps_the_original_attempt_and_never_replays() { - use onestore::TextAttribute as A; - for fault in [Fault::UnknownBefore, Fault::UnknownAfter] { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("unknown-format.sqlite"); - let source = onestore::create_section("format.one", "abcdef", "Author").unwrap(); - let (sid, id, _) = text(&source); - let cache = Replica::create(&path, &source).unwrap(); - let local_id = cache - .format(&source, sid, id, 1..5, &[A::Bold(true)]) - .unwrap() - .unwrap(); - let mut server = Server::new(&source); - server.fault = fault; - assert!( - matches!(cache.sync_once(&mut server),Err(Error::Remote(e))if e.state==CommitState::Unknown) - ); - let status = cache.status(local_id).unwrap(); - drop(cache); - let cache = Replica::open(&path).unwrap(); - if matches!(fault, Fault::UnknownBefore) { - for _ in 0..4 { - assert_eq!( - cache.sync_once(&mut server).unwrap(), - status.map(|s| (local_id, s)) - ); - } - assert_eq!(server.confirmations, 0); - } else { - assert!(matches!( - cache.sync_once(&mut server).unwrap(), - Some((_, EditStatus::Published { .. })) - )); - assert_eq!(server.confirmations, 1); - } - assert_eq!(server.publications, 1); - } -} - -#[test] -fn reviewed_format_conflicts_preserve_dependent_edits_and_recheck_later_remote_changes() { - use onestore::TextAttribute as A; - for changed_again in [false, true] { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("review-format.sqlite"); - let source = onestore::create_section("format.one", "abcdef", "Author").unwrap(); - let (sid, id, _) = text(&source); - let cache = Replica::create(&path, &source).unwrap(); - let first = cache - .format(&source, sid, id, 1..5, &[A::FontSize(14.0)]) - .unwrap() - .unwrap(); - let second = cache - .edit_text(&cache.snapshot().unwrap(), sid, id, 2..2, "X") - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let pending = cache.pending().unwrap(); - let remote = PreparedEdit::format(&source, sid, id, 1..5, &[A::FontSize(18.0)]).unwrap(); - let mut server = Server::new(remote.as_bytes()); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((first, EditStatus::Conflict(ConflictKind::FormattingChanged))) - ); - assert!( - matches!(cache.rebase_conflict(first,&source,remote.as_bytes(),1..5),Err(Error::Io(e))if e.kind()==io::ErrorKind::ResourceBusy) - ); - cache - .rebase_conflict(first, &local, remote.as_bytes(), 1..5) - .unwrap(); - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(cache.pending().unwrap()[1], pending[1]); - drop(cache); - let cache = Replica::open(&path).unwrap(); - if changed_again { - let newer = - PreparedEdit::format(&server.visible, sid, id, 1..5, &[A::FontSize(20.0)]).unwrap(); - server = Server::new(newer.as_bytes()); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((first, EditStatus::Conflict(ConflictKind::FormattingChanged))) - ); - assert_eq!(server.publications, 0); - assert_eq!(cache.snapshot().unwrap(), local); - } else { - for expected in [first, second] { - assert!( - matches!(cache.sync_once(&mut server).unwrap(),Some((id,EditStatus::Published{..}))if id==expected) - ); - } - assert_eq!(text(&server.durable).2, "abXcdef"); - assert!(cache.pending().unwrap().is_empty()); - } - } -} - -#[test] -fn superscript_reconciliation_checks_the_implicit_subscript_change() { - use onestore::TextAttribute as A; - let directory = tempfile::tempdir().unwrap(); - let source = onestore::create_section("script.one", "abc", "Author").unwrap(); - let (sid, id, _) = text(&source); - let cache = Replica::create(directory.path().join("script.sqlite"), &source).unwrap(); - let local = cache - .format(&source, sid, id, 0..3, &[A::Superscript(true)]) - .unwrap() - .unwrap(); - let remote = PreparedEdit::format(&source, sid, id, 0..3, &[A::Subscript(true)]).unwrap(); - let mut server = Server::new(remote.as_bytes()); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((local, EditStatus::Conflict(ConflictKind::FormattingChanged))) - ); - assert_eq!(server.publications, 0); -} - -#[test] -fn seeded_formatting_reconciliation_matches_a_character_model() { - use onestore::TextAttribute as A; - #[derive(Clone, Debug, PartialEq)] - struct Style { - size: f32, - color: u32, - bold: bool, - italic: bool, - } - let mut conflicts = 0; - let mut published = 0; - let mut satisfied_parts = 0; - for seed in 1_u64..=128 { - let mut random = seed; - let mut next = || { - random = random - .wrapping_mul(6364136223846793005) - .wrapping_add(1442695040888963407); - random >> 32 - }; - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("model.sqlite"); - let mut source = - onestore::create_section("model.one", "abcdefghijklmnopqrstuvwxyz012345", "Author") - .unwrap(); - let (sid, object, original_text) = text(&source); - let mut baseline = vec![ - Style { - size: 11.0, - color: 0xff000000, - bold: false, - italic: false - }; - 32 - ]; - for _ in 0..6 { - let start = next() as usize % 32; - let end = start + 1 + next() as usize % (32 - start); - let size = [12.0, 14.0, 18.0][next() as usize % 3]; - let color: u32 = [0xabcdef, 0x987654, 0xff000000][next() as usize % 3]; - let bold = next() % 2 == 0; - source = PreparedEdit::format( - &source, - sid, - object, - start as u32..end as u32, - &[ - A::FontSize(size), - A::Color((color != 0xff000000).then(|| { - let b = color.to_le_bytes(); - [b[0], b[1], b[2]] - })), - A::Bold(bold), - ], - ) - .unwrap() - .as_bytes() - .to_vec(); - for style in &mut baseline[start..end] { - style.size = size; - style.color = color; - style.bold = bold; - } - } - let start = next() as usize % 24; - let end = start + 2 + next() as usize % (31 - start); - let attribute = match seed % 3 { - 0 => A::FontSize(21.0), - 1 => A::Color(Some([0x12, 0x34, 0x56])), - _ => A::Bold(!baseline[start].bold), - }; - let cache = Replica::create(&path, &source).unwrap(); - let id = cache - .format( - &source, - sid, - object, - start as u32..end as u32, - std::slice::from_ref(&attribute), - ) - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let mut remote = source.clone(); - let mut expected = baseline.clone(); - for step in 0..8 { - let left = next() as usize % 32; - let right = left + 1 + next() as usize % (32 - left); - let update = if step % 3 == 0 { - attribute.clone() - } else { - match next() % 4 { - 0 => A::FontSize([11.0, 14.0, 18.0, 21.0][next() as usize % 4]), - 1 => A::Color(Some([0x99, 0x88, 0x77])), - 2 => A::Bold(next() % 2 == 0), - _ => A::Italic(true), - } - }; - remote = PreparedEdit::format( - &remote, - sid, - object, - left as u32..right as u32, - std::slice::from_ref(&update), - ) - .unwrap() - .as_bytes() - .to_vec(); - for style in &mut expected[left..right] { - match update { - A::FontSize(value) => style.size = value, - A::Color(Some([r, g, b])) => style.color = u32::from_le_bytes([r, g, b, 0]), - A::Bold(value) => style.bold = value, - A::Italic(value) => style.italic = value, - _ => unreachable!(), - } - } - } - let conflict = (start..end).any(|i| match attribute { - A::FontSize(value) => expected[i].size != baseline[i].size && expected[i].size != value, - A::Color(_) => expected[i].color != baseline[i].color && expected[i].color != 0x563412, - A::Bold(value) => expected[i].bold != baseline[i].bold && expected[i].bold != value, - _ => unreachable!(), - }); - drop(cache); - let cache = Replica::open(&path).unwrap(); - let mut server = Server::new(&remote); - let result = cache.sync_once(&mut server).unwrap().unwrap(); - assert_eq!(result.0, id, "seed {seed}"); - if conflict { - conflicts += 1; - assert_eq!( - result.1, - EditStatus::Conflict(ConflictKind::FormattingChanged), - "seed {seed}" - ); - assert_eq!(server.publications, 0, "seed {seed}"); - assert_eq!(cache.snapshot().unwrap(), local, "seed {seed}"); - continue; - } - published += 1; - assert!( - matches!(result.1, EditStatus::Published { .. }), - "seed {seed}: {result:?}" - ); - for style in &mut expected[start..end] { - match attribute { - A::FontSize(value) => { - satisfied_parts += usize::from(style.size == value); - style.size = value; - } - A::Color(_) => { - satisfied_parts += usize::from(style.color == 0x563412); - style.color = 0x563412; - } - A::Bold(value) => { - satisfied_parts += usize::from(style.bold == value); - style.bold = value; - } - _ => unreachable!(), - } - } - assert_eq!(text(&server.durable).2, original_text, "seed {seed}"); - let store = Store::parse(&server.durable).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let space = &doc.spaces[&sid]; - let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; - let actual: Vec<_> = revision - .text_runs(object) - .unwrap() - .iter() - .flat_map(|run| { - std::iter::repeat_n( - Style { - size: run.format.font_size.unwrap(), - color: run.format.color.unwrap_or(0xff000000), - bold: run.format.bold.unwrap_or(false), - italic: run.format.italic.unwrap_or(false), - }, - run.text.chars().count(), - ) - }) - .collect(); - assert_eq!(actual, expected, "seed {seed}"); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.status(id).unwrap(), Some(result.1)); - let snapshot = cache.snapshot().unwrap(); - assert_eq!(snapshot.len(), server.durable.len(), "seed {seed}"); - assert!( - snapshot - .iter() - .zip(&server.durable) - .enumerate() - .all(|(offset, (a, b))| a == b || (212..252).contains(&offset)), - "seed {seed}: only confirmation version metadata may change" - ); - assert_eq!(cache.sync_once(&mut server).unwrap(), None); - assert!( - cache.snapshot().unwrap() == server.durable, - "seed {seed}: refreshed snapshot" - ); - } - assert!( - conflicts >= 16 && published >= 32 && satisfied_parts >= 128, - "conflicts={conflicts}, published={published}, already desired characters={satisfied_parts}" - ); - println!( - "128 seeds: {conflicts} conflicts, {published} publications, {satisfied_parts} already desired characters" - ); -} - -#[test] -fn inserted_empty_text_retains_formatting_and_dependent_text_across_sync() { - use onestore::{Insertion, TextAttribute as A}; - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("empty.sqlite"); - let source = onestore::create_section("empty.one", "Original", "Author").unwrap(); - let store = Store::parse(&source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let (sid, page) = doc.pages().unwrap()[0]; - let insertion = Insertion::outline(page, 144.0, 144.0, "", "Author").unwrap(); - let cache = Replica::create(&path, &source).unwrap(); - let first = cache.insert(&source, sid, &insertion).unwrap().unwrap(); - let second = cache - .format( - &cache.snapshot().unwrap(), - sid, - insertion.text_object(), - 0..0, - &[A::Bold(true), A::FontSize(18.0)], - ) - .unwrap() - .unwrap(); - let third = cache - .edit_text( - &cache.snapshot().unwrap(), - sid, - insertion.text_object(), - 0..0, - "Typed 🦀", - ) - .unwrap() - .unwrap(); - let pending = cache.pending().unwrap(); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.pending().unwrap(), pending); - let mut server = Server::new(&source); - for id in [first, second, third] { - let result = cache.sync_once(&mut server).unwrap(); - assert!( - matches!(result,Some((actual,EditStatus::Published{..}))if actual==id), - "{result:?}" - ); - } - let store = Store::parse(&server.durable).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let space = &doc.spaces[&sid]; - let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; - let runs = revision.text_runs(insertion.text_object()).unwrap(); - assert_eq!(runs.iter().map(|r| r.text).collect::(), "Typed 🦀"); - assert!( - runs.iter() - .all(|r| r.format.bold == Some(true) && r.format.font_size == Some(18.0)) - ); - assert_eq!(cache.snapshot().unwrap(), server.durable); -} - -#[test] -fn every_visual_attribute_rebases_with_an_independent_remote_attribute() { - use onestore::TextAttribute as A; - use serde_json::json; - let mut cases = Vec::new(); - for value in [false, true] { - cases.extend([ - (A::Bold(!value), A::Bold(value), "bold", json!(value)), - (A::Italic(!value), A::Italic(value), "italic", json!(value)), - ( - A::Underline(!value), - A::Underline(value), - "underline", - json!(value), - ), - (A::Strike(!value), A::Strike(value), "strike", json!(value)), - ( - A::Superscript(!value), - A::Superscript(value), - "superscript", - json!(value), - ), - ( - A::Subscript(!value), - A::Subscript(value), - "subscript", - json!(value), - ), - ]); - } - cases.extend([ - ( - A::Font("Georgia".into()), - A::Font("Arial".into()), - "font", - json!("Arial"), - ), - ( - A::FontSize(11.0), - A::FontSize(18.0), - "font_size", - json!(18.0), - ), - ( - A::Color(None), - A::Color(Some([1, 2, 3])), - "color", - json!(0x030201), - ), - ( - A::Color(Some([1, 2, 3])), - A::Color(None), - "color", - json!(0xff000000_u32), - ), - ( - A::Highlight(None), - A::Highlight(Some([4, 5, 6])), - "highlight", - json!(0x060504), - ), - ( - A::Highlight(Some([4, 5, 6])), - A::Highlight(None), - "highlight", - json!(0xff000000_u32), - ), - ]); - for (baseline, desired, field, value) in cases { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("attribute.sqlite"); - let source = onestore::create_section("attribute.one", "abc", "Author").unwrap(); - let (sid, object, _) = text(&source); - let source = PreparedEdit::format(&source, sid, object, 0..3, &[baseline]) - .unwrap() - .as_bytes() - .to_vec(); - let cache = Replica::create(&path, &source).unwrap(); - let id = cache - .format(&source, sid, object, 0..3, &[desired]) - .unwrap() - .unwrap(); - let moved = PreparedEdit::text(&source, sid, object, 0..0, "Z").unwrap(); - let (other, other_field, other_value) = if field == "font_size" { - (A::Italic(true), "italic", json!(true)) - } else { - (A::FontSize(22.0), "font_size", json!(22.0)) - }; - let remote = PreparedEdit::format(moved.as_bytes(), sid, object, 1..4, &[other]).unwrap(); - let mut server = Server::new(remote.as_bytes()); - drop(cache); - let cache = Replica::open(&path).unwrap(); - let result = cache.sync_once(&mut server).unwrap(); - assert!( - matches!(result,Some((observed,EditStatus::Published{..}))if observed==id), - "{field}={value}: {result:?}" - ); - assert_eq!(text(&server.durable).2, "Zabc"); - let store = Store::parse(&server.durable).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let space = &doc.spaces[&sid]; - let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; - let mut position = 0; - for run in revision.text_runs(object).unwrap() { - let format = serde_json::to_value(run.format).unwrap(); - for _ in run.text.chars() { - if position > 0 { - assert_eq!(format[field], value, "{field}, character {position}"); - assert_eq!( - format[other_field], other_value, - "{field}, character {position}" - ); - } - position += 1; - } - } - assert_eq!(position, 4); - } -} - -#[test] -fn reviewed_insertion_placements_preserve_identity_and_dependent_operations() { - use onestore::{Insertion, TextAttribute as A}; - use onestore_offline::Operation; - for outline_case in [false, true] { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("placement.sqlite"); - let base = onestore::create_section("placement.one", "Original", "Author").unwrap(); - let (sid, _, _) = text(&base); - let store = Store::parse(&base).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let (_, page) = doc.pages().unwrap()[0]; - let space = &doc.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let parent = *view.nodes[&page] - .children - .iter() - .find(|id| matches!(view.nodes[id].kind, Kind::Outline { .. })) - .unwrap(); - let anchor = Insertion::paragraph(parent, None, "Temporary anchor", "Author").unwrap(); - let source = PreparedEdit::insert(&base, sid, &anchor) - .unwrap() - .as_bytes() - .to_vec(); - let insertion = if outline_case { - Insertion::outline(page, 144.0, 144.0, "Offline", "Author").unwrap() - } else { - Insertion::paragraph(parent, Some(anchor.object()), "Offline", "Author").unwrap() - } - .with_formatting(0..7, &[A::Italic(true)]) - .unwrap(); - let cache = Replica::create(&path, &source).unwrap(); - let first = cache.insert(&source, sid, &insertion).unwrap().unwrap(); - let second = cache - .format( - &cache.snapshot().unwrap(), - sid, - insertion.text_object(), - 0..7, - &[A::Bold(true)], - ) - .unwrap() - .unwrap(); - let third = cache - .edit_text( - &cache.snapshot().unwrap(), - sid, - insertion.text_object(), - 7..7, - " 🦀", - ) - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let pending = cache.pending().unwrap(); - let mut server = Server::new(&base); - if outline_case { - drop(cache); - let db = rusqlite::Connection::open(&path).unwrap(); - db.execute( - "INSERT INTO conflicts VALUES (?1,2)", - [i64::try_from(first).unwrap()], - ) - .unwrap(); - db.execute("UPDATE replica SET base=?1", [&base]).unwrap(); - drop(db); - } else { - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((first, EditStatus::Conflict(ConflictKind::UnsupportedEdit))) - ); - drop(cache); - } - let cache = Replica::open(&path).unwrap(); - if outline_case { - assert!( - cache - .rebase_paragraph_conflict(first, &local, &base, parent, None) - .is_err() - ); - assert!( - cache - .rebase_outline_conflict(first, &local, &base, page, f32::NAN, 288.0) - .is_err() - ); - assert!( - matches!(cache.rebase_outline_conflict(first,&source,&base,page,288.0,360.0),Err(Error::Io(e))if e.kind()==io::ErrorKind::ResourceBusy) - ); - cache - .rebase_outline_conflict(first, &local, &base, page, 288.0, 360.0) - .unwrap(); - } else { - assert!( - cache - .rebase_outline_conflict(first, &local, &base, page, 288.0, 360.0) - .is_err() - ); - assert!( - cache - .rebase_paragraph_conflict(first, &local, &base, parent, Some(anchor.object())) - .is_err() - ); - assert!( - matches!(cache.rebase_paragraph_conflict(first,&source,&base,parent,None),Err(Error::Io(e))if e.kind()==io::ErrorKind::ResourceBusy) - ); - cache - .rebase_paragraph_conflict(first, &local, &base, parent, None) - .unwrap(); - } - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(cache.pending().unwrap()[1..], pending[1..]); - let Operation::Insert(rebased) = cache.pending().unwrap().remove(0).operation else { - panic!() - }; - assert_eq!(rebased.object(), insertion.object()); - assert_eq!(rebased.text_object(), insertion.text_object()); - assert_eq!( - serde_json::to_value(&rebased).unwrap()["formats"], - serde_json::to_value(&insertion).unwrap()["formats"] - ); - assert_eq!(cache.status(first).unwrap(), Some(EditStatus::Pending)); - drop(cache); - let cache = Replica::open(&path).unwrap(); - for id in [first, second, third] { - let result = cache.sync_once(&mut server).unwrap(); - assert!( - matches!(result,Some((actual,EditStatus::Published{..}))if actual==id), - "{result:?}" - ); - } - let store = Store::parse(&server.durable).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let space = &doc.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let runs = view.text_runs(insertion.text_object()).unwrap(); - assert_eq!( - runs.iter().map(|r| r.text).collect::(), - "Offline 🦀" - ); - assert!(runs.iter().all(|r| r.format.bold == Some(true))); - assert!(runs.iter().all(|r| r.format.italic == Some(true))); - if outline_case { - assert_eq!( - ( - view.nodes[&insertion.object()].layout.x, - view.nodes[&insertion.object()].layout.y - ), - (Some(288.0), Some(360.0)) - ); - } else { - assert_eq!( - view.nodes[&parent].children.last(), - Some(&insertion.object()) - ); - } - assert!(!view.nodes.contains_key(&anchor.object())); - assert_eq!(server.publications, 3); - assert!(cache.pending().unwrap().is_empty()); - assert_eq!(cache.snapshot().unwrap(), server.durable); - } -} - -#[test] -fn placement_reviews_cannot_replace_pending_or_uncertain_attempts() { - use onestore::Insertion; - for outline_case in [false, true] { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("attempt-placement.sqlite"); - let source = onestore::create_section("placement.one", "Original", "Author").unwrap(); - let store = Store::parse(&source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let doc = Document::parse(&index).unwrap(); - let (sid, page) = doc.pages().unwrap()[0]; - let space = &doc.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let parent = *view.nodes[&page] - .children - .iter() - .find(|id| matches!(view.nodes[id].kind, Kind::Outline { .. })) - .unwrap(); - let insertion = if outline_case { - Insertion::outline(page, 144.0, 144.0, "Offline", "Author").unwrap() - } else { - Insertion::paragraph(parent, None, "Offline", "Author").unwrap() - }; - let cache = Replica::create(&path, &source).unwrap(); - let id = cache.insert(&source, sid, &insertion).unwrap().unwrap(); - let local = cache.snapshot().unwrap(); - let pending = cache.pending().unwrap(); - let mut server = Server::new(&source); - for attempted in [false, true] { - if attempted { - server.fault = Fault::UnknownBefore; - assert!( - matches!(cache.sync_once(&mut server),Err(Error::Remote(e))if e.state==CommitState::Unknown) - ); - } - let state = cache.status(id).unwrap(); - let result = if outline_case { - cache.rebase_outline_conflict(id, &local, &source, page, 288.0, 360.0) - } else { - cache.rebase_paragraph_conflict(id, &local, &source, parent, None) - }; - assert!(matches!(result,Err(Error::Io(e))if e.kind()==io::ErrorKind::InvalidInput)); - assert_eq!(cache.pending().unwrap(), pending); - assert_eq!(cache.status(id).unwrap(), state); - assert_eq!(cache.snapshot().unwrap(), local); - } - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.pending().unwrap(), pending); - assert!(matches!( - cache.sync_once(&mut server).unwrap(), - Some((_, EditStatus::AwaitingConfirmation { .. })) - )); - assert_eq!(server.publications, 1); - } -} diff --git a/crates/onestore-offline/tests/sync/outline.rs b/crates/onestore-offline/tests/sync/outline.rs deleted file mode 100644 index 112402c85421ca57f5fe842fd3baf0e358b2df0d..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/tests/sync/outline.rs +++ /dev/null @@ -1,719 +0,0 @@ -use super::*; -use onestore::OutlineEdit as Change; - -pub(super) fn fixture() -> (Vec, ExGuid, ExGuid, ExGuid, ExGuid) { - let source = onestore::create_section("layout.one", "Original 🦀 é", "Author").unwrap(); - let (sid, text, _) = text(&source); - let store = Store::parse(&source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let space = &document.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let paragraph = *view - .nodes - .iter() - .find(|(_, node)| node.content == [text]) - .unwrap() - .0; - let outline = *view - .nodes - .iter() - .find(|(_, node)| node.children.contains(¶graph)) - .unwrap() - .0; - (source, sid, outline, paragraph, text) -} - -pub(super) fn node(source: &[u8], sid: ExGuid, object: ExGuid) -> serde_json::Value { - let store = Store::parse(source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - index.validate_current().unwrap(); - let document = Document::parse(&index).unwrap(); - let space = &document.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - serde_json::to_value(&view.nodes[&object]).unwrap() -} - -#[test] -fn layout_and_dependent_text_survive_reopen_and_independent_remote_formatting() { - let (source, sid, outline, paragraph, text_id) = fixture(); - for (object, change) in [ - (outline, Change::Position { x: 180.0, y: 216.0 }), - ( - outline, - Change::Width { - points: 144.0, - user_set: true, - }, - ), - (paragraph, Change::Collapsed(true)), - ] { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("cache.sqlite"); - let cache = Replica::create(&path, &source).unwrap(); - let id = cache - .outline(&source, sid, object, change) - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let expected = node(&local, sid, object); - let dependent = cache - .edit_text(&local, sid, text_id, 0..0, "Local ") - .unwrap() - .unwrap(); - let pending = cache.pending().unwrap(); - let local = cache.snapshot().unwrap(); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.pending().unwrap(), pending); - assert!(cache.snapshot().unwrap() == local); - let remote = PreparedEdit::format( - &source, - sid, - text_id, - 0..8, - &[onestore::TextAttribute::Bold(true)], - ) - .unwrap(); - let mut server = Server::new(remote.as_bytes()); - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n==id) - ); - let actual = node(&server.durable, sid, object); - assert_eq!(actual["layout"], expected["layout"]); - assert_eq!(actual["kind"], expected["kind"]); - assert_eq!(cache.pending().unwrap().len(), 1); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n==dependent) - ); - assert_eq!(text(&server.durable).2, "Local Original 🦀 é"); - assert!(cache.pending().unwrap().is_empty()); - let store = Store::parse(&server.durable).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let space = &document.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - assert!(view.text_runs(text_id).unwrap()[0].format.bold.unwrap()); - assert_eq!(server.publications, 2); - drop(cache); - let cache = Replica::open(&path).unwrap(); - for id in [id, dependent] { - assert!(matches!( - cache.status(id).unwrap(), - Some(EditStatus::Published { .. }) - )); - } - } -} - -#[test] -fn rebased_outline_movement_uses_remote_title_text_and_confirms_after_reopen() { - let (source, sid, outline, _, text_id) = fixture(); - let store = Store::parse(&source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let (_, page) = document.pages().unwrap()[0]; - let space = &document.spaces[&sid]; - let metadata = space.revisions[&space.contexts[&ExGuid::default()]].roots[&2]; - let second = onestore::Insertion::outline(page, 144.0, 36.0, "Second", "Author").unwrap(); - let source = PreparedEdit::insert(&source, sid, &second) - .unwrap() - .as_bytes() - .to_vec(); - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("titles.sqlite"); - let cache = Replica::create(&path, &source).unwrap(); - let movement = cache - .outline( - &source, - sid, - outline, - Change::Position { x: 216.0, y: 72.0 }, - ) - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - assert_eq!(node(&local, sid, metadata)["kind"]["title"], "Second"); - let dependent = cache - .edit_text(&local, sid, text_id, 0..0, "Local ") - .unwrap() - .unwrap(); - drop(cache); - let remote = - PreparedEdit::text(&source, sid, second.text_object(), 0..6, "Remote second 🐈").unwrap(); - let mut server = Server::new(remote.as_bytes()); - server.fault = Fault::UnknownAfter; - let cache = Replica::open(&path).unwrap(); - assert!(cache.sync_once(&mut server).is_err()); - assert!(matches!( - cache.status(movement).unwrap(), - Some(EditStatus::AwaitingConfirmation { .. }) - )); - assert_eq!( - node(&server.durable, sid, metadata)["kind"]["title"], - "Original 🦀 é" - ); - assert_eq!( - node(&server.visible, sid, metadata)["kind"]["title"], - "Remote second 🐈" - ); - drop(cache); - for id in [movement, dependent] { - let cache = Replica::open(&path).unwrap(); - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) - ); - assert_eq!( - node(&server.durable, sid, metadata)["kind"]["title"], - "Remote second 🐈" - ); - assert_eq!( - node(&server.durable, sid, page)["kind"]["alternate_title"], - "Remote second 🐈" - ); - } - let cache = Replica::open(&path).unwrap(); - assert!(cache.pending().unwrap().is_empty()); - assert_eq!( - node(&cache.snapshot().unwrap(), sid, text_id)["kind"]["text"], - "Local Original 🦀 é" - ); - assert_eq!(server.publications, 2); -} - -#[test] -fn competing_layout_requires_current_review_and_preserves_dependent_edits() { - let (source, sid, outline, _, text_id) = fixture(); - for (local_change, remote_change) in [ - ( - Change::Position { x: 180.0, y: 216.0 }, - Change::Position { x: 288.0, y: 216.0 }, - ), - ( - Change::Width { - points: 144.0, - user_set: true, - }, - Change::Width { - points: 216.0, - user_set: false, - }, - ), - ] { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("cache.sqlite"); - let cache = Replica::create(&path, &source).unwrap(); - let id = cache - .outline(&source, sid, outline, local_change) - .unwrap() - .unwrap(); - let dependent = cache - .edit_text(&cache.snapshot().unwrap(), sid, text_id, 0..0, "Local ") - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let queue = cache.pending().unwrap(); - let remote = PreparedEdit::outline(&source, sid, outline, remote_change).unwrap(); - let mut server = Server::new(remote.as_bytes()); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((id, EditStatus::Conflict(ConflictKind::LayoutChanged))) - ); - assert_eq!(server.publications, 0); - assert_eq!(cache.pending().unwrap(), queue); - assert!(cache.snapshot().unwrap() == local); - assert!( - cache - .rebase_layout_conflict(id, &source, &server.visible) - .is_err() - ); - assert!(cache.rebase_layout_conflict(id, &local, &source).is_err()); - assert!( - cache - .rebase_conflict(id, &local, &server.visible, 0..0) - .is_err() - ); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!( - cache.status(id).unwrap(), - Some(EditStatus::Conflict(ConflictKind::LayoutChanged)) - ); - cache - .rebase_layout_conflict(id, &local, &server.visible) - .unwrap(); - let reviewed = cache.pending().unwrap(); - assert_eq!(reviewed[1], queue[1]); - assert!(cache.snapshot().unwrap() == local); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.pending().unwrap(), reviewed); - for id in [id, dependent] { - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == id) - ); - } - assert_eq!( - node(&server.durable, sid, outline)["layout"], - node(&local, sid, outline)["layout"] - ); - assert_eq!(text(&server.durable).2, "Local Original 🦀 é"); - } -} - -#[test] -fn independent_axes_converge_without_overwriting_competing_values() { - let (source, sid, outline, _, _) = fixture(); - let directory = tempfile::tempdir().unwrap(); - let cache = Replica::create(directory.path().join("cache.sqlite"), &source).unwrap(); - let original = node(&source, sid, outline); - let local = Change::Position { x: 180.0, y: 216.0 }; - let id = cache - .outline(&source, sid, outline, local) - .unwrap() - .unwrap(); - let remote = PreparedEdit::outline( - &source, - sid, - outline, - Change::Position { - x: original["layout"]["x"].as_f64().unwrap() as f32, - y: 216.0, - }, - ) - .unwrap(); - let mut server = Server::new(remote.as_bytes()); - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n==id) - ); - let actual = node(&server.durable, sid, outline); - assert_eq!(actual["layout"]["x"], 180.0); - assert_eq!(actual["layout"]["y"], 216.0); -} - -#[test] -fn twelve_offline_writers_preserve_all_layout_intents_through_review_and_restarts() { - let (source, sid, outline, paragraph, _) = fixture(); - let directory = tempfile::tempdir().unwrap(); - std::thread::scope(|scope| { - let mut writers = Vec::new(); - for actor in 0..12 { - let source = &source; - let path = directory.path().join(format!("{actor}.sqlite")); - writers.push(scope.spawn(move || { - let cache = Replica::create(&path, source).unwrap(); - for round in 0..4 { - for (object, change) in [ - ( - outline, - Change::Position { - x: (actor + 2) as f32 * 36.0, - y: (round + 2) as f32 * 36.0, - }, - ), - ( - outline, - Change::Width { - points: (actor + round + 2) as f32 * 36.0, - user_set: true, - }, - ), - (paragraph, Change::Collapsed((actor + round) % 2 != 0)), - ] { - assert!( - cache - .outline(&cache.snapshot().unwrap(), sid, object, change) - .unwrap() - .is_some() - ); - } - } - assert_eq!(cache.pending().unwrap().len(), 12); - })); - } - for writer in writers { - writer.join().unwrap(); - } - }); - let mut server = Server::new(&source); - let mut expected_layout = node(&source, sid, outline)["layout"].clone(); - let mut expected_collapse = None; - let mut reviewed = 0; - for round in 0..12 { - for actor in 0..12 { - let path = directory.path().join(format!("{actor}.sqlite")); - let cache = Replica::open(&path).unwrap(); - let first = cache.pending().unwrap()[0].clone(); - let onestore_offline::Operation::Outline(edit) = first.operation else { - panic!() - }; - let local = cache.snapshot().unwrap(); - let result = cache.sync_once(&mut server).unwrap(); - if result == Some((first.id, EditStatus::Conflict(ConflictKind::LayoutChanged))) { - let remote = cache.remote_snapshot().unwrap(); - cache - .rebase_layout_conflict(first.id, &local, &remote) - .unwrap(); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((id, EditStatus::Published { .. })) if id == first.id) - ); - reviewed += 1; - } else { - assert!( - matches!(result, Some((id, EditStatus::Published { .. })) if id == first.id) - ); - } - match edit.change { - Change::Position { x, y } => { - expected_layout["x"] = x.into(); - expected_layout["y"] = y.into(); - } - Change::Width { points, user_set } => { - expected_layout["max_width"] = points.into(); - expected_layout["width_set_by_user"] = user_set.into(); - } - Change::Collapsed(value) => expected_collapse = Some(u8::from(value)), - } - assert_eq!( - node(&server.durable, sid, outline)["layout"], - expected_layout - ); - assert_eq!( - node(&server.durable, sid, paragraph)["kind"]["collapse_state"], - serde_json::json!(expected_collapse) - ); - assert_eq!(text(&server.durable).2, "Original 🦀 é"); - } - for actor in 0..12 { - let cache = Replica::open(directory.path().join(format!("{actor}.sqlite"))).unwrap(); - assert_eq!(cache.pending().unwrap().len(), 11 - round); - } - } - assert!(reviewed >= 12); - for actor in 0..12 { - let cache = Replica::open(directory.path().join(format!("{actor}.sqlite"))).unwrap(); - assert!(cache.pending().unwrap().is_empty()); - for id in 1..=12 { - assert!(matches!( - cache.status(id).unwrap(), - Some(EditStatus::Published { .. }) - )); - } - } -} - -#[test] -fn uncertain_layout_is_confirmed_by_revision_and_never_by_converged_values() { - let (source, sid, outline, _, text_id) = fixture(); - for fault in [Fault::UnknownBefore, Fault::UnknownAfter] { - let before = matches!(fault, Fault::UnknownBefore); - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("cache.sqlite"); - let cache = Replica::create(&path, &source).unwrap(); - let change = Change::Width { - points: 144.0, - user_set: true, - }; - let id = cache - .outline(&source, sid, outline, change) - .unwrap() - .unwrap(); - cache - .edit_text(&cache.snapshot().unwrap(), sid, text_id, 0..0, "Local ") - .unwrap(); - let local = cache.snapshot().unwrap(); - let mut server = Server::new(&source); - server.fault = fault; - assert!(cache.sync_once(&mut server).is_err()); - let status = cache.status(id).unwrap().unwrap(); - assert!(matches!(status, EditStatus::AwaitingConfirmation { .. })); - drop(cache); - let cache = Replica::open(&path).unwrap(); - if before { - let converged = PreparedEdit::outline(&source, sid, outline, change).unwrap(); - server.visible = converged.as_bytes().to_vec(); - assert_eq!(cache.sync_once(&mut server).unwrap(), Some((id, status))); - assert!( - cache - .rebase_layout_conflict(id, &local, &server.visible) - .is_err() - ); - assert_eq!(cache.pending().unwrap().len(), 2); - assert!(cache.snapshot().unwrap() == local); - let archive = directory.path().join("recovery.sqlite"); - cache.export_recovery(&archive).unwrap(); - let recovery = onestore_offline::Recovery::open(&archive).unwrap(); - assert_eq!(recovery.pending().unwrap(), cache.pending().unwrap()); - assert_eq!(recovery.status(id).unwrap(), Some(status)); - assert_eq!(server.confirmations, 0); - } else { - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n==id) - ); - assert_eq!( - node(&server.durable, sid, outline)["layout"]["max_width"], - 144.0 - ); - assert_eq!(server.confirmations, 1); - } - assert_eq!(server.publications, 1); - } -} - -#[test] -fn native_moves_deletions_and_layout_changes_merge_or_retain_explicit_conflicts() { - let source = include_bytes!("../../../../corpus/outline-edit/before/notebook/synthetic.one"); - let native = include_bytes!("../../../../corpus/outline-edit/after/notebook/synthetic.one"); - let store = Store::parse(source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let mut server = Server::new(native); - let mut counts = [0; 3]; - let mut records = Vec::new(); - for (sid, page) in document.pages().unwrap() { - let space = &document.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let Kind::Metadata { - title: Some(name), .. - } = &view.nodes[&view.roots[&2]].kind - else { - continue; - }; - if name == "Unicode rich text" { - continue; - } - let outlines: Vec<_> = view.nodes[&page] - .children - .iter() - .filter(|id| matches!(view.nodes[id].kind, Kind::Outline { .. })) - .copied() - .collect(); - if outlines.len() != 2 { - continue; - } - let outline = outlines[0]; - let mut pending = vec![outline]; - let mut target = None; - while let Some(id) = pending.pop() { - let node = &view.nodes[&id]; - pending.extend(&node.children); - if node.content.first().is_some_and(|text| matches!(&view.nodes[text].kind, Kind::RichText { text, .. } if text.starts_with("Target "))) { - target = Some(id); - } - } - let target = target.unwrap(); - let text_id = view.nodes[&view.nodes[&outlines[1]].children[0]].content[0]; - let (object, change, conflict) = match name.as_str() { - "Move outline" => ( - outline, - Change::Position { x: 252.0, y: 288.0 }, - Some(ConflictKind::LayoutChanged), - ), - "Resize outline" | "Automatic outline size" => ( - outline, - Change::Width { - points: 252.0, - user_set: true, - }, - Some(ConflictKind::LayoutChanged), - ), - "Delete outline" => ( - outline, - Change::Width { - points: 252.0, - user_set: true, - }, - Some(ConflictKind::TargetUnavailable), - ), - "Delete leaf" | "Delete subtree" | "Delete only paragraph" => ( - target, - Change::Collapsed(true), - Some(ConflictKind::TargetUnavailable), - ), - "Indent subtree" | "Outdent subtree" => ( - target, - Change::Collapsed(true), - Some(ConflictKind::StructureChanged), - ), - "Expand subtree" => (target, Change::Collapsed(false), None), - "Collapse subtree" | "Move leaf down" | "Move subtree down" | "Move subtree up" => { - (target, Change::Collapsed(true), None) - } - _ => panic!("Unexpected native case {name}"), - }; - let expected_layout = if conflict == Some(ConflictKind::TargetUnavailable) { - None - } else { - let mut layout = node(&server.visible, sid, object)["layout"].clone(); - match change { - Change::Position { x, y } => { - layout["x"] = x.into(); - layout["y"] = y.into(); - } - Change::Width { points, user_set } => { - layout["max_width"] = points.into(); - layout["width_set_by_user"] = user_set.into(); - } - Change::Collapsed(_) => {} - } - Some(layout) - }; - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("cache.sqlite"); - let cache = Replica::create(&path, source).unwrap(); - let id = cache.outline(source, sid, object, change).unwrap().unwrap(); - let dependent = cache - .edit_text(&cache.snapshot().unwrap(), sid, text_id, 0..0, "Local ") - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let queue = cache.pending().unwrap(); - let result = cache.sync_once(&mut server).unwrap().unwrap(); - let mut retained = false; - if let Some(kind) = conflict { - assert_eq!(result, (id, EditStatus::Conflict(kind)), "{name}"); - assert_eq!(cache.pending().unwrap(), queue); - assert!(cache.snapshot().unwrap() == local); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Conflict(kind))); - if kind == ConflictKind::TargetUnavailable { - assert!( - cache - .rebase_layout_conflict(id, &local, &server.visible) - .is_err() - ); - counts[2] += 1; - retained = true; - } else { - cache - .rebase_layout_conflict(id, &local, &server.visible) - .unwrap(); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == id) - ); - counts[1] += 1; - } - } else { - assert!( - matches!(result, (n, EditStatus::Published { .. }) if n == id), - "{name}" - ); - counts[0] += 1; - drop(cache); - } - let cache = Replica::open(&path).unwrap(); - if retained { - assert_eq!(cache.pending().unwrap(), queue); - assert!(cache.snapshot().unwrap() == local); - } else { - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == dependent) - ); - assert_eq!( - node(&server.durable, sid, object)["layout"], - expected_layout.unwrap(), - "{name}" - ); - if matches!(change, Change::Collapsed(_)) { - assert_eq!( - node(&server.durable, sid, object)["kind"]["collapse_state"], - node(&local, sid, object)["kind"]["collapse_state"] - ); - } - assert!( - node(&server.durable, sid, text_id)["kind"]["text"] - .as_str() - .unwrap() - .starts_with("Local ") - ); - drop(cache); - let cache = Replica::open(&path).unwrap(); - for id in [id, dependent] { - assert!(matches!( - cache.status(id).unwrap(), - Some(EditStatus::Published { .. }) - )); - } - } - records.push(serde_json::json!({"name": name, "space": sid, "object": object, "change": change, "retained": retained, "dependent_text": text_id})); - } - assert_eq!(counts, [5, 5, 4]); - assert_eq!(server.publications, 18); - if let Some(output) = std::env::var_os("ONESTORE_OFFLINE_OUTLINE_OUTPUT") { - let output = std::path::PathBuf::from(output); - assert!(output.is_absolute()); - std::fs::create_dir(&output).unwrap(); - std::fs::write(output.join("synthetic.one"), &server.durable).unwrap(); - std::fs::write( - output.with_extension("json"), - serde_json::to_vec_pretty(&records).unwrap(), - ) - .unwrap(); - } -} - -#[test] -fn a_new_native_wrap_reservation_requires_review_before_width_replacement() { - let source = include_bytes!("../../../../corpus/outline-edit/before/notebook/synthetic.one"); - let native = include_bytes!("../../../../corpus/outline-edit/after/notebook/synthetic.one"); - let store = Store::parse(source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let (sid, page) = document.pages().unwrap().into_iter().find(|(sid, _)| { - let space = &document.spaces[sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - matches!(&view.nodes[&view.roots[&2]].kind, Kind::Metadata { title: Some(name), .. } if name=="Move outline") - }).unwrap(); - let space = &document.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let outline = *view.nodes[&page] - .children - .iter() - .find(|id| matches!(view.nodes[id].kind, Kind::Outline { .. })) - .unwrap(); - let old = node(source, sid, outline); - let remote = node(native, sid, outline); - assert_eq!(old["layout"]["max_width"], remote["layout"]["max_width"]); - assert_eq!( - old["layout"]["width_set_by_user"], - remote["layout"]["width_set_by_user"] - ); - assert!(remote["layout"]["reserved_width"].is_number()); - let directory = tempfile::tempdir().unwrap(); - let cache = Replica::create(directory.path().join("cache.sqlite"), source).unwrap(); - let id = cache - .outline( - source, - sid, - outline, - Change::Width { - points: 144.0, - user_set: true, - }, - ) - .unwrap() - .unwrap(); - let mut server = Server::new(native); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((id, EditStatus::Conflict(ConflictKind::LayoutChanged))) - ); - cache - .rebase_layout_conflict(id, &cache.snapshot().unwrap(), &server.visible) - .unwrap(); - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n==id) - ); - let after = node(&server.durable, sid, outline); - assert_eq!(after["layout"]["x"], remote["layout"]["x"]); - assert_eq!(after["layout"]["y"], remote["layout"]["y"]); - assert_eq!(after["layout"]["max_width"], 144.0); - assert!(after["layout"]["reserved_width"].is_null()); -} diff --git a/crates/onestore-offline/tests/sync/page.rs b/crates/onestore-offline/tests/sync/page.rs deleted file mode 100644 index 711facd5b6e82585dbcad80b9ea9630dfeb42dc4..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/tests/sync/page.rs +++ /dev/null @@ -1,350 +0,0 @@ -use super::*; -use onestore::{Insertion, PageCreation}; -use onestore_offline::{Operation, Recovery}; -use std::collections::BTreeMap; - -#[test] -fn twelve_replica_page_schedules_retain_acknowledged_pages_through_interruptions() { - for seed in 0..24_u64 { - let mut random = seed + 1956; - let mut input = Vec::new(); - for step in 0..48 { - random ^= random << 13; - random ^= random >> 7; - random ^= random << 17; - let mut action = random.to_le_bytes(); - if step < 12 { - action[0] = step; - action[1] = 1; - } - input.extend_from_slice(&action); - } - page_schedule::run(&input); - } -} - -#[test] -fn version_two_uncertain_text_migrates_without_replay_or_lost_receipts() { - let source = onestore::create_section("pages.one", "Original", "Author").unwrap(); - let (sid, oid, _) = text(&source); - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("legacy.sqlite"); - let cache = Replica::create(&path, &source).unwrap(); - let id = cache - .edit_text(&source, sid, oid, 0..0, "Old ") - .unwrap() - .unwrap(); - let snapshot = cache.snapshot().unwrap(); - let pending = cache.pending().unwrap(); - let store = Store::parse(&snapshot).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let revision = index.spaces[&sid].labels[&(ExGuid::default(), 1)]; - drop(cache); - let db = rusqlite::Connection::open(&path).unwrap(); - db.execute_batch( - "DROP TABLE edits; DROP TABLE assets; - CREATE TABLE edits ( - id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(id>0), space TEXT NOT NULL, - object TEXT NOT NULL, before_text TEXT NOT NULL, - start INTEGER NOT NULL CHECK(start BETWEEN 0 AND 4294967295), - end INTEGER NOT NULL CHECK(end BETWEEN start AND 4294967295), replacement TEXT NOT NULL - ) STRICT; - ALTER TABLE attempt RENAME COLUMN revisions TO revision; - PRAGMA user_version=2;", - ) - .unwrap(); - db.execute( - "INSERT INTO edits VALUES (?1,?2,?3,'Original',0,0,'Old ')", - rusqlite::params![i64::try_from(id).unwrap(), sid.to_string(), oid.to_string()], - ) - .unwrap(); - db.execute( - "INSERT INTO attempt VALUES (1,?1,?2)", - rusqlite::params![i64::try_from(id).unwrap(), revision.to_string()], - ) - .unwrap(); - db.execute( - "INSERT INTO receipts VALUES (1000,?1)", - [revision.to_string()], - ) - .unwrap(); - db.execute("UPDATE sqlite_sequence SET seq=1000 WHERE name='edits'", []) - .unwrap(); - drop(db); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.pending().unwrap(), pending); - assert_eq!( - cache.status(id).unwrap(), - Some(EditStatus::AwaitingConfirmation { revision }) - ); - assert_eq!( - cache.status(1000).unwrap(), - Some(EditStatus::Published { revision }) - ); - assert!(cache.snapshot().unwrap() == snapshot); - let mut server = Server::new(&snapshot); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((id, EditStatus::Published { revision })) - ); - assert_eq!(server.publications, 0); - assert_eq!(server.confirmations, 1); - assert_eq!( - cache - .edit_text(&cache.snapshot().unwrap(), sid, oid, 0..0, "Next ") - .unwrap(), - Some(1001) - ); -} - -#[test] -fn offline_page_creation_rebases_with_dependent_edits_and_duplicate_titles() { - let source = onestore::create_section("pages.one", "Original", "Author").unwrap(); - let directory = tempfile::tempdir().unwrap(); - let mut server = Server::new(&source); - let mut expected = Vec::new(); - for actor in 0..12 { - let path = directory.path().join(format!("{actor}.sqlite")); - let cache = Replica::create(&path, &source).unwrap(); - let page = PageCreation::new(None, Some("Same 🦋 é"), "Offline author").unwrap(); - let id = cache.create_page(&source, &page).unwrap().unwrap(); - let insertion = Insertion::outline(page.object(), 36.0, 36.0, "Body", "Author").unwrap(); - cache - .insert(&cache.snapshot().unwrap(), page.space(), &insertion) - .unwrap() - .unwrap(); - cache - .edit_text( - &cache.snapshot().unwrap(), - page.space(), - insertion.text_object(), - 4..4, - &format!(" {actor}"), - ) - .unwrap() - .unwrap(); - let original = cache.pending().unwrap(); - assert!( - matches!(&original[0].operation, Operation::CreatePage(retained) if retained == &page) - ); - let local = cache.snapshot().unwrap(); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.pending().unwrap(), original); - assert_eq!(cache.snapshot().unwrap(), local); - for edit in original { - assert!(matches!(cache.sync_once(&mut server).unwrap(), - Some((published, EditStatus::Published { .. })) if published == edit.id)); - } - assert!(matches!( - cache.status(id).unwrap(), - Some(EditStatus::Published { .. }) - )); - expected.push(( - page.space(), - page.object(), - insertion.text_object(), - format!("Body {actor}"), - )); - } - let store = Store::parse(&server.durable).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let pages = document.pages().unwrap(); - assert_eq!(pages.len(), 13); - for (actual, (sid, page, text, expected)) in pages[1..].iter().zip(&expected) { - assert_eq!(*actual, (*sid, *page)); - let space = &document.spaces[sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - assert!(matches!(&view.nodes[text].kind, Kind::RichText { text, .. } if text == expected)); - } - assert_eq!(server.publications, 36); - if let Some(output) = std::env::var_os("ONESTORE_OFFLINE_PAGE_OUTPUT") { - std::fs::create_dir(&output).unwrap(); - std::fs::write( - std::path::Path::new(&output).join("pages.one"), - &server.durable, - ) - .unwrap(); - } -} - -#[test] -fn uncertain_page_publication_retains_both_revisions_and_never_replays() { - for fault in [ - Fault::UnknownBefore, - Fault::UnknownAfter, - Fault::PanicBefore, - Fault::PanicAfter, - ] { - let source = onestore::create_section("pages.one", "Original", "Author").unwrap(); - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("pages.sqlite"); - let cache = Replica::create(&path, &source).unwrap(); - let page = PageCreation::new(None, Some("Created"), "Author").unwrap(); - let id = cache.create_page(&source, &page).unwrap().unwrap(); - let local = cache.snapshot().unwrap(); - let mut server = Server::new(&source); - server.fault = fault; - let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - cache.sync_once(&mut server) - })); - let attempted = cache.status(id).unwrap().unwrap(); - assert!(matches!(attempted, EditStatus::AwaitingConfirmation { .. })); - let archive = directory.path().join("recovery.sqlite"); - cache.export_recovery(&archive).unwrap(); - assert_eq!( - Recovery::open(&archive).unwrap().status(id).unwrap(), - Some(attempted) - ); - drop(cache); - let db = rusqlite::Connection::open(&path).unwrap(); - let encoded: String = db - .query_row("SELECT revisions FROM attempt", [], |row| row.get(0)) - .unwrap(); - let revisions: BTreeMap = serde_json::from_str(&encoded).unwrap(); - assert_eq!(revisions.len(), 2); - assert!(revisions.contains_key(&page.space())); - drop(db); - let cache = Replica::open(&path).unwrap(); - let observed = server.visible.clone(); - let result = cache.sync_once(&mut server).unwrap().unwrap(); - let visible = matches!(fault, Fault::UnknownAfter | Fault::PanicAfter); - assert_eq!(server.publications, 1); - assert_eq!(server.confirmations, usize::from(visible)); - if visible { - assert!(matches!(result.1, EditStatus::Published { .. })); - assert!(cache.snapshot().unwrap() == observed); - assert!(observed[..212] == server.durable[..212]); - assert!(observed[252..] == server.durable[252..]); - } else { - assert_eq!(result, (id, attempted)); - assert_eq!(cache.snapshot().unwrap(), local); - } - } -} - -#[test] -fn surviving_page_revision_alone_does_not_confirm_section_publication() { - let source = onestore::create_section("pages.one", "Original", "Author").unwrap(); - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("pages.sqlite"); - let cache = Replica::create(&path, &source).unwrap(); - let page = PageCreation::new(None, None, "Author").unwrap(); - let id = cache.create_page(&source, &page).unwrap().unwrap(); - let mut server = Server::new(&source); - server.fault = Fault::UnknownAfter; - assert!(cache.sync_once(&mut server).is_err()); - let attempted = cache.status(id).unwrap().unwrap(); - drop(cache); - let db = rusqlite::Connection::open(&path).unwrap(); - let encoded: String = db - .query_row("SELECT revisions FROM attempt", [], |row| row.get(0)) - .unwrap(); - let revisions: BTreeMap = serde_json::from_str(&encoded).unwrap(); - drop(db); - let complete = server.visible.clone(); - let (§ion, &retired) = revisions - .iter() - .find(|(sid, _)| **sid != page.space()) - .unwrap(); - let store = Store::parse(&complete).unwrap(); - let mut positions = Vec::new(); - for list in store.lists.values() { - for node in &list.nodes { - if node.id == 0x1e - && node.payload[..16] == retired.guid - && node.payload[16..20] == retired.n.to_le_bytes() - { - positions.push(node.offset); - } - } - } - assert_eq!(positions.len(), 1); - // Replacing only the revision identity models maintenance retiring the section proof. - let at = positions[0] + 4; - server.visible[at] ^= 0x40; - let store = Store::parse(&server.visible).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - index.validate_current().unwrap(); - assert!(!index.spaces[§ion].revisions.contains_key(&retired)); - assert!( - index.spaces[&page.space()] - .revisions - .contains_key(&revisions[&page.space()]) - ); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.sync_once(&mut server).unwrap(), Some((id, attempted))); - assert_eq!(server.confirmations, 0); - assert_eq!(server.publications, 1); - server.visible = complete; - assert!(matches!( - cache.sync_once(&mut server).unwrap(), - Some((_, EditStatus::Published { .. })) - )); - assert_eq!(server.publications, 1); -} - -#[test] -fn changed_page_anchor_requires_review_without_regenerating_dependent_identities() { - let source = - include_bytes!("../../../../corpus/page-lifecycle/03-renamed/notebook/Lifecycle.one"); - let remote = - include_bytes!("../../../../corpus/page-lifecycle/04-nested/notebook/Lifecycle.one"); - let store = Store::parse(source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let pages = Document::parse(&index).unwrap().pages().unwrap(); - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("pages.sqlite"); - let cache = Replica::create(&path, source).unwrap(); - let page = PageCreation::new(Some(pages[4].0), Some("Created"), "Author").unwrap(); - let id = cache.create_page(source, &page).unwrap().unwrap(); - let insertion = - Insertion::outline(page.object(), 36.0, 36.0, "Retained body", "Author").unwrap(); - cache - .insert(&cache.snapshot().unwrap(), page.space(), &insertion) - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let pending = cache.pending().unwrap(); - let mut server = Server::new(remote); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((id, EditStatus::Conflict(ConflictKind::StructureChanged))) - ); - assert!( - cache - .rebase_page_creation_conflict(id, source, remote, None) - .is_err() - ); - assert!( - cache - .rebase_page_creation_conflict(id, &local, source, None) - .is_err() - ); - cache - .rebase_page_creation_conflict(id, &local, remote, None) - .unwrap(); - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(cache.pending().unwrap()[1], pending[1]); - let Operation::CreatePage(reviewed) = &cache.pending().unwrap()[0].operation else { - panic!() - }; - assert_eq!(*reviewed, page.reposition(None).unwrap()); - drop(cache); - let cache = Replica::open(&path).unwrap(); - for _ in 0..2 { - assert!(matches!( - cache.sync_once(&mut server).unwrap(), - Some((_, EditStatus::Published { .. })) - )); - } - assert_eq!(server.publications, 2); - let store = Store::parse(&server.durable).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let space = &document.spaces[&page.space()]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - assert!(matches!(&view.nodes[&insertion.text_object()].kind, - Kind::RichText { text, .. } if text == "Retained body")); -} diff --git a/crates/onestore-offline/tests/sync/pages.rs b/crates/onestore-offline/tests/sync/pages.rs deleted file mode 100644 index 45d447d7cada7a959c5c7cb8f48e2dc474867af2..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/tests/sync/pages.rs +++ /dev/null @@ -1,631 +0,0 @@ -use super::*; -use onestore::{PageEdit, PagePosition}; -use onestore_offline::{Operation, Recovery}; -use std::collections::BTreeMap; - -const SOURCE: &[u8] = - include_bytes!("../../../../corpus/page-lifecycle/04-nested/notebook/Lifecycle.one"); - -fn order(source: &[u8]) -> Vec<(ExGuid, u32)> { - let store = Store::parse(source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - index.validate_current().unwrap(); - let document = Document::parse(&index).unwrap(); - document - .pages() - .unwrap() - .iter() - .map(|(sid, _)| { - let space = &document.spaces[sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let Kind::Metadata { level, .. } = view.nodes[&view.roots[&2]].kind else { - panic!() - }; - (*sid, level.unwrap_or(1)) - }) - .collect() -} - -fn texts(source: &[u8]) -> BTreeMap<(ExGuid, ExGuid), String> { - let store = Store::parse(source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let mut texts = BTreeMap::new(); - for (sid, space) in &document.spaces { - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - for (oid, node) in &view.nodes { - if let Kind::RichText { text, .. } = &node.kind { - texts.insert((*sid, *oid), text.clone()); - } - } - } - texts -} - -#[test] -fn atomic_page_batches_survive_reopen_with_dependent_text() { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("pages.sqlite"); - let cache = Replica::create(&path, SOURCE).unwrap(); - let before = order(SOURCE); - let edits: Vec<_> = before[3..6] - .iter() - .map(|(sid, level)| PageEdit::move_to(*sid, None, *level).unwrap()) - .collect(); - let id = cache.pages(SOURCE, &edits).unwrap().unwrap(); - assert!(cache.pages(SOURCE, &edits).is_err()); - let mut expected_text = texts(SOURCE); - let (&(sid, oid), original) = expected_text - .iter() - .find(|(_, text)| text.starts_with("Body parent")) - .unwrap(); - let changed = format!("Offline {original}"); - expected_text.insert((sid, oid), changed); - cache - .edit_text(&cache.snapshot().unwrap(), sid, oid, 0..0, "Offline ") - .unwrap() - .unwrap(); - let queue = cache.pending().unwrap(); - assert!(matches!(&queue[0].operation, Operation::Pages(batch) if batch.edits == edits)); - let local = cache.snapshot().unwrap(); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.pending().unwrap(), queue); - assert_eq!(cache.snapshot().unwrap(), local); - let mut server = Server::new(SOURCE); - for edit in queue { - assert!(matches!(cache.sync_once(&mut server).unwrap(), - Some((published, EditStatus::Published { .. })) if published == edit.id)); - } - let expected = [ - before[..3].to_vec(), - before[6..].to_vec(), - before[3..6].to_vec(), - ] - .concat(); - assert_eq!(order(&server.durable), expected); - assert_eq!(texts(&server.durable), expected_text); - assert!(matches!( - cache.status(id).unwrap(), - Some(EditStatus::Published { .. }) - )); - assert_eq!(server.publications, 2); -} - -#[test] -fn indentation_only_edits_preserve_remote_page_movement() { - let directory = tempfile::tempdir().unwrap(); - let cache = Replica::create(directory.path().join("pages.sqlite"), SOURCE).unwrap(); - let before = order(SOURCE); - let sid = before[4].0; - let id = cache - .pages(SOURCE, &[PageEdit::set_level(sid, 3).unwrap()]) - .unwrap() - .unwrap(); - let mut server = Server::new(SOURCE); - PreparedEdit::pages( - SOURCE, - &[PageEdit::move_to(sid, Some(before[7].0), 2).unwrap()], - ) - .unwrap() - .commit(&mut server) - .unwrap(); - let mut expected = order(&server.durable); - expected.iter_mut().find(|p| p.0 == sid).unwrap().1 = 3; - assert!(matches!(cache.sync_once(&mut server).unwrap(), - Some((published, EditStatus::Published { .. })) if published == id)); - assert_eq!(order(&server.durable), expected); -} - -#[test] -fn competing_page_moves_require_current_review_and_retain_intent_identities() { - let directory = tempfile::tempdir().unwrap(); - let cache = Replica::create(directory.path().join("pages.sqlite"), SOURCE).unwrap(); - let pages = order(SOURCE); - let sid = pages[6].0; - let original = PageEdit::move_to(sid, None, 1).unwrap(); - let id = cache - .pages(SOURCE, std::slice::from_ref(&original)) - .unwrap() - .unwrap(); - let mut server = Server::new(SOURCE); - PreparedEdit::pages( - SOURCE, - &[PageEdit::move_to(sid, Some(pages[0].0), 1).unwrap()], - ) - .unwrap() - .commit(&mut server) - .unwrap(); - let remote = server.visible.clone(); - let local = cache.snapshot().unwrap(); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((id, EditStatus::Conflict(ConflictKind::StructureChanged))) - ); - assert_eq!(server.publications, 0); - assert_eq!(cache.snapshot().unwrap(), local); - let revised = original - .reposition(PagePosition::Before(Some(pages[2].0)), 1) - .unwrap(); - let fresh = PageEdit::move_to(sid, Some(pages[2].0), 1).unwrap(); - assert!( - cache - .rebase_pages_conflict(id, &local, &remote, &[fresh]) - .is_err() - ); - assert!( - cache - .rebase_pages_conflict(id, SOURCE, &remote, std::slice::from_ref(&revised)) - .is_err() - ); - assert!( - cache - .rebase_pages_conflict(id, &local, SOURCE, std::slice::from_ref(&revised)) - .is_err() - ); - cache - .rebase_pages_conflict(id, &local, &remote, std::slice::from_ref(&revised)) - .unwrap(); - assert!(matches!(&cache.pending().unwrap()[0].operation, - Operation::Pages(batch) if batch.edits == [revised.clone()])); - let expected = PreparedEdit::pages(&remote, &[revised]).unwrap(); - assert!(matches!(cache.sync_once(&mut server).unwrap(), - Some((published, EditStatus::Published { .. })) if published == id)); - assert_eq!(order(&server.durable), order(expected.as_bytes())); -} - -#[test] -fn one_competing_level_prevents_publication_of_the_whole_batch() { - let directory = tempfile::tempdir().unwrap(); - let cache = Replica::create(directory.path().join("pages.sqlite"), SOURCE).unwrap(); - let pages = order(SOURCE); - let id = cache - .pages( - SOURCE, - &[ - PageEdit::set_level(pages[4].0, 3).unwrap(), - PageEdit::set_level(pages[5].0, 2).unwrap(), - ], - ) - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let mut server = Server::new(SOURCE); - PreparedEdit::pages(SOURCE, &[PageEdit::set_level(pages[5].0, 1).unwrap()]) - .unwrap() - .commit(&mut server) - .unwrap(); - let remote = server.durable.clone(); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((id, EditStatus::Conflict(ConflictKind::StructureChanged))) - ); - assert_eq!(server.publications, 0); - assert_eq!(server.durable, remote); - assert_eq!(cache.snapshot().unwrap(), local); -} - -#[test] -fn uncertain_page_batches_survive_recovery_and_do_not_replay() { - for fault in [ - Fault::UnknownBefore, - Fault::UnknownAfter, - Fault::PanicBefore, - Fault::PanicAfter, - ] { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("pages.sqlite"); - let mut cache = Replica::create(&path, SOURCE).unwrap(); - let pages = order(SOURCE); - let edits = [ - PageEdit::set_level(pages[4].0, 3).unwrap(), - PageEdit::set_level(pages[5].0, 2).unwrap(), - ]; - let id = cache.pages(SOURCE, &edits).unwrap().unwrap(); - let local = cache.snapshot().unwrap(); - let queue = cache.pending().unwrap(); - let mut server = Server::new(SOURCE); - server.fault = fault; - let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - cache.sync_once(&mut server) - })); - drop(cache); - cache = Replica::open(&path).unwrap(); - let attempted = cache.status(id).unwrap().unwrap(); - assert!(matches!(attempted, EditStatus::AwaitingConfirmation { .. })); - let archive = directory.path().join("recovery.sqlite"); - cache.export_recovery(&archive).unwrap(); - let recovery = Recovery::open(&archive).unwrap(); - assert_eq!(recovery.pending().unwrap(), queue); - assert_eq!(recovery.status(id).unwrap(), Some(attempted)); - assert_eq!(recovery.snapshot().unwrap(), local); - assert!( - cache - .rebase_pages_conflict(id, &local, SOURCE, &edits) - .is_err() - ); - let result = cache.sync_once(&mut server).unwrap().unwrap(); - assert_eq!(server.publications, 1); - if matches!(fault, Fault::UnknownAfter | Fault::PanicAfter) { - assert!(matches!(result.1, EditStatus::Published { .. })); - assert_eq!(server.confirmations, 1); - } else { - assert_eq!(result, (id, attempted)); - assert_eq!(server.confirmations, 0); - } - } -} - -#[test] -fn an_unchanged_section_revision_cannot_confirm_a_changed_page() { - let before = order(SOURCE); - let sid = before[4].0; - let store = Store::parse(SOURCE).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let root = index.root; - let doc = Document::parse(&index).unwrap(); - let space = &doc.spaces[&root]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let copy = *view.nodes.iter().find(|(_, node)| - matches!(&node.kind, Kind::Metadata { title: Some(title), .. } if title == "child")) - .unwrap().0; - let source = - onestore::replace_property_bytes(SOURCE, root, copy, 0x14001dff, &3_u32.to_le_bytes()) - .unwrap(); - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("pages.sqlite"); - let cache = Replica::create(&path, &source).unwrap(); - let id = cache - .pages(&source, &[PageEdit::set_level(sid, 3).unwrap()]) - .unwrap() - .unwrap(); - let mut server = Server::new(&source); - server.fault = Fault::UnknownAfter; - assert!(cache.sync_once(&mut server).is_err()); - let attempted = cache.status(id).unwrap().unwrap(); - drop(cache); - let db = rusqlite::Connection::open(&path).unwrap(); - let encoded: String = db - .query_row("SELECT revisions FROM attempt", [], |r| r.get(0)) - .unwrap(); - let proofs: BTreeMap = serde_json::from_str(&encoded).unwrap(); - assert_eq!(proofs.len(), 2); - let store = Store::parse(&source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - assert_eq!( - proofs[&root], - index.spaces[&root].labels[&(ExGuid::default(), 1)] - ); - assert!(!index.spaces[&sid].revisions.contains_key(&proofs[&sid])); - drop(db); - let cache = Replica::open(&path).unwrap(); - let complete = server.visible.clone(); - server.visible = source; - assert_eq!(cache.sync_once(&mut server).unwrap(), Some((id, attempted))); - assert_eq!(server.confirmations, 0); - server.visible = complete; - assert!(matches!( - cache.sync_once(&mut server).unwrap(), - Some((_, EditStatus::Published { .. })) - )); - assert_eq!(server.publications, 1); - assert_eq!(server.confirmations, 1); -} - -#[test] -fn an_explicit_anchor_is_retained_even_when_originally_in_place() { - let directory = tempfile::tempdir().unwrap(); - let cache = Replica::create(directory.path().join("pages.sqlite"), SOURCE).unwrap(); - let pages = order(SOURCE); - let id = cache - .pages( - SOURCE, - &[ - PageEdit::move_to(pages[6].0, Some(pages[7].0), 1).unwrap(), - PageEdit::set_level(pages[4].0, 3).unwrap(), - ], - ) - .unwrap() - .unwrap(); - let mut server = Server::new(SOURCE); - PreparedEdit::pages( - SOURCE, - &[PageEdit::move_to(pages[6].0, Some(pages[0].0), 1).unwrap()], - ) - .unwrap() - .commit(&mut server) - .unwrap(); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((id, EditStatus::Conflict(ConflictKind::StructureChanged))) - ); - assert_eq!(server.publications, 0); -} - -#[test] -fn convergent_page_indentation_requires_confirmation_without_republishing() { - let directory = tempfile::tempdir().unwrap(); - let cache = Replica::create(directory.path().join("pages.sqlite"), SOURCE).unwrap(); - let sid = order(SOURCE)[4].0; - let id = cache - .pages(SOURCE, &[PageEdit::set_level(sid, 1).unwrap()]) - .unwrap() - .unwrap(); - let mut server = Server::new(SOURCE); - PreparedEdit::pages(SOURCE, &[PageEdit::set_level(sid, 1).unwrap()]) - .unwrap() - .commit(&mut server) - .unwrap(); - server.fault = Fault::Confirm; - assert!(cache.sync_once(&mut server).is_err()); - assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); - assert_eq!(server.publications, 0); - assert!(matches!(cache.sync_once(&mut server).unwrap(), - Some((published, EditStatus::Published { .. })) if published == id)); - assert_eq!(server.publications, 0); - assert_eq!(server.confirmations, 2); -} - -#[test] -fn schema_ten_migration_preserves_multi_space_uncertainty() { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("pages.sqlite"); - let cache = Replica::create(&path, SOURCE).unwrap(); - let page = onestore::PageCreation::new(None, Some("Migration"), "Author").unwrap(); - let id = cache.create_page(SOURCE, &page).unwrap().unwrap(); - let mut server = Server::new(SOURCE); - server.fault = Fault::UnknownBefore; - assert!(cache.sync_once(&mut server).is_err()); - let local = cache.snapshot().unwrap(); - let queue = cache.pending().unwrap(); - let status = cache.status(id).unwrap(); - drop(cache); - let db = rusqlite::Connection::open(&path).unwrap(); - db.pragma_update(None, "user_version", 10).unwrap(); - let proofs: String = db - .query_row("SELECT revisions FROM attempt", [], |r| r.get(0)) - .unwrap(); - drop(db); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(cache.pending().unwrap(), queue); - assert_eq!(cache.status(id).unwrap(), status); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((id, status.unwrap())) - ); - assert_eq!(server.publications, 1); - drop(cache); - let db = rusqlite::Connection::open(&path).unwrap(); - assert_eq!( - db.query_row("SELECT revisions FROM attempt", [], |r| r - .get::<_, String>(0)) - .unwrap(), - proofs - ); - assert_eq!( - db.pragma_query_value(None, "user_version", |r| r.get::<_, u32>(0)) - .unwrap(), - 11 - ); -} - -#[test] -fn native_page_changes_reconcile_with_atomic_offline_batches_and_review() { - let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../../corpus/page-lifecycle/movement"); - let provenance: serde_json::Value = - serde_json::from_slice(&std::fs::read(root.join("provenance.json")).unwrap()).unwrap(); - let phases = provenance["cold"].as_object().unwrap(); - assert_eq!(phases.len(), 9); - let original = order(SOURCE); - let original_text = texts(SOURCE); - let (&(text_space, text_object), body) = original_text - .iter() - .find(|(_, text)| text.starts_with("Body parent")) - .unwrap(); - let mut expected_text = original_text.clone(); - expected_text.insert((text_space, text_object), format!("Offline {body}")); - let mut counts = [0; 2]; - for mode in ["indent", "group"] { - for phase in phases.keys() { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("pages.sqlite"); - let cache = Replica::create(&path, SOURCE).unwrap(); - let edits = if mode == "indent" { - vec![PageEdit::set_level(original[4].0, 3).unwrap()] - } else { - original[3..6] - .iter() - .map(|(sid, level)| PageEdit::move_to(*sid, None, *level).unwrap()) - .collect() - }; - let id = cache.pages(SOURCE, &edits).unwrap().unwrap(); - let text_id = cache - .edit_text( - &cache.snapshot().unwrap(), - text_space, - text_object, - 0..0, - "Offline ", - ) - .unwrap() - .unwrap(); - let native = std::fs::read(root.join(phase).join("notebook/Lifecycle.one")).unwrap(); - let mut server = Server::new(&native); - let local = cache.snapshot().unwrap(); - let expected_conflict = if mode == "indent" { - matches!( - phase.as_str(), - "04-subpage-move" | "07-promoted-root" | "08-collapsed-group-move" - ) - } else { - !matches!( - phase.as_str(), - "02-promoted-parent" | "03-selected-group-move" | "06-promoted-level-two" - ) - }; - let first = cache.sync_once(&mut server).unwrap().unwrap(); - assert_eq!(first.0, id); - let mut reviewed = edits.clone(); - if expected_conflict { - assert_eq!( - first.1, - EditStatus::Conflict(ConflictKind::StructureChanged), - "{mode}:{phase}" - ); - assert_eq!(server.publications, 0); - assert_eq!(cache.snapshot().unwrap(), local); - if mode == "indent" && phase == "08-collapsed-group-move" { - reviewed[0] = reviewed[0].reposition(PagePosition::Keep, 1).unwrap(); - } - cache - .rebase_pages_conflict(id, &local, &native, &reviewed) - .unwrap(); - assert!(matches!(cache.sync_once(&mut server).unwrap(), - Some((published, EditStatus::Published { .. })) if published == id)); - } else { - assert!( - matches!(first.1, EditStatus::Published { .. }), - "{mode}:{phase}" - ); - } - counts[usize::from(expected_conflict)] += 1; - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert!(matches!(cache.sync_once(&mut server).unwrap(), - Some((published, EditStatus::Published { .. })) if published == text_id)); - assert!(cache.pending().unwrap().is_empty()); - let mut expected = order(&native); - for edit in &reviewed { - let at = expected.iter().position(|p| p.0 == edit.space()).unwrap(); - expected[at].1 = edit.level(); - if let PagePosition::Before(before) = edit.position() { - let page = expected.remove(at); - let at = before.map_or(expected.len(), |sid| { - expected.iter().position(|p| p.0 == sid).unwrap() - }); - expected.insert(at, page); - } - } - assert_eq!(order(&server.durable), expected, "{mode}:{phase}"); - assert_eq!(texts(&server.durable), expected_text, "{mode}:{phase}"); - assert_eq!( - server.publications, - 1 + usize::from(expected != order(&native)) - ); - if let Some(output) = std::env::var_os("ONESTORE_OFFLINE_PAGE_EDIT_OUTPUT") { - let output = std::path::Path::new(&output); - assert!(output.is_absolute()); - let path = output.join(format!("{mode}-{phase}")); - std::fs::create_dir_all(&path).unwrap(); - std::fs::write(path.join("Lifecycle.one"), &server.durable).unwrap(); - std::fs::write( - path.join("manifest.json"), - serde_json::to_vec_pretty( - &serde_json::json!({"mode": mode, "phase": phase, "original": edits, - "reviewed": reviewed, "conflict": expected_conflict, - "publications": server.publications, "confirmations": server.confirmations, - "page_receipt": format!("{:?}", cache.status(id).unwrap()), - "text_receipt": format!("{:?}", cache.status(text_id).unwrap())}), - ) - .unwrap(), - ) - .unwrap(); - } - } - } - assert_eq!(counts, [9, 9]); -} - -#[test] -fn twelve_disjoint_page_batches_merge_with_dependent_bodies_without_review() { - let mut source = onestore::create_section("pages.one", "Sentinel", "Author").unwrap(); - let mut created = Vec::new(); - for _ in 0..36 { - let page = onestore::PageCreation::new(None, Some("Same title"), "Author").unwrap(); - source = PreparedEdit::create_page(&source, &page) - .unwrap() - .as_bytes() - .to_vec(); - created.push(page); - } - let mut expected = vec![order(&source)[0]]; - let mut expected_text = texts(&source); - let directory = tempfile::tempdir().unwrap(); - let mut queues = Vec::new(); - for (actor, group) in created.chunks_exact(3).enumerate() { - let path = directory.path().join(format!("{actor}.sqlite")); - let cache = Replica::create(&path, &source).unwrap(); - let edits = [ - PageEdit::move_to(group[1].space(), Some(group[0].space()), 1).unwrap(), - PageEdit::set_level(group[2].space(), 2).unwrap(), - ]; - cache.pages(&source, &edits).unwrap().unwrap(); - let body = format!("Actor {actor} 🦀 é"); - let insertion = - onestore::Insertion::outline(group[1].object(), 36.0, 36.0, &body, "Author").unwrap(); - cache - .insert(&cache.snapshot().unwrap(), group[1].space(), &insertion) - .unwrap() - .unwrap(); - expected.extend([ - (group[1].space(), 1), - (group[0].space(), 1), - (group[2].space(), 2), - ]); - expected_text.insert((group[1].space(), insertion.text_object()), body); - queues.push((path, cache.pending().unwrap())); - } - let mut server = Server::new(&source); - for (path, queue) in &queues { - let cache = Replica::open(path).unwrap(); - assert_eq!(cache.pending().unwrap(), *queue); - for edit in queue { - assert!(matches!(cache.sync_once(&mut server).unwrap(), - Some((published, EditStatus::Published { .. })) if published == edit.id)); - } - assert!(cache.pending().unwrap().is_empty()); - } - assert_eq!(server.publications, 24); - assert_eq!(order(&server.durable), expected); - assert_eq!(texts(&server.durable), expected_text); - for (path, queue) in &queues { - let cache = Replica::open(path).unwrap(); - assert_eq!(cache.sync_once(&mut server).unwrap(), None); - assert_eq!(order(&cache.snapshot().unwrap()), expected); - for edit in queue { - assert!(matches!( - cache.status(edit.id).unwrap(), - Some(EditStatus::Published { .. }) - )); - } - } - if let Some(output) = std::env::var_os("ONESTORE_OFFLINE_PAGE_CLIENT_OUTPUT") { - let output = std::path::Path::new(&output); - assert!(output.is_absolute()); - std::fs::create_dir_all(output).unwrap(); - std::fs::write(output.join("pages.one"), &server.durable).unwrap(); - let operations: Vec<_> = queues - .iter() - .map(|(_, queue)| { - queue - .iter() - .map(|edit| { - serde_json::json!({"id": edit.id, "space": edit.space, - "operation": edit.operation}) - }) - .collect::>() - }) - .collect(); - std::fs::write( - output.join("manifest.json"), - serde_json::to_vec_pretty( - &serde_json::json!({"operations": operations, "publications": server.publications, - "expected_order": expected, "pages": 37}), - ) - .unwrap(), - ) - .unwrap(); - } -} diff --git a/crates/onestore-offline/tests/sync/tree.rs b/crates/onestore-offline/tests/sync/tree.rs deleted file mode 100644 index b55b304ca3921551c45b1d17173faaed27c1fd60..0000000000000000000000000000000000000000 --- a/crates/onestore-offline/tests/sync/tree.rs +++ /dev/null @@ -1,821 +0,0 @@ -use super::*; -use onestore::{Insertion, TreeEdit}; - -#[test] -fn twelve_offline_clients_reconcile_tree_text_and_interrupted_publication() { - let mut random = 1940_u64; - for _ in 0..60 { - let input: Vec<_> = (0..384) - .map(|_| { - random ^= random << 13; - random ^= random >> 7; - random ^= random << 17; - random as u8 - }) - .collect(); - tree_schedule::run(&input); - } -} - -fn fixture() -> (Vec, ExGuid, ExGuid, [ExGuid; 4], [ExGuid; 4]) { - let (mut source, sid, outline, first, text) = super::outline::fixture(); - let mut paragraphs = [first; 4]; - let mut texts = [text; 4]; - for at in 1..4 { - let insertion = - Insertion::paragraph(outline, None, &format!("Sibling {at}"), "Author").unwrap(); - source = PreparedEdit::insert(&source, sid, &insertion) - .unwrap() - .as_bytes() - .to_vec(); - paragraphs[at] = insertion.object(); - texts[at] = insertion.text_object(); - } - (source, sid, outline, paragraphs, texts) -} - -fn children(source: &[u8], sid: ExGuid, object: ExGuid) -> Vec { - let store = Store::parse(source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - index.validate_current().unwrap(); - let document = Document::parse(&index).unwrap(); - let space = &document.spaces[&sid]; - space.revisions[&space.contexts[&ExGuid::default()]].nodes[&object] - .children - .clone() -} - -#[test] -fn move_preserves_remote_content_and_dependent_edits_through_reopen() { - let (source, sid, outline, paragraphs, texts) = fixture(); - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("cache.sqlite"); - let cache = Replica::create(&path, &source).unwrap(); - let intent = TreeEdit::move_to(paragraphs[0], outline, None, "Offline").unwrap(); - let id = cache.tree(&source, sid, &intent).unwrap().unwrap(); - let dependent = cache - .edit_text(&cache.snapshot().unwrap(), sid, texts[0], 0..0, "Local ") - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let queue = cache.pending().unwrap(); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.pending().unwrap(), queue); - assert_eq!(cache.snapshot().unwrap(), local); - let edited = PreparedEdit::format( - &source, - sid, - texts[0], - 0..8, - &[onestore::TextAttribute::Bold(true)], - ) - .unwrap(); - let descendant = - Insertion::paragraph(paragraphs[0], None, "New remote child", "Remote").unwrap(); - let remote = PreparedEdit::insert(edited.as_bytes(), sid, &descendant).unwrap(); - let mut server = Server::new(remote.as_bytes()); - for expected in [id, dependent] { - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == expected) - ); - } - assert_eq!( - children(&server.durable, sid, outline), - [paragraphs[1], paragraphs[2], paragraphs[3], paragraphs[0]] - ); - assert_eq!( - children(&server.durable, sid, paragraphs[0]), - [descendant.object()] - ); - let node = super::outline::node(&server.durable, sid, texts[0]); - assert_eq!(node["kind"]["text"], "Local Original 🦀 é"); - let store = Store::parse(&server.durable).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let space = &document.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - assert_eq!(view.text_runs(texts[0]).unwrap()[0].format.bold, Some(true)); - assert!(cache.pending().unwrap().is_empty()); - assert_eq!(server.publications, 2); -} - -#[test] -fn queued_format_split_and_delete_reconcile_equivalent_immutable_style_identities() { - let (source, sid, _, _, texts) = fixture(); - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("cache.sqlite"); - let cache = Replica::create(&path, &source).unwrap(); - cache - .format( - &source, - sid, - texts[0], - 0..8, - &[onestore::TextAttribute::Bold(true)], - ) - .unwrap(); - let split = onestore::ParagraphSplit::new(texts[0], 4, "Author").unwrap(); - cache - .split(&cache.snapshot().unwrap(), sid, &split) - .unwrap(); - cache - .tree( - &cache.snapshot().unwrap(), - sid, - &TreeEdit::delete(split.object(), "Author").unwrap(), - ) - .unwrap(); - cache - .edit_text(&cache.snapshot().unwrap(), sid, texts[0], 0..0, "Local ") - .unwrap(); - let queue = cache.pending().unwrap(); - drop(cache); - let mut server = Server::new(&source); - for intent in queue { - let cache = Replica::open(&path).unwrap(); - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((id, EditStatus::Published { .. })) if id == intent.id) - ); - } - assert_eq!( - super::outline::node(&server.durable, sid, texts[0])["kind"]["text"], - "Local Orig" - ); -} - -#[test] -fn native_empty_child_list_normalization_preserves_deletion_and_its_dependents() { - let source = include_bytes!("../../../../corpus/outline-edit/empty-children/before.one"); - let remote = include_bytes!("../../../../corpus/outline-edit/empty-children/remote.one"); - let (sid, intent): (ExGuid, TreeEdit) = serde_json::from_str(include_str!( - "../../../../corpus/outline-edit/empty-children/intent.json" - )) - .unwrap(); - let store = Store::parse(source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let space = &document.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let tail = view.nodes[&intent.object()].content[0]; - let (outline_id, outline) = view - .nodes - .iter() - .find(|(_, node)| node.children.contains(&intent.object())) - .unwrap(); - let left = view.nodes[&outline.children[0]].content[0]; - for changed in [false, true] { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("cache.sqlite"); - let cache = Replica::create(&path, source).unwrap(); - let deletion = cache.tree(source, sid, &intent).unwrap().unwrap(); - let dependent = cache - .edit_text(&cache.snapshot().unwrap(), sid, left, 0..0, "Local ") - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - drop(cache); - let cache = Replica::open(&path).unwrap(); - let mut server = if changed { - let edited = PreparedEdit::text(remote, sid, tail, 0..2, "Changed").unwrap(); - Server::new(edited.as_bytes()) - } else { - Server::new(remote) - }; - if changed { - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((deletion, EditStatus::Conflict(ConflictKind::ContentChanged))) - ); - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(cache.pending().unwrap().len(), 2); - assert_eq!(server.publications, 0); - } else { - for expected in [deletion, dependent] { - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((id, EditStatus::Published { .. })) if id == expected) - ); - } - assert!(cache.pending().unwrap().is_empty()); - assert_eq!(server.publications, 2); - assert!(!children(&server.durable, sid, *outline_id).contains(&intent.object())); - let node = super::outline::node(&server.durable, sid, left); - assert!(node["kind"]["text"].as_str().unwrap().starts_with("Local ")); - } - } -} - -#[test] -fn deletion_requires_review_of_remote_content_and_preserves_later_work() { - let (source, sid, outline, paragraphs, texts) = fixture(); - let child = Insertion::paragraph(paragraphs[0], None, "Descendant", "Author").unwrap(); - let source = PreparedEdit::insert(&source, sid, &child) - .unwrap() - .as_bytes() - .to_vec(); - for remote in [ - PreparedEdit::text(&source, sid, texts[0], 0..0, "Remote ") - .unwrap() - .as_bytes() - .to_vec(), - PreparedEdit::text(&source, sid, child.text_object(), 0..0, "Remote ") - .unwrap() - .as_bytes() - .to_vec(), - PreparedEdit::format( - &source, - sid, - child.text_object(), - 0..10, - &[onestore::TextAttribute::Italic(true)], - ) - .unwrap() - .as_bytes() - .to_vec(), - PreparedEdit::insert( - &source, - sid, - &Insertion::paragraph(paragraphs[0], None, "New child", "Remote").unwrap(), - ) - .unwrap() - .as_bytes() - .to_vec(), - ] { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("cache.sqlite"); - let cache = Replica::create(&path, &source).unwrap(); - let intent = TreeEdit::delete(paragraphs[0], "Offline").unwrap(); - let id = cache.tree(&source, sid, &intent).unwrap().unwrap(); - let dependent = cache - .edit_text(&cache.snapshot().unwrap(), sid, texts[1], 0..0, "Local ") - .unwrap() - .unwrap(); - let queue = cache.pending().unwrap(); - let local = cache.snapshot().unwrap(); - let mut server = Server::new(&remote); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))) - ); - assert_eq!(server.publications, 0); - assert_eq!(cache.pending().unwrap(), queue); - assert_eq!(cache.snapshot().unwrap(), local); - assert!(cache.rebase_tree_conflict(id, &source, &remote).is_err()); - assert!(cache.rebase_tree_conflict(id, &local, &source).is_err()); - assert!( - cache - .rebase_tree_conflict(dependent, &local, &remote) - .is_err() - ); - assert!(cache.rebase_conflict(id, &local, &remote, 0..0).is_err()); - let archive = directory.path().join("recovery.sqlite"); - cache.export_recovery(&archive).unwrap(); - let recovery = onestore_offline::Recovery::open(&archive).unwrap(); - assert_eq!(recovery.pending().unwrap(), queue); - assert_eq!(recovery.status(id).unwrap(), cache.status(id).unwrap()); - assert!(recovery.snapshot().unwrap() == local); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!( - cache.status(id).unwrap(), - Some(EditStatus::Conflict(ConflictKind::ContentChanged)) - ); - cache.rebase_tree_conflict(id, &local, &remote).unwrap(); - assert_eq!(cache.pending().unwrap()[1], queue[1]); - assert_eq!(cache.snapshot().unwrap(), local); - for expected in [id, dependent] { - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == expected) - ); - } - assert_eq!(children(&server.durable, sid, outline), paragraphs[1..]); - assert_eq!( - super::outline::node(&server.durable, sid, texts[1])["kind"]["text"], - "Local Sibling 1" - ); - } -} - -#[test] -fn sibling_and_ancestry_changes_have_explicit_merge_or_conflict() { - let (source, sid, outline, p, texts) = fixture(); - let insertion = Insertion::paragraph(outline, Some(p[0]), "New sibling", "Remote").unwrap(); - let cases = [ - ( - PreparedEdit::insert(&source, sid, &insertion) - .unwrap() - .as_bytes() - .to_vec(), - None, - ), - ( - PreparedEdit::tree(&source, sid, &TreeEdit::delete(p[1], "Remote").unwrap()) - .unwrap() - .as_bytes() - .to_vec(), - None, - ), - ( - PreparedEdit::tree( - &source, - sid, - &TreeEdit::move_to(p[1], outline, None, "Remote").unwrap(), - ) - .unwrap() - .as_bytes() - .to_vec(), - None, - ), - ( - PreparedEdit::tree( - &source, - sid, - &TreeEdit::move_to(p[0], outline, Some(p[2]), "Remote").unwrap(), - ) - .unwrap() - .as_bytes() - .to_vec(), - Some(ConflictKind::StructureChanged), - ), - ( - PreparedEdit::tree( - &source, - sid, - &TreeEdit::move_to(p[0], p[1], None, "Remote").unwrap(), - ) - .unwrap() - .as_bytes() - .to_vec(), - Some(ConflictKind::StructureChanged), - ), - ( - PreparedEdit::tree(&source, sid, &TreeEdit::delete(p[0], "Remote").unwrap()) - .unwrap() - .as_bytes() - .to_vec(), - Some(ConflictKind::TargetUnavailable), - ), - ]; - for (remote, expected) in cases { - let directory = tempfile::tempdir().unwrap(); - let cache = Replica::create(directory.path().join("cache.sqlite"), &source).unwrap(); - let intent = TreeEdit::move_to(p[0], outline, None, "Offline").unwrap(); - let id = cache.tree(&source, sid, &intent).unwrap().unwrap(); - let mut server = Server::new(&remote); - let result = cache.sync_once(&mut server).unwrap().unwrap(); - assert_eq!(result.0, id); - if let Some(kind) = expected { - assert_eq!(result.1, EditStatus::Conflict(kind)); - assert_eq!(server.publications, 0); - } else { - assert!(matches!(result.1, EditStatus::Published { .. })); - assert_eq!(children(&server.durable, sid, outline).last(), Some(&p[0])); - assert_eq!( - super::outline::node(&server.durable, sid, texts[0])["kind"]["text"], - "Original 🦀 é" - ); - } - } -} - -#[test] -fn moved_destination_and_missing_anchor_retain_conflicts_but_unrelated_text_does_not() { - let (source, sid, outline, p, texts) = fixture(); - let child = Insertion::paragraph(p[1], None, "Anchor", "Author").unwrap(); - let source = PreparedEdit::insert(&source, sid, &child) - .unwrap() - .as_bytes() - .to_vec(); - let intent = TreeEdit::move_to(p[0], p[1], Some(child.object()), "Offline").unwrap(); - let cases = [ - ( - PreparedEdit::tree( - &source, - sid, - &TreeEdit::move_to(p[1], p[2], None, "Remote").unwrap(), - ) - .unwrap() - .as_bytes() - .to_vec(), - ConflictKind::StructureChanged, - ), - ( - PreparedEdit::tree( - &source, - sid, - &TreeEdit::move_to(child.object(), outline, None, "Remote").unwrap(), - ) - .unwrap() - .as_bytes() - .to_vec(), - ConflictKind::UnsupportedEdit, - ), - ]; - for (remote, kind) in cases { - let directory = tempfile::tempdir().unwrap(); - let cache = Replica::create(directory.path().join("cache.sqlite"), &source).unwrap(); - let id = cache.tree(&source, sid, &intent).unwrap().unwrap(); - let mut server = Server::new(&remote); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((id, EditStatus::Conflict(kind))) - ); - assert_eq!(server.publications, 0); - } - let directory = tempfile::tempdir().unwrap(); - let cache = Replica::create(directory.path().join("cache.sqlite"), &source).unwrap(); - let id = cache - .tree(&source, sid, &TreeEdit::delete(p[0], "Offline").unwrap()) - .unwrap() - .unwrap(); - let remote = PreparedEdit::text(&source, sid, texts[1], 0..0, "Remote ").unwrap(); - let mut server = Server::new(remote.as_bytes()); - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == id) - ); - assert_eq!( - super::outline::node(&server.durable, sid, texts[1])["kind"]["text"], - "Remote Sibling 1" - ); -} - -#[test] -fn unknown_tree_attempts_require_revision_evidence_even_when_effect_is_visible() { - let (source, sid, outline, p, _) = fixture(); - for deletion in [false, true] { - for fault in [ - Fault::UnknownBefore, - Fault::UnknownAfter, - Fault::Before, - Fault::Committed, - ] { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("cache.sqlite"); - let cache = Replica::create(&path, &source).unwrap(); - let intent = if deletion { - TreeEdit::delete(p[0], "Offline") - } else { - TreeEdit::move_to(p[0], outline, None, "Offline") - } - .unwrap(); - let id = cache.tree(&source, sid, &intent).unwrap().unwrap(); - let mut server = Server::new(&source); - server.fault = fault; - assert!(cache.sync_once(&mut server).is_err()); - let local = cache.snapshot().unwrap(); - let before = cache.status(id).unwrap(); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.status(id).unwrap(), before); - if matches!(fault, Fault::UnknownBefore) { - let independent = if deletion { - TreeEdit::delete(p[0], "Other") - } else { - TreeEdit::move_to(p[0], outline, None, "Other") - } - .unwrap(); - server = Server::new( - PreparedEdit::tree(&source, sid, &independent) - .unwrap() - .as_bytes(), - ); - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::AwaitingConfirmation { .. })) if n == id) - ); - assert!( - cache - .rebase_tree_conflict(id, &local, &server.visible) - .is_err() - ); - assert_eq!(server.publications, 0); - assert_eq!(cache.snapshot().unwrap(), local); - } else if matches!(fault, Fault::Committed) { - assert!(matches!( - cache.status(id).unwrap(), - Some(EditStatus::Published { .. }) - )); - } else { - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == id) - ); - assert_eq!( - server.publications, - if matches!(fault, Fault::Before) { 2 } else { 1 } - ); - let saved = cache.snapshot().unwrap(); - assert!(saved[..212] == server.durable[..212]); - assert!(saved[252..] == server.durable[252..]); - assert_eq!( - children(&saved, sid, outline), - children(&server.durable, sid, outline) - ); - } - } - } -} - -#[test] -fn independently_satisfied_move_confirms_without_republication() { - let (source, sid, outline, p, _) = fixture(); - let directory = tempfile::tempdir().unwrap(); - let cache = Replica::create(directory.path().join("cache.sqlite"), &source).unwrap(); - let id = cache - .tree( - &source, - sid, - &TreeEdit::move_to(p[0], outline, None, "Offline").unwrap(), - ) - .unwrap() - .unwrap(); - let remote = PreparedEdit::tree( - &source, - sid, - &TreeEdit::move_to(p[0], outline, None, "Remote").unwrap(), - ) - .unwrap(); - let mut server = Server::new(remote.as_bytes()); - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == id) - ); - assert_eq!(server.publications, 0); - assert_eq!(server.confirmations, 1); -} - -#[test] -fn emptied_cell_replacement_is_durable_and_cannot_be_silently_omitted_on_replay() { - let source = - include_bytes!("../../../../corpus/outline-edit/tree/before/notebook/synthetic.one"); - let store = Store::parse(source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let (sid, page) = document.pages().unwrap().into_iter().find(|(sid, _)| { - let space = &document.spaces[sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - matches!(&view.nodes[&view.roots[&2]].kind, Kind::Metadata { title: Some(title), .. } if title == "Delete sole cell paragraph") - }).unwrap(); - let space = &document.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let mut pending = vec![page]; - let mut selected = None; - let mut other = None; - while let Some(id) = pending.pop() { - let node = &view.nodes[&id]; - pending.extend(&node.children); - pending.extend(&node.content); - if matches!(node.kind, Kind::Cell { .. }) { - let paragraph = node.children[0]; - let text = view.nodes[¶graph].content[0]; - if matches!(&view.nodes[&text].kind, Kind::RichText { text, .. } if text.starts_with("Target ")) - { - selected = Some((id, paragraph)); - } else { - other = Some((id, text)); - } - } - } - let (cell, target) = selected.unwrap(); - let (other_cell, other_text) = other.unwrap(); - for move_out in [false, true] { - for competing_child in [false, true] { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("cell.sqlite"); - let cache = Replica::create(&path, source).unwrap(); - let intent = if move_out { - TreeEdit::move_to(target, other_cell, None, "Offline") - } else { - TreeEdit::delete(target, "Offline") - } - .unwrap(); - let id = cache.tree(source, sid, &intent).unwrap().unwrap(); - let local = cache.snapshot().unwrap(); - let replacement = children(&local, sid, cell); - assert_eq!(replacement.len(), 1); - assert_ne!(replacement[0], target); - let replacement_text: ExGuid = - super::outline::node(&local, sid, replacement[0])["content"][0] - .as_str() - .unwrap() - .parse() - .unwrap(); - let dependent = cache - .edit_text(&local, sid, replacement_text, 0..0, "Local replacement") - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let queue = cache.pending().unwrap(); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.pending().unwrap(), queue); - assert_eq!(cache.snapshot().unwrap(), local); - let remote = if competing_child { - PreparedEdit::insert( - source, - sid, - &Insertion::paragraph(cell, None, "Remote sibling", "Remote").unwrap(), - ) - .unwrap() - .as_bytes() - .to_vec() - } else { - PreparedEdit::text(source, sid, other_text, 0..0, "Remote ") - .unwrap() - .as_bytes() - .to_vec() - }; - let mut server = Server::new(&remote); - if competing_child { - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((id, EditStatus::Conflict(ConflictKind::StructureChanged))) - ); - assert_eq!(server.publications, 0); - assert!(cache.rebase_tree_conflict(id, &local, &remote).is_err()); - assert_eq!(cache.pending().unwrap(), queue); - assert_eq!(cache.snapshot().unwrap(), local); - } else { - for expected in [id, dependent] { - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == expected) - ); - } - assert_eq!(children(&server.durable, sid, cell), replacement); - assert_eq!( - super::outline::node(&server.durable, sid, replacement_text)["kind"]["text"], - "Local replacement" - ); - assert_eq!( - super::outline::node(&server.durable, sid, other_text)["kind"]["text"], - "Remote Other cell" - ); - if let Some(output) = std::env::var_os("ONESTORE_OFFLINE_TREE_OUTPUT") { - let output = std::path::PathBuf::from(output) - .join(if move_out { "cell-move" } else { "cell-delete" }) - .join("candidate"); - std::fs::create_dir_all(&output).unwrap(); - std::fs::write(output.join("synthetic.one"), &server.durable).unwrap(); - } - } - } - } -} - -#[test] -fn native_tree_and_layout_changes_reconcile_without_discarding_unreviewed_content() { - let source = include_bytes!("../../../../corpus/outline-edit/before/notebook/synthetic.one"); - let native = include_bytes!("../../../../corpus/outline-edit/after/notebook/synthetic.one"); - let store = Store::parse(source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let mut server = Server::new(native); - let mut counts = [0; 3]; - let mut records = Vec::new(); - for (sid, page) in document.pages().unwrap() { - let space = &document.spaces[&sid]; - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - let Kind::Metadata { - title: Some(name), .. - } = &view.nodes[&view.roots[&2]].kind - else { - continue; - }; - if name == "Unicode rich text" { - continue; - } - let outlines: Vec<_> = view.nodes[&page] - .children - .iter() - .copied() - .filter(|id| matches!(view.nodes[id].kind, Kind::Outline { .. })) - .collect(); - if outlines.len() != 2 { - continue; - } - let outline = outlines[0]; - let other = view.nodes[&outlines[1]].children[0]; - let dependent_text = view.nodes[&other].content[0]; - let mut pending = vec![outline]; - let mut paragraphs = std::collections::BTreeMap::new(); - while let Some(id) = pending.pop() { - let node = &view.nodes[&id]; - pending.extend(&node.children); - if let Some(text) = node.content.first() - && let Kind::RichText { text, .. } = &view.nodes[text].kind - { - paragraphs.insert(text.as_str(), id); - } - } - let target = *paragraphs - .iter() - .find(|(text, _)| text.starts_with("Target ")) - .unwrap() - .1; - let (intent, conflict) = match name.as_str() { - "Move leaf down" | "Move subtree down" => { - (TreeEdit::move_to(target, outline, None, "Offline"), None) - } - "Move subtree up" => ( - TreeEdit::move_to(target, outline, Some(paragraphs["Anchor"]), "Offline"), - None, - ), - "Indent subtree" | "Outdent subtree" => ( - TreeEdit::delete(target, "Offline"), - Some(ConflictKind::StructureChanged), - ), - "Collapse subtree" | "Expand subtree" => ( - TreeEdit::delete(target, "Offline"), - Some(ConflictKind::ContentChanged), - ), - "Delete leaf" | "Delete subtree" | "Delete only paragraph" => ( - TreeEdit::delete(target, "Offline"), - Some(ConflictKind::TargetUnavailable), - ), - "Delete outline" => ( - TreeEdit::delete(outline, "Offline"), - Some(ConflictKind::TargetUnavailable), - ), - "Move outline" | "Resize outline" | "Automatic outline size" => ( - TreeEdit::delete(outline, "Offline"), - Some(ConflictKind::ContentChanged), - ), - _ => panic!("{name}"), - }; - let intent = intent.unwrap(); - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("native.sqlite"); - let cache = Replica::create(&path, source).unwrap(); - let id = cache.tree(source, sid, &intent).unwrap().unwrap(); - let dependent = cache - .edit_text( - &cache.snapshot().unwrap(), - sid, - dependent_text, - 0..0, - "Offline ", - ) - .unwrap() - .unwrap(); - let local = cache.snapshot().unwrap(); - let queue = cache.pending().unwrap(); - let old_publications = server.publications; - let (actual, status) = cache.sync_once(&mut server).unwrap().unwrap(); - assert_eq!(actual, id); - if let Some(kind) = conflict { - assert_eq!(status, EditStatus::Conflict(kind), "{name}"); - assert_eq!(server.publications, old_publications); - assert_eq!(cache.pending().unwrap(), queue); - assert!(cache.snapshot().unwrap() == local); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.status(id).unwrap(), Some(status)); - if kind == ConflictKind::TargetUnavailable { - counts[2] += 1; - assert!( - cache - .rebase_tree_conflict(id, &local, &server.visible) - .is_err() - ); - records.push(serde_json::json!({"page": name, "space": sid, "result": "retained"})); - continue; - } - counts[1] += 1; - cache - .rebase_tree_conflict(id, &local, &server.visible) - .unwrap(); - assert_eq!(cache.pending().unwrap()[1], queue[1]); - for expected in [id, dependent] { - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == expected) - ); - } - } else { - counts[0] += 1; - assert!(matches!(status, EditStatus::Published { .. }), "{name}"); - assert_eq!(server.publications, old_publications); - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == dependent) - ); - } - assert_eq!( - super::outline::node(&server.durable, sid, dependent_text)["kind"]["text"], - format!( - "Offline {}", - match &view.nodes[&dependent_text].kind { - Kind::RichText { text, .. } => text, - _ => panic!(), - } - ) - ); - records.push(serde_json::json!({"page": name, "space": sid, "result": if conflict.is_some() { "reviewed" } else { "converged" }})); - } - assert_eq!(counts, [3, 7, 4]); - if let Some(output) = std::env::var_os("ONESTORE_OFFLINE_TREE_OUTPUT") { - let output = std::path::PathBuf::from(output); - std::fs::create_dir_all(output.join("candidate")).unwrap(); - std::fs::write(output.join("candidate/synthetic.one"), &server.durable).unwrap(); - std::fs::write( - output.join("manifest.json"), - serde_json::to_vec_pretty(&records).unwrap(), - ) - .unwrap(); - } -} diff --git a/crates/onestore-smb/Cargo.toml b/crates/onestore-smb/Cargo.toml deleted file mode 100644 index 65a3eb6a84d2f313991f0ce6247ee7d1399d2c0c..0000000000000000000000000000000000000000 --- a/crates/onestore-smb/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "onestore-smb" -version = "0.1.0" -edition = "2024" -publish = false - -[dependencies] -onestore = { path = "../onestore" } -smb2 = "=0.21.0" -tokio = { version = "1", features = ["rt-multi-thread", "time"] } - -[dev-dependencies] -serde_json = "1" diff --git a/crates/onestore-smb/README.md b/crates/onestore-smb/README.md deleted file mode 100644 index f13eb815e9f14c23e83fa10a1e3a17b74b41a642..0000000000000000000000000000000000000000 --- a/crates/onestore-smb/README.md +++ /dev/null @@ -1,63 +0,0 @@ -# onestore-smb - -Optional blocking SMB access for OneNote sections and table-of-contents files. -The core `onestore` crate remains independent of network runtimes. This is an -experimental Rust API with native interoperability evidence in the repository's -[Milestone 9](../../evidence/MILESTONE9.md). - -```no_run -use onestore_smb::{Client, Credentials}; -use std::time::Duration; - -let client = Client::connect( - "server:445", - "notes", - Credentials { username: "user", password: "password", domain: "" }, - Duration::from_secs(5), -)?; -let snapshot = client.read("Personal/Video.one", 64 * 1024 * 1024)?; -let store = onestore::Store::parse(&snapshot)?; -let revisions = onestore::RevisionIndex::parse(&store)?; -let document = onestore::document::Document::parse(&revisions)?; -# Ok::<(), Box>(()) -``` - -Paths are relative to the share. The read limit bounds the complete physical -snapshot. Call from a background thread outside a Tokio runtime. Use identities -from the document and the same snapshot with `Client::commit_text` or -`Client::commit_property_bytes`; their errors retain `onestore::CommitState`. -`PreparedEdit::{text,insert,format}` separate preparation from I/O: inspect the immutable image -and persist the intended revision identity before `Client::commit_prepared`. -`Client::confirm_snapshot` compares and flushes an observed image, then refreshes -its header version metadata without adding a revision. The caller must first -establish which intents that image contains and reread before another commit. - -Readers use shared native guards while writers publish under native write-open -and byte-lock exclusion. Maintenance is excluded during each operation; pathname -identity is checked after acquiring the guards. Connection loss retires the -client. Reconnect for subsequent operations, and reconcile an `Unknown` edit -before retrying it. The transport does not automatically replay requests. - -`Client::read_dir(path, entry_limit)` enumerates a directory, including the share -root with an empty path. It follows every response page and returns no partial -list on interruption, entry-limit overflow or close failure. Entries retain exact -Unicode names, observed sizes and MS-FSCC attributes, including directory/reparse -flags. Concurrent directory changes are not an atomic snapshot; repeated names -are rejected with `ResourceBusy`. Notebook identities come from the files, not -directory names or sizes. Missing paths, denied access and non-directory paths -have distinct I/O error kinds. - -`Client::read_asset(path, byte_limit)` reads an external payload under a read-only -share handle that excludes writes and deletion. Empty files succeed; limits, -interrupted reads and failed close never return partial bytes. This payload read -does not parse a OneStore header or acquire its reader-coordination bytes. - -`python3 tools/test_smb_directory.py VM OUTPUT` checks a caller-owned disposable -Linux lab VM against its filesystem listing and interrupts directory requests, -responses and close. It creates synthetic files in that VM; the caller retains -responsibility for VM teardown. The parser also has bounded-record/truncation -tests independent of the server. - -Device and simulator builds link for iOS. Native acceptance uses disposable -OneNote 2010 clients and Samba; it does not establish on-device execution or -physical power-loss durability. diff --git a/crates/onestore-smb/examples/smb_concurrent_client.rs b/crates/onestore-smb/examples/smb_concurrent_client.rs deleted file mode 100644 index c54cc91159f7125667d07638bc77ebc4f148f9ab..0000000000000000000000000000000000000000 --- a/crates/onestore-smb/examples/smb_concurrent_client.rs +++ /dev/null @@ -1,22 +0,0 @@ -#[path = "../../onestore/examples/support/concurrent.rs"] -mod concurrent; - -use onestore_smb::{Client, Credentials}; -use std::{env, time::Duration}; - -fn main() -> Result<(), Box> { - let client = Client::connect( - &env::var("ONESTORE_SMB_LAB")?, - &env::var("ONESTORE_SMB_SHARE")?, - Credentials::default(), - Duration::from_secs(10), - )?; - let args: Vec<_> = env::args().skip(1).collect(); - concurrent::run( - &args, - |path| client.read(path, 256 * 1024 * 1024), - |path, source, space, object, range, replacement| { - client.commit_text(path, source, space, object, range, replacement) - }, - ) -} diff --git a/crates/onestore-smb/examples/smb_reconnect_client.rs b/crates/onestore-smb/examples/smb_reconnect_client.rs deleted file mode 100644 index 1c88874791fe610c2b3fb6f7221014bbd0de45ae..0000000000000000000000000000000000000000 --- a/crates/onestore-smb/examples/smb_reconnect_client.rs +++ /dev/null @@ -1,267 +0,0 @@ -#[path = "../../onestore/examples/support/concurrent.rs"] -mod concurrent; - -use onestore::{ - CommitError, CommitState, ExGuid, RevisionIndex, Store, - document::{Document, Kind}, -}; -use onestore_smb::{Client, Credentials}; -use serde_json::json; -use std::{ - cell::RefCell, - env, io, thread, - time::{Duration, Instant, SystemTime, UNIX_EPOCH}, -}; - -fn paragraph( - bytes: &[u8], - space: ExGuid, - object: ExGuid, -) -> Result> { - let store = Store::parse(bytes)?; - let index = RevisionIndex::parse(&store)?; - index.validate_current()?; - let document = Document::parse(&index)?; - let space = &document.spaces[&space]; - let revision = &space.revisions[&space.contexts[&ExGuid::default()]]; - let Kind::RichText { text, .. } = &revision.nodes[&object].kind else { - return Err("The append target is no longer text.".into()); - }; - Ok(text.clone()) -} - -// Only the owned append workload guarantees unique tokens that no writer removes. -fn retained(before: &str, token: &str, current: &str, state: CommitState) -> io::Result { - let count = current.matches(token).count(); - if count == 0 && state != CommitState::Committed && current.starts_with(before) { - return Ok(false); - } - if count == 1 - && state != CommitState::NotCommitted - && current.starts_with(&format!("{before}{token}")) - { - return Ok(true); - } - Err(io::Error::other( - "The append history contradicts the commit outcome.", - )) -} - -fn main() -> Result<(), Box> { - let args: Vec<_> = env::args().skip(1).collect(); - if args - .first() - .is_none_or(|mode| !["read", "write"].contains(&mode.as_str())) - { - return Err("Reconnect testing requires the append-only workload.".into()); - } - let address = env::var("ONESTORE_SMB_LAB")?; - let share = env::var("ONESTORE_SMB_SHARE")?; - let client = RefCell::new(Some(Client::connect( - &address, - &share, - Credentials::default(), - Duration::from_secs(5), - )?)); - let read = |path: &str| { - let mut session = client.borrow_mut(); - if session.is_none() { - match Client::connect( - &address, - &share, - Credentials::default(), - Duration::from_secs(5), - ) { - Ok(fresh) => { - *session = Some(fresh); - println!( - "{}", - json!({"event":"transport_connected", "at_us":SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_micros()}) - ); - } - Err(error) => { - println!( - "{}", - json!({"event":"transport_connect_error","error":error.to_string()}) - ); - return Err(io::ErrorKind::WouldBlock.into()); - } - } - } - let started = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_micros(); - match session.as_ref().unwrap().read(path, 256 * 1024 * 1024) { - Ok(bytes) => { - if args.get(2).is_some_and(|actor| actor == "r0") - && let Some(folder) = env::var_os("ONESTORE_OFFLINE_FORMAT_REPLY_DIR") - { - let folder = std::path::PathBuf::from(folder); - let captured = folder.join("offline-retired.one"); - if !captured.exists() - && let Ok(marker) = std::fs::read(folder.join("offline-paused-w0.isolate")) - && let Ok(watched) = serde_json::from_slice::(&marker) - && watched["after_us"] - .as_u64() - .is_some_and(|after| started > u128::from(after)) - { - let sid: ExGuid = watched["space"] - .as_str() - .ok_or_else(|| io::Error::other("Missing watched space"))? - .parse() - .map_err(io::Error::other)?; - let rid: ExGuid = watched["revision"] - .as_str() - .ok_or_else(|| io::Error::other("Missing watched revision"))? - .parse() - .map_err(io::Error::other)?; - let store = Store::parse(&bytes).map_err(io::Error::other)?; - let index = RevisionIndex::parse(&store).map_err(io::Error::other)?; - if index - .spaces - .get(&sid) - .is_some_and(|space| !space.revisions.contains_key(&rid)) - { - index.validate_current().map_err(io::Error::other)?; - std::fs::write(captured.with_extension("tmp"), &bytes)?; - std::fs::rename(captured.with_extension("tmp"), captured)?; - println!( - "{}", - json!({"event":"revision_retired", "space":sid.to_string(), "revision":rid.to_string(), "started_us":started, "finished_us":SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_micros()}) - ); - } - } - } - Ok(bytes) - } - Err(error) - if matches!( - error.kind(), - io::ErrorKind::Other | io::ErrorKind::TimedOut | io::ErrorKind::NotConnected - ) => - { - println!( - "{}", - json!({"event":"transport_read_error","error":error.to_string()}) - ); - *session = None; - Err(io::ErrorKind::WouldBlock.into()) - } - result => result, - } - }; - concurrent::run( - &args, - read, - |path, source, space, object, range, replacement| { - let outcome = client.borrow().as_ref().unwrap().commit_text( - path, - source, - space, - object, - range, - replacement, - ); - let Err(error) = outcome else { - return Ok(()); - }; - if error.state == CommitState::NotCommitted - && matches!( - error.error.kind(), - io::ErrorKind::WouldBlock - | io::ErrorKind::ResourceBusy - | io::ErrorKind::PermissionDenied - | io::ErrorKind::NotFound - ) - { - return Err(error); - } - println!( - "{}", - json!({"event":"transport_commit_error","state":format!("{:?}",error.state),"token":replacement,"error":error.error.to_string()}) - ); - *client.borrow_mut() = None; - let deadline = Instant::now() + Duration::from_secs(60); - loop { - if Instant::now() >= deadline { - return Err(error); - } - let current = match read(path) { - Ok(bytes) => bytes, - Err(retry) if retry.kind() == io::ErrorKind::WouldBlock => { - thread::sleep(Duration::from_millis(100)); - continue; - } - Err(_) => return Err(error), - }; - let published = paragraph(source, space, object) - .and_then(|before| { - Ok(retained( - &before, - replacement, - ¶graph(¤t, space, object)?, - error.state, - )?) - }) - .map_err(|failure| CommitError { - state: error.state, - error: io::Error::other(failure.to_string()), - })?; - if published { - let confirmation = client.borrow().as_ref().unwrap().commit_text( - path, - ¤t, - space, - object, - 0..0, - "", - ); - if let Err(failure) = confirmation - && failure.state != CommitState::Committed - { - println!( - "{}", - json!({"event":"transport_confirmation_error","state":format!("{:?}", failure.state),"error":failure.error.to_string()}) - ); - *client.borrow_mut() = None; - thread::sleep(Duration::from_millis(100)); - continue; - } - println!( - "{}", - json!({"event":"transport_reconciled","token":replacement,"published":true,"flush_confirmed":true}) - ); - return Ok(()); - } - println!( - "{}", - json!({"event":"transport_reconciled","token":replacement,"published":false}) - ); - return Err(CommitError { - state: CommitState::NotCommitted, - error: io::ErrorKind::ResourceBusy.into(), - }); - } - }, - ) -} - -#[test] -fn uncertain_append_requires_one_retained_token_and_its_predecessor() { - for state in [CommitState::Unknown, CommitState::Committed] { - assert!(retained("before", " [w0:0]", "before [w0:0] [w1:0]", state).unwrap()); - } - for state in [CommitState::Unknown, CommitState::NotCommitted] { - assert!(!retained("before", " [w0:0]", "before [w1:0]", state).unwrap()); - } - for (current, state) in [ - ("before [w0:0] [w0:0]", CommitState::Unknown), - ("changed [w0:0]", CommitState::Unknown), - ("befor", CommitState::Unknown), - ("before", CommitState::Committed), - ("before [w0:0]", CommitState::NotCommitted), - ] { - assert!(retained("before", " [w0:0]", current, state).is_err()); - } -} diff --git a/crates/onestore-smb/src/directory.rs b/crates/onestore-smb/src/directory.rs deleted file mode 100644 index 4997431a09200a47462e6573c515ead57de50eb9..0000000000000000000000000000000000000000 --- a/crates/onestore-smb/src/directory.rs +++ /dev/null @@ -1,219 +0,0 @@ -use super::*; -use smb2::msg::query_directory::{ - FileInformationClass, QueryDirectoryFlags, QueryDirectoryRequest, QueryDirectoryResponse, -}; -use std::collections::BTreeMap; - -/// Observed directory metadata, not a stable notebook identity or a file snapshot. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DirectoryEntry { - pub name: String, - pub size: u64, - /// MS-FSCC file attributes; directory is 0x10 and reparse point is 0x400. - pub attributes: u32, -} - -impl Client { - /// Enumerates a share-relative directory completely or returns an error without a partial list. - /// An empty path selects the share root. The limit excludes `.` and `..`. - /// Concurrent directory changes are not an atomic snapshot; repeated names return ResourceBusy. - pub fn read_dir(&self, path: &str, limit: usize) -> io::Result> { - if path.contains('\0') || path.encode_utf16().count() > 32767 { - return Err(io::ErrorKind::InvalidInput.into()); - } - let response: CreateResponse = self.request( - Command::Create, - CreateRequest { - requested_oplock_level: OplockLevel::None, - impersonation_level: ImpersonationLevel::Impersonation, - desired_access: FileAccessMask::new(0x80000000), - file_attributes: 0, - share_access: ShareAccess(7), - create_disposition: CreateDisposition::FileOpen, - create_options: 0x1, - name: smb2::encode_path(&path.replace('\\', "/")), - create_contexts: Vec::new(), - }, - )?; - let file = File { - client: self, - id: Some(response.file_id), - }; - let mut entries = BTreeMap::new(); - loop { - let response: io::Result = - self.request_with(Command::QueryDirectory, |connection| { - ( - QueryDirectoryRequest { - file_information_class: FileInformationClass::FileDirectoryInformation, - flags: QueryDirectoryFlags(if entries.is_empty() { 1 } else { 0 }), - file_index: 0, - file_id: file.id.unwrap(), - output_buffer_length: connection - .params() - .expect("connected SMB session is negotiated") - .max_transact_size - .min(65536), - file_name: "*".into(), - }, - CreditCharge(1), - ) - }); - let response = match response { - Ok(response) => response, - Err(error) - if matches!( - error.get_ref().and_then(|error| error.downcast_ref::()), - Some(smb2::Error::Protocol { status, command: Command::QueryDirectory }) - if status.0 == 0x80000006 || (entries.is_empty() && status.0 == 0xc000000f) - ) => - { - break; - } - Err(error) => return Err(error), - }; - for entry in decode(&response.output_buffer)? { - if entries - .insert(entry.name, (entry.size, entry.attributes)) - .is_some() - { - return Err(io::ErrorKind::ResourceBusy.into()); - } - if entries.len() - - usize::from(entries.contains_key(".")) - - usize::from(entries.contains_key("..")) - > limit - { - return Err(io::ErrorKind::FileTooLarge.into()); - } - } - } - file.close()?; - Ok(entries - .into_iter() - .filter(|(name, _)| name != "." && name != "..") - .map(|(name, (size, attributes))| DirectoryEntry { - name, - size, - attributes, - }) - .collect()) - } -} - -fn decode(mut bytes: &[u8]) -> io::Result> { - if bytes.len() > 65536 { - return Err(io::ErrorKind::InvalidData.into()); - } - let mut entries = Vec::new(); - loop { - if bytes.len() < 64 { - return Err(io::ErrorKind::InvalidData.into()); - } - let next = u32::from_le_bytes(bytes[..4].try_into().unwrap()) as usize; - let length = u32::from_le_bytes(bytes[60..64].try_into().unwrap()) as usize; - let end = 64usize - .checked_add(length) - .ok_or(io::ErrorKind::InvalidData)?; - if length == 0 - || !length.is_multiple_of(2) - || end > bytes.len() - || (next != 0 && (next < end || !next.is_multiple_of(8) || next >= bytes.len())) - || (next == 0 && bytes.len() - end > 7) - { - return Err(io::ErrorKind::InvalidData.into()); - } - let units: Vec<_> = bytes[64..end] - .chunks_exact(2) - .map(|unit| u16::from_le_bytes([unit[0], unit[1]])) - .collect(); - let name = String::from_utf16(&units).map_err(|_| io::ErrorKind::InvalidData)?; - if name.contains(['\0', '/', '\\']) { - return Err(io::ErrorKind::InvalidData.into()); - } - let size = i64::from_le_bytes(bytes[40..48].try_into().unwrap()); - entries.push(DirectoryEntry { - name, - size: size.try_into().map_err(|_| io::ErrorKind::InvalidData)?, - attributes: u32::from_le_bytes(bytes[56..60].try_into().unwrap()), - }); - if next == 0 { - return Ok(entries); - } - bytes = &bytes[next..]; - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn record(name: &str, size: u64, attributes: u32) -> Vec { - let mut bytes = vec![0; 64]; - bytes[40..48].copy_from_slice(&size.to_le_bytes()); - bytes[56..60].copy_from_slice(&attributes.to_le_bytes()); - let name: Vec<_> = name.encode_utf16().flat_map(u16::to_le_bytes).collect(); - bytes[60..64].copy_from_slice(&(name.len() as u32).to_le_bytes()); - bytes.extend(name); - bytes - } - - #[test] - fn directory_records_preserve_names_sizes_and_attributes() { - let mut bytes = Vec::new(); - let mut expected = Vec::new(); - for i in 0..300u32 { - let name = format!("Section {i} 🦀 e\u{301}.one"); - let size = u64::from(i) * 100_000_000; - let attributes = if i % 3 == 0 { 0x410 } else { 0x20 }; - let mut entry = record(&name, size, attributes); - if i != 299 { - entry.resize(entry.len().next_multiple_of(8), 0xa5); - let length = entry.len() as u32; - entry[..4].copy_from_slice(&length.to_le_bytes()); - } - bytes.extend(entry); - expected.push(DirectoryEntry { - name, - size, - attributes, - }); - } - assert_eq!(decode(&bytes).unwrap(), expected); - for end in 0..bytes.len() { - assert!(decode(&bytes[..end]).is_err(), "accepted prefix {end}"); - } - } - - #[test] - fn invalid_directory_records_never_become_partial_results() { - assert!(decode(&vec![0; 65537]).is_err()); - for name in ["", "bad\0name", "a/b", "a\\b"] { - assert!(decode(&record(name, 0, 0)).is_err()); - } - let valid = record("a.one", 12, 0x20); - for (at, value) in [ - (0, 8), - (0, 65), - (0, 72), - (0, u32::MAX), - (60, 0), - (60, 1), - (60, u32::MAX), - (44, u32::MAX), - ] { - let mut bytes = valid.clone(); - bytes[at..at + 4].copy_from_slice(&value.to_le_bytes()); - assert!(decode(&bytes).is_err(), "offset={at} value={value}"); - } - let mut bytes = valid.clone(); - bytes[64..66].copy_from_slice(&0xd800u16.to_le_bytes()); - assert!(decode(&bytes).is_err()); - let mut bytes = valid; - bytes.extend_from_slice(&[0; 8]); - assert!(decode(&bytes).is_err()); - for name in [".", ".."] { - assert_eq!(decode(&record(name, 0, 0x10)).unwrap()[0].name, name); - } - } -} diff --git a/crates/onestore-smb/src/lib.rs b/crates/onestore-smb/src/lib.rs deleted file mode 100644 index 7f0d3dc562aa53b9e0c7f60707e8355c247847ee..0000000000000000000000000000000000000000 --- a/crates/onestore-smb/src/lib.rs +++ /dev/null @@ -1,517 +0,0 @@ -#![forbid(unsafe_code)] -#![doc = include_str!("../README.md")] - -use onestore::{CommitError, CommitIo, CommitState, ExGuid}; -use smb2::{ - Session, Tree, - client::connection::{Connection, NegotiatedParams}, - msg::{ - close::{CloseRequest, CloseResponse}, - create::{ - CreateDisposition, CreateRequest, CreateResponse, ImpersonationLevel, ShareAccess, - }, - flush::{FlushRequest, FlushResponse}, - lock::{LockElement, LockRequest, LockResponse}, - query_info::{InfoType, QueryInfoRequest, QueryInfoResponse}, - read::{ReadRequest, ReadResponse}, - write::{WriteRequest, WriteResponse}, - }, - pack::{Pack, ReadCursor, Unpack}, - types::{ - Command, CreditCharge, Dialect, FileId, OplockLevel, - flags::{Capabilities, FileAccessMask}, - }, -}; -use std::{io, ops::Range, sync::Mutex, time::Duration}; -use tokio::runtime::{Handle, Runtime}; - -mod directory; -pub use directory::DirectoryEntry; - -#[derive(Default)] -pub struct Credentials<'a> { - pub username: &'a str, - pub password: &'a str, - pub domain: &'a str, -} - -/// Blocking connection with no automatic request replay or cached file contents. -/// Call from a background thread outside a Tokio runtime. -/// Paths are relative to the share; both `/` and `\` are separators. -pub struct Client { - connection: Mutex>, - tree: Tree, - timeout: Duration, - runtime: Mutex>, -} - -impl Client { - pub fn connect( - address: &str, - share: &str, - credentials: Credentials<'_>, - timeout: Duration, - ) -> io::Result { - if Handle::try_current().is_ok() || timeout.is_zero() { - return Err(io::ErrorKind::InvalidInput.into()); - } - let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(1) - .enable_all() - .build()?; - let (connection, tree) = runtime - .block_on(async { - tokio::time::timeout(timeout, async { - let mut connection = Connection::connect(address, timeout).await?; - connection.set_compression_requested(false); - connection.negotiate().await?; - Session::setup( - &mut connection, - credentials.username, - credentials.password, - credentials.domain, - ) - .await?; - let tree = Tree::connect(&mut connection, share).await?; - Ok::<_, smb2::Error>((connection, tree)) - }) - .await - }) - .map_err(|_| io::Error::from(io::ErrorKind::TimedOut))? - .map_err(io::Error::other)?; - Ok(Self { - connection: Mutex::new(Some(connection)), - tree, - timeout, - runtime: Mutex::new(Some(runtime)), - }) - } - - fn retire(&self) { - if let Some(connection) = self - .connection - .lock() - .unwrap_or_else(|error| error.into_inner()) - .take() - { - connection.mark_dead(); - } - if let Some(runtime) = self - .runtime - .lock() - .unwrap_or_else(|error| error.into_inner()) - .take() - { - runtime.shutdown_background(); - } - } - - fn request(&self, command: Command, body: impl Pack) -> io::Result { - self.request_with(command, |_| (body, CreditCharge(1))) - } - - fn request_with( - &self, - command: Command, - prepare: impl FnOnce(&Connection) -> (B, CreditCharge), - ) -> io::Result { - if Handle::try_current().is_ok() { - return Err(io::ErrorKind::InvalidInput.into()); - } - let connection = self - .connection - .lock() - .map_err(|_| io::ErrorKind::Other)? - .clone() - .ok_or(io::ErrorKind::NotConnected)?; - let frame = { - let runtime = self.runtime.lock().map_err(|_| io::ErrorKind::Other)?; - let (body, charge) = prepare(&connection); - runtime - .as_ref() - .ok_or(io::ErrorKind::NotConnected)? - .block_on(async { - tokio::time::timeout( - self.timeout, - connection.execute_with_credits( - command, - &body, - Some(self.tree.tree_id), - charge, - ), - ) - .await - }) - }; - let frame = frame - .map_err(|_| io::Error::from(io::ErrorKind::TimedOut)) - .and_then(|result| result.map_err(io::Error::other)) - .inspect_err(|_| self.retire())?; - if frame.header.command != command { - self.retire(); - return Err(io::ErrorKind::InvalidData.into()); - } - if frame.header.status.0 != 0 { - let kind = match frame.header.status.0 { - 0xc0000043 | 0xc0000054 | 0xc0000055 => io::ErrorKind::WouldBlock, - 0xc0000011 => io::ErrorKind::UnexpectedEof, - 0xc0000034 | 0xc000003a => io::ErrorKind::NotFound, - 0xc0000022 => io::ErrorKind::PermissionDenied, - 0xc0000103 => io::ErrorKind::NotADirectory, - _ => io::ErrorKind::Other, - }; - return Err(io::Error::new( - kind, - smb2::Error::Protocol { - status: frame.header.status, - command, - }, - )); - } - T::unpack(&mut ReadCursor::new(&frame.body)).map_err(|error| { - self.retire(); - io::Error::new(io::ErrorKind::InvalidData, error) - }) - } - - fn open(&self, path: &str, write: bool) -> io::Result> { - self.open_shared(path, write, if write { 5 } else { 7 }) - } - - fn open_shared(&self, path: &str, write: bool, sharing: u32) -> io::Result> { - if path.is_empty() || path.contains('\0') || path.encode_utf16().count() > 32767 { - return Err(io::ErrorKind::InvalidInput.into()); - } - let response: CreateResponse = self.request( - Command::Create, - CreateRequest { - requested_oplock_level: OplockLevel::None, - impersonation_level: ImpersonationLevel::Impersonation, - desired_access: FileAccessMask::new(if write { 0xc0000000 } else { 0x80000000 }), - file_attributes: 0, - share_access: ShareAccess(sharing), - create_disposition: CreateDisposition::FileOpen, - create_options: 0x42, - name: smb2::encode_path(&path.replace('\\', "/")), - create_contexts: Vec::new(), - }, - )?; - Ok(File { - client: self, - id: Some(response.file_id), - }) - } - - /// Reads a bounded external payload while denying concurrent writes and deletion. - /// Empty files succeed; limits, sharing contention and failed close return no payload. - pub fn read_asset(&self, path: &str, limit: usize) -> io::Result> { - let mut file = self.open_shared(path, false, 1)?; - let mut bytes = Vec::new(); - let mut block = [0; 65536]; - loop { - let count = (limit - bytes.len()).min(block.len() - 1) + 1; - let read = file.read_at( - u64::try_from(bytes.len()).map_err(|_| io::ErrorKind::InvalidInput)?, - &mut block[..count], - )?; - if read == 0 { - break; - } - bytes.extend_from_slice(&block[..read]); - if bytes.len() > limit { - return Err(io::ErrorKind::FileTooLarge.into()); - } - } - file.close()?; - Ok(bytes) - } - - /// Reads one bounded, consistent snapshot; contention returns WouldBlock. - pub fn read(&self, path: &str, limit: usize) -> io::Result> { - self.read_with(path, |file| { - onestore::read_snapshot(|offset, output| file.read_at(offset, output), limit) - }) - } - - /// Reads consistent storage, including encrypted or incomplete document graphs. - /// Storage validation alone does not establish edit readiness. - pub fn read_storage(&self, path: &str, limit: usize) -> io::Result> { - self.read_with(path, |file| { - onestore::read_storage_snapshot(|offset, output| file.read_at(offset, output), limit) - }) - } - - fn read_with( - &self, - path: &str, - snapshot: impl FnOnce(&mut File<'_>) -> io::Result>>, - ) -> io::Result> { - let mut file = self.open(path, false)?.coordinate(path, false)?; - let result = snapshot(&mut file) - .and_then(|snapshot| snapshot.ok_or_else(|| io::ErrorKind::WouldBlock.into())); - let closed = file.close(); - let snapshot = result?; - closed?; - Ok(snapshot) - } - - pub fn commit_text( - &self, - path: &str, - source: &[u8], - space: ExGuid, - object: ExGuid, - range: Range, - replacement: &str, - ) -> Result<(), CommitError> { - self.commit(path, |file| { - onestore::commit_text(file, source, space, object, range, replacement) - }) - } - - pub fn commit_property_bytes( - &self, - path: &str, - source: &[u8], - space: ExGuid, - object: ExGuid, - property: u32, - value: &[u8], - ) -> Result<(), CommitError> { - self.commit(path, |file| { - onestore::commit_property_bytes(file, source, space, object, property, value) - }) - } - - /// Publishes a prepared edit using the same native writer coordination as text commits. - pub fn commit_prepared( - &self, - path: &str, - edit: &onestore::PreparedEdit<'_>, - ) -> Result<(), CommitError> { - self.commit(path, |file| edit.commit(file)) - } - - /// Confirms an observed snapshot's durability under native writer coordination. - pub fn confirm_snapshot(&self, path: &str, source: &[u8]) -> Result<(), CommitError> { - self.commit(path, |file| onestore::confirm_snapshot(file, source)) - } - - fn commit( - &self, - path: &str, - operation: impl FnOnce(&mut File<'_>) -> Result<(), CommitError>, - ) -> Result<(), CommitError> { - let mut file = self - .open(path, true) - .and_then(|file| file.coordinate(path, true)) - .map_err(|error| CommitError { - state: CommitState::NotCommitted, - error, - })?; - let result = operation(&mut file); - let closed = file.close(); - result?; - closed.map_err(|error| CommitError { - state: CommitState::Committed, - error, - }) - } -} - -impl Drop for Client { - fn drop(&mut self) { - self.retire(); - } -} - -struct File<'a> { - client: &'a Client, - id: Option, -} -impl File<'_> { - fn coordinate(self, path: &str, write: bool) -> io::Result { - self.lock(0xfffffffb, 0x11)?; - if write { - self.lock(0xfffffffd, 0x12)?; - } - let current = self.client.open(path, false)?; - let same = self.identity()? == current.identity()?; - current.close()?; - if !same { - return Err(io::ErrorKind::ResourceBusy.into()); - } - Ok(self) - } - - fn lock(&self, offset: u64, flags: u32) -> io::Result<()> { - let _: LockResponse = self.client.request( - Command::Lock, - LockRequest { - file_id: self.id.unwrap(), - lock_sequence: 0, - locks: vec![LockElement { - offset, - length: 1, - flags, - }], - }, - )?; - Ok(()) - } - - fn identity(&self) -> io::Result<(u64, u32)> { - let index: QueryInfoResponse = self.client.request( - Command::QueryInfo, - QueryInfoRequest { - info_type: InfoType::File, - file_info_class: 6, - output_buffer_length: 8, - additional_information: 0, - flags: 0, - file_id: self.id.unwrap(), - input_buffer: Vec::new(), - }, - )?; - let index = u64::from_le_bytes( - index - .output_buffer - .try_into() - .map_err(|_| io::ErrorKind::InvalidData)?, - ); - if index == 0 { - return Err(io::ErrorKind::Unsupported.into()); - } - let volume: QueryInfoResponse = self.client.request( - Command::QueryInfo, - QueryInfoRequest { - info_type: InfoType::Filesystem, - file_info_class: 1, - output_buffer_length: 1024, - additional_information: 0, - flags: 0, - file_id: self.id.unwrap(), - input_buffer: Vec::new(), - }, - )?; - let serial = volume - .output_buffer - .get(8..12) - .ok_or(io::ErrorKind::InvalidData)?; - Ok((index, u32::from_le_bytes(serial.try_into().unwrap()))) - } - - fn close(mut self) -> io::Result<()> { - self.release() - } - - fn release(&mut self) -> io::Result<()> { - if let Some(file_id) = self.id.take() { - let result: io::Result = self - .client - .request(Command::Close, CloseRequest { file_id, flags: 0 }); - if result.is_err() { - self.client.retire(); - } - result?; - } - Ok(()) - } -} -impl Drop for File<'_> { - fn drop(&mut self) { - let _ = self.release(); - } -} -fn read_size(params: &NegotiatedParams, credits: u16, requested: usize) -> usize { - let budget = if params.dialect != Dialect::Smb2_0_2 - && params.capabilities.contains(Capabilities::LARGE_MTU) - { - usize::from(credits.max(1)) * 65536 - } else { - 65536 - }; - requested - .min(params.max_read_size as usize) - .min(budget) - .min(1024 * 1024) -} - -impl CommitIo for File<'_> { - fn read_at(&mut self, offset: u64, output: &mut [u8]) -> io::Result { - if output.is_empty() { - return Ok(0); - } - let mut size = 0; - let response: io::Result = - self.client.request_with(Command::Read, |connection| { - size = read_size( - &connection - .params() - .expect("connected SMB session is negotiated"), - connection.credits(), - output.len(), - ); - ( - ReadRequest { - file_id: self.id.unwrap(), - offset, - length: size as u32, - minimum_count: 1, - flags: 0, - padding: 0, - channel: 0, - remaining_bytes: 0, - read_channel_info: Vec::new(), - }, - CreditCharge(size.div_ceil(65536) as u16), - ) - }); - let response = match response { - Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(0), - result => result?, - }; - if response.data.len() > size { - self.client.retire(); - return Err(io::ErrorKind::InvalidData.into()); - } - output[..response.data.len()].copy_from_slice(&response.data); - Ok(response.data.len()) - } - fn write_at(&mut self, offset: u64, bytes: &[u8]) -> io::Result { - if bytes.is_empty() { - return Ok(0); - } - let size = bytes.len().min(65536); - let response: WriteResponse = self.client.request( - Command::Write, - WriteRequest { - file_id: self.id.unwrap(), - offset, - data: bytes[..size].to_vec(), - data_offset: 112, - flags: 1, - channel: 0, - remaining_bytes: 0, - write_channel_info_offset: 0, - write_channel_info_length: 0, - }, - )?; - if response.count as usize > size { - return Err(io::ErrorKind::InvalidData.into()); - } - Ok(response.count as usize) - } - fn flush(&mut self) -> io::Result<()> { - let _: FlushResponse = self.client.request( - Command::Flush, - FlushRequest { - file_id: self.id.unwrap(), - }, - )?; - Ok(()) - } -} - -#[cfg(test)] -mod tests; diff --git a/crates/onestore-smb/src/tests.rs b/crates/onestore-smb/src/tests.rs deleted file mode 100644 index 94b3dbd9638288fc2ea2c0fc69b6a8a6f13a3f22..0000000000000000000000000000000000000000 --- a/crates/onestore-smb/src/tests.rs +++ /dev/null @@ -1,507 +0,0 @@ -use super::*; -use onestore::{ - RevisionIndex, Store, - document::{Document, Kind}, -}; -use std::{ - fs, - time::{Instant, SystemTime, UNIX_EPOCH}, -}; - -mod faults; - -#[test] -#[ignore = "requires an owned Samba directory and ONESTORE_SMB_DIRECTORY_ORACLE from its local filesystem"] -fn live_directory() { - let client = client(); - let bytes = fs::read(std::env::var("ONESTORE_SMB_DIRECTORY_ORACLE").unwrap()).unwrap(); - let oracle: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); - let root = oracle["path"].as_str().unwrap(); - let expected = oracle["entries"].as_array().unwrap(); - let entries = client.read_dir(root, expected.len()).unwrap(); - assert_eq!(entries.len(), expected.len()); - for expected in expected { - let entry = entries - .iter() - .find(|entry| entry.name == expected["name"]) - .unwrap(); - let directory = expected["directory"].as_bool().unwrap(); - assert_eq!(entry.attributes & 0x10 != 0, directory, "{}", entry.name); - if !directory { - assert_eq!( - entry.size, - expected["size"].as_u64().unwrap(), - "{}", - entry.name - ); - } - } - assert_eq!( - client.read_dir(root, entries.len() - 1).unwrap_err().kind(), - io::ErrorKind::FileTooLarge - ); - assert_eq!(client.read_dir(root, entries.len()).unwrap(), entries); - assert!( - client - .read_dir(&format!("{root}/empty"), 0) - .unwrap() - .is_empty() - ); - assert_eq!( - client - .read_dir(&format!("{root}/missing"), 1) - .unwrap_err() - .kind(), - io::ErrorKind::NotFound - ); - assert_eq!( - client - .read_dir(&format!("{root}/file.one"), 1) - .unwrap_err() - .kind(), - io::ErrorKind::NotADirectory - ); - assert_eq!( - client - .read_dir(&format!("{root}/denied"), 1) - .unwrap_err() - .kind(), - io::ErrorKind::PermissionDenied - ); - assert!( - client - .read_dir("", 10_000) - .unwrap() - .iter() - .any(|entry| entry.name == root) - ); -} - -#[test] -#[ignore = "requires an owned Samba directory through a proxy that interrupts a later directory page or close"] -fn live_directory_interruption() { - let client = client(); - let root = std::env::var("ONESTORE_SMB_DIRECTORY").unwrap(); - let started = Instant::now(); - assert!(client.read_dir(&root, 10_000).is_err()); - assert!(started.elapsed() < Duration::from_secs(10)); - assert_eq!( - client.read_dir(&root, 10_000).unwrap_err().kind(), - io::ErrorKind::NotConnected - ); -} - -fn client() -> Client { - Client::connect( - &std::env::var("ONESTORE_SMB_LAB").unwrap(), - "agent", - Credentials::default(), - Duration::from_secs(5), - ) - .unwrap() -} -fn create(client: &Client, path: &str, bytes: &[u8]) { - let response: CreateResponse = client - .request( - Command::Create, - CreateRequest { - requested_oplock_level: OplockLevel::None, - impersonation_level: ImpersonationLevel::Impersonation, - desired_access: FileAccessMask::new(0xc0000000), - file_attributes: 0, - share_access: ShareAccess(7), - create_disposition: CreateDisposition::FileCreate, - create_options: 0x42, - name: path.to_owned(), - create_contexts: Vec::new(), - }, - ) - .unwrap(); - let mut file = File { - client, - id: Some(response.file_id), - }; - let mut offset = 0; - while offset < bytes.len() { - offset += file.write_at(offset as u64, &bytes[offset..]).unwrap(); - } - file.flush().unwrap(); - file.close().unwrap(); -} -fn text(bytes: &[u8]) -> (ExGuid, ExGuid, String) { - let store = Store::parse(bytes).unwrap(); - assert!(store.checksum_mismatches.is_empty()); - let index = RevisionIndex::parse(&store).unwrap(); - index.validate_current().unwrap(); - let doc = Document::parse(&index).unwrap(); - doc.spaces - .iter() - .find_map(|(sid, space)| { - let revision = space.active().unwrap(); - revision - .nodes - .iter() - .find_map(|(oid, node)| match &node.kind { - Kind::RichText { text, .. } => Some((*sid, *oid, text.clone())), - _ => None, - }) - }) - .unwrap() -} -#[test] -#[ignore = "requires disposable Samba and an existing ONESTORE_SMB_EVIDENCE directory"] -fn live_coordination() { - let output = std::env::var("ONESTORE_SMB_EVIDENCE").unwrap(); - let writer = client(); - let path = format!( - "adapter-{}.one", - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos() - ); - let source = onestore::create_section(&path, "Before café 🦀", "Fixture").unwrap(); - create(&writer, &path, &source); - assert_eq!(writer.read(&path, 1 << 20).unwrap(), source); - let readers: Vec<_> = (0..12).map(|_| client()).collect(); - let mut held: Vec<_> = readers - .iter() - .map(|client| { - client - .open(&path, false) - .unwrap() - .coordinate(&path, false) - .unwrap() - }) - .collect(); - let (sid, oid, before) = text(&source); - let replacement = "After café 🦀"; - writer - .commit_text( - &path, - &source, - sid, - oid, - 0..before.encode_utf16().count() as u32, - replacement, - ) - .unwrap(); - let after = writer.read(&path, 1 << 20).unwrap(); - assert_eq!(text(&after).2, replacement); - for file in &mut held { - let snapshot = onestore::read_snapshot(|offset, out| file.read_at(offset, out), 1 << 20) - .unwrap() - .unwrap(); - assert_eq!(snapshot, after); - } - let error = writer - .commit_text(&path, &source, sid, oid, 0..1, "X") - .unwrap_err(); - assert_eq!(error.state, CommitState::NotCommitted); - assert_eq!(error.error.kind(), io::ErrorKind::ResourceBusy); - assert_eq!(writer.read(&path, 1 << 20).unwrap(), after); - drop(held); - - let stale = writer.open(&path, true).unwrap(); - let maintenance = client(); - let guard = maintenance.open(&path, false).unwrap(); - let _: LockResponse = maintenance - .request( - Command::Lock, - LockRequest { - file_id: guard.id.unwrap(), - lock_sequence: 0, - locks: vec![ - LockElement { - offset: 0xfffffffc, - length: 1, - flags: 0x12, - }, - LockElement { - offset: 0xffffeffc, - length: 4096, - flags: 0x12, - }, - ], - }, - ) - .unwrap(); - let replacement_path = format!("{path}.replacement"); - create(&maintenance, &replacement_path, &source); - let mut connection = maintenance - .connection - .lock() - .unwrap() - .as_ref() - .unwrap() - .clone(); - maintenance - .runtime - .lock() - .unwrap() - .as_ref() - .unwrap() - .block_on( - maintenance - .tree - .rename(&mut connection, &path, &format!("{path}.old")), - ) - .unwrap(); - maintenance - .runtime - .lock() - .unwrap() - .as_ref() - .unwrap() - .block_on( - maintenance - .tree - .rename(&mut connection, &replacement_path, &path), - ) - .unwrap(); - drop(connection); - guard.close().unwrap(); - let error = stale.coordinate(&path, true).err().unwrap(); - assert_eq!(error.kind(), io::ErrorKind::ResourceBusy); - assert_eq!(writer.read(&path, 1 << 20).unwrap(), source); - - let retiring = client(); - let file = retiring - .open(&path, true) - .unwrap() - .coordinate(&path, true) - .unwrap(); - retiring.retire(); - assert_eq!( - retiring.read(&path, 1 << 20).unwrap_err().kind(), - io::ErrorKind::NotConnected - ); - drop(file); - let deadline = Instant::now() + Duration::from_secs(5); - loop { - match writer - .open(&path, true) - .and_then(|file| file.coordinate(&path, true)) - { - Ok(file) => { - file.close().unwrap(); - break; - } - Err(error) - if error.kind() == io::ErrorKind::WouldBlock && Instant::now() < deadline => - { - std::thread::sleep(Duration::from_millis(10)) - } - Err(error) => panic!("retired connection retained locks: {error}"), - } - } - - let mut unfinished = writer - .open(&path, true) - .unwrap() - .coordinate(&path, true) - .unwrap(); - assert_eq!( - unfinished - .write_at(source.len() as u64, b"unpublished") - .unwrap(), - 11 - ); - unfinished.flush().unwrap(); - unfinished.close().unwrap(); - let snapshot = writer.read(&path, 1 << 20).unwrap(); - assert_eq!(snapshot.len(), source.len() + 11); - assert_eq!(text(&snapshot).2, before); - writer - .commit_text( - &path, - &snapshot, - sid, - oid, - 0..before.encode_utf16().count() as u32, - "Recovered café 🦀", - ) - .unwrap(); - let recovered = writer.read(&path, 1 << 20).unwrap(); - assert_eq!(text(&recovered).2, "Recovered café 🦀"); - let spare = client(); - tokio::runtime::Builder::new_current_thread() - .build() - .unwrap() - .block_on(async { - drop(spare); - }); - fs::write(std::path::Path::new(&output).join("source.one"), &source).unwrap(); - fs::write(std::path::Path::new(&output).join("committed.one"), &after).unwrap(); - fs::write( - std::path::Path::new(&output).join("recovered.one"), - &recovered, - ) - .unwrap(); - println!( - "{}", - serde_json::json!({"path":path,"readers":12,"committed_while_readers_held":true,"fresh_reads":12,"stale_snapshot_rejected":true,"replaced_handle_rejected":true,"retirement_releases_locks":true,"unpublished_tail_recovered":true,"drop_inside_runtime":true}) - ); -} - -#[test] -#[ignore = "requires ONESTORE_SMB_LAB pointing to disposable Samba"] -fn live_storage_inspection() { - let reader = client(); - for (index, fixture) in [ - "native-encrypted/encrypted-01/notebook/synthetic.one", - "native-encrypted/cold-encrypted-02/notebook/Open Notebook.one", - ] - .into_iter() - .enumerate() - { - let source = fs::read(format!("../../corpus/{fixture}")).unwrap(); - let path = format!( - "inspection-{index}-{}.one", - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos() - ); - create(&reader, &path, &source); - assert_eq!(reader.read_storage(&path, source.len()).unwrap(), source); - assert_eq!( - reader.read(&path, source.len()).unwrap_err().kind(), - io::ErrorKind::WouldBlock - ); - let maintenance = client(); - let guard = maintenance.open(&path, false).unwrap(); - guard.lock(0xfffffffb, 0x12).unwrap(); - assert_eq!( - reader.read_storage(&path, source.len()).unwrap_err().kind(), - io::ErrorKind::WouldBlock - ); - guard.close().unwrap(); - assert_eq!(reader.read_storage(&path, source.len()).unwrap(), source); - } -} - -#[test] -#[ignore = "requires ONESTORE_SMB_LAB pointing to disposable Samba"] -fn live_assets() { - let reader = client(); - let writer = client(); - for size in [0, 1, 65535, 65536, 65537, 1048577] { - let bytes: Vec<_> = (0..size).map(|index| (index % 251) as u8).collect(); - let path = format!( - "asset-{size}-{}.onebin", - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos() - ); - create(&writer, &path, &bytes); - assert_eq!(reader.read_asset(&path, size).unwrap(), bytes); - if size != 0 { - assert_eq!( - reader.read_asset(&path, size - 1).unwrap_err().kind(), - io::ErrorKind::FileTooLarge - ); - assert_eq!(reader.read_asset(&path, size).unwrap(), bytes); - } - let writing = writer.open(&path, true).unwrap(); - assert_eq!( - reader.read_asset(&path, size).unwrap_err().kind(), - io::ErrorKind::WouldBlock - ); - writing.close().unwrap(); - let asset = reader.open_shared(&path, false, 1).unwrap(); - assert_eq!( - writer.open(&path, true).err().unwrap().kind(), - io::ErrorKind::WouldBlock - ); - let other = writer.open_shared(&path, false, 1).unwrap(); - other.close().unwrap(); - asset.close().unwrap(); - writer.open(&path, true).unwrap().close().unwrap(); - assert_eq!(reader.read_asset(&path, size).unwrap(), bytes); - } - assert_eq!( - reader - .read_asset("absent-payload.onebin", 0) - .unwrap_err() - .kind(), - io::ErrorKind::NotFound - ); -} - -#[test] -#[ignore = "requires an owned Samba fixture and maintenance controller"] -fn live_reader_hold() { - let client = client(); - let path = std::env::var("ONESTORE_SMB_PATH").unwrap(); - let output = std::path::PathBuf::from(std::env::var("ONESTORE_SMB_HOLD").unwrap()); - assert!(!output.join("ready").exists() && !output.join("release").exists()); - let mut file = client - .open(&path, false) - .unwrap() - .coordinate(&path, false) - .unwrap(); - fs::write(output.join("ready"), b"held").unwrap(); - let deadline = Instant::now() + Duration::from_secs(300); - let mut accepted = 0; - let mut retries = 0; - while !output.join("release").exists() { - assert!( - Instant::now() < deadline, - "maintenance controller timed out" - ); - match onestore::read_snapshot(|offset, out| file.read_at(offset, out), 256 << 20).unwrap() { - Some(_) => accepted += 1, - None => retries += 1, - } - std::thread::sleep(Duration::from_millis(5)); - } - file.close().unwrap(); - assert!(accepted > 0); - fs::write( - output.join("released.json"), - serde_json::to_vec(&serde_json::json!({"accepted": accepted, "retries": retries})).unwrap(), - ) - .unwrap(); -} - -#[test] -fn read_limits_respect_negotiation_and_available_credits() { - for dialect in Dialect::ALL { - for large_mtu in [false, true] { - for max_read_size in [65536, 65537, 131072, 1048576, u32::MAX] { - let params = NegotiatedParams { - dialect: *dialect, - max_read_size, - max_write_size: 65536, - max_transact_size: 65536, - server_guid: Default::default(), - signing_required: false, - capabilities: Capabilities(if large_mtu { - Capabilities::LARGE_MTU - } else { - 0 - }), - gmac_negotiated: false, - cipher: None, - compression_supported: false, - }; - for credits in [0, 1, 2, 3, 15, 16, 17, u16::MAX] { - for requested in [1, 1024, 65535, 65536, 65537, 131072, 1048576, usize::MAX] { - let size = read_size(¶ms, credits, requested); - assert!(size > 0 && size <= requested && size <= max_read_size as usize); - assert!(size <= 1048576); - assert!(size.div_ceil(65536) <= usize::from(credits.max(1))); - if *dialect == Dialect::Smb2_0_2 || !large_mtu { - assert!(size <= 65536); - } else if credits >= 16 && max_read_size >= 1048576 && requested >= 1048576 - { - assert_eq!(size, 1048576); - } - } - } - } - } - } -} diff --git a/crates/onestore-smb/src/tests/faults.rs b/crates/onestore-smb/src/tests/faults.rs deleted file mode 100644 index 99641ee062cb06e9a4dad56f284ca954293b65fc..0000000000000000000000000000000000000000 --- a/crates/onestore-smb/src/tests/faults.rs +++ /dev/null @@ -1,511 +0,0 @@ -use super::*; -use serde_json::{Value, json}; -use std::{collections::BTreeMap, path::Path, process::Child}; - -#[derive(Clone)] -struct Snapshot { - content: BTreeMap, - modified: BTreeMap<(ExGuid, ExGuid), u32>, -} - -fn snapshot(bytes: &[u8]) -> Snapshot { - let store = Store::parse(bytes).unwrap(); - assert!(store.checksum_mismatches.is_empty()); - let index = RevisionIndex::parse(&store).unwrap(); - index.validate_current().unwrap(); - Document::parse(&index).unwrap(); - let mut content = BTreeMap::new(); - let mut modified = BTreeMap::new(); - for (sid, space) in &index.spaces { - let revision = index - .resolve(*sid, space.labels[&(ExGuid::default(), 1)]) - .unwrap(); - let mut objects = BTreeMap::new(); - for (oid, object) in &revision.objects { - if let Some(onestore::FileDataReference::Internal(guid)) = - object.file_reference().unwrap() - { - store.file_data(guid).unwrap(); - } - let data = match object.data { - onestore::ObjectData::Properties(bytes) => { - let mut properties = onestore::PropertySets::parse(bytes).unwrap(); - for property in &mut properties.sets[0] { - if property.id == 0x14001d7a { - let onestore::Value::Bytes(value) = property.value else { - panic!() - }; - assert!( - modified - .insert( - (*sid, *oid), - u32::from_le_bytes(value.try_into().unwrap()) - ) - .is_none() - ); - property.value = onestore::Value::Bytes(&[0; 4]); - } - } - format!("{properties:?}") - } - other => format!("{other:?}"), - }; - objects.insert( - *oid, - ( - object.jcid, - object.reference_count, - data, - format!("{:?}", object.references().unwrap()), - ), - ); - } - content.insert(*sid, format!("{:?} {objects:?}", revision.roots)); - } - Snapshot { content, modified } -} - -fn stamp() -> u32 { - (SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_secs() - - 315532800) - .try_into() - .unwrap() -} - -fn published( - actual: &Snapshot, - old: &Snapshot, - new: &Snapshot, - time: std::ops::RangeInclusive, -) -> bool { - if actual.content == old.content { - assert_eq!( - actual.modified, old.modified, - "Unpublished modification time changed" - ); - return false; - } - assert_eq!( - actual.content, new.content, - "Partial or unexpected publication" - ); - assert!(actual.modified.keys().eq(new.modified.keys())); - for (key, value) in &actual.modified { - if old.modified.get(key) != new.modified.get(key) { - assert!( - time.contains(value), - "Modification time outside the commit interval" - ); - } else { - assert_eq!( - *value, new.modified[key], - "Unrelated modification time changed" - ); - } - } - true -} - -#[test] -fn publication_oracle_bounds_changed_timestamps_and_preserves_others() { - let id = ExGuid::default(); - let other = ExGuid { n: 1, ..id }; - let old = Snapshot { - content: BTreeMap::from([(id, "old".into())]), - modified: BTreeMap::from([((id, id), 10), ((id, other), 7)]), - }; - let new = Snapshot { - content: BTreeMap::from([(id, "new".into())]), - modified: BTreeMap::from([((id, id), 20), ((id, other), 7)]), - }; - let mut actual = new.clone(); - actual.modified.insert((id, id), 30); - assert!(published(&actual, &old, &new, 25..=35)); - assert!(!published(&old, &old, &new, 25..=35)); - for (key, value) in [((id, id), 24), ((id, id), 36), ((id, other), 30)] { - let mut changed = actual.clone(); - changed.modified.insert(key, value); - assert!(std::panic::catch_unwind(|| published(&changed, &old, &new, 25..=35)).is_err()); - } - actual.content = old.content.clone(); - assert!(std::panic::catch_unwind(|| published(&actual, &old, &new, 25..=35)).is_err()); -} - -struct Proxy(Child); -impl Drop for Proxy { - fn drop(&mut self) { - let _ = self.0.kill(); - let _ = self.0.wait(); - } -} - -fn records(output: &Path) -> Vec { - let data = fs::read_to_string(output.join("proxy.jsonl")).unwrap(); - data.rsplit_once('\n') - .map_or("", |(complete, _)| complete) - .lines() - .map(|line| serde_json::from_str(line).unwrap()) - .collect() -} - -fn configure(output: &Path, state: Value) -> usize { - fs::write(output.join("control.tmp"), state.to_string()).unwrap(); - fs::rename(output.join("control.tmp"), output.join("control.json")).unwrap(); - let deadline = Instant::now() + Duration::from_secs(5); - loop { - let events = records(output); - if let Some(index) = events - .iter() - .rposition(|event| event.get("control") == Some(&state)) - { - return index + 1; - } - assert!( - Instant::now() < deadline, - "proxy did not acknowledge its control state" - ); - std::thread::sleep(Duration::from_millis(5)); - } -} - -fn proxy(output: &Path) -> (Proxy, String) { - let address = std::env::var("ONESTORE_SMB_LAB").unwrap(); - let (host, port) = address.rsplit_once(':').unwrap(); - let mut proxy = Proxy( - std::process::Command::new("python3") - .arg(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tools/smb-proxy.py")) - .arg(output.join("control.json")) - .args(["--port", "0", "--server", host, "--server-port", port]) - .stdout(fs::File::create(output.join("proxy.jsonl")).unwrap()) - .stderr(fs::File::create(output.join("proxy.stderr")).unwrap()) - .spawn() - .unwrap(), - ); - let deadline = Instant::now() + Duration::from_secs(5); - let port = loop { - if let Some(port) = records(output) - .iter() - .find_map(|event| event["listening"].as_u64()) - { - break port; - } - assert!(proxy.0.try_wait().unwrap().is_none()); - assert!(Instant::now() < deadline, "proxy did not start"); - std::thread::sleep(Duration::from_millis(5)); - }; - (proxy, format!("127.0.0.1:{port}")) -} - -#[test] -#[ignore = "requires an owned Samba share and a new ONESTORE_SMB_EVIDENCE directory"] -fn live_asset_loss() { - let output = std::path::PathBuf::from(std::env::var("ONESTORE_SMB_EVIDENCE").unwrap()); - fs::create_dir(&output).unwrap(); - let (_proxy, address) = proxy(&output); - let observer = client(); - let bytes: Vec<_> = (0..1048577).map(|index| (index % 251) as u8).collect(); - let path = format!( - "asset-loss-{}.onebin", - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos() - ); - create(&observer, &path, &bytes); - for (command, occurrence) in [(8, 1), (8, 9), (8, 17), (8, 18), (6, 1)] { - for direction in ["request", "response"] { - let reader = Client::connect( - &address, - "agent", - Credentials::default(), - Duration::from_secs(5), - ) - .unwrap(); - // Response occurrences count only replies with the selected status. - let begin = configure( - &output, - json!({"cut":command,"occurrence":if command == 8 && occurrence == 18 && direction == "response" { 1 } else { occurrence }, - "direction":direction,"status":if command == 8 && occurrence == 18 { "0xc0000011" } else { "0x0" }}), - ); - assert!( - reader.read_asset(&path, bytes.len()).is_err(), - "{command}/{occurrence}/{direction}" - ); - assert_eq!( - reader.read_asset(&path, bytes.len()).unwrap_err().kind(), - io::ErrorKind::NotConnected - ); - assert_eq!( - records(&output)[begin..] - .iter() - .filter(|row| row.get("cut").is_some()) - .count(), - 1 - ); - configure( - &output, - json!({"phase":format!("{command}-{occurrence}-{direction}-reconnected")}), - ); - let reconnected = Client::connect( - &address, - "agent", - Credentials::default(), - Duration::from_secs(5), - ) - .unwrap(); - assert_eq!(reconnected.read_asset(&path, bytes.len()).unwrap(), bytes); - } - } - assert_eq!(observer.read_asset(&path, bytes.len()).unwrap(), bytes); -} - -#[test] -#[ignore = "requires an owned Samba share and a new ONESTORE_SMB_EVIDENCE directory"] -fn live_message_loss() { - let output = std::path::PathBuf::from(std::env::var("ONESTORE_SMB_EVIDENCE").unwrap()); - fs::create_dir(&output).unwrap(); - fs::create_dir(output.join("interrupted")).unwrap(); - fs::create_dir(output.join("recovered")).unwrap(); - fs::create_dir(output.join("source")).unwrap(); - let (_proxy, proxied) = proxy(&output); - let observer = client(); - let prefix = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos(); - let fixtures = [ - ( - "chunked", - onestore::create_section("fault.one", "Before café 🦀", "Fault test").unwrap(), - ), - ( - "carry", - fs::read( - Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../../corpus/append/round-01/tx-255/notebook/synthetic.one"), - ) - .unwrap(), - ), - ]; - let mut results = Vec::new(); - for (fixture, source) in fixtures { - fs::write( - output.join("source").join(format!("{fixture}.one")), - &source, - ) - .unwrap(); - let (sid, oid, before) = text(&source); - let after = if fixture == "chunked" { - "After café 🦀 ".repeat(6000) - } else { - "After café 🦀".to_owned() - }; - let suffix = " [reconnected]"; - fs::write( - output.join(format!("{fixture}-intent.json")), - serde_json::to_vec(&json!({"before":before,"after":after,"suffix":suffix})).unwrap(), - ) - .unwrap(); - let old = snapshot(&source); - let deadline = Instant::now() + Duration::from_secs(2); - while old.modified.values().any(|value| *value >= stamp()) { - assert!( - Instant::now() < deadline, - "source timestamps did not precede the test" - ); - std::thread::sleep(Duration::from_millis(5)); - } - let expected = onestore::replace_text( - &source, - sid, - oid, - 0..before.encode_utf16().count() as u32, - &after, - ) - .unwrap(); - let new = snapshot(&expected); - let baseline_path = format!("fault-{prefix}-{fixture}-baseline.one"); - create(&observer, &baseline_path, &source); - configure(&output, json!({"phase":format!("{fixture}-setup")})); - let baseline = Client::connect( - &proxied, - "agent", - Credentials::default(), - Duration::from_secs(5), - ) - .unwrap(); - let start = configure(&output, json!({"phase":format!("{fixture}-baseline")})); - let began = stamp(); - baseline - .commit_text( - &baseline_path, - &source, - sid, - oid, - 0..before.encode_utf16().count() as u32, - &after, - ) - .unwrap(); - let events = records(&output); - let mut occurrences = BTreeMap::new(); - let mut cuts = Vec::new(); - for event in &events[start..] { - let Some(direction) = event["direction"].as_str() else { - continue; - }; - if event["status"] == "0x103" { - continue; - } - let command = event["command"].as_u64().unwrap(); - let status = event["status"].as_str().map(str::to_owned); - let count = occurrences - .entry((direction.to_owned(), command, status.clone())) - .or_insert(0); - *count += 1; - cuts.push((direction.to_owned(), command, status, *count)); - } - assert!( - cuts.iter() - .any(|(direction, command, _, count)| direction == "response" - && *command == 7 - && *count == 3) - ); - assert!(published( - &snapshot(&observer.read(&baseline_path, 1 << 20).unwrap()), - &old, - &new, - began..=stamp() - )); - drop(baseline); - for (case, (direction, command, status, occurrence)) in cuts.iter().enumerate() { - let name = format!("{fixture}-{case:03}"); - let path = format!("fault-{prefix}-{name}.one"); - create(&observer, &path, &source); - configure(&output, json!({"phase":format!("{name}-setup")})); - let interrupted = Client::connect( - &proxied, - "agent", - Credentials::default(), - Duration::from_secs(5), - ) - .unwrap(); - let start = configure( - &output, - json!({"phase":name, "direction":direction, "cut":command, "status":status, "occurrence":occurrence}), - ); - let began = stamp(); - let error = interrupted - .commit_text( - &path, - &source, - sid, - oid, - 0..before.encode_utf16().count() as u32, - &after, - ) - .unwrap_err(); - assert_eq!( - interrupted.read(&path, 1 << 20).unwrap_err().kind(), - io::ErrorKind::NotConnected - ); - let events = records(&output); - assert_eq!( - events[start..] - .iter() - .filter(|event| event.get("cut").is_some()) - .count(), - 1 - ); - assert!( - !events - .iter() - .any(|event| event.get("trace_error").is_some()) - ); - let fresh = client(); - let deadline = Instant::now() + Duration::from_secs(5); - loop { - match fresh - .open(&path, true) - .and_then(|file| file.coordinate(&path, true)) - { - Ok(file) => { - file.close().unwrap(); - break; - } - Err(error) - if error.kind() == io::ErrorKind::WouldBlock - && Instant::now() < deadline => - { - std::thread::sleep(Duration::from_millis(5)) - } - Err(error) => panic!("{name}: retired session retained exclusion: {error}"), - } - } - let saved = fresh.read(&path, 1 << 20).unwrap(); - fs::write( - output.join("interrupted").join(format!("{name}.one")), - &saved, - ) - .unwrap(); - let visible = published(&snapshot(&saved), &old, &new, began..=stamp()); - match error.state { - CommitState::NotCommitted => assert!(!visible, "{name}"), - CommitState::Committed => assert!(visible, "{name}"), - CommitState::Unknown => {} - } - if !visible { - fresh - .commit_text( - &path, - &saved, - sid, - oid, - 0..before.encode_utf16().count() as u32, - &after, - ) - .unwrap(); - } - let saved = fresh.read(&path, 1 << 20).unwrap(); - assert!( - published(&snapshot(&saved), &old, &new, began..=stamp()), - "{name}: replay did not publish" - ); - let end = after.encode_utf16().count() as u32; - fresh - .commit_text(&path, &saved, sid, oid, end..end, suffix) - .unwrap(); - let recovered = fresh.read(&path, 1 << 20).unwrap(); - assert_eq!(text(&recovered).2, format!("{after}{suffix}")); - fs::write( - output.join("recovered").join(format!("{name}.one")), - recovered, - ) - .unwrap(); - results.push(json!({"case":name,"path":path,"direction":direction,"command":command,"status":status,"occurrence":occurrence, - "state":format!("{:?}",error.state),"visible":if visible {"after"} else {"before"}, - "retired_session_rejected":true,"exclusion_released":true,"fresh_commit_succeeded":true})); - fs::write( - output.join("results.json"), - serde_json::to_vec_pretty(&results).unwrap(), - ) - .unwrap(); - } - } - for state in ["NotCommitted", "Unknown", "Committed"] { - assert!(results.iter().any(|result| result["state"] == state)); - } - assert!( - results - .iter() - .any(|result| result["state"] == "Unknown" && result["visible"] == "before") - ); - assert!( - results - .iter() - .any(|result| result["state"] == "Unknown" && result["visible"] == "after") - ); - println!("{} message-loss cases passed", results.len()); -} diff --git a/crates/onestore/README.md b/crates/onestore/README.md index b650d80518cf468679e985b4e6cc7b94c41bb86e..ca543ea889f24a1c7c5d763f3f8792775dbc9881 100644 --- a/crates/onestore/README.md +++ b/crates/onestore/README.md @@ -23,10 +23,9 @@ OneNote verification of all 3200 editing intents. prototypes belong in sibling directories under `crates/` and depend on `onestore = { path = "../onestore" }`. The root manifest discovers these crates. The consumer boundary and API tradeoffs are recorded in [API-AUDIT.md](../../evidence/API-AUDIT.md). -`crates/onestore-diagnostic` backs the [HTML diagnostic editor](../../evidence/DIAGNOSTIC.md). -[`onestore-smb`](../onestore-smb/README.md) provides optional embedded network -access; [`onestore-offline`](../onestore-offline/README.md) provides local -SQLite persistence and reconnect reconciliation for text, insertion and formatting. +`onestore-diagnostic` in the `notebook` crate backs the [HTML diagnostic editor](../../evidence/DIAGNOSTIC.md). +[`notebook`](../notebook/README.md) provides notebook discovery, optional embedded +network access (feature `smb`) and local SQLite persistence and reconnect reconciliation for text, insertion and formatting. Shared native fixtures, specifications, evidence and Python/VM tools stay at the repository root; `fuzz/` remains an independent cargo-fuzz workspace. @@ -143,7 +142,7 @@ Ambiguous ancestry, unsupported indentation transitions and unknown implicit font/language inheritance reject before I/O. This is a logical join, so keyboard actions that only change list or indentation state remain separate operations. Generated fields, protected targets and unsupported run-data boundary changes are -rejected before publication. The [offline crate](../onestore-offline/README.md) +rejected before publication. The [notebook crate](../notebook/README.md) documents durable local operations and reconciliation. The [document-writer acceptance](../../evidence/MILESTONE9.md#document-writer-and-offline-acceptance) includes twelve mixed native/Rust clients, outages, lost replies and native revision retirement. @@ -161,7 +160,7 @@ cargo run --example create_notebook -- /tmp/one-demo 'Hello from Rust.' 'Example cargo run --example inventory -- /tmp/one-demo/synthetic.one cargo run --example inspect -- /tmp/one-demo/synthetic.one cargo run --example document -- /tmp/one-demo/synthetic.one /tmp/one-model -cargo build -p onestore-diagnostic +cargo build -p notebook --bin onestore-diagnostic python3 tools/notebook_report.py /tmp/one-demo /tmp/one-report --timezone America/Los_Angeles ``` @@ -249,7 +248,7 @@ The evidence covers transport failures, not physical server power loss or every filesystem's lock implementation. For shared network notebooks, use the optional -[`onestore-smb`](../onestore-smb/README.md) crate. It uses native share modes, +[`notebook::smb`](../notebook/README.md) module. It uses native share modes, shared reader guards, writer exclusion and fresh pathname identity checks without an OS-mounted share. Its [coordination acceptance](../../evidence/MILESTONE9.md) covers native maintenance, mixed readers/writers, reconnects and uncertain publication. The diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 71d81cf9569121fd47613819f1e67dc553d67584..aeb18075979bba53c2030be9bfa8db908cc91fd4 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -577,6 +577,19 @@ dependencies = [ "libc", ] +[[package]] +name = "notebook" +version = "0.1.0" +dependencies = [ + "onestore", + "rusqlite", + "serde", + "serde_json", + "sha2", + "tempfile", + "thiserror", +] + [[package]] name = "objc2" version = "0.6.4" @@ -651,35 +664,12 @@ dependencies = [ "canvas", "libfuzzer-sys", "md5", + "notebook", "onestore", - "onestore-offline", "serde_json", "tempfile", ] -[[package]] -name = "onestore-notebook" -version = "0.1.0" -dependencies = [ - "onestore", - "serde", - "thiserror", -] - -[[package]] -name = "onestore-offline" -version = "0.1.0" -dependencies = [ - "onestore", - "onestore-notebook", - "rusqlite", - "serde", - "serde_json", - "sha2", - "tempfile", - "thiserror", -] - [[package]] name = "parlance" version = "0.1.0" diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 702328b67f6bb61b70d2b31a8e824ca2d7a4e9bb..48acc1b606f5b318cda25c8baa4bfdfc13f00624 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -10,7 +10,7 @@ cargo-fuzz = true [dependencies] libfuzzer-sys = "0.4" onestore = { path = "../crates/onestore", features = ["protected"] } -onestore-offline = { path = "../crates/onestore-offline" } +notebook = { path = "../crates/notebook" } canvas = { path = "../crates/canvas" } tempfile = "3" md5 = "0.8.1" diff --git a/fuzz/fuzz_targets/offline_page.rs b/fuzz/fuzz_targets/offline_page.rs index 103617b5c15807aef9f01363225b4384d4d366e8..ef91755840c0c6ddee9eff5d1d69750fca87ad99 100644 --- a/fuzz/fuzz_targets/offline_page.rs +++ b/fuzz/fuzz_targets/offline_page.rs @@ -3,7 +3,7 @@ use libfuzzer_sys::fuzz_target; #[path = "../../crates/onestore/tests/support/disk.rs"] mod disk; -#[path = "../../crates/onestore-offline/tests/support/page_schedule.rs"] +#[path = "../../crates/notebook/tests/support/page_schedule.rs"] mod page_schedule; fuzz_target!(|input: &[u8]| page_schedule::run(input)); diff --git a/fuzz/fuzz_targets/offline_paragraph.rs b/fuzz/fuzz_targets/offline_paragraph.rs index 3ad34c04feb29be8e5f91212f66b9df55d80a380..0d8bf5e5dfc6e567c7ee9922f6a3f552c43f7d64 100644 --- a/fuzz/fuzz_targets/offline_paragraph.rs +++ b/fuzz/fuzz_targets/offline_paragraph.rs @@ -5,7 +5,7 @@ use onestore::{ Store, document::{Document, Kind}, }; -use onestore_offline::{EditStatus, Operation, Remote, Replica}; +use notebook::{EditStatus, Operation, Remote, Replica}; use std::{io, sync::LazyLock}; #[path = "../../crates/onestore/tests/support/disk.rs"] diff --git a/fuzz/fuzz_targets/offline_tree.rs b/fuzz/fuzz_targets/offline_tree.rs index 5463e3709d22564ad0e2e12b0e5ee14aa6cb4d6a..2ab663e639f852eb8bb94e8d5dba9b75acac8aec 100644 --- a/fuzz/fuzz_targets/offline_tree.rs +++ b/fuzz/fuzz_targets/offline_tree.rs @@ -3,7 +3,7 @@ use libfuzzer_sys::fuzz_target; #[path = "../../crates/onestore/tests/support/disk.rs"] mod disk; -#[path = "../../crates/onestore-offline/tests/support/tree_schedule.rs"] +#[path = "../../crates/notebook/tests/support/tree_schedule.rs"] mod tree_schedule; fuzz_target!(|input: &[u8]| tree_schedule::run(input)); diff --git a/tools/TESTING.md b/tools/TESTING.md index 90be63e2462e80ffc6de0fe3462c38b7dfbadaf2..6b96a7e66780c72867c2a5d57c974cc2ea2a6504 100644 --- a/tools/TESTING.md +++ b/tools/TESTING.md @@ -39,7 +39,7 @@ the run's commands, inputs, outputs and teardown evidence. | Native authoring and cold reopen | `native_runner.py --help` | Independent OneNote capture; exact expected page count when known; owned clone teardown | | Mixed native/Rust/offline writers | `native_collaboration.py --help` | Recorded intents, durable receipts, independent server state and cold native comparison | | SMB directory pagination | `test_smb_directory.py --help` | Caller-owned Linux VM, native filesystem oracle, interrupted-page rejection | -| SMB publication and payload interruptions | Ignored tests in `onestore-smb` | Explicit `ONESTORE_SMB_*` lab inputs, retained protocol traces and independent recovery checks | +| SMB publication and payload interruptions | Ignored tests in `notebook` (feature `smb`) | Explicit `ONESTORE_SMB_*` lab inputs, retained protocol traces and independent recovery checks | | Retained cache migration | Ignored `migrate_retained_cache_copy` test | New destination, unchanged source, images, intent IDs, attempts and receipts | To compare an additional **already captured** notebook hierarchy without running diff --git a/tools/native_maintenance.py b/tools/native_maintenance.py index c7b4d61b7e31f94eaafb8877a8ef6104a5ee238f..ac27df8db79f43b5025e08c0faa68375c802f2b2 100644 --- a/tools/native_maintenance.py +++ b/tools/native_maintenance.py @@ -125,7 +125,7 @@ Sleep(1000) hold.mkdir() config = json.loads((output / 'linux.json').read_text()) with (hold / 'run.log').open('w') as guardian_log: - guardian = subprocess.Popen(['cargo', 'test', '-p', 'onestore-smb', 'live_reader_hold', '--', '--ignored', '--nocapture'], + guardian = subprocess.Popen(['cargo', 'test', '-p', 'notebook', '--features', 'smb', 'live_reader_hold', '--', '--ignored', '--nocapture'], cwd=ROOT, stdout=guardian_log, stderr=subprocess.STDOUT, env={**os.environ, 'ONESTORE_SMB_LAB': f'127.0.0.1:{config["samba_port"]}', 'ONESTORE_SMB_PATH': 'm6-collaboration/synthetic.one', 'ONESTORE_SMB_HOLD': str(hold)}) diff --git a/tools/smb_faults.py b/tools/smb_faults.py index 32e897b2edac2c8410ea9c9e056a0619f5d0dec8..f271b9ca01bec4675f3169a7e2ff5e5891a9c10e 100644 --- a/tools/smb_faults.py +++ b/tools/smb_faults.py @@ -20,8 +20,8 @@ def run(output, server): output = output.resolve() output.mkdir(parents=True, exist_ok=False) sources = ['tools/smb_faults.py', 'tools/smb-proxy.py', 'Cargo.lock', - 'crates/onestore-smb/src/lib.rs', 'crates/onestore-smb/src/tests.rs', - 'crates/onestore-smb/src/tests/faults.rs', 'crates/onestore/src/commit.rs', + 'crates/notebook/src/smb/mod.rs', 'crates/notebook/src/smb/tests.rs', + 'crates/notebook/src/smb/tests/faults.rs', 'crates/onestore/src/commit.rs', 'crates/onestore/src/snapshot.rs'] for name in sources: target = output / 'harness' / name @@ -37,7 +37,7 @@ def run(output, server): config = linux_vm.load_instance(server) (output / 'linux.json').write_text(json.dumps(config, indent=2)) with (output / 'test.log').open('w') as log: - process = subprocess.Popen(['cargo', 'test', '-p', 'onestore-smb', 'live_message_loss', '--', '--ignored', '--nocapture'], + process = subprocess.Popen(['cargo', 'test', '-p', 'notebook', '--features', 'smb', 'live_message_loss', '--', '--ignored', '--nocapture'], cwd=ROOT, stdout=log, stderr=subprocess.STDOUT, start_new_session=True, env={**os.environ, 'ONESTORE_SMB_LAB': f'127.0.0.1:{config["samba_port"]}', 'ONESTORE_SMB_EVIDENCE': str(output / 'cases')}) diff --git a/tools/test_offline_document_history.py b/tools/test_offline_document_history.py index 59e33f449ab2da2dc9c33542a36acb61fa74e56c..95698e545b9bd17d42c3a7d9e558df3282cb49ae 100644 --- a/tools/test_offline_document_history.py +++ b/tools/test_offline_document_history.py @@ -141,7 +141,7 @@ class DocumentHistoryTests(unittest.TestCase): document_history(logs, 2) def test_production_cache_workload_matches_the_independent_history_model(self): - result = subprocess.run(['cargo', 'test', '--locked', '-p', 'onestore-offline', '--features', 'smb', + result = subprocess.run(['cargo', 'test', '--locked', '-p', 'notebook', '--features', 'smb', '--example', 'smb_offline_client', 'tests::document_workload_retains_dependencies_and_receipts_across_reopen', '--', '--exact', '--nocapture'], diff --git a/tools/test_smb_directory.py b/tools/test_smb_directory.py index 6b91864b0582385ff14b534e78253ad1a2ae5213..b1bdcee2e4e18a19bd5eb40f9c2216aadf9aaa49 100644 --- a/tools/test_smb_directory.py +++ b/tools/test_smb_directory.py @@ -67,9 +67,9 @@ print(json.dumps(dict(path=root.name, entries=entries), ensure_ascii=False)) time.sleep(.05) env = dict(os.environ, ONESTORE_SMB_LAB=f'127.0.0.1:{port}', ONESTORE_SMB_DIRECTORY=root, ONESTORE_SMB_DIRECTORY_ORACLE=str(output / 'oracle.json')) - test = 'tests::live_directory_interruption' if control else 'tests::live_directory' + test = 'tests::live_directory_interruption' if control else 'smb::tests::live_directory' with (output / f'{name}.log').open('w') as result: - subprocess.run(['cargo', 'test', '-p', 'onestore-smb', test, '--', '--ignored', '--exact'], + subprocess.run(['cargo', 'test', '-p', 'notebook', '--features', 'smb', test, '--', '--ignored', '--exact'], env=env, stdout=result, stderr=result, check=True, timeout=120) finally: proxy.terminate() @@ -84,7 +84,7 @@ print(json.dumps(dict(path=root.name, entries=entries), ensure_ascii=False)) env = dict(os.environ, ONESTORE_SMB_LAB=f'127.0.0.1:{config["samba_port"]}', ONESTORE_SMB_DIRECTORY_ORACLE=str(output / 'oracle.json')) with (output / 'reconnected.log').open('w') as result: - subprocess.run(['cargo', 'test', '-p', 'onestore-smb', 'tests::live_directory', '--', '--ignored', '--exact'], + subprocess.run(['cargo', 'test', '-p', 'notebook', '--features', 'smb', 'smb::tests::live_directory', '--', '--ignored', '--exact'], env=env, stdout=result, stderr=result, check=True, timeout=120) print('reconnected: passed', flush=True)