diff --git a/crates/notebook/Cargo.toml b/crates/notebook/Cargo.toml index 6c7a47eadcdb9156907dd56c131fe16884961f10..798548ff3a48fc39ae90b621a5de01e91b23c191 100644 --- a/crates/notebook/Cargo.toml +++ b/crates/notebook/Cargo.toml @@ -21,6 +21,7 @@ tokio = { version = "1", features = ["rt-multi-thread", "time"], optional = true zeroize = { version = "1.9.0", optional = true } [dev-dependencies] +libc = "0.2" serde_json = "1" tempfile = "3" diff --git a/crates/notebook/README.md b/crates/notebook/README.md index 15f7670c6620391bedfb27561208998dd227ce33..02f47cf807b27b685101f351a8ee6d8c1992fc9c 100644 --- a/crates/notebook/README.md +++ b/crates/notebook/README.md @@ -2,184 +2,97 @@ The application-facing crate: notebook discovery, a durable local replica with reconnect reconciliation, external-asset caching, recovery export and optional -embedded SMB access. The replica stores the last observed remote image of one section, -the working image as its difference from that, and the editor's intents in a -local SQLite database; a save writes the revision it appended, not the section. `sync_once` provides a -reconciliation step and `start_sync` owns automatic polling and reconnects. Local -success does not acknowledge publication to a shared notebook. +embedded SMB access. A replica keeps one section's queue in a local SQLite database: +the image the queued edits apply to (the base, in 16 KiB chunks), the edits as +`onestore::op` ops in publication batches, and picture and attachment bytes once +each by hash. Local success does not acknowledge publication to a shared notebook. -## Saving pages +```text +editor ── Edit ──► section thread ─ Section::apply, INSERT edit ─┐ + (the parsed section: base + sealed + open) │ one fsync per burst +sync thread: remote.stamp() == base ? publish(sealed batch) ──────┤ base chunks += transaction + otherwise read once, replay the queue on it ─────────┘ conflict, or base := remote +``` -The editor works on `onestore::page::Page` values and saves whole pages. `save` -diffs the supplied model against the page stored in the supplied local snapshot, -writes the difference into the working image and queues one -`Operation::Page(PageIntent { before, after, author })`; `before` is the page the -edit started from and is the precondition reconciliation checks. Text, styles, -paragraph formatting, hyperlinks (external and internal), bullets, numbering, note tags, table rows, columns, cell shading, nested tables, pictures (insertion in paragraphs or on the page, position, size, description), attachments, ink strokes, equations, paragraph structure, -outline layout, insertions and deletions are all differences between `before` and -`after`; the library never sees editor operations. +## Edits + +`Replica::apply(author, edit)` applies an `onestore::op::Edit` to the section the +queue leaves and returns its id once written; a refused edit returns `Rejected` +and changes nothing. `page(space)` and `pages()` read what the queue leaves, for +opening and reloading. Edits collect in an open batch until the sync thread seals +it, so edits arriving while a publication is in flight publish together. ```no_run -use onestore::{ExGuid, page::Page}; +use onestore::{ExGuid, op::{Edit, Op, PageOp}}; use notebook::Replica; -# fn example(path: &std::path::Path, source: &[u8], space: ExGuid, edited: &Page) +# fn example(path: &std::path::Path, source: &[u8], space: ExGuid, text: ExGuid) # -> Result<(), Box> { -// `space` identifies the page; `edited` is the editor's current model of it. let cache = Replica::create(path, source)?; -let snapshot = cache.snapshot()?; -let local_id = cache.save(&snapshot, space, edited, "Author")?; +let typed = PageOp::Text { text, range: 0..0, with: "Hello ".into() }; +let id = cache.apply("Author", Edit { at: 133_000_000_000_000_000, ops: vec![Op::Page { space, op: typed }] })?; drop(cache); let reopened = Replica::open(path)?; -let pending = reopened.pending()?; -assert_eq!(pending.last().map(|edit| edit.id), local_id); +assert_eq!(reopened.pending()?.last().map(|edit| edit.id), Some(id)); # Ok(()) # } ``` -Saves coalesce the way OneNote's own autosave does: while the newest queued intent -for the same page has not been attempted, a later save replaces its `after` model -under the same ID. Once an intent has been attempted or holds a conflict it is -never rewritten, nor is the intent a synchronization step has selected for -publication; the next save then queues a new intent. A save whose model equals -the stored page returns `None`. `PageIntent::text_change` describes an intent that -changes exactly one paragraph's text as the text object, its previous text, the -replaced UTF-16 range and the replacement. - -Share one `Replica` between application threads. Each save compares its supplied -snapshot under the cache transaction; stale snapshots return `Io(ResourceBusy)`. -The intent and its resulting image commit together. Keep the cache on a local -filesystem: the connection holds exclusive ownership between transactions, and a -second open fails busy. No network wait occurs in a local save. After a database -error, reopen and inspect the durable state before retrying. +Share one `Replica` between threads. Keep the cache on a local filesystem: the +connection holds exclusive ownership between transactions, and a second open fails +busy. No network wait occurs in a local edit. After a database error the section +thread rereads the queue. ## Sessions `session::Notebook::open(root, cache_dir)` discovers a notebook directory and `session::Section::open(file, cache_dir, notify)` opens one section file through -a replica stored under the cache directory, named by the section's document -identity so the same file reopens the same queue after a relaunch. A section -publishes in the background to the file itself under OneNote-compatible -exclusion. `pages()` lists page spaces and titles from the local image, -`page(space)` returns the model to edit (its `identity` and the section's -`identity()` feed `onestore::page::link::internal_link`), and `save(space, before, after, -author)` queues the edited model: `Save::Queued(id)` is durable locally, -`Save::Unchanged` means the model equals the stored page, and `Save::Stale` -means the stored page no longer matches `before` because the section changed -underneath the editor, so the page must be reloaded before saving again. -`queue_save(space, before, after, author)` does the same on the section's save -thread and returns at once, so an editor's frame never waits on the write; -`saved()` drains the outcomes. A queued save whose `before` is the previous -one's `after` continues it, needing no reload in between, and a run of such -saves waiting together is written once. -`events()` drains what the synchronization thread reported since the last -poll: refreshes, attempt outcomes and unreachable files; `notify` runs -whenever an event or save outcome is available so the application can wake its -event loop. `close()` finishes queued saves and stops publication. +a replica named by the section's document identity, so the same file reopens the +same queue after a relaunch. A section is `Send + Sync`: `apply(author, edit)` +returns at once and reports a refusal as `Event::Rejected`; `events()` drains +remote changes (`Changed(spaces)`), publication outcomes, unreachable files and +failures; `notify` runs on a background thread whenever an event waits. +`conflict()`, `remote_page(space)` and `resolve(id, Resolution)` present and end a +conflict; `release(id, archive, Resolution)` ends an uncertain attempt. +`import_page` creates a page holding a copy of another; `delete_pages` removes pages. -`Event::Unreachable` retains an `io::Error`: callers can distinguish permission -denial, missing targets, timeouts, and connection failures through `kind()` without -parsing display text. Document/cache failures are reported as `Event::Failed` and -stop the worker. Durable publication outcomes remain available through `status`. +Until the application emits ops, `save`, `queue_save`, `saved`, `saving`, +`conflicts`, `queue`, `review` and `Event::Refreshed` keep the whole-page API: a +save lowers the page it is given against the stored page (`onestore::op::lower_page`) +and applies the ops. -`Section::resume(file, replica, notify)` starts a session from an owned -`Replica::open(cache_file)` without consulting the remote file. The caller retains -the cache location and publication path for offline relaunch. Local pages and -saves remain available while the target is absent; synchronization verifies the -document identity before adopting or publishing remote content. A different -document stops the worker and retains the pending local edits. `Section::open` -remains the online convenience constructor that discovers the identity from the -file and creates or reopens its cache. - -With the `smb` feature, `Section::resume_smb(path, replica, limit, connect, -notify)` binds the same session operations to a share-relative path. `connect` -returns a new SMB client on the synchronization worker and is called again after -transport failure. The caller supplies credentials there, outside the replica -schema; construction and local saving do not wait for a network connection. - -## Pages of a section - -`create_page` accepts the core `PageCreation` intent and queues its page space -and section entry as one publication. Subsequent saves of the new page use the -intent's stable identities immediately after local acknowledgement. Independent -page additions rebase against the current section order; duplicate titles remain -distinct. An unavailable or no-longer-leading insertion anchor produces -`StructureChanged`. `rebase_page_creation_conflict(id, local, remote, before)` -reviews a replacement anchor against both cache images while retaining the new -page and dependent object identities. Existing page identities require -reconciliation; a matching page alone does not establish a receipt. - -`pages(snapshot, edits)` queues a slice of core `PageEdit` intents as one atomic -publication. `PagePosition::Keep` preserves remote movement during indentation-only -edits. Every requested level remains explicit: a competing remote level produces -`StructureChanged` unless it already matches the requested level. Moves compare -the selected page's position relative to surviving observed pages; an unchanged -or already-satisfied position can proceed, and new remote pages remain present. -Indistinguishable competing moves retain a conflict and the complete local image. -`rebase_pages_conflict(id, local, remote, edits)` reviews the batch against both -cache images. Use `PageEdit::reposition(position, level)` on the retained intents -to revise anchors or indentation; replacing page or allocated series identities -is rejected. Dependent edits remain queued. Attempt evidence includes every -changed space plus the receipt space when that space is unchanged. An existing -section revision alone cannot confirm a page edit. The -[offline page corpus](../../corpus/page-lifecycle/offline-edits/README.md) -contains native comparisons, reproduction commands and stateful sanitizer seeds. - -`delete_pages(snapshot, pages)` queues the permanent removal of page spaces; -a page already absent remotely produces `TargetUnavailable`. +`Section::resume(file, replica, notify)` starts from an owned `Replica` without +consulting the remote file; with the `smb` feature, `Section::resume_smb(path, +replica, limit, connect, notify)` binds a share-relative path, `connect` running on +the worker again after transport failure. ## Reconciliation -`sync_once(&mut remote)` processes the oldest pending intent through a `Remote` -implementation, returning its ID and `EditStatus`. When the remote page still -equals `before`, the intent publishes as prepared. When the remote page changed, -a three-way merge keeps everything the remote changed and re-applies the local -changes wherever the two sides touched different objects, fields or text ranges: -a local text edit merges with a remote edit elsewhere in the same paragraph, an -outline move merges with a remote text change, a new paragraph survives a remote -deletion elsewhere. Any overlap retains `Conflict(ContentChanged)` together with -the complete local image and the last observed remote image returned by -`remote_snapshot`; a page removed remotely retains `TargetUnavailable`, and a -page the writer can no longer express retains `UnsupportedEdit`. +`sync_once(&mut remote)` returns what one step did as `Synced { edit, changed }`: +the state the step's batch reached, named by its newest edit, and the pages a remote +change replaced. While the remote's stamp (header and length) equals the base's, the +oldest sealed batch publishes as its `Transaction`, and the base's chunks take its +writes; nothing is read. When the stamp moved, the remote image is read once and the +queue replays on it (`merge.rs`): an op whose objects the remote left alone applies +as it is; a text op shifts past the remote's changes to the same text when its +range stays clear of them; a move, deletion or property the remote already made is +done; a page the remote already holds as the local edits leave it drops their ops. +Anything else is a `Conflict` on the batch: the queue stays on its base and the +remote image is kept for review. `resolve(id, Mine)` rewrites the remote page to the +local one (`lower_page`, once), `Theirs` drops the local ops on that page; either +applies from the conflicted batch on. -Publication attempts are recorded before network I/O. A retained attempt -requires its recorded revision to remain present in the remote, or the remote -page to equal the intent's `after` model exactly, followed by comparison, -flushing and refreshed header version metadata before acknowledgement. Otherwise -the attempt remains `AwaitingConfirmation`; an uncertain publication is never -replayed. A durable `Published` receipt survives reopening. Transport errors -return `Error::Remote` or `Error::RemoteIo`; inspect `status(id)` after the error -to distinguish a retained attempt from a pending edit or receipt. The error return -does not roll back a locally acknowledged intent. +Publication attempts are recorded before network I/O. An attempt confirms only +when the remote holds its revisions, or every page it changed as it changed them, +followed by `Remote::confirm`; otherwise it stays `AwaitingConfirmation` and is never +replayed. `release(id, archive, Resolution)` exports the queue first, then +publishes the batch again (`Mine`) or abandons every unpublished edit (`Theirs`). +A durable `Published` receipt survives reopening. -The current operation processes one queue head; while edits remain, `snapshot` -preserves the complete local working image. An empty queue can refresh from the -remote image. Synchronization holds a separate owner lock, so local saves can -continue during network waits; competing synchronization calls return `WouldBlock`. - -`review_page(id, local, remote, after)` resolves the oldest page conflict with a -model the user reviewed against the current remote page: the intent's `before` -becomes that remote page and its `after` the reviewed model, under the same ID, -in one local transaction. The supplied images must still match `snapshot()` and -`remote_snapshot()`. Review performs no network I/O, clears the conflict to -`Pending` and wakes the worker; publication still reads the latest remote image, -so another overlapping remote edit can produce a new conflict. Pending intents -and uncertain attempts cannot be reviewed. - -An attempt that stays `AwaitingConfirmation` is released by review, never by -replay: `release_attempt(id, local, remote, archive, after)` on the replica -(`release` on a session) exports the branch to a new archive, then retires -the oldest edit's attempt. `Some(page)` continues from a page reviewed -against the current remote image as a fresh intent under the same id, -keeping later edits; `None` abandons the local branch, whose ids report -`EditStatus::Archived { archive }` from then on, and the working image -returns to the remote image. Neither writes a receipt: the uncertain -publication may or may not have landed, and the archive is the record of -what was attempted. - -Opening recognizes the application identity and the current schema version only; -caches and archives written by other versions are refused unchanged. Until the -application is usable end to end there are no migrations. +A schema-14 cache is converted when opened: it is exported to +`.v14-recovery`, each queued page is lowered against the page the conversion +has so far, and every converted page must equal the page in the old working image, +or the conversion rolls back and the open fails naming the archive. With the optional `smb` feature, `SmbRemote::new(client, path, limit)` binds an `notebook::smb::Client` to one share-relative file and snapshot limit. Remote identity uses the logical root @@ -188,18 +101,15 @@ object space, which survives the tested native compaction that replaces the file An `Arc` can own one background worker. Supply a connection factory, poll interval and observer; successful publications drain immediately, durable local edits wake the worker, and `wake()` requests an immediate reachability retry. -While nothing is queued, a remote whose `Remote::stamp` (its header and length, read -without coordination) equals the last observed image's is not read again; publishing -against an unchanged stamp needs no read either. +While nothing is queued, or the queue waits on a remote that has not changed since, +a remote whose `Remote::stamp` holds is not read again. Transport failures discard the old connection and retry through the factory; Read contention and `NotCommitted` operations with `WouldBlock` or `ResourceBusy` reuse the connection. Contended `NotCommitted` operations use randomized backoff, capped at one second, to separate competing retry cycles. Cancellation interrupts this delay; local wake notifications remain coalesced until its end. `RemoteIo` distinguishes connection/read failures from -local `Io` errors. Cache/document errors stop the worker. Observers receive every -attempt's result on the worker thread, including unchanged conflict/uncertain -statuses; durable edit state remains available through `status`. +local `Io` errors. Cache/document errors stop the worker. ```no_run # #[cfg(feature = "smb")] @@ -229,35 +139,25 @@ worker.stop()?; ``` `stop()` cancels future steps and joins the worker, returning a fatal cache error -or worker panic. Dropping it requests cancellation without waiting. An in-flight -step completes before releasing ownership; a replacement worker cannot start -while the old one still owns the replica. Remote operations and callbacks must -have bounded execution times if shutdown latency matters. A dropped worker can -briefly retain the cache; use `stop()` before requiring an immediate reopen. -Cancellation and application restart retain pending edits and publication attempts. -Credentials belong to the factory, not the cache database. +or worker panic. Dropping it requests cancellation without waiting. Remote +operations and callbacks must have bounded execution times if shutdown latency +matters. Credentials belong to the factory, not the cache database. -Creation refuses existing paths. Opening recognizes the application identity and -schema version, validates database integrity and both notebook images, and rejects -unsupported journal modes without converting them. Failed initialization preserves -the file for inspection. SQLite uses DELETE journaling, EXTRA synchronization and -fullfsync; each required setting is queried back. - -The cache and its pending intents contain notebook content. The core `onestore` -crate stays independent of SQLite and network runtimes. Device and simulator -examples compile and link for iOS; recorded process-interruption tests do not -establish physical power-loss durability. Evidence is tracked in [Milestone 9](../../evidence/MILESTONE9.md). +Creation refuses existing paths. Opening validates database integrity and replays +the queue on its base. SQLite runs in WAL mode under exclusive locking with FULL +synchronization and fullfsync, each queried back: every commit is durable, and the only +file beside the cache is `-wal`, which holds committed pages until a checkpoint +and must travel with the cache when it is copied. Recovery archives are single files. +The cache contains notebook content. The SMB-enabled `smb_offline_client` example is an owned-lab workload for -`tools/native_collaboration.py --offline --embedded-smb`. It separates local -acknowledgements, remote publication attempts, persisted receipts and cache reopen -checks. Its append-specific conflict review policy lives in the test client; -the library continues to preserve conflicts requiring an explicit decision. +`tools/native_collaboration.py --offline --embedded-smb`; its append-specific +conflict policy lives in the client. ## Recovery archives -`export_recovery(new_path)` captures both complete notebook images, the intent -queue, uncertain attempts, conflicts, receipts, downloaded media and the edit-ID sequence in one +`export_recovery(new_path)` captures the base and remote images, the edit queue, +uncertain attempts, conflicts, receipts, downloaded media and the edit-ID sequence in one SQLite snapshot. It refuses existing destinations and leaves the live queue unchanged. Export to a local directory from a background thread: copying holds the cache mutex while capturing the database. Failure after the final rename can @@ -448,9 +348,6 @@ physical power-loss durability. ## Queue measurement `cargo run -p notebook --release --example queue_scale -- NEW_DIRECTORY 1000` -measures alternating-page saves that retain separate queue IDs, cache reopen, -recovery export and local-file publication. JSON lines record each acknowledgement -and publication latency plus image/cache sizes. The workload verifies every ID -and both final page titles, then independently reopens the recovery archive. -Use an external resource monitor for peak memory; these local-file timings do not -measure SMB or phone performance. +measures alternating-page edits, cache reopen, recovery export and one local-file +publication of the queue. `keystroke_probe PARAGRAPHS KEYSTROKES` types into a large +page and reports the bytes the cache and the section file write per keystroke. diff --git a/crates/notebook/examples/cache_probe.rs b/crates/notebook/examples/cache_probe.rs index f4cc24d21d3d0aeca47d816a700e41af600c6fb7..0e1df5bad63a10f9728981594fcb4ff3c34eeeb9 100644 --- a/crates/notebook/examples/cache_probe.rs +++ b/crates/notebook/examples/cache_probe.rs @@ -1,8 +1,9 @@ -use notebook::{Operation, Replica}; +use notebook::Replica; +use onestore::op::{Op, PageOp}; use onestore::{ ExGuid, RevisionIndex, Store, document::{Document, Kind}, - page::{Page, PageObject, Paragraph, text::Edit}, + page::{PageObject, Paragraph}, }; use std::{ io::{self, BufRead, Write}, @@ -86,34 +87,25 @@ fn main() -> Result<(), Box> { let mut ids = Vec::new(); let mut font_size = None; for pending in cache.pending()? { - let Operation::Page(intent) = &pending.operation else { - panic!("Unexpected operation for this fixture") + let [Op::Page { op, .. }] = &pending.edit.ops[..] else { + panic!("Unexpected edit for this fixture") }; - let operation = match operation_kind.as_str() { - "text" => { - let (object, before, range, replacement) = - intent.text_change().expect("one replaced paragraph"); - assert_eq!(object, target); - assert_eq!(before, expected); - assert_eq!(range, 0..u32::try_from(expected.encode_utf16().count())?); - let operation: u64 = replacement.split_once(':').unwrap().0.parse()?; + let operation = match (operation_kind.as_str(), op) { + ("text", PageOp::Text { text, range, with }) => { + assert_eq!(*text, target); + assert_eq!(*range, 0..u32::try_from(expected.encode_utf16().count())?); + let operation: u64 = with.split_once(':').unwrap().0.parse()?; expected = payload(operation, size); - assert_eq!(replacement, expected); + assert_eq!(*with, expected); operation } - "insert" => { - let added: Vec<_> = intent - .after - .objects - .iter() - .filter(|object| { - !intent.before.objects.iter().any(|o| o.id() == object.id()) - }) - .collect(); - // A save that replaced a pending one carries its outlines too. - let Some(PageObject::Outline(outline)) = added.last() else { - panic!("Expected an added outline") - }; + ( + "insert", + PageOp::Add { + object: PageObject::Outline(outline), + .. + }, + ) => { let text = outline.paragraphs[0].text().unwrap().text.text(); let operation: u64 = text.split_once(':').unwrap().0.parse()?; assert_eq!(text, payload(operation, size)); @@ -124,33 +116,16 @@ fn main() -> Result<(), Box> { assert!(revision.nodes.contains_key(&outline.id)); operation } - "format" => { - let paragraph = intent - .after - .objects - .iter() - .find_map(|object| match object { - PageObject::Outline(outline) => outline - .paragraphs - .iter() - .find(|p| p.text().is_some_and(|t| t.id == target)), - _ => None, - }) - .expect("formatted paragraph"); - let text = paragraph.text().unwrap(); - assert_eq!(text.text.text(), expected); - let value = text.text.spans()[0].format.font_size.expect("font size"); - assert!( - text.text - .spans() - .iter() - .all(|s| s.format.font_size == Some(value)) - ); + ("format", PageOp::Format { text, set, .. }) => { + assert_eq!(*text, target); + let [onestore::TextAttribute::FontSize(value)] = set[..] else { + panic!("Expected a font size") + }; assert!((7.0..=130.0).contains(&value) && value.fract() == 0.0); font_size = Some(value); value as u64 - 6 } - _ => unreachable!(), + other => panic!("Unexpected op {other:?}"), }; assert!(operations.last().is_none_or(|last| *last < operation)); assert!(ids.last().is_none_or(|last| *last < pending.id)); @@ -167,10 +142,7 @@ fn main() -> Result<(), Box> { ); } if operations.is_empty() { - assert!( - snapshot == base, - "An empty local queue changed its working image" - ); + assert!(snapshot == base, "An empty local queue changed its image"); } if let Some(output) = args.get(3) { std::fs::write(output, &snapshot)?; @@ -197,51 +169,43 @@ fn main() -> Result<(), Box> { let replacement = payload(operation, size); println!("editing {operation}"); io::stdout().flush()?; - let (space, page) = { - let store = Store::parse(&source)?; - let index = RevisionIndex::parse(&store)?; - let document = Document::parse(&index)?; - let mut page = Page::from_space(&document, sid)?; - fn target_paragraph(page: &mut Page, oid: ExGuid) -> &mut onestore::page::PageParagraph { - page.objects - .iter_mut() + let end = u32::try_from(text.encode_utf16().count())?; + let op = match operation_kind.as_str() { + "text" => PageOp::Text { + text: oid, + range: 0..end, + with: replacement.clone(), + }, + "insert" => { + let page = cache.page(sid)?; + let template = page + .objects + .iter() .find_map(|object| match object { PageObject::Outline(outline) => outline .paragraphs - .iter_mut() + .iter() .find(|p| p.text().is_some_and(|t| t.id == oid)), _ => None, }) .expect("target paragraph") - } - let end = u32::try_from(text.encode_utf16().count())?; - match operation_kind.as_str() { - "text" => { - let target = target_paragraph(&mut page, oid).text_mut().unwrap(); - let format = target.text.format_at(0)?.clone(); - target.text.apply(Edit { - range: 0..end, - replacement: Paragraph::new(replacement.clone(), format), - })?; - } - "insert" => { - let template = target_paragraph(&mut page, oid).clone(); - let mut fresh = template.clone(); - fresh.id = onestore::page::text::new_id()?; - fresh.parent = None; - fresh.level = 1; - fresh.lists.clear(); - fresh.tags.clear(); - fresh.style = None; - let format = template.text().unwrap().text.format_at(0)?.clone(); - fresh.content = - onestore::page::ParagraphContent::Text(onestore::page::TextObject { - id: onestore::page::text::new_id()?, - date_field: None, - text: Paragraph::new(replacement.clone(), format), - tags: Vec::new(), - }); - let outline = onestore::page::Outline { + .clone(); + let mut fresh = template.clone(); + fresh.id = onestore::page::text::new_id()?; + fresh.parent = None; + fresh.level = 1; + fresh.lists.clear(); + fresh.tags.clear(); + fresh.style = None; + let format = template.text().unwrap().text.format_at(0)?.clone(); + fresh.content = onestore::page::ParagraphContent::Text(onestore::page::TextObject { + id: onestore::page::text::new_id()?, + date_field: None, + text: Paragraph::new(replacement.clone(), format), + tags: Vec::new(), + }); + PageOp::Add { + object: PageObject::Outline(onestore::page::Outline { id: onestore::page::text::new_id()?, title: false, min_width: None, @@ -253,29 +217,32 @@ fn main() -> Result<(), Box> { indents: Vec::new(), paragraphs: vec![fresh], unsupported: Vec::new(), - }; - let at = page + }), + before: page .objects .iter() - .position(|o| matches!(o, PageObject::Title(_))) - .unwrap_or(page.objects.len()); - page.objects.insert(at, PageObject::Outline(outline)); + .find(|o| matches!(o, PageObject::Title(_))) + .map(PageObject::id), } - "format" => { - assert!((1..=124).contains(&operation)); - let target = target_paragraph(&mut page, oid).text_mut().unwrap(); - let mut format = target.text.format_at(0)?.clone(); - format.font_size = Some(6.0 + operation as f32); - target.text.apply(Edit { - range: 0..end, - replacement: Paragraph::new(text.clone(), format), - })?; + } + "format" => { + assert!((1..=124).contains(&operation)); + PageOp::Format { + text: oid, + range: 0..end, + set: vec![onestore::TextAttribute::FontSize(6.0 + operation as f32)], + clear: Vec::new(), } - _ => unreachable!(), } - (sid, page) + _ => unreachable!(), }; - let id = cache.save(&source, space, &page, "Fixture")?.unwrap(); + let id = cache.apply( + "Fixture", + onestore::op::Edit { + at: 133_000_000_000_000_000, + ops: vec![Op::Page { space: sid, op }], + }, + )?; if acknowledgement == "unack" { println!("durable {operation} {id}"); } else { diff --git a/crates/notebook/examples/keystroke_probe.rs b/crates/notebook/examples/keystroke_probe.rs new file mode 100644 index 0000000000000000000000000000000000000000..b7829a358d5851bbc136d47c7f89db3861f717b1 --- /dev/null +++ b/crates/notebook/examples/keystroke_probe.rs @@ -0,0 +1,260 @@ +//! `keystroke_probe PARAGRAPHS KEYSTROKES`: types into the middle paragraph of a page +//! of PARAGRAPHS paragraphs and reports, per keystroke, the time, the bytes the cache +//! writes to disk and the bytes the section file sees: queued alone, then each one +//! published at once, then idle polls. The section file stays in memory so the +//! process's disk writes (macOS `proc_pid_rusage`) are the cache's. + +use notebook::{Remote, Replica}; +use onestore::{ + CommitError, CommitIo, ExGuid, Stamp, Transaction, + document::{Format, Layout}, + op::{Edit, Op, PageOp, SectionOp}, + page::{ + Outline, Page, PageObject, PageParagraph, Paragraph, ParagraphContent, TextObject, + text::new_id, + }, +}; +use std::{ + io, + time::{Duration, Instant}, +}; + +fn disk_written() -> u64 { + // SAFETY: proc_pid_rusage fills the zeroed structure for this process. + unsafe { + let mut info: libc::rusage_info_v2 = std::mem::zeroed(); + libc::proc_pid_rusage( + libc::getpid(), + libc::RUSAGE_INFO_V2, + &mut info as *mut _ as *mut _, + ); + info.ri_diskio_byteswritten + } +} + +/// The section file in memory, counting what synchronization reads and writes, so the +/// process's disk writes are the cache's alone. +struct Memory { + bytes: Vec, + read: u64, + written: u64, +} + +impl CommitIo for Memory { + fn read_at(&mut self, offset: u64, output: &mut [u8]) -> io::Result { + let offset = offset as usize; + let size = output.len().min(self.bytes.len().saturating_sub(offset)); + output[..size].copy_from_slice(&self.bytes[offset..offset + size]); + self.read += size as u64; + Ok(size) + } + fn write_at(&mut self, offset: u64, bytes: &[u8]) -> io::Result { + let offset = offset as usize; + self.bytes + .resize(self.bytes.len().max(offset + bytes.len()), 0); + self.bytes[offset..offset + bytes.len()].copy_from_slice(bytes); + self.written += bytes.len() as u64; + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl Remote for Memory { + fn read(&mut self) -> io::Result> { + self.read += self.bytes.len() as u64; + Ok(self.bytes.clone()) + } + fn stamp(&mut self) -> io::Result> { + self.read += 1024; + Ok(Stamp::of(&self.bytes).ok()) + } + fn publish(&mut self, transaction: &Transaction) -> Result<(), CommitError> { + transaction.commit(self) + } + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + onestore::confirm_snapshot(self, snapshot) + } +} + +fn paragraph(i: usize) -> PageParagraph { + let words = [ + "quick", "brown", "fox", "jumps", "over", "the", "lazy", "dog", + ]; + let text: Vec<&str> = (0..1 + i % 17).map(|j| words[(i * 7 + j) % 8]).collect(); + PageParagraph { + id: new_id().unwrap(), + parent: None, + level: 1, + style: None, + format: Format::default(), + content: ParagraphContent::Text(TextObject { + id: new_id().unwrap(), + date_field: None, + text: Paragraph::new(text.join(" "), Format::default()), + tags: Vec::new(), + }), + lists: Vec::new(), + tags: Vec::new(), + media: Default::default(), + collapsed: false, + } +} + +fn percentiles(label: &str, mut times: Vec, bytes: &[u64]) { + times.sort(); + let at = |q: f64| times[((times.len() - 1) as f64 * q) as usize]; + let mean = bytes.iter().sum::() as f64 / bytes.len() as f64; + let mut sorted = bytes.to_vec(); + sorted.sort(); + println!( + "{label}: time p50 {:.2?} p90 {:.2?} max {:.2?}; bytes mean {:.0} p50 {} max {}", + at(0.5), + at(0.9), + at(1.0), + mean, + sorted[sorted.len() / 2], + sorted[sorted.len() - 1] + ); +} + +fn main() -> Result<(), Box> { + let mut args = std::env::args().skip(1); + let count: usize = args.next().ok_or("PARAGRAPHS")?.parse()?; + let keystrokes: usize = args.next().map_or(Ok(200), |n| n.parse())?; + let directory = tempfile::tempdir()?; + // The probe page, stored in the section file before the cache opens. + let source = { + let arena = onestore::Arena::default(); + let mut section = + onestore::Section::open(&arena, onestore::create_section("probe.one", "", "Probe")?)?; + let creation = onestore::PageCreation::new(None, Some("Probe"), "Probe")?; + let page = Page { + title: "Probe".into(), + identity: None, + created: None, + margin_origin: [0.0; 2], + objects: vec![PageObject::Outline(Outline { + id: new_id()?, + title: false, + min_width: None, + layout: Layout { + x: Some(36.0), + y: Some(86.4), + ..Layout::default() + }, + indents: Vec::new(), + paragraphs: (0..count).map(paragraph).collect(), + unsupported: Vec::new(), + })], + definitions: Default::default(), + }; + section.apply( + "Probe", + &Edit { + at: 133_000_000_000_000_000, + ops: vec![Op::Section(SectionOp::Import { creation, page })], + }, + )?; + section.seal()?; + section.image() + }; + println!("section {} bytes, {count} paragraphs", source.len()); + let cache = Replica::create(directory.path().join("cache.sqlite"), &source)?; + let (space, _, _) = cache + .pages()? + .into_iter() + .find(|(_, title, _)| title == "Probe") + .ok_or("the probe page")?; + let target: ExGuid = cache + .page(space)? + .objects + .iter() + .find_map(|object| match object { + PageObject::Outline(outline) if outline.paragraphs.len() == count => { + outline.paragraphs[count / 2].text().map(|text| text.id) + } + _ => None, + }) + .ok_or("the probe paragraph")?; + let mut remote = Memory { + bytes: source.clone(), + read: 0, + written: 0, + }; + let mut at = 0_u32; + let mut keystroke = |cache: &Replica| -> Result<(), notebook::Error> { + cache.apply( + "Probe", + Edit { + at: 133_000_000_000_000_000, + ops: vec![Op::Page { + space, + op: PageOp::Text { + text: target, + range: at..at, + with: "x".into(), + }, + }], + }, + )?; + at += 1; + Ok(()) + }; + + let (mut times, mut bytes) = (Vec::new(), Vec::new()); + for _ in 0..keystrokes { + let (disk, start) = (disk_written(), Instant::now()); + keystroke(&cache)?; + times.push(start.elapsed()); + bytes.push(disk_written() - disk); + } + percentiles("queued keystroke (apply + SQLite commit)", times, &bytes); + + let (disk, read, written, start) = + (disk_written(), remote.read, remote.written, Instant::now()); + cache.sync_once(&mut remote)?; + println!( + "publish the {keystrokes} queued keystrokes as one batch: {:.2?}; cache {} B, file writes {} B, file reads {} B", + start.elapsed(), + disk_written() - disk, + remote.written - written, + remote.read - read + ); + + let (mut times, mut cache_bytes, mut file_bytes, mut reads) = + (Vec::new(), Vec::new(), Vec::new(), Vec::new()); + for _ in 0..keystrokes { + let (disk, read, written, start) = + (disk_written(), remote.read, remote.written, Instant::now()); + keystroke(&cache)?; + cache.sync_once(&mut remote)?; + times.push(start.elapsed()); + file_bytes.push(remote.written - written); + cache_bytes.push(disk_written() - disk); + reads.push(remote.read - read); + } + percentiles( + "keystroke published at once: cache", + times.clone(), + &cache_bytes, + ); + percentiles( + "keystroke published at once: file writes", + times.clone(), + &file_bytes, + ); + percentiles("keystroke published at once: file reads", times, &reads); + + let (disk, read) = (disk_written(), remote.read); + for _ in 0..100 { + assert!(cache.sync_once(&mut remote)?.edit.is_none()); + } + println!( + "100 idle polls: cache {} B, file reads {} B", + disk_written() - disk, + remote.read - read + ); + Ok(()) +} diff --git a/crates/notebook/examples/migrate_probe.rs b/crates/notebook/examples/migrate_probe.rs new file mode 100644 index 0000000000000000000000000000000000000000..c26e8032a9628e230bad606ae71ce34f6ca22908 --- /dev/null +++ b/crates/notebook/examples/migrate_probe.rs @@ -0,0 +1,118 @@ +//! migrate_probe CACHE...: converts copies of schema-14 caches, reports what they hold +//! and, where nothing blocks the queue, publishes it to an in-memory copy of the remote. +use notebook::{Remote, Replica}; +use onestore::{CommitError, CommitIo, Stamp, Transaction}; +use std::io; + +struct Memory(Vec); + +impl CommitIo for Memory { + fn read_at(&mut self, offset: u64, output: &mut [u8]) -> io::Result { + let offset = offset as usize; + let size = output.len().min(self.0.len().saturating_sub(offset)); + output[..size].copy_from_slice(&self.0[offset..offset + size]); + Ok(size) + } + fn write_at(&mut self, offset: u64, bytes: &[u8]) -> io::Result { + let offset = offset as usize; + self.0.resize(self.0.len().max(offset + bytes.len()), 0); + self.0[offset..offset + bytes.len()].copy_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +impl Remote for Memory { + fn read(&mut self) -> io::Result> { + Ok(self.0.clone()) + } + fn stamp(&mut self) -> io::Result> { + Ok(Stamp::of(&self.0).ok()) + } + fn publish(&mut self, transaction: &Transaction) -> Result<(), CommitError> { + transaction.commit(self) + } + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + onestore::confirm_snapshot(self, snapshot) + } +} + +fn pages(replica: &Replica) -> Result, notebook::Error> { + replica + .pages()? + .into_iter() + .map(|(space, ..)| replica.page(space)) + .collect() +} + +fn main() -> Result<(), Box> { + for path in std::env::args().skip(1) { + let copy = std::env::temp_dir().join(format!( + "migrate-probe-{}-{}", + std::process::id(), + std::path::Path::new(&path) + .file_name() + .unwrap() + .to_string_lossy() + )); + std::fs::copy(&path, ©)?; + // A cache open or crashed in WAL mode keeps committed pages in its `-wal` file. + let wal = format!("{path}-wal"); + if std::path::Path::new(&wal).exists() { + let mut target = copy.as_os_str().to_owned(); + target.push("-wal"); + std::fs::copy(&wal, target)?; + } + let started = std::time::Instant::now(); + match Replica::open(©) { + Ok(replica) => { + let elapsed = started.elapsed(); + let pending = replica.pending()?; + let summary = replica.recovery_summary()?; + println!( + "{path}: converted in {:.1} ms; {} edits; {:?}; conflict {:?}", + elapsed.as_secs_f64() * 1e3, + pending.len(), + summary, + replica.conflict()? + ); + for edit in &pending { + println!( + " {} {:?} {} ops", + edit.id, + replica.status(edit.id)?, + edit.edit.ops.len() + ); + } + for (space, title, level) in replica.pages()? { + println!(" page {space} {level} {title:?}"); + } + let blocked = replica.conflict()?.is_some() + || replica.recovery_summary()?.uncertain_edits > 0; + if !pending.is_empty() && !blocked { + let local = pages(&replica)?; + let mut remote = Memory(replica.remote_snapshot()?); + let synced = replica.sync_once(&mut remote)?; + let published = { + let arena = onestore::Arena::default(); + let mut section = onestore::Section::open(&arena, remote.0.clone())?; + section + .pages()? + .into_iter() + .map(|(space, ..)| section.page(space)) + .collect::, _>>()? + }; + println!( + " published {:?}; remote pages equal the converted pages: {}", + synced.edit.map(|(id, _)| id), + published == local + ); + } + } + Err(error) => println!("{path}: {error}"), + } + } + Ok(()) +} diff --git a/crates/notebook/examples/queue_scale.rs b/crates/notebook/examples/queue_scale.rs index 84e424f42352dcbe4e0941bac01577904f90a9b1..4b5e4393cc08bab0482a7c7d57993a199014e4eb 100644 --- a/crates/notebook/examples/queue_scale.rs +++ b/crates/notebook/examples/queue_scale.rs @@ -47,13 +47,9 @@ fn main() -> Result<(), Box> { let start = Instant::now(); for n in 0..count { let edit_start = Instant::now(); - let source = cache.snapshot()?; - let store = Store::parse(&source)?; - let index = RevisionIndex::parse(&store)?; - let document = Document::parse(&index)?; let slot = n % 2; let space = pages[slot].0; - let mut page = Page::from_space(&document, space)?; + let mut page = cache.page(space)?; let text = page .objects .iter_mut() @@ -76,13 +72,24 @@ fn main() -> Result<(), Box> { range: 0..end, replacement: Paragraph::new(expected[slot].clone(), format), })?; - let id = cache.save(&source, space, &page, "Fixture")?.unwrap(); + let before = cache.page(space)?; + let ops = onestore::op::lower_page(&before, &page)?; + let id = cache.apply( + "Fixture", + onestore::op::Edit { + at: 133_000_000_000_000_000, + ops: ops + .into_iter() + .map(|op| onestore::op::Op::Page { space, op }) + .collect(), + }, + )?; assert!(ids.last().is_none_or(|previous| *previous < id)); ids.push(id); println!( "{}", serde_json::json!({"phase":"ack","n":n,"id":id, - "ms":edit_start.elapsed().as_secs_f64()*1000.0,"working_bytes":source.len()}) + "ms":edit_start.elapsed().as_secs_f64()*1000.0}) ); } println!( @@ -114,16 +121,16 @@ fn main() -> Result<(), Box> { serde_json::json!({"phase":"export","seconds":start.elapsed().as_secs_f64()}) ); let start = Instant::now(); - for (n, id) in ids.iter().enumerate() { - let step = Instant::now(); - assert!( - matches!(cache.sync_once(&mut remote)?, Some((actual, EditStatus::Published { .. })) if actual == *id) - ); - println!( - "{}", - serde_json::json!({"phase":"publish","n":n,"id":id,"ms":step.elapsed().as_secs_f64()*1000.0}) - ); - } + // The queue publishes as one batch. + let step = Instant::now(); + assert!(matches!( + cache.sync_once(&mut remote)?.edit, + Some((actual, EditStatus::Published { .. })) if Some(&actual) == ids.last() + )); + println!( + "{}", + serde_json::json!({"phase":"publish","count":ids.len(),"ms":step.elapsed().as_secs_f64()*1000.0}) + ); assert!(cache.pending()?.is_empty()); let published = cache.snapshot()?; assert_eq!(onestore::read_file(&remote.0)?, published); @@ -145,7 +152,7 @@ fn main() -> Result<(), Box> { .collect::>(), ids ); - assert_eq!(archive.snapshot()?, source); + verify(&archive.snapshot()?, &pages, &expected)?; println!( "{}", serde_json::json!({"phase":"complete","count":count, diff --git a/crates/notebook/examples/recovery_probe.rs b/crates/notebook/examples/recovery_probe.rs index 0335d17dacaf0a81ac2a2ddb70d6ec8977056001..6a14689bbdde8f6ab1b9263621432439cbc10a44 100644 --- a/crates/notebook/examples/recovery_probe.rs +++ b/crates/notebook/examples/recovery_probe.rs @@ -98,12 +98,10 @@ fn report(cache: &Replica, root: &Path) -> Result<(), Box "{}", json!({"event":"state", "status":status, "revision":revision, "local_text":local.text, "remote_text":remote.text, "remote_revision":remote.revision.to_string(), - "pending":cache.pending()?.iter().map(|pending|match &pending.operation { - notebook::Operation::Page(intent) => match intent.text_change() { - Some((_, before, range, replacement)) => json!({"id":pending.id,"before":before,"replacement":replacement,"range":[range.start,range.end]}), - None => json!({"id":pending.id,"operation":"page"}), - }, - operation => json!({"id":pending.id,"operation":operation}), + "pending":cache.pending()?.iter().map(|pending| match &pending.edit.ops[..] { + [onestore::op::Op::Page { op: onestore::op::PageOp::Text { range, with, .. }, .. }] => + json!({"id":pending.id,"replacement":with,"range":[range.start,range.end]}), + _ => json!({"id":pending.id,"edit":pending.edit}), }).collect::>() }) ); Ok(()) @@ -128,34 +126,18 @@ fn main() -> Result<(), Box> { drop(remote); let cache = Replica::create(root.join("cache.sqlite"), &source)?; let at = u32::try_from(target.text.encode_utf16().count())?; - let mut page = { - let store = onestore::Store::parse(&source)?; - let index = onestore::RevisionIndex::parse(&store)?; - let document = onestore::document::Document::parse(&index)?; - onestore::page::Page::from_space(&document, target.space)? + let edit = onestore::op::Edit { + at: 133_000_000_000_000_000, + ops: vec![onestore::op::Op::Page { + space: target.space, + op: onestore::op::PageOp::Text { + text: target.object, + range: at..at, + with: " [offline-recovery]".into(), + }, + }], }; - let paragraph = page - .objects - .iter_mut() - .find_map(|object| match object { - onestore::page::PageObject::Outline(outline) => outline - .paragraphs - .iter_mut() - .find(|p| p.text().is_some_and(|t| t.id == target.object)), - _ => None, - }) - .ok_or("Target paragraph is not on its page")? - .text_mut() - .unwrap(); - let format = paragraph.text.format_at(at)?.clone(); - paragraph.text.apply(onestore::page::text::Edit { - range: at..at, - replacement: onestore::page::Paragraph::new(" [offline-recovery]".into(), format), - })?; - assert_eq!( - cache.save(&source, target.space, &page, "Fixture")?, - Some(1) - ); + assert_eq!(cache.apply("Fixture", edit)?, 1); phase("local-after"); report(&cache, root)?; } else if args.len() == 2 && ["inspect", "sync"].contains(&mode.as_str()) { diff --git a/crates/notebook/examples/session_client.rs b/crates/notebook/examples/session_client.rs index 2dfc33c0b1269f87b703ceaf6d36e2990ddbb0a8..2cae639fd49e77790f6457afab66f52ee6067608 100644 --- a/crates/notebook/examples/session_client.rs +++ b/crates/notebook/examples/session_client.rs @@ -159,9 +159,9 @@ fn main() -> Result<(), Box> { for event in section.events() { match event { Event::Attempt { - id, - status: EditStatus::Published { revision }, - } => receipts.push((id, revision)), + status: EditStatus::Published { .. }, + .. + } => {} Event::Attempt { id, status } => { println!( "{}", @@ -175,7 +175,16 @@ fn main() -> Result<(), Box> { ); } Event::Failed(error) => return Err(error.into()), - Event::Refreshed => {} + Event::Rejected { error, .. } => return Err(error.into()), + Event::Refreshed | Event::Changed(_) => {} + } + } + // A batch reports its newest edit; each edit's receipt is its own. + for (id, ..) in &queued { + if !receipts.iter().any(|(n, _)| n == id) + && let Some(EditStatus::Published { revision }) = section.status(*id)? + { + receipts.push((*id, revision)); } } } diff --git a/crates/notebook/examples/smb_offline_client.rs b/crates/notebook/examples/smb_offline_client.rs index 3867a0f2b18b07e95c4a81829d9490cb9fa43055..73e90b9eb2bfb40e097ca0813d8695c521943bb3 100644 --- a/crates/notebook/examples/smb_offline_client.rs +++ b/crates/notebook/examples/smb_offline_client.rs @@ -3,9 +3,7 @@ mod concurrent; use notebook::smb::{Client, Credentials}; use notebook::{EditStatus, Error, Remote, Replica, SmbRemote}; -use onestore::{ - CommitError, CommitState, ExGuid, RevisionIndex, Store, Transaction, page::Paragraph, -}; +use onestore::{CommitError, CommitState, ExGuid, RevisionIndex, Store, Transaction}; use serde_json::json; use std::{ env, @@ -252,91 +250,19 @@ impl Remote for Traced { } } -fn tokens(text: &str) -> Option> { - let mut remaining = text.strip_prefix("Concurrent edits:")?; - let mut tokens = Vec::new(); - while !remaining.is_empty() { - let body = remaining.strip_prefix(" [w")?; - let (token, tail) = body.split_once(']')?; - let (actor, operation) = token.split_once(':')?; - if actor.is_empty() - || operation.is_empty() - || !actor - .bytes() - .chain(operation.bytes()) - .all(|b| b.is_ascii_digit()) - || tokens.contains(&token) - { - return None; - } - tokens.push(token); - remaining = tail; +/// The edit appending `token` at `at` of a text. +fn append(space: ExGuid, text: ExGuid, at: u32, token: &str) -> onestore::op::Edit { + onestore::op::Edit { + at: u64::try_from(now()).unwrap_or_default() * 10 + 116_444_736_000_000_000, + ops: vec![onestore::op::Op::Page { + space, + op: onestore::op::PageOp::Text { + text, + range: at..at, + with: token.to_owned(), + }, + }], } - Some(tokens) -} - -// This owned workload explicitly resolves append conflicts after all retained tokens. -fn page_with( - bytes: &[u8], - space: ExGuid, - text: ExGuid, -) -> Result> { - let store = Store::parse(bytes)?; - let index = RevisionIndex::parse(&store)?; - let document = onestore::document::Document::parse(&index)?; - let page = onestore::page::Page::from_space(&document, space)?; - if paragraph_mut(&mut page.clone(), text).is_none() { - return Err("The target text is not on its page".into()); - } - Ok(page) -} - -fn paragraph_mut( - page: &mut onestore::page::Page, - text: ExGuid, -) -> Option<&mut onestore::page::TextObject> { - for object in &mut page.objects { - let outlines: Vec<&mut onestore::page::Outline> = match object { - onestore::page::PageObject::Outline(outline) => vec![outline], - onestore::page::PageObject::Title(title) => title.outlines.iter_mut().collect(), - _ => Vec::new(), - }; - for outline in outlines { - if let Some(paragraph) = outline - .paragraphs - .iter_mut() - .find(|p| p.text().is_some_and(|t| t.id == text)) - { - return paragraph.text_mut(); - } - } - } - None -} - -fn append_to(page: &mut onestore::page::Page, text: ExGuid, at: u32, token: &str) { - let target = paragraph_mut(page, text).expect("target text is on its page"); - let format = target.text.format_at(at).unwrap().clone(); - target - .text - .apply(onestore::page::text::Edit { - range: at..at, - replacement: Paragraph::new(token.into(), format), - }) - .unwrap(); -} - -fn append_position(before: &str, current: &str, token: &str) -> Option { - let original = tokens(before)?; - let present = tokens(current)?; - let mut retained = present.iter(); - for token in original { - retained.find(|&&candidate| candidate == token)?; - } - if current.contains(token) { - return None; - } - u32::try_from(current.encode_utf16().count()).ok() } fn main() -> Result<(), Box> { @@ -459,10 +385,11 @@ fn main() -> Result<(), Box> { let at = u32::try_from(target.text.encode_utf16().count())?; let token = format!(" [{}:{}]", args[2], generated); let started = now(); - let mut page = page_with(&source, target.space, target.object)?; - append_to(&mut page, target.object, at, &token); - match cache.save(&source, target.space, &page, "Offline document writer") { - Ok(Some(id)) => { + match cache.apply( + "Offline document writer", + append(target.space, target.object, at, &token), + ) { + Ok(id) => { println!( "{}", json!({"event":"local_commit", "id":id, "operation":generated, "space":target.space.to_string(), "object":target.object.to_string(), "before":target.text, "token":token, "started_us":started, "finished_us":now()}) @@ -470,43 +397,44 @@ fn main() -> Result<(), Box> { ids.push(id); generated += 1; } - Err(Error::Io(error)) if error.kind() == io::ErrorKind::ResourceBusy => {} other => return Err(format!("Unexpected local result: {other:?}").into()), } } - if let Some(intent) = pending.first() - && matches!(cache.status(intent.id)?, Some(EditStatus::Conflict(_))) - { - let local = cache.snapshot()?; - let remote = cache.remote_snapshot()?; - let current = view(&remote)?; - let notebook::Operation::Page(edit) = &intent.operation else { - return Err(format!( - "Intent {} requires review: {:?}", - intent.id, - cache.status(intent.id)? - ) - .into()); - }; - let (object, before, _, token) = edit - .text_change() - .ok_or("A review expects one appended token")?; - let at = append_position(&before, ¤t.text, &token) - .ok_or("Append model disagrees with retained history")?; - let mut reviewed = page_with(&remote, intent.space, object)?; - append_to(&mut reviewed, object, at, &token); - match cache.review_page(intent.id, &local, &remote, &reviewed) { - Ok(()) => println!( - "{}", - json!({"event":"reviewed_append", "id":intent.id, "before":before, "remote":current.text, "token":token, "at_us":now()}) - ), + if let Some(conflict) = cache.conflict()? { + // Concurrent appends meet at the end of the text: take the remote text and + // append the queued tokens it lacks after it again. + let remote = view(&cache.remote_snapshot()?)?; + let tokens: Vec = cache + .pending()? + .iter() + .flat_map(|queued| queued.edit.ops.iter()) + .filter_map(|op| match op { + onestore::op::Op::Page { + op: onestore::op::PageOp::Text { with, .. }, + .. + } => Some(with.clone()), + _ => None, + }) + .filter(|token| !remote.text.contains(token.as_str())) + .collect(); + match cache.resolve(conflict.id, notebook::Resolution::Theirs) { + Ok(()) => { + let mut at = u32::try_from(remote.text.encode_utf16().count())?; + for token in &tokens { + cache.apply( + "Offline document writer", + append(remote.space, remote.object, at, token), + )?; + at += u32::try_from(token.encode_utf16().count())?; + } + println!( + "{}", + json!({"event":"reviewed_append", "id":conflict.id, "remote":remote.text, "tokens":tokens, "at_us":now()}) + ); + } Err(Error::Io(error)) - if [ - io::ErrorKind::ResourceBusy, - io::ErrorKind::WouldBlock, - io::ErrorKind::InvalidInput, - ] - .contains(&error.kind()) => {} + if [io::ErrorKind::WouldBlock, io::ErrorKind::InvalidInput] + .contains(&error.kind()) => {} Err(error) => return Err(error.into()), } } @@ -546,74 +474,3 @@ fn main() -> Result<(), Box> { ); Ok(()) } - -#[test] -fn append_review_requires_a_unique_ordered_history_and_an_absent_new_token() { - assert_eq!( - append_position( - "Concurrent edits: [w0:0]", - "Concurrent edits: [w1:0] [w0:0]", - " [w0:1]" - ), - Some(31) - ); - for current in [ - "Concurrent edits:", - "Concurrent edits: [w0:0] [w0:0]", - "Concurrent edits: [w0:1] [w0:0]", - "Concurrent edits: changed [w0:0]", - ] { - assert_eq!( - append_position("Concurrent edits: [w0:0]", current, " [w0:1]"), - None - ); - } - assert_eq!( - append_position( - "Concurrent edits: [w0:0] [w1:0]", - "Concurrent edits: [w1:0] [w0:0]", - " [w0:1]" - ), - None - ); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn publication_differences_retain_removed_text_and_graph_identities() { - let before = DocumentView { - texts: serde_json::from_value(json!({"left":{"text":"ab"}, "right":{"text":"cd"}, "untouched":{"text":"ef"}})).unwrap(), - graph: serde_json::from_value(json!({"outline":{"children":["a","b"]}, "a":{"content":["left"]}, "b":{"content":["right"]}})).unwrap(), - }; - let after = DocumentView { - texts: serde_json::from_value( - json!({"left":{"text":"abcd"}, "untouched":{"text":"ef"}}), - ) - .unwrap(), - graph: serde_json::from_value( - json!({"outline":{"children":["a"]}, "a":{"content":["left"]}}), - ) - .unwrap(), - }; - let changes = after.changes(&before); - assert_eq!( - changes.texts, - serde_json::from_value::>( - json!({"left":{"text":"abcd"}, "right":null}) - ) - .unwrap() - ); - assert_eq!( - changes.graph, - serde_json::from_value::>( - json!({"outline":{"children":["a"]}, "b":null}) - ) - .unwrap() - ); - assert_eq!(before.changes(&after).texts["right"], before.texts["right"]); - assert_eq!(before.changes(&before), DocumentView::default()); - } -} diff --git a/crates/notebook/src/assets.rs b/crates/notebook/src/assets.rs index 91f8a5b32f0d429f1ea4e5be248ca15e522b3dd6..bc789dc0f27baa5924c53bd788c51654a2785037 100644 --- a/crates/notebook/src/assets.rs +++ b/crates/notebook/src/assets.rs @@ -1,5 +1,9 @@ use super::*; use onestore::FileDataReference; +use onestore::{ + RevisionIndex, Store, + document::{Document, Kind}, +}; use rusqlite::OptionalExtension; use sha2::{Digest, Sha256}; @@ -7,12 +11,7 @@ impl Replica { /// Reads a previously downloaded external payload without network access. /// Absence is distinct from an empty payload; every returned buffer passes its stored checksum. pub fn cached_asset(&self, filename: &str, limit: usize) -> Result>> { - let key = key(filename)?; - let connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - cached(&connection, &key, limit) + cached(&*self.lock()?, &key(filename)?, limit) } /// Fetches a declared external payload and durably retains it without changing the edit queue. @@ -26,32 +25,23 @@ impl Replica { limit: usize, ) -> Result> { let key = key(filename)?; - { - let connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - if !referenced(&connection, &key)? { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "The retained document images do not reference this external payload", - ) - .into()); - } + if !self.references(&key)? { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "The retained document images do not reference this external payload", + ) + .into()); } let bytes = crate::discover::read_external_asset(source, section, filename, limit)?; - let mut connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; - if !referenced(&transaction, &key)? { + if !self.references(&key)? { return Err(io::Error::new( io::ErrorKind::ResourceBusy, "The external payload reference changed during download", ) .into()); } + let mut connection = self.lock()?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; let previous = match cached(&transaction, &key, bytes.len()) { Err(Error::Io(error)) if error.kind() == io::ErrorKind::FileTooLarge => { return Err(Error::AssetChanged); @@ -78,21 +68,22 @@ pub(crate) fn key(filename: &str) -> Result { Ok(filename.to_ascii_lowercase()) } -fn referenced(connection: &Connection, key: &str) -> Result { - // One image at a time: the working image usually answers. - for read in [crate::images::working, crate::images::base] { - let image = read(connection)?; - let store = Store::parse(&image)?; - let index = RevisionIndex::parse(&store)?; - let document = Document::parse(&index)?; - if document.spaces.values().flat_map(|space| space.revisions.values()) - .flat_map(|revision| revision.nodes.values()).any(|node| { - matches!(&node.kind, Kind::File { reference: FileDataReference::External(name), .. } if name.eq_ignore_ascii_case(key)) - }) { - return Ok(true); +impl Replica { + /// Whether the local pages or the remote image declare the external payload `key`. + fn references(&self, key: &str) -> Result { + for image in [self.snapshot()?, self.remote_snapshot()?] { + let store = Store::parse(&image)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + if document.spaces.values().flat_map(|space| space.revisions.values()) + .flat_map(|revision| revision.nodes.values()).any(|node| { + matches!(&node.kind, Kind::File { reference: FileDataReference::External(name), .. } if name.eq_ignore_ascii_case(key)) + }) { + return Ok(true); + } } + Ok(false) } - Ok(false) } pub(crate) fn cached(connection: &Connection, key: &str, limit: usize) -> Result>> { diff --git a/crates/notebook/src/base.rs b/crates/notebook/src/base.rs new file mode 100644 index 0000000000000000000000000000000000000000..070c665191a3bdb64808dc633d43e1f349098934 --- /dev/null +++ b/crates/notebook/src/base.rs @@ -0,0 +1,231 @@ +//! Section images kept as 16 KiB chunk rows, so a publication rewrites the chunks its +//! transaction touches instead of the image: `base`, the image the queued edits apply to, +//! and `remote`, the last observed remote image while it is not the base. + +use crate::Result; +use onestore::{Stamp, Transaction}; +use rusqlite::{Connection, OptionalExtension, params}; +use std::io; + +const CHUNK: usize = 16 * 1024; + +#[derive(Clone, Copy)] +pub(crate) enum Image { + Base, + Remote, +} + +impl Image { + fn table(self) -> &'static str { + match self { + Self::Base => "base", + Self::Remote => "remote", + } + } +} + +fn damaged() -> crate::Error { + io::Error::new(io::ErrorKind::InvalidData, "Damaged cached image").into() +} + +/// The whole image, or `None` when none is stored. +pub(crate) fn read(connection: &Connection, image: Image) -> Result>> { + let mut query = connection.prepare_cached(&format!( + "SELECT chunk, bytes FROM {} ORDER BY chunk", + image.table() + ))?; + let mut rows = query.query([])?; + let mut bytes = Vec::new(); + let mut count = 0; + while let Some(row) = rows.next()? { + let chunk: i64 = row.get(0)?; + let part = row.get_ref(1)?.as_blob().map_err(|_| damaged())?; + if chunk != count || bytes.len() % CHUNK != 0 || part.is_empty() || part.len() > CHUNK { + return Err(damaged()); + } + bytes.extend_from_slice(part); + count += 1; + } + Ok((count > 0).then_some(bytes)) +} + +/// The stored image's header and length, reading its first and last chunks. +pub(crate) fn stamp(connection: &Connection, image: Image) -> Result> { + let table = image.table(); + let Some((header, last)): Option<(Vec, i64)> = connection + .query_row( + &format!("SELECT substr(bytes, 1, 1024), (SELECT max(chunk) FROM {table}) FROM {table} WHERE chunk=0"), + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional()? + else { + return Ok(None); + }; + let tail: i64 = connection.query_row( + &format!("SELECT length(bytes) FROM {table} WHERE chunk=?1"), + [last], + |row| row.get(0), + )?; + Ok(Some(Stamp { + header: header.try_into().map_err(|_| damaged())?, + length: u64::try_from(last).map_err(|_| damaged())? * CHUNK as u64 + + u64::try_from(tail).map_err(|_| damaged())?, + })) +} + +/// Stores `bytes` as the image, rewriting only the chunks that differ. +pub(crate) fn write(connection: &Connection, image: Image, bytes: &[u8]) -> Result<()> { + let table = image.table(); + let count = bytes.len().div_ceil(CHUNK); + connection.execute( + &format!("DELETE FROM {table} WHERE chunk>=?1"), + [count as i64], + )?; + let mut stored = + connection.prepare_cached(&format!("SELECT bytes FROM {table} WHERE chunk=?1"))?; + let mut replace = connection.prepare_cached(&format!( + "INSERT INTO {table}(chunk, bytes) VALUES (?1, ?2) ON CONFLICT(chunk) DO UPDATE SET bytes=excluded.bytes" + ))?; + for (index, part) in bytes.chunks(CHUNK).enumerate() { + let current: Option> = stored + .query_row([index as i64], |row| row.get(0)) + .optional()?; + if current.as_deref() != Some(part) { + replace.execute(params![index as i64, part])?; + } + } + Ok(()) +} + +pub(crate) fn clear(connection: &Connection, image: Image) -> Result<()> { + connection.execute(&format!("DELETE FROM {}", image.table()), [])?; + Ok(()) +} + +/// Commits `transaction` to the base image the way it commits to the file, rewriting only +/// the chunks it writes. +pub(crate) fn publish(connection: &Connection, transaction: &Transaction) -> Result<()> { + if stamp(connection, Image::Base)?.as_ref() != Some(transaction.base()) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "The published transaction is not on the cached base image", + ) + .into()); + } + let mut stored = connection.prepare_cached("SELECT bytes FROM base WHERE chunk=?1")?; + let mut replace = connection.prepare_cached( + "INSERT INTO base(chunk, bytes) VALUES (?1, ?2) ON CONFLICT(chunk) DO UPDATE SET bytes=excluded.bytes", + )?; + let mut chunks: std::collections::BTreeMap> = Default::default(); + for (offset, bytes) in transaction.writes() { + let mut offset = usize::try_from(offset).map_err(|_| damaged())?; + let mut bytes = bytes; + while !bytes.is_empty() { + let index = offset / CHUNK; + let chunk = match chunks.entry(index) { + std::collections::btree_map::Entry::Occupied(entry) => entry.into_mut(), + std::collections::btree_map::Entry::Vacant(entry) => entry.insert( + stored + .query_row([index as i64], |row| row.get(0)) + .optional()? + .unwrap_or_default(), + ), + }; + let at = offset % CHUNK; + let take = bytes.len().min(CHUNK - at); + if chunk.len() < at { + return Err(damaged()); + } + if chunk.len() < at + take { + chunk.resize(at + take, 0); + } + chunk[at..at + take].copy_from_slice(&bytes[..take]); + offset += take; + bytes = &bytes[take..]; + } + } + for (index, bytes) in chunks { + replace.execute(params![index as i64, bytes])?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn connection() -> Connection { + let connection = Connection::open_in_memory().unwrap(); + connection.execute_batch(crate::schema::QUEUE).unwrap(); + connection + } + + /// Types `text` at the start of the page's first body paragraph. + pub(crate) fn typed(section: &mut onestore::Section<'_>, space: onestore::ExGuid, text: &str) { + use onestore::op::{Edit, Op, PageOp}; + let page = section.page(space).unwrap(); + let target = page + .objects + .iter() + .find_map(|object| match object { + onestore::page::PageObject::Outline(outline) => outline + .paragraphs + .iter() + .find_map(|p| p.text().map(|t| t.id)), + _ => None, + }) + .unwrap(); + section + .apply( + "Author", + &Edit { + at: 133_000_000_000_000_000, + ops: vec![Op::Page { + space, + op: PageOp::Text { + text: target, + range: 0..0, + with: text.into(), + }, + }], + }, + ) + .unwrap(); + } + + #[test] + fn a_published_transaction_rewrites_only_its_chunks_and_matches_the_file() { + let source = + include_bytes!("../../../corpus/outline-edit/before/notebook/synthetic.one").to_vec(); + assert!(source.len() > 4 * CHUNK); + let arena = onestore::Arena::default(); + let mut section = onestore::Section::open(&arena, source.clone()).unwrap(); + let space = section.pages().unwrap()[0].0; + let connection = connection(); + write(&connection, Image::Base, &source).unwrap(); + assert_eq!(read(&connection, Image::Base).unwrap().unwrap(), source); + assert_eq!( + stamp(&connection, Image::Base).unwrap().unwrap(), + Stamp::of(&source).unwrap() + ); + let mut image = source.clone(); + for round in 0..3 { + typed(&mut section, space, &format!("{round}")); + let transaction = section.seal().unwrap().unwrap(); + publish(&connection, &transaction).unwrap(); + transaction.apply(&mut image).unwrap(); + assert_eq!(read(&connection, Image::Base).unwrap().unwrap(), image); + assert_eq!( + stamp(&connection, Image::Base).unwrap().unwrap(), + Stamp::of(&image).unwrap() + ); + assert!(publish(&connection, &transaction).is_err()); + } + let shorter = &image[..CHUNK + 5]; + write(&connection, Image::Base, shorter).unwrap(); + assert_eq!(read(&connection, Image::Base).unwrap().unwrap(), shorter); + clear(&connection, Image::Remote).unwrap(); + assert!(read(&connection, Image::Remote).unwrap().is_none()); + } +} diff --git a/crates/notebook/src/images.rs b/crates/notebook/src/images.rs deleted file mode 100644 index 963d76933beb2618d937bd1ac61d6441cd24d439..0000000000000000000000000000000000000000 --- a/crates/notebook/src/images.rs +++ /dev/null @@ -1,198 +0,0 @@ -//! The cache's two section images. `base` is stored whole; `working` is stored as the byte -//! ranges where it differs from `base`, so saving an edit writes the revision it appended -//! instead of the section. - -use crate::Result; -use onestore::Stamp; -use rusqlite::{Connection, params}; -use std::io; - -const BLOCK: usize = 4096; - -/// `image` as its length and the `(offset, bytes)` runs of blocks that differ from `base`; -/// empty when the images are equal. -fn difference(base: &[u8], image: &[u8]) -> Vec { - if base == image { - return Vec::new(); - } - let mut patch = (image.len() as u64).to_le_bytes().to_vec(); - let mut run: Option = None; - let close = |patch: &mut Vec, start: usize, end: usize| { - patch.extend_from_slice(&(start as u64).to_le_bytes()); - patch.extend_from_slice(&((end - start) as u64).to_le_bytes()); - patch.extend_from_slice(&image[start..end]); - }; - for start in (0..image.len()).step_by(BLOCK) { - let end = (start + BLOCK).min(image.len()); - if base.get(start..end) == Some(&image[start..end]) { - if let Some(from) = run.take() { - close(&mut patch, from, start); - } - } else { - run.get_or_insert(start); - } - } - if let Some(from) = run { - close(&mut patch, from, image.len()); - } - patch -} - -fn restore(mut base: Vec, patch: &[u8]) -> Result> { - let damaged = || io::Error::new(io::ErrorKind::InvalidData, "Damaged working image"); - let Some((length, mut runs)) = patch.split_first_chunk::<8>() else { - return if patch.is_empty() { - Ok(base) - } else { - Err(damaged().into()) - }; - }; - // Every block past the base is a run, so a whole patch bounds the image. - let length = usize::try_from(u64::from_le_bytes(*length)) - .ok() - .filter(|length| *length <= base.len() + patch.len()) - .ok_or_else(damaged)?; - base.resize(length, 0); - while let Some((header, rest)) = runs.split_first_chunk::<16>() { - let offset = usize::try_from(u64::from_le_bytes(header[..8].try_into().unwrap())) - .map_err(|_| damaged())?; - let size = usize::try_from(u64::from_le_bytes(header[8..].try_into().unwrap())) - .map_err(|_| damaged())?; - let (bytes, rest) = rest.split_at_checked(size).ok_or_else(damaged)?; - base.get_mut(offset..) - .and_then(|target| target.get_mut(..size)) - .ok_or_else(damaged)? - .copy_from_slice(bytes); - runs = rest; - } - if runs.is_empty() { - Ok(base) - } else { - Err(damaged().into()) - } -} - -pub(crate) fn base(connection: &Connection) -> Result> { - Ok(connection.query_row("SELECT base FROM replica WHERE id=1", [], |row| row.get(0))?) -} - -/// `(base, working)`. -pub(crate) fn both(connection: &Connection) -> Result<(Vec, Vec)> { - let (base, patch): (Vec, Vec) = - connection.query_row("SELECT base, working FROM replica WHERE id=1", [], |row| { - Ok((row.get(0)?, row.get(1)?)) - })?; - let working = restore(base.clone(), &patch)?; - Ok((base, working)) -} - -/// The working image's length, without restoring it. -pub(crate) fn working_length(connection: &Connection) -> Result { - Ok(connection.query_row( - "SELECT length(base), substr(working, 1, 8) FROM replica WHERE id=1", - [], - |row| { - // SQLite's substr of an empty blob is NULL. - let header: Option> = row.get(1)?; - Ok(match header.as_deref().and_then(<[u8]>::first_chunk::<8>) { - Some(length) => u64::from_le_bytes(*length), - None => row.get::<_, i64>(0)? as u64, - }) - }, - )?) -} - -/// Whether no edit is queued and the working image is the base image, without reading -/// either image. -pub(crate) fn settled(connection: &Connection) -> Result { - Ok(connection.query_row( - "SELECT length(working) = 0 AND NOT EXISTS (SELECT 1 FROM edits) FROM replica WHERE id=1", - [], - |row| row.get(0), - )?) -} - -/// The base image's stamp, without reading the image. -pub(crate) fn stamp(connection: &Connection) -> Result { - let (header, length): (Vec, i64) = connection.query_row( - "SELECT substr(base, 1, 1024), length(base) FROM replica WHERE id=1", - [], - |row| Ok((row.get(0)?, row.get(1)?)), - )?; - Ok(Stamp { - header: header - .try_into() - .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "Damaged base image"))?, - length: u64::try_from(length).map_err(io::Error::other)?, - }) -} - -pub(crate) fn working(connection: &Connection) -> Result> { - let (base, patch): (Vec, Vec) = - connection.query_row("SELECT base, working FROM replica WHERE id=1", [], |row| { - Ok((row.get(0)?, row.get(1)?)) - })?; - restore(base, &patch) -} - -/// Replaces the working image; `base` is the stored base image. -pub(crate) fn set_working(connection: &Connection, base: &[u8], working: &[u8]) -> Result<()> { - connection.execute( - "UPDATE replica SET working=?1 WHERE id=1", - [difference(base, working)], - )?; - Ok(()) -} - -/// Replaces the base image, keeping `working` as the working image. An unchanged base -/// leaves its stored bytes alone. -pub(crate) fn set_base( - connection: &Connection, - old: &[u8], - base: &[u8], - working: &[u8], -) -> Result<()> { - if old == base { - return set_working(connection, base, working); - } - connection.execute( - "UPDATE replica SET base=?1, working=?2 WHERE id=1", - params![base, difference(base, working)], - )?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn a_working_image_survives_as_its_difference_from_any_base() { - let base: Vec = (0..3 * BLOCK + 17).map(|i| i as u8).collect(); - let mut edited = base.clone(); - edited[5] ^= 1; - edited[2 * BLOCK + 1] ^= 1; - edited.extend_from_slice(b"appended revision"); - for image in [ - base.clone(), - edited.clone(), - base[..BLOCK + 3].to_vec(), - Vec::new(), - vec![7; 5 * BLOCK], - ] { - let patch = difference(&base, &image); - assert_eq!(restore(base.clone(), &patch).unwrap(), image); - } - assert!(difference(&base, &base).is_empty()); - assert!(difference(&base, &edited).len() < 3 * BLOCK); - let mut huge = difference(&base, &edited); - huge[..8].copy_from_slice(&u64::MAX.to_le_bytes()); - assert!(restore(base.clone(), &huge).is_err()); - let patch = difference(&base, &edited); - for cut in 1..patch.len() { - if let Ok(image) = restore(base.clone(), &patch[..cut]) { - assert_ne!(image, edited); - } - } - } -} diff --git a/crates/notebook/src/lib.rs b/crates/notebook/src/lib.rs index d60f120a5a131ca4ea7e4cab2c0041e0d2a5ef6b..15ed5979a9d09bc29ffcd15f6b9164fc52f14c76 100644 --- a/crates/notebook/src/lib.rs +++ b/crates/notebook/src/lib.rs @@ -6,32 +6,33 @@ pub mod discover; pub mod smb; use onestore::{ - ExGuid, PageCreation, PreparedEdit, RevisionIndex, Store, - document::{Document, Kind}, + ExGuid, + op::{Edit, OpError}, page::Page, }; -use rusqlite::{Connection, OpenFlags, OptionalExtension, TransactionBehavior, params}; +use rusqlite::{Connection, OpenFlags, TransactionBehavior, params}; use std::{ fs::OpenOptions, io, path::Path, - sync::{Mutex, atomic::AtomicI64}, + sync::{Arc, Mutex, MutexGuard, mpsc}, + thread::JoinHandle, time::Duration, }; mod assets; -mod images; +mod base; mod merge; -mod pages; -mod rebase; +mod migrate; +mod queue; mod recovery; mod schema; pub mod session; -pub use pages::PageEdits; pub use recovery::{Recovery, RecoverySummary}; mod sync; -pub use sync::{ConflictKind, EditStatus, Remote}; +pub use sync::{ConflictKind, EditStatus, Remote, Synced}; mod worker; +mod working; #[cfg(feature = "smb")] pub use smb::SmbRemote; pub use worker::SyncWorker; @@ -50,6 +51,9 @@ pub enum Error { RemoteIo(io::Error), #[error(transparent)] Discovery(#[from] discover::Error), + /// The section refused an edit; the pages it names are as they were. + #[error(transparent)] + Rejected(#[from] OpError), #[error("External payload identity now refers to different bytes")] AssetChanged, #[cfg(feature = "protected")] @@ -60,114 +64,49 @@ pub enum Error { type Result = std::result::Result; const APPLICATION_ID: u32 = 0x4f4e454f; -const SCHEMA_VERSION: u32 = 14; -/// An edited page model together with the stored model it was edited from. -/// `before` is the precondition reconciliation checks against the remote page. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PageIntent { - pub before: Page, - pub after: Page, - pub author: String, -} - -impl PageIntent { - /// Describes the intent as one paragraph's text replacement, when that is all it changes: - /// the text object, its text before, the replaced UTF-16 range and the replacement. - pub fn text_change(&self) -> Option<(ExGuid, String, std::ops::Range, String)> { - fn texts(page: &Page) -> Vec<(ExGuid, &onestore::page::Paragraph)> { - let mut out = Vec::new(); - for object in &page.objects { - let outlines: Vec<&onestore::page::Outline> = match object { - onestore::page::PageObject::Outline(outline) => vec![outline], - onestore::page::PageObject::Title(title) => title.outlines.iter().collect(), - _ => Vec::new(), - }; - for outline in outlines { - for paragraph in &outline.paragraphs { - if let Some(text) = paragraph.text() { - out.push((text.id, &text.text)); - } - } - } - } - out - } - let (before, after) = (texts(&self.before), texts(&self.after)); - if before.len() != after.len() { - return None; - } - let mut changed = None; - for ((id, x), (other, y)) in before.iter().zip(&after) { - if id != other { - return None; - } - if x.text() != y.text() { - if changed.is_some() { - return None; - } - changed = Some((*id, *x, *y)); - } - } - let (id, x, y) = changed?; - let (b, o) = (x.text(), y.text()); - let prefix = b - .char_indices() - .zip(o.chars()) - .take_while(|((_, c), d)| c == d) - .map(|((i, c), _)| i + c.len_utf8()) - .last() - .unwrap_or(0); - let suffix = b[prefix..] - .chars() - .rev() - .zip(o[prefix..].chars().rev()) - .take_while(|(c, d)| c == d) - .map(|(c, _)| c.len_utf8()) - .sum::(); - let start = x.utf16_offset(prefix).ok()?; - let end = x.utf16_offset(b.len() - suffix).ok()?; - Some(( - id, - b.to_owned(), - start..end, - o[prefix..o.len() - suffix].to_owned(), - )) - } -} - -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub enum Operation { - /// An edited page model; body content is edited only through this intent. - Page(PageIntent), - CreatePage(PageCreation), - Pages(PageEdits), - /// Permanent removal of explicitly selected page spaces from the section. - DeletePages(Vec), -} - -/// A locally acknowledged intent; its ID remains stable across cache reopen. +/// A locally durable edit; its ID remains stable across cache reopen. #[derive(Debug, Clone, PartialEq)] pub struct PendingEdit { + pub id: u64, + pub author: String, + pub edit: Edit, +} + +/// Unpublished work the remote section no longer accepts: the batch holding edit `id` +/// changed page `space` where the remote changed it too. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Conflict { pub id: u64, pub space: ExGuid, - pub operation: Operation, + pub kind: ConflictKind, +} + +/// How a conflict or an uncertain attempt ends. `Mine` keeps the local pages: a conflicted +/// page is rewritten from the remote page to the local one; a released attempt publishes +/// again. `Theirs` drops the local edits to the conflicted page, or the whole unpublished +/// branch of a released attempt. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Resolution { + Mine, + Theirs, } /// Owns one local cache. Share this handle between threads; a second open fails busy. -/// SQLite's exclusive connection retains ownership between local transactions. +/// Edits apply on the cache's section thread, which keeps the section parsed; SQLite's +/// exclusive connection retains ownership between local transactions. pub struct Replica { - connection: Mutex, + connection: Arc>, + working: Option>, + thread: Option>, synchronization: Mutex<()>, - /// The intent a synchronization step selected for publication, or zero. - in_flight: AtomicI64, - worker: Mutex>, + worker: Arc>>, + /// The section's root object space, which names the document. + root: ExGuid, } impl Replica { - /// Seeds a new cache from a validated notebook image, refusing any existing path. + /// Seeds a new cache from a validated section image, refusing any existing path. /// An initialization error preserves the created file for inspection. pub fn create(path: impl AsRef, source: &[u8]) -> Result { validate(source)?; @@ -179,364 +118,224 @@ impl Replica { options.mode(0o600); } drop(options.open(path.as_ref())?); - Self::connect(path.as_ref(), Some(source)) + let mut connection = cache_connection(path.as_ref())?; + write_ahead(&connection)?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Exclusive)?; + let tables: i64 = + transaction.query_row("SELECT count(*) FROM sqlite_schema", [], |row| row.get(0))?; + let application: u32 = + transaction.pragma_query_value(None, "application_id", |row| row.get(0))?; + if application != 0 || tables != 0 { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "Cache initialization found an existing database", + ) + .into()); + } + transaction.pragma_update(None, "application_id", APPLICATION_ID)?; + transaction.pragma_update(None, "user_version", schema::VERSION)?; + schema::create(&transaction)?; + base::write(&transaction, base::Image::Base, source)?; + transaction.commit()?; + Self::start(connection) } /// Reopens an existing cache and its durable pending edits without network access. - /// Unrecognized databases and unsupported journal modes are rejected without conversion. + /// A schema-14 cache is converted once, after exporting it to `.v14-recovery`; + /// a conversion that cannot reproduce every queued page leaves it untouched. + /// Unrecognized databases and unsupported journal modes are rejected. pub fn open(path: impl AsRef) -> Result { - Self::connect(path.as_ref(), None) - } - - fn connect(path: &Path, source: Option<&[u8]>) -> Result { + let path = path.as_ref(); let mut connection = cache_connection(path)?; - let transaction = connection.transaction_with_behavior(TransactionBehavior::Exclusive)?; let application: u32 = - transaction.pragma_query_value(None, "application_id", |row| row.get(0))?; - let version: u32 = - transaction.pragma_query_value(None, "user_version", |row| row.get(0))?; - if let Some(source) = source { - let tables: i64 = - transaction - .query_row("SELECT count(*) FROM sqlite_schema", [], |row| row.get(0))?; - if application != 0 || version != 0 || tables != 0 { - return Err(io::Error::new( - io::ErrorKind::AlreadyExists, - "Cache initialization found an existing database", - ) - .into()); - } - transaction.pragma_update(None, "application_id", APPLICATION_ID)?; - transaction.pragma_update(None, "user_version", SCHEMA_VERSION)?; - transaction.execute_batch( - " - CREATE TABLE replica ( - id INTEGER PRIMARY KEY CHECK(id=1), - base BLOB NOT NULL, - working BLOB NOT NULL - ) STRICT; - ", - )?; - schema::create(&transaction)?; - transaction.execute("INSERT INTO replica VALUES (1, ?1, x'')", [source])?; - } else { - if application != APPLICATION_ID { - return Err( - io::Error::new(io::ErrorKind::InvalidData, "Not a notebook cache").into(), - ); - } - if version != SCHEMA_VERSION { - // Older caches are refused rather than migrated until the application is - // usable end to end; the queue must be drained by the build that wrote it. + connection.pragma_query_value(None, "application_id", |row| row.get(0))?; + if application != APPLICATION_ID { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Not a notebook cache").into()); + } + let integrity: String = connection.query_row("PRAGMA quick_check", [], |row| row.get(0))?; + if integrity != "ok" { + return Err( + io::Error::new(io::ErrorKind::InvalidData, "Cache integrity check failed").into(), + ); + } + let version: u32 = connection.pragma_query_value(None, "user_version", |row| row.get(0))?; + // A schema-14 cache converts in its rollback journal, so a failed conversion leaves + // the file as it was. + match version { + migrate::VERSION => migrate::migrate(&mut connection, path)?, + schema::VERSION => {} + _ => { return Err(io::Error::new( io::ErrorKind::InvalidData, - format!("Cache schema version {version} is not the supported version {SCHEMA_VERSION}"), + format!( + "Cache schema version {version} is not the supported version {}", + schema::VERSION + ), ) .into()); } - validate_images(&transaction)?; - pending(&transaction)?; } - transaction.commit()?; + write_ahead(&connection)?; + Self::start(connection) + } + + fn start(connection: Connection) -> Result { + let connection = Arc::new(Mutex::new(connection)); + let worker = Arc::new(Mutex::new(std::sync::Weak::new())); + let (sender, thread, root) = working::spawn(Arc::clone(&connection), Arc::clone(&worker))?; Ok(Self { - connection: Mutex::new(connection), + connection, + working: Some(sender), + thread: Some(thread), synchronization: Mutex::new(()), - in_flight: AtomicI64::new(0), - worker: Mutex::new(std::sync::Weak::new()), + worker, + root, }) } - /// Returns the latest complete locally committed image, including pending edits. - pub fn snapshot(&self) -> Result> { - let connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - images::working(&connection) + fn lock(&self) -> Result> { + lock(&self.connection) } - /// The page in `space` as the working image stores it. + /// Hands `request` to the section thread. + fn send(&self, request: working::Request) -> Result<()> { + self.working + .as_ref() + .and_then(|working| working.send(request).ok()) + .ok_or_else(|| io::Error::other("The section thread stopped").into()) + } + + /// Asks the section thread and waits for its answer. + fn ask( + &self, + request: impl FnOnce(working::Reply) -> working::Request, + ) -> Result { + let (sender, receiver) = mpsc::sync_channel(1); + self.send(request(Box::new(move |result| { + let _ = sender.send(result); + })))?; + receiver + .recv() + .map_err(|_| io::Error::other("The section thread stopped"))? + } + + /// Applies an edit and queues it durably for publication, returning its id once + /// written. A refused edit returns `Rejected` and leaves every page as it was. + pub fn apply(&self, author: &str, edit: Edit) -> Result { + self.ask(|reply| working::Request::Apply { + author: author.to_owned(), + edit, + reply, + }) + } + + /// `apply` without waiting: `reply` runs on the section thread once the edit is + /// durable or refused. + pub(crate) fn submit( + &self, + author: &str, + edit: Edit, + reply: working::Reply, + ) -> Result<()> { + self.send(working::Request::Apply { + author: author.to_owned(), + edit, + reply, + }) + } + + /// The page in `space` as the queued edits leave it; O(page), for opening and reloading. pub fn page(&self, space: ExGuid) -> Result { - let snapshot = self.snapshot()?; - let store = Store::parse(&snapshot)?; - let index = RevisionIndex::parse(&store)?; - Ok(Page::from_space(&Document::parse(&index)?, space)?) + self.ask(|reply| working::Request::Page { space, reply }) } - /// Whether no edit is queued and the remote still has the last observed image's stamp, - /// reading neither image; the cache is not locked during remote I/O. - pub(crate) fn settled(&self, remote: &mut impl Remote) -> Result { - let lock = || { - self.connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked")) - }; - if !images::settled(&*lock()?)? { - return Ok(false); - } - let Some(stamp) = remote.stamp().map_err(Error::RemoteIo)? else { - return Ok(false); - }; - Ok(images::stamp(&*lock()?)? == stamp) + /// Page spaces, titles and outline levels (1 at the top) in section order. + pub fn pages(&self) -> Result> { + self.ask(|reply| working::Request::Pages { reply }) } + /// The section image the queued edits leave, the unsealed ones sealed as one more + /// revision whose identities differ per call: O(section), for tests and diagnostics. + pub fn snapshot(&self) -> Result> { + self.ask(|reply| working::Request::Image { reply }) + } + + /// The section file's identity, which internal links name as `section-id`. + pub fn identity(&self) -> Result<[u8; 16]> { + let stamp = base::stamp(&*self.lock()?, base::Image::Base)? + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Damaged cached image"))?; + Ok(onestore::Header::parse(&stamp.header)?.file_id) + } + + /// Queued edits, oldest first. pub fn pending(&self) -> Result> { - let connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - pending(&connection) - } - - /// Identities and page spaces of the pending edits, oldest first, without reading their - /// operations. - pub(crate) fn queued(&self) -> Result> { - let connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - let mut query = connection.prepare("SELECT id, space FROM edits ORDER BY id")?; - let mut rows = query.query([])?; - let mut edits = Vec::new(); - while let Some(row) = rows.next()? { - edits.push(( - u64::try_from(row.get::<_, i64>(0)?).map_err(io::Error::other)?, - row.get::<_, String>(1)?.parse()?, - )); - } - Ok(edits) - } - - /// Durably queues an edited page model using the supplied local snapshot. - /// While the newest pending edit is an unattempted save of the same page, a new save - /// replaces its result instead of queueing another publication, as OneNote does - /// within its own save interval. An unchanged model returns `None`. - pub fn save( - &self, - source: &[u8], - space: ExGuid, - after: &Page, - author: &str, - ) -> Result> { - Ok(self.save_image(source, space, after, author)?.0) - } - - /// `save`, with the working image it leaves. - pub(crate) fn save_image( - &self, - source: &[u8], - space: ExGuid, - after: &Page, - author: &str, - ) -> Result<(Option, Vec)> { - let prepared = PreparedEdit::page(source, space, after, author)?; - if prepared.as_bytes() == source { - return Ok((None, source.to_vec())); - } - let before = page_of(source, space)?.ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "The saved page is not in the local image", - ) - })?; - { - let mut connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - let transaction = - connection.transaction_with_behavior(TransactionBehavior::Immediate)?; - let (base, current) = images::both(&transaction)?; - if current != source { - return Err(io::Error::new( - io::ErrorKind::ResourceBusy, - "The local snapshot changed before this edit", - ) - .into()); - } - let newest: Option<(i64, String, String)> = transaction - .query_row( - "SELECT id, space, operation FROM edits WHERE id=(SELECT max(id) FROM edits)", - [], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - ) - .optional()?; - // The intent a step is publishing has no attempt row yet; rewriting it would lose - // this save under the published bytes. - if let Some((id, sid, operation)) = newest - && id != self.in_flight.load(std::sync::atomic::Ordering::Acquire) - && sid == space.to_string() - && let Ok(Operation::Page(mut head)) = serde_json::from_str::(&operation) - { - let attempted: bool = transaction.query_row( - "SELECT EXISTS(SELECT 1 FROM attempt WHERE edit_id=?1) OR EXISTS(SELECT 1 FROM conflicts WHERE edit_id=?1)", - [id], - |row| row.get(0), - )?; - if !attempted { - head.after = after.clone(); - transaction.execute( - "UPDATE edits SET operation=?1 WHERE id=?2", - params![ - serde_json::to_string(&Operation::Page(head)) - .map_err(io::Error::other)?, - id - ], - )?; - images::set_working(&transaction, &base, prepared.as_bytes())?; - transaction.commit()?; - drop(connection); - self.wake_sync(); - let id = u64::try_from(id).map_err(io::Error::other)?; - return Ok((Some(id), prepared.as_bytes().to_vec())); - } - } - } - let id = self.record( - source, - space, - Operation::Page(PageIntent { - before, - after: after.clone(), - author: author.to_owned(), - }), - &prepared, - )?; - Ok((id, prepared.as_bytes().to_vec())) - } - - /// Queues a new page and its section entry with stable identities for dependent edits. - pub fn create_page(&self, source: &[u8], page: &PageCreation) -> Result> { - let edit = PreparedEdit::create_page(source, page)?; - self.record( - source, - page.space(), - Operation::CreatePage(page.clone()), - &edit, - ) - } - - /// Queues the permanent removal of explicitly selected pages; the batch is republished - /// only while every selected page still exists remotely. - pub fn delete_pages(&self, source: &[u8], pages: &[ExGuid]) -> Result> { - let edit = PreparedEdit::delete_pages_permanently(source, pages)?; - let space = validate(source)?; - self.record(source, space, Operation::DeletePages(pages.to_vec()), &edit) - } - - fn record( - &self, - source: &[u8], - space: ExGuid, - operation: Operation, - edit: &PreparedEdit<'_>, - ) -> Result> { - let mut connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; - let (base, current) = images::both(&transaction)?; - if current != source { - return Err(io::Error::new( - io::ErrorKind::ResourceBusy, - "The local snapshot changed before this edit", - ) - .into()); - } - if edit.as_bytes() == source { - return Ok(None); - } - transaction.execute( - "INSERT INTO edits(space, operation) VALUES (?1, ?2)", - params![ - space.to_string(), - serde_json::to_string(&operation).map_err(io::Error::other)? - ], - )?; - let id = u64::try_from(transaction.last_insert_rowid()).map_err(io::Error::other)?; - images::set_working(&transaction, &base, edit.as_bytes())?; - transaction.commit()?; - drop(connection); - self.wake_sync(); - Ok(Some(id)) + pending(&*self.lock()?) } fn wake_sync(&self) { - if let Ok(worker) = self.worker.lock() - && let Some(worker) = worker.upgrade() - { - worker.wake(); + wake(&self.worker); + } +} + +impl Drop for Replica { + fn drop(&mut self) { + drop(self.working.take()); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); } } } -fn page_of(source: &[u8], space: ExGuid) -> Result> { - let store = Store::parse(source)?; - let index = RevisionIndex::parse(&store)?; - let document = Document::parse(&index)?; - Ok(Page::from_space(&document, space).ok()) +fn lock(connection: &Mutex) -> Result> { + connection + .lock() + .map_err(|_| io::Error::other("Cache owner panicked").into()) } +fn wake(worker: &Mutex>) { + if let Ok(worker) = worker.lock() + && let Some(worker) = worker.upgrade() + { + worker.wake(); + } +} + +/// Fully validates a section image, returning its root object space. fn validate(source: &[u8]) -> Result { - let store = Store::parse(source)?; - if !store.checksum_mismatches.is_empty() { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Notebook transaction checksum damage", - ) - .into()); - } - let index = RevisionIndex::parse(&store)?; - index.validate_current()?; - Document::parse(&index)?; - Ok(index.root) + let arena = onestore::Arena::default(); + Ok(onestore::Section::open(&arena, source.to_vec())?.root()) } fn pending(connection: &Connection) -> Result> { - let mut query = connection.prepare("SELECT id, space, operation FROM edits ORDER BY id")?; - let mut rows = query.query([])?; - let mut edits = Vec::new(); - while let Some(row) = rows.next()? { - edits.push(pending_edit(row)?); - } - Ok(edits) + queue::load(connection, None) } -fn pending_edit(row: &rusqlite::Row<'_>) -> Result { - let edit = PendingEdit { - id: u64::try_from(row.get::<_, i64>(0)?).map_err(io::Error::other)?, - space: row.get::<_, String>(1)?.parse()?, - operation: serde_json::from_str(&row.get::<_, String>(2)?) - .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?, - }; - if matches!(&edit.operation, Operation::CreatePage(page) if page.space() != edit.space) { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Cached page identity differs from its creation intent", - ) - .into()); - } - Ok(edit) +fn unsigned(value: i64) -> Result { + Ok(u64::try_from(value).map_err(|_| io::Error::from(io::ErrorKind::InvalidData))?) } +fn signed(value: u64) -> Result { + Ok(i64::try_from(value).map_err(|_| io::Error::from(io::ErrorKind::InvalidInput))?) +} + +/// Opens a cache without writing to it: exclusive locking, a full sync of every commit +/// (`F_FULLFSYNC` on macOS, the WAL's syncs included) and foreign keys, each queried back. fn cache_connection(path: &Path) -> Result { let connection = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_WRITE)?; connection.busy_timeout(Duration::ZERO)?; connection.execute_batch( - "PRAGMA locking_mode=EXCLUSIVE; PRAGMA synchronous=EXTRA; PRAGMA fullfsync=ON; PRAGMA foreign_keys=ON;", + "PRAGMA locking_mode=EXCLUSIVE; PRAGMA synchronous=FULL; PRAGMA fullfsync=ON; PRAGMA foreign_keys=ON;", )?; - for (name, expected) in [("locking_mode", "exclusive"), ("journal_mode", "delete")] { - let actual: String = connection.pragma_query_value(None, name, |row| row.get(0))?; - if actual != expected { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Unsupported cache locking or journal mode", - ) - .into()); - } + let locking: String = connection.pragma_query_value(None, "locking_mode", |row| row.get(0))?; + let journal: String = connection.pragma_query_value(None, "journal_mode", |row| row.get(0))?; + if locking != "exclusive" || !["wal", "delete"].contains(&journal.as_str()) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Unsupported cache locking or journal mode", + ) + .into()); } - for (name, expected) in [("synchronous", 3), ("fullfsync", 1), ("foreign_keys", 1)] { + for (name, expected) in [("synchronous", 2), ("fullfsync", 1), ("foreign_keys", 1)] { let actual: i64 = connection.pragma_query_value(None, name, |row| row.get(0))?; if actual != expected { return Err(io::Error::new( @@ -549,20 +348,27 @@ fn cache_connection(path: &Path) -> Result { Ok(connection) } -fn validate_images(connection: &Connection) -> Result<()> { - let integrity: String = connection.query_row("PRAGMA quick_check", [], |row| row.get(0))?; - if integrity != "ok" { - return Err( - io::Error::new(io::ErrorKind::InvalidData, "Cache integrity check failed").into(), - ); - } - let (base, working) = images::both(connection)?; - if validate(&base)? != validate(&working)? { +/// Switches a cache to write-ahead logging: a commit appends its pages to `-wal` +/// instead of copying the originals to a rollback journal. Under exclusive locking the WAL +/// index lives in memory, so the WAL is the only file beside the cache; it is checkpointed +/// and removed on close, and replayed on the next open after a crash. +fn write_ahead(connection: &Connection) -> Result<()> { + let mode: String = + connection.pragma_update_and_check(None, "journal_mode", "WAL", |row| row.get(0))?; + if mode != "wal" { return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Cache images belong to different documents", + io::ErrorKind::Unsupported, + "The cache cannot use write-ahead logging", ) .into()); } Ok(()) } + +/// FILETIME now, as an edit's `at`. +pub(crate) fn now() -> u64 { + let unix = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default(); + (unix.as_secs() + 11_644_473_600) * 10_000_000 + u64::from(unix.subsec_nanos() / 100) +} diff --git a/crates/notebook/src/merge.rs b/crates/notebook/src/merge.rs index dfb7d4f455fa78f7f5c5008e610064ee8a5f8f3d..1d22b0fd051443e7c5906bb349faad6e909ce4d3 100644 --- a/crates/notebook/src/merge.rs +++ b/crates/notebook/src/merge.rs @@ -1,554 +1,1180 @@ -//! Three-way merge of page models: the remote page keeps everything it changed, and the -//! local changes (`base` → `ours`) are re-applied wherever the two sides touched -//! different objects, fields or text ranges. Any overlap is a conflict for review. +//! Rebasing queued edits onto a changed remote section. Each op replays on the remote +//! section when the objects it reads are as the base image had them; text ops on a text +//! the remote also changed shift past the remote's changes when they stay clear of them; +//! a move, deletion or outline setting the remote already made is dropped as done. +//! Anything else is a conflict: the queue stays on its base until it is resolved. +//! Objects the queue itself created are not in the base, so nothing remote can have +//! changed them. +use crate::{ConflictKind, PendingEdit, Result}; use onestore::{ - ExGuid, - page::{Outline, Page, PageObject, PageParagraph, ParagraphContent, text::Edit}, + ExGuid, OutlineEdit, PageCreation, PageEdit, PagePosition, Section, + document::{Format, Layout, Tag}, + op::{Edit, Op, OpError, PageOp, SectionOp, TableEdit}, + page::{ + Attachment, Image, Ink, MediaIndex, Outline, Page, PageObject, PageParagraph, Paragraph, + ParagraphContent, TableColumn, TextObject, Unsupported, + }, +}; +use std::{ + collections::{BTreeMap, BTreeSet}, + ops::Range, }; -use std::collections::{BTreeMap, BTreeSet}; -pub(crate) fn merge(base: &Page, ours: &Page, theirs: &Page) -> Option { - if ours == base { - return Some(theirs.clone()); - } - if theirs == base || ours == theirs { - return Some(ours.clone()); - } - if ours.created != base.created || ours.margin_origin != base.margin_origin { - return None; - } - fn by_id(page: &Page) -> BTreeMap { - page.objects - .iter() - .map(|object| (object.id(), object)) - .collect() - } - let (b, o, t) = (by_id(base), by_id(ours), by_id(theirs)); - let mut result: Vec = Vec::new(); - for object in &theirs.objects { - let id = object.id(); - match (b.get(&id), o.get(&id)) { - (Some(before), Some(after)) => match (before, after, object) { - (PageObject::Outline(x), PageObject::Outline(y), PageObject::Outline(z)) => { - result.push(PageObject::Outline(merge_outline(x, y, z)?)); +/// A conflict being resolved: from edit `first` on, ops on `space` are dropped. Keeping +/// mine replaces the first of them by the ops rewriting the remote page to `target` (or +/// putting `target` back as a page where the remote removed it), or, for the section's +/// page list, keeps what of each section op still applies. +pub(crate) struct Resolving<'p> { + pub first: u64, + pub space: ExGuid, + pub mine: bool, + pub target: Option<&'p Page>, +} + +pub(crate) enum Outcome { + /// Every edit applied; these edits' ops changed. An edit whose ops were all dropped + /// stays, empty, to be published and acknowledged with its batch. + Applied(Vec<(u64, Edit)>), + Conflict { + id: u64, + space: ExGuid, + kind: ConflictKind, + }, +} + +/// Replays `edits` on `new`, the remote section; `old` is the base they were made on. +pub(crate) fn rebase( + old: &mut Section<'_>, + new: &mut Section<'_>, + edits: &[PendingEdit], + converged: &BTreeSet, + resolving: Option>, +) -> Result { + let root = new.root(); + let before: BTreeMap = old.revisions().collect(); + let after: BTreeMap = new.revisions().collect(); + let orders = (order(old)?, order(new)?); + let mut state = Replay { + before, + after, + orders, + created: BTreeSet::new(), + diffs: BTreeMap::new(), + }; + let mut rewritten = Vec::new(); + let mut resolved = false; + for queued in edits { + let branch = resolving + .as_ref() + .filter(|resolving| queued.id >= resolving.first); + let mut ops = Vec::new(); + let mut changed = false; + for op in &queued.edit.ops { + let conflict = |kind, space| Outcome::Conflict { + id: queued.id, + space, + kind, + }; + if let Some(resolving) = branch { + let dropped = match op { + Op::Page { space, .. } => *space == resolving.space, + Op::Section(_) => resolving.space == root, + }; + if dropped { + changed = true; + let rewrite = match op { + Op::Page { space, .. } if !resolved && resolving.mine => { + resolving.target.map(|target| (*space, target)) + } + _ => None, + }; + resolved |= matches!(op, Op::Page { .. }); + if let Some((space, target)) = rewrite { + if !state.listed(space) { + // Kept while the remote removed it: the page comes back. + match state.restore( + new, + space, + target, + &queued.author, + queued.edit.at, + )? { + Ok(restored) => ops.extend(restored), + Err(kind) => return Ok(conflict(kind, space)), + } + continue; + } + let current = new.page(space)?; + for op in onestore::op::lower_page(¤t, target) + .map_err(|error| OpError::Unsupported(error.message))? + { + let op = Op::Page { space, op }; + if let Some(kind) = + state.apply(new, &queued.author, queued.edit.at, &op)? + { + return Ok(conflict(kind, space)); + } + ops.push(op); + } + } + if let (Op::Section(op), true) = (op, resolving.mine) { + // Keeping mine for the section list: the first form that applies. + for op in state.salvage(new, op) { + let op = Op::Section(op); + if state + .apply(new, &queued.author, queued.edit.at, &op)? + .is_none() + { + ops.push(op); + break; + } + } + } + continue; } - _ => result.push(pick(before, after, &object)?.clone()), - }, - (Some(before), None) => { - // Removed locally; keep only if the remote left it untouched. - if object != *before { - return None; + } + let space = match op { + Op::Page { space, .. } => *space, + Op::Section(_) => root, + }; + if matches!(op, Op::Page { .. }) && converged.contains(&space) { + changed = true; + continue; + } + let mapped = match state.check(old, new, op)? { + Ok(Some(mapped)) => mapped, + Ok(None) => { + changed = true; + continue; } + Err(kind) => return Ok(conflict(kind, space)), + }; + changed |= mapped != *op; + if let Some(kind) = state.apply(new, &queued.author, queued.edit.at, &mapped)? { + return Ok(conflict(kind, space)); } - (None, _) => result.push(object.clone()), + ops.push(mapped); + } + if changed { + rewritten.push(( + queued.id, + Edit { + at: queued.edit.at, + ops, + }, + )); } } - for (index, object) in ours.objects.iter().enumerate() { - let id = object.id(); - if b.contains_key(&id) { - if !t.contains_key(&id) { - // Removed remotely; a local edit to it cannot be placed. - if b[&id] != object { - return None; + Ok(Outcome::Applied(rewritten)) +} + +fn order(section: &mut Section<'_>) -> Result { + Ok(section + .pages()? + .into_iter() + .map(|(space, _, level)| (space, level)) + .collect()) +} + +/// Page spaces in section order with their levels. +type Order = Vec<(ExGuid, u32)>; + +struct Replay { + /// Active revisions of the base and the remote. + before: BTreeMap, + after: BTreeMap, + /// Page order of the base and the remote. + orders: (Order, Order), + /// Page spaces the replayed edits created. + created: BTreeSet, + /// Per page the remote changed, what it changed; `None` when it left the page alone. + diffs: BTreeMap>, +} + +impl Replay { + /// Applies one op to the remote section; `Some` names why it did not apply. + fn apply( + &mut self, + new: &mut Section<'_>, + author: &str, + at: u64, + op: &Op, + ) -> Result> { + match new.apply( + author, + &Edit { + at, + ops: vec![op.clone()], + }, + ) { + Ok(()) => { + if let Op::Section( + SectionOp::Create(creation) | SectionOp::Import { creation, .. }, + ) = op + { + self.created.insert(creation.space()); } + Ok(None) } - continue; + Err(OpError::Failed(error)) => Err(error.into()), + Err(error) => Ok(Some(match op { + // A page list the remote reordered refuses the op, however it says so. + Op::Section(_) => ConflictKind::StructureChanged, + Op::Page { .. } if matches!(error, OpError::Unsupported(_)) => { + ConflictKind::UnsupportedEdit + } + Op::Page { space, .. } if !self.listed(*space) => ConflictKind::TargetUnavailable, + Op::Page { .. } => ConflictKind::ContentChanged, + })), } - if t.contains_key(&id) { - return None; + } + + /// The op as it replays on the remote section, or why it cannot. + fn check( + &mut self, + old: &mut Section<'_>, + new: &mut Section<'_>, + op: &Op, + ) -> Result, ConflictKind>> { + match op { + Op::Page { space, op } => { + if !self.listed(*space) { + return Ok(Err(ConflictKind::TargetUnavailable)); + } + let diff = match self.diffs.get_mut(space) { + Some(diff) => diff, + None => { + let diff = match (self.before.get(space), self.after.get(space)) { + (Some(before), Some(after)) if before != after => Some(Diff { + old: Index::of(&old.page(*space)?), + new: Index::of(&new.page(*space)?), + regions: BTreeMap::new(), + }), + _ => None, + }; + self.diffs.entry(*space).or_insert(diff) + } + }; + Ok(match diff { + None => Ok(Some(Op::Page { + space: *space, + op: op.clone(), + })), + Some(diff) => diff + .check(op) + .map(|op| op.map(|op| Op::Page { space: *space, op })), + }) + } + Op::Section(section) => Ok(self.section(section).map(|()| Some(op.clone()))), } - let PageObject::Outline(_) = object else { - return None; - }; - let predecessor = ours.objects[..index] - .iter() - .rev() - .map(PageObject::id) - .find(|other| result.iter().any(|r| r.id() == *other)); - let at = match predecessor { - Some(other) => result.iter().position(|r| r.id() == other).unwrap() + 1, - None => 0, - }; - let at = at.min( - result - .iter() - .position(|r| matches!(r, PageObject::Title(_))) - .unwrap_or(result.len()), - ); - result.insert(at, object.clone()); } - let order = |objects: &[PageObject], keep: &dyn Fn(ExGuid) -> bool| -> Vec { - objects + + /// Whether the remote section lists the page, or the replayed edits created it; a page + /// removed from the list keeps its stored revisions. + fn listed(&self, space: ExGuid) -> bool { + self.created.contains(&space) || self.orders.1.iter().any(|(page, _)| *page == space) + } + + /// Puts `page`, which the remote removed, back into the section and returns the ops + /// that did: a copy under fresh identities at its level, before the first page after it + /// that the remote still has and that can take a page before it, or last. + fn restore( + &mut self, + new: &mut Section<'_>, + space: ExGuid, + page: &Page, + author: &str, + at: u64, + ) -> Result, ConflictKind>> { + let old = self.orders.0.clone(); + let position = old.iter().position(|(listed, _)| *listed == space); + let anchors = position + .map_or(&[][..], |position| &old[position + 1..]) .iter() - .filter(|object| matches!(object, PageObject::Outline(_)) && keep(object.id())) - .map(PageObject::id) - .collect() - }; - let common = |id: ExGuid| b.contains_key(&id) && o.contains_key(&id) && t.contains_key(&id); - let (base_order, our_order, their_order) = ( - order(&base.objects, &common), - order(&ours.objects, &common), - order(&theirs.objects, &common), - ); - if our_order != base_order { - if their_order != base_order && their_order != our_order { - return None; + .map(|(next, _)| Some(*next)) + .filter(|next| next.is_some_and(|next| self.listed(next))) + .chain([None]) + .collect::>(); + let copy = page.copy()?; + let mut refusal = ConflictKind::StructureChanged; + for before in anchors { + let creation = PageCreation::new(before, Some(&page.title), author)?; + let import = Op::Section(SectionOp::Import { + creation: creation.clone(), + page: copy.clone(), + }); + match self.apply(new, author, at, &import)? { + Some(kind) => refusal = kind, + None => { + let mut ops = vec![import]; + if let Some(level) = position.map(|at| old[at].1).filter(|level| *level > 1) { + let indent = Op::Section(SectionOp::Pages(vec![PageEdit::set_level( + creation.space(), + level, + )?])); + if let Some(kind) = self.apply(new, author, at, &indent)? { + return Ok(Err(kind)); + } + ops.push(indent); + } + return Ok(Ok(ops)); + } + } } - result = reorder(result, &our_order); + Ok(Err(refusal)) } - // Definitions we added or changed travel with our paragraphs; theirs win elsewhere. - let mut definitions = theirs.definitions.clone(); - for (id, definition) in &ours.definitions { - match (base.definitions.get(id), theirs.definitions.get(id)) { - (Some(before), Some(after)) if before != definition => { - if after != before && after != definition { - return None; + + fn section(&self, op: &SectionOp) -> std::result::Result<(), ConflictKind> { + let (old, new) = &self.orders; + let find = + |list: &[(ExGuid, u32)], space: ExGuid| list.iter().position(|(s, _)| *s == space); + match op { + SectionOp::Create(_) | SectionOp::Import { .. } => Ok(()), + SectionOp::Delete(pages) => { + for page in pages { + if find(new, *page).is_none() && !self.created.contains(page) { + return Err(ConflictKind::TargetUnavailable); + } + if self.before.get(page) != self.after.get(page) { + return Err(ConflictKind::ContentChanged); + } } - definitions.insert(*id, definition.clone()); + Ok(()) } - (None, None) => { - definitions.insert(*id, definition.clone()); + SectionOp::Pages(edits) => { + // Where the batch puts each page, from the base order. + let mut wanted: Vec = old.iter().map(|(space, _)| *space).collect(); + for edit in edits { + if let PagePosition::Before(before) = edit.position() + && let Some(at) = wanted.iter().position(|space| *space == edit.space()) + { + let space = wanted.remove(at); + let at = before + .and_then(|before| wanted.iter().position(|space| *space == before)) + .unwrap_or(wanted.len()); + wanted.insert(at, space); + } + } + let wanted: Vec<(ExGuid, u32)> = + wanted.into_iter().map(|space| (space, 0)).collect(); + for edit in edits { + let space = edit.space(); + let Some(at) = find(old, space) else { + continue; + }; + let Some(now) = find(new, space) else { + return Err(ConflictKind::TargetUnavailable); + }; + if old[at].1 != new[now].1 && new[now].1 != edit.level() { + return Err(ConflictKind::StructureChanged); + } + if edit.position() != PagePosition::Keep { + // Each page the remote also has must keep its side of the moved page, + // or already be on the side the batch wants. + let side = |list: &[(ExGuid, u32)], peer: ExGuid| { + Some(find(list, peer)? < find(list, space)?) + }; + let conflicting = old.iter().any(|(peer, _)| { + *peer != space + && find(new, *peer).is_some() + && side(new, *peer) != side(old, *peer) + && side(new, *peer) != side(&wanted, *peer) + }); + if conflicting { + return Err(ConflictKind::StructureChanged); + } + } + } + Ok(()) } - (None, Some(after)) if after != definition => return None, - _ => {} } } - Some(Page { - title: theirs.title.clone(), - identity: theirs.identity, - created: theirs.created, - margin_origin: theirs.margin_origin, - objects: result, - definitions, - }) -} -/// Places the objects named in `order` into the slots those objects occupy in `objects`, -/// leaving every other object where it is. -fn reorder(objects: Vec, order: &[ExGuid]) -> Vec { - let mut slots: Vec> = objects.into_iter().map(Some).collect(); - let positions: Vec = slots - .iter() - .enumerate() - .filter(|(_, slot)| { - slot.as_ref() - .is_some_and(|object| order.contains(&object.id())) - }) - .map(|(i, _)| i) - .collect(); - let mut taken: BTreeMap = positions - .iter() - .map(|&i| { - let object = slots[i].take().unwrap(); - (object.id(), object) - }) - .collect(); - for (slot, id) in positions.into_iter().zip(order) { - slots[slot] = taken.remove(id); + /// Forms of a section op to try on the remote section list, in order: as it is, a + /// creation appended instead, moves and removals of the pages still there. + fn salvage(&self, new: &mut Section<'_>, op: &SectionOp) -> Vec { + let pages: BTreeSet = new + .pages() + .map(|pages| pages.into_iter().map(|(space, ..)| space).collect()) + .unwrap_or_default(); + let present = |space: &ExGuid| pages.contains(space) || self.created.contains(space); + match op { + SectionOp::Create(creation) => std::iter::once(creation.clone()) + .chain(creation.reposition(None).ok()) + .map(SectionOp::Create) + .collect(), + SectionOp::Import { .. } => vec![op.clone()], + SectionOp::Pages(edits) => { + let edits: Vec = edits + .iter() + .filter(|edit| present(&edit.space())) + .cloned() + .collect(); + if edits.is_empty() { + Vec::new() + } else { + vec![SectionOp::Pages(edits)] + } + } + SectionOp::Delete(deleted) => { + let deleted: Vec = deleted + .iter() + .filter(|space| present(space)) + .copied() + .collect(); + if deleted.is_empty() { + Vec::new() + } else { + vec![SectionOp::Delete(deleted)] + } + } + } } - slots.into_iter().flatten().collect() } -fn pick(base: &T, ours: &T, theirs: &T) -> Option { - if ours == base { - Some(theirs.clone()) - } else if theirs == base || theirs == ours { - Some(ours.clone()) - } else { - None - } +/// One page as the base stored it and as the remote does, with the remote's text changes +/// in the coordinates of the replayed edits. +struct Diff { + old: Index, + new: Index, + regions: BTreeMap, } -fn merge_outline(base: &Outline, ours: &Outline, theirs: &Outline) -> Option { - let title = pick(&base.title, &ours.title, &theirs.title)?; - let min_width = pick(&base.min_width, &ours.min_width, &theirs.min_width)?; - let indents = pick(&base.indents, &ours.indents, &theirs.indents)?; - let unsupported = pick(&base.unsupported, &ours.unsupported, &theirs.unsupported)?; - let mut layout = theirs.layout.clone(); - let (x, y) = pick( - &(base.layout.x, base.layout.y), - &(ours.layout.x, ours.layout.y), - &(theirs.layout.x, theirs.layout.y), - )?; - // A user-set width clears the wrap reservation, so the three travel together. - let width = |layout: &onestore::document::Layout| { - ( - layout.max_width, - layout.width_set_by_user, - layout.reserved_width, - ) - }; - let (max_width, user_set, reserved) = pick( - &width(&base.layout), - &width(&ours.layout), - &width(&theirs.layout), - )?; - layout.x = x; - layout.y = y; - layout.max_width = max_width; - layout.width_set_by_user = user_set; - layout.reserved_width = reserved; - Some(Outline { - id: theirs.id, - title, - min_width, - layout, - indents, - paragraphs: merge_paragraphs(&base.paragraphs, &ours.paragraphs, &theirs.paragraphs)?, - unsupported, - }) -} +impl Diff { + fn ours(&self, id: ExGuid) -> bool { + !self.old.nodes.contains_key(&id) + } -type Children = BTreeMap, Vec>; + /// Whether the remote left the object's own data as the base had it. + fn kept(&self, id: ExGuid) -> bool { + self.ours(id) + || matches!((self.old.nodes.get(&id), self.new.nodes.get(&id)), (Some(a), Some(b)) if a.own == b.own) + } -fn children(list: &[PageParagraph]) -> Children { - let mut children: Children = BTreeMap::new(); - for paragraph in list { - children - .entry(paragraph.parent) - .or_default() - .push(paragraph.id); + /// Whether the remote left the object and everything under it as the base had it. + fn untouched(&self, id: ExGuid) -> bool { + if self.ours(id) { + return true; + } + let (Some(a), Some(b)) = (self.old.nodes.get(&id), self.new.nodes.get(&id)) else { + return false; + }; + a.own == b.own + && a.parent == b.parent + && a.children == b.children + && a.below().all(|child| self.untouched(child)) } - children -} -fn merge_paragraphs( - base: &[PageParagraph], - ours: &[PageParagraph], - theirs: &[PageParagraph], -) -> Option> { - fn index(list: &[PageParagraph]) -> BTreeMap { - list.iter().map(|p| (p.id, p)).collect() + /// Whether the remote kept the object under the same parent, in the same order among + /// the siblings both sides have. + fn placed(&self, id: ExGuid) -> bool { + if self.ours(id) { + return true; + } + let (Some(a), Some(b)) = (self.old.nodes.get(&id), self.new.nodes.get(&id)) else { + return false; + }; + if a.parent != b.parent { + return false; + } + let siblings = |index: &Index| index.children(a.parent).to_vec(); + let (old, new) = (siblings(&self.old), siblings(&self.new)); + let at = |list: &[ExGuid], id: ExGuid| list.iter().position(|x| *x == id); + let (Some(i), Some(j)) = (at(&old, id), at(&new, id)) else { + return false; + }; + old.iter() + .enumerate() + .all(|(k, peer)| *peer == id || at(&new, *peer).is_none_or(|l| (k < i) == (l < j))) } - let (b, o, t) = (index(base), index(ours), index(theirs)); - let mut merged: BTreeMap = BTreeMap::new(); - let mut removed = BTreeSet::new(); - for id in b.keys().chain(o.keys()).chain(t.keys()) { - if merged.contains_key(id) || removed.contains(id) { - continue; + + /// Where the remote changed a text's characters and formats; its tags and date field + /// are not positions an op names. + fn region(&mut self, text: ExGuid) -> std::result::Result, ConflictKind> { + if !self.regions.contains_key(&text) { + if self.ours(text) { + return Ok(None); + } + let (Some(Own::Text(a)), Some(Own::Text(b))) = ( + self.old.nodes.get(&text).map(|node| &node.own), + self.new.nodes.get(&text).map(|node| &node.own), + ) else { + return Err(ConflictKind::ContentChanged); + }; + if a.text == b.text { + return Ok(None); + } + let region = Changes::between(&a.text, &b.text); + self.regions.insert(text, region); } - let paragraph = match (b.get(id), o.get(id), t.get(id)) { - (Some(x), Some(y), Some(z)) => merge_paragraph(x, y, z)?, - (Some(x), None, Some(z)) => { - if z != x { - return None; + Ok(self.regions.get_mut(&text)) + } + + /// The op as it replays on the remote page; `None` when the remote already made it. + fn check(&mut self, op: &PageOp) -> std::result::Result, ConflictKind> { + use ConflictKind::ContentChanged as Changed; + let require = |ok: bool| if ok { Ok(()) } else { Err(Changed) }; + let mut op = op.clone(); + match &mut op { + PageOp::Text { text, range, with } => { + if let Some(region) = self.region(*text)? { + let mapped = region.map(range).ok_or(Changed)?; + region.replaced(range, with.encode_utf16().count() as u32); + *range = mapped; } - removed.insert(*id); - continue; } - (Some(x), Some(y), None) => { - if y != x { - return None; + PageOp::Format { text, range, .. } => { + if let Some(region) = self.region(*text)? { + *range = region.map(range).ok_or(Changed)?; } - removed.insert(*id); - continue; } - (Some(_), None, None) => { - removed.insert(*id); - continue; + PageOp::Link { text, .. } | PageOp::Equation { text, .. } => { + require(self.region(*text)?.is_none())?; } - (None, Some(_), Some(_)) => return None, - (None, Some(y), None) => (*y).clone(), - (None, None, Some(z)) => (*z).clone(), - (None, None, None) => unreachable!(), - }; - merged.insert(*id, paragraph); - } - // Child order per container: the remote order with our repositioned paragraphs re-placed. - let (bc, oc, tc) = (children(base), children(ours), children(theirs)); - // A paragraph we added after one they gave new children would sit after or among those - // children; Enter splitting a paragraph carries its children below, so neither is assumed. - for pair in oc.values().flat_map(|list| list.windows(2)) { - if b.contains_key(&pair[0]) - && !b.contains_key(&pair[1]) - && tc - .get(&Some(pair[0])) - .is_some_and(|children| children.iter().any(|id| !b.contains_key(id))) - { - return None; + PageOp::Split { + text, at, right, .. + } => { + // The new paragraph copies the holder's placement and formats; lists and tags + // stay with the holder unless the split names copies of them. + let holder = self.holder(*text); + require(holder.is_none_or(|holder| self.splits_alike(holder)))?; + if let Some(region) = self.region(*text)? { + // The cut moves with the character after it: text the remote typed at + // the cut stays left of it. + let cut = region.map(&(*at..*at + 1)).ok_or(Changed)?; + let right_changes = region.split(*at); + if !right_changes.0.is_empty() { + self.regions.insert(*right, right_changes); + } + *at = cut.start; + } + } + PageOp::Join { left, right } => { + require(self.untouched(*right))?; + require( + self.holder_text(*right) + .is_none_or(|text| !self.regions.contains_key(&text)), + )?; + if let Some(text) = self.holder_text(*left) { + self.region(text)?; + } + } + PageOp::Insert { .. } | PageOp::Add { .. } => {} + PageOp::Move { + object, + parent, + before, + } => require(self.placed(*object) || self.placed_at(*object, *parent, *before))?, + PageOp::Delete { object } => { + if !self.ours(*object) && !self.new.nodes.contains_key(object) { + return Ok(None); + } + require(self.untouched(*object))? + } + PageOp::Level { paragraph, .. } => require( + self.ours(*paragraph) || { + let level = |index: &Index| { + index.nodes.get(paragraph).map(|node| { + ( + node.parent, + match &node.own { + Own::Paragraph { level, .. } => *level, + _ => 0, + }, + ) + }) + }; + level(&self.old).is_some() && level(&self.old) == level(&self.new) + }, + )?, + PageOp::Outline { object, edit } => { + if self.ours(*object) { + return Ok(Some(op)); + } + let edit: &OutlineEdit = edit; + // What the edit sets, as each side has it. + let part = |index: &Index| { + index.nodes.get(object).map(|node| match (&node.own, edit) { + (Own::Outline { layout, .. }, OutlineEdit::Position { .. }) => { + (layout.x, layout.y, None, None) + } + (Own::Outline { layout, .. }, OutlineEdit::Width { .. }) => ( + layout.max_width, + layout.reserved_width, + layout.width_set_by_user, + None, + ), + (Own::Paragraph { collapsed, .. }, OutlineEdit::Collapsed(_)) => { + (None, None, None, Some(*collapsed)) + } + _ => (None, None, None, None), + }) + }; + let (before, after) = (part(&self.old), part(&self.new)); + if before.is_some() && before != after { + // The remote set the same value: the edit is done. + let near = |a: Option, b: f32| a.is_some_and(|a| (a - b).abs() < 1e-3); + let done = match (after, edit) { + (Some((x, y, ..)), OutlineEdit::Position { x: px, y: py }) => { + near(x, *px) && near(y, *py) + } + ( + Some((width, reserved, set, _)), + OutlineEdit::Width { points, user_set }, + ) => near(width, *points) && reserved.is_none() && set == Some(*user_set), + (Some((.., collapsed)), OutlineEdit::Collapsed(value)) => { + collapsed == Some(*value) + } + _ => false, + }; + return if done { Ok(None) } else { Err(Changed) }; + } + require(before.is_some())? + } + PageOp::Paragraph { paragraph, .. } + | PageOp::Style { paragraph, .. } + | PageOp::List { paragraph, .. } => { + require(self.kept(*paragraph) && self.new_has(*paragraph))? + } + PageOp::Tags { target, .. } => require(self.same_tags(*target))?, + PageOp::Picture { + picture: object, .. + } + | PageOp::Attachment { + attachment: object, .. + } + | PageOp::Strokes { ink: object, .. } => { + require(self.kept(*object) && self.new_has(*object))? + } + PageOp::Table { table, edit } => match edit { + TableEdit::Cell { cell, .. } => require(self.kept(*cell) && self.new_has(*cell))?, + TableEdit::DeleteRow(row) => require(self.kept(*table) && self.untouched(*row))?, + TableEdit::DeleteColumn(_) => require(self.untouched(*table))?, + TableEdit::Rows { .. } + | TableEdit::Column { .. } + | TableEdit::Columns(_) + | TableEdit::Borders(_) => require(self.kept(*table) && self.new_has(*table))?, + }, } + Ok(Some(op)) } - let mut order: BTreeMap, Vec> = BTreeMap::new(); - let containers: BTreeSet> = bc - .keys() - .chain(oc.keys()) - .chain(tc.keys()) - .copied() - .collect(); - for container in containers { - let live = |ids: Option<&Vec>| -> Vec { - ids.map(|ids| { - ids.iter() - .copied() - .filter(|id| merged.contains_key(id)) - .collect() - }) - .unwrap_or_default() + + /// Whether the remote put the object where a move places it: under `parent` (the page + /// when `None`), before `before` or last. + fn placed_at(&self, id: ExGuid, parent: Option, before: Option) -> bool { + let Some(node) = self.new.nodes.get(&id) else { + return false; }; - let (our_list, their_list) = (live(oc.get(&container)), live(tc.get(&container))); - let of_base = |ids: Option<&Vec>| -> Vec { - ids.map(|ids| { - ids.iter() - .copied() - .filter(|id| b.contains_key(id)) - .collect() + let container = parent.unwrap_or(PAGE); + let siblings = self.new.children(container); + node.parent == container + && siblings + .iter() + .position(|sibling| *sibling == id) + .is_some_and(|at| siblings.get(at + 1).copied() == before) + } + + /// Whether the remote kept a paragraph's, text's or table's tags. + fn same_tags(&self, id: ExGuid) -> bool { + if self.ours(id) { + return true; + } + let tags = |index: &Index| { + index.nodes.get(&id).map(|node| match &node.own { + Own::Paragraph { tags, .. } | Own::Table { tags, .. } => Some(tags.clone()), + Own::Text(text) => Some(text.tags.clone()), + _ => None, }) - .unwrap_or_default() - }; - let (base_ids, our_ids, their_ids) = ( - of_base(bc.get(&container)), - of_base(oc.get(&container)), - of_base(tc.get(&container)), - ); - let ours_moved = if our_ids == base_ids || our_ids == their_ids { - BTreeSet::new() - } else { - moved(&base_ids, &our_ids) }; - // A paragraph we repositioned that the remote also moved, reparented or removed - // has two destinations; neither side's placement can be assumed. - let mut theirs_touched = moved(&base_ids, &their_ids); - theirs_touched.extend(base_ids.iter().filter(|id| !their_ids.contains(id))); - if ours_moved.iter().any(|id| theirs_touched.contains(id)) { - return None; + let before = tags(&self.old); + before.is_some() && before == tags(&self.new) + } + + fn new_has(&self, id: ExGuid) -> bool { + self.ours(id) || self.new.nodes.contains_key(&id) + } + + /// The paragraph holding a text, as the base stored it. + fn holder(&self, text: ExGuid) -> Option { + self.old.nodes.get(&text).map(|node| node.parent) + } + + fn holder_text(&self, paragraph: ExGuid) -> Option { + match self.old.nodes.get(¶graph).map(|node| &node.own) { + Some(Own::Paragraph { content, .. }) => Some(*content), + _ => None, } - let mut list: Vec = their_list.clone(); - list.retain(|id| merged[id].parent == container); - // Paragraphs we repositioned, added or brought here follow their predecessor in our list. - for (at, id) in our_list.iter().enumerate() { - if merged[id].parent != container || (list.contains(id) && !ours_moved.contains(id)) { - continue; - } - list.retain(|other| other != id); - let predecessor = our_list[..at] - .iter() - .rev() - .find(|other| list.contains(other)); - let position = predecessor.map_or(0, |p| list.iter().position(|x| x == p).unwrap() + 1); - list.insert(position, *id); + } + + /// Whether a split of the paragraph makes the same new paragraph on both sides: its + /// container, style, format and children are as the base had them. + fn splits_alike(&self, id: ExGuid) -> bool { + if self.ours(id) { + return true; } - order.insert(container, list); + let shape = |index: &Index| { + index.nodes.get(&id).map(|node| match &node.own { + Own::Paragraph { style, format, .. } => { + Some((node.parent, *style, format.clone(), node.children.clone())) + } + _ => None, + }) + }; + let before = shape(&self.old); + before.is_some() && before == shape(&self.new) } +} + +/// Where the remote replaced parts of a text, in order: each `start..end` of the text as +/// the replayed edits see it holds what the remote replaced by `end - start + delta` UTF-16 +/// units. Outside them both texts hold the same characters in the same formats. +#[derive(Debug, Clone, Default, PartialEq)] +struct Changes(Vec); + +#[derive(Debug, Clone, Copy, PartialEq)] +struct Region { + start: u32, + end: u32, + delta: i64, +} + +/// The characters of a text with their formats and UTF-16 widths. +fn units(paragraph: &Paragraph) -> Vec<(char, Format)> { let mut out = Vec::new(); - let mut pending: Vec = order.get(&None).cloned().unwrap_or_default(); - pending.reverse(); - while let Some(id) = pending.pop() { - let paragraph = merged.remove(&id)?; - out.push(paragraph); - if let Some(children) = order.get(&Some(id)) { - pending.extend(children.iter().rev().copied()); + let mut spans = paragraph.spans().iter(); + let mut span = spans.next(); + for (byte, character) in paragraph.text().char_indices() { + while span.is_some_and(|span| span.end <= byte) { + span = spans.next(); } + out.push(( + character, + span.map(|span| span.format.clone()).unwrap_or_default(), + )); } - if !merged.is_empty() { - return None; - } - Some(out) + out } -/// The members of `list` outside a longest common subsequence with `base`: the paragraphs -/// a reorder of the shared members had to move. -fn moved(base: &[ExGuid], list: &[ExGuid]) -> BTreeSet { - let a: Vec = base - .iter() - .copied() - .filter(|id| list.contains(id)) - .collect(); - let b: Vec = list - .iter() - .copied() - .filter(|id| base.contains(id)) - .collect(); - let mut table = vec![vec![0usize; b.len() + 1]; a.len() + 1]; - for i in (0..a.len()).rev() { - for j in (0..b.len()).rev() { - table[i][j] = if a[i] == b[j] { - table[i + 1][j + 1] + 1 +/// Beyond this many cells of the alignment table the changed middle counts as one change. +const CELLS: usize = 4_000_000; + +impl Changes { + /// The runs where `new` differs from `old` in characters or formats, from a longest + /// common subsequence of the two: where several align equally, one is chosen, which + /// places an op among repeated characters one way of the equally valid ones. + fn between(old: &Paragraph, new: &Paragraph) -> Self { + let (a, b) = (units(old), units(new)); + let prefix = a.iter().zip(&b).take_while(|(x, y)| x == y).count(); + let suffix = a[prefix..] + .iter() + .rev() + .zip(b[prefix..].iter().rev()) + .take_while(|(x, y)| x == y) + .count(); + let (x, y) = (&a[prefix..a.len() - suffix], &b[prefix..b.len() - suffix]); + // Matched pairs of the middles, in order. + let mut matched = Vec::new(); + if x.len().saturating_mul(y.len()) <= CELLS { + let width = y.len() + 1; + let mut table = vec![0_u32; (x.len() + 1) * width]; + for i in (0..x.len()).rev() { + for j in (0..y.len()).rev() { + table[i * width + j] = if x[i] == y[j] { + table[(i + 1) * width + j + 1] + 1 + } else { + table[(i + 1) * width + j].max(table[i * width + j + 1]) + }; + } + } + let (mut i, mut j) = (0, 0); + while i < x.len() && j < y.len() { + if x[i] == y[j] { + matched.push((i, j)); + i += 1; + j += 1; + } else if table[(i + 1) * width + j] >= table[i * width + j + 1] { + i += 1; + } else { + j += 1; + } + } + } + matched.push((x.len(), y.len())); + let width = |units: &[(char, Format)]| -> u32 { + units + .iter() + .map(|(character, _)| character.len_utf16() as u32) + .sum() + }; + let mut regions = Vec::new(); + let (mut at, mut i, mut j) = (width(&a[..prefix]), 0, 0); + for (k, l) in matched { + if (k, l) != (i, j) { + let (removed, inserted) = (width(&x[i..k]), width(&y[j..l])); + regions.push(Region { + start: at, + end: at + removed, + delta: i64::from(inserted) - i64::from(removed), + }); + at += removed; + } + if k < x.len() { + at += width(&x[k..k + 1]); + } + (i, j) = (k + 1, l + 1); + } + Self(regions) + } + + /// The range on the remote text, when it lies clear of every change: a range may end + /// where a change starts or start where one ends, an insertion point may not. + fn map(&self, range: &Range) -> Option> { + let empty = range.start == range.end; + let mut shift = 0; + for region in &self.0 { + if range.end < region.start || (!empty && range.end == region.start) { + break; + } + if range.start > region.end || (!empty && range.start == region.end) { + shift += region.delta; } else { - table[i + 1][j].max(table[i][j + 1]) - }; + return None; + } } + let shifted = |at: u32| u32::try_from(i64::from(at) + shift).ok(); + Some(shifted(range.start)?..shifted(range.end)?) } - let mut kept = BTreeSet::new(); - let (mut i, mut j) = (0, 0); - while i < a.len() && j < b.len() { - if a[i] == b[j] { - kept.insert(a[i]); - i += 1; - j += 1; - } else if table[i + 1][j] >= table[i][j + 1] { - i += 1; + + /// Follows a replacement of `range`, clear of every change, by `length` units. + fn replaced(&mut self, range: &Range, length: u32) { + let moved = i64::from(length) - i64::from(range.end - range.start); + for region in &mut self.0 { + if region.start >= range.end { + region.start = (i64::from(region.start) + moved) as u32; + region.end = (i64::from(region.end) + moved) as u32; + } + } + } + + /// Cuts the text at `at`, clear of every change, keeping the changes left of it (text + /// typed at the cut included) and returning those right of it, measured from the cut. + fn split(&mut self, at: u32) -> Self { + let right = self + .0 + .iter() + .filter(|region| region.end > at) + .map(|region| Region { + start: region.start - at, + end: region.end - at, + delta: region.delta, + }) + .collect(); + self.0.retain(|region| region.end <= at); + Self(right) + } +} + +/// What an object holds apart from the objects under it. +#[derive(Debug, Clone, PartialEq)] +enum Own { + Outline { + title: bool, + min_width: Option, + layout: Layout, + indents: Vec, + unsupported: Vec, + }, + Title { + date: Option, + layout: Layout, + }, + Paragraph { + level: u32, + style: Option, + format: Format, + lists: Vec, + tags: Vec, + media: MediaIndex, + collapsed: bool, + content: ExGuid, + }, + Text(TextObject), + Table { + columns: Vec, + borders: Option, + layout: Layout, + tags: Vec, + }, + Row, + Cell { + layout: Layout, + indents: Vec, + shading: Option, + unsupported: Vec, + }, + Image(Image), + Attachment(Attachment), + Ink(Ink), + Unsupported(Unsupported), +} + +#[derive(Debug)] +struct Node { + own: Own, + /// The containing object; the page is the default identity. + parent: ExGuid, + children: Vec, +} + +impl Node { + /// Everything directly under the object, a paragraph's content included. + fn below(&self) -> impl Iterator + '_ { + let content = match &self.own { + Own::Paragraph { content, .. } => Some(*content), + _ => None, + }; + content.into_iter().chain(self.children.iter().copied()) + } +} + +#[derive(Default)] +struct Index { + nodes: BTreeMap, + page: Vec, +} + +const PAGE: ExGuid = ExGuid { + guid: [0; 16], + n: 0, +}; + +impl Index { + fn of(page: &Page) -> Self { + let mut index = Self::default(); + for object in &page.objects { + index.page.push(object.id()); + match object { + PageObject::Outline(outline) => index.outline(outline, PAGE), + PageObject::Title(title) => { + index.add( + title.id, + PAGE, + Own::Title { + date: title.date, + layout: title.layout.clone(), + }, + title.outlines.iter().map(|outline| outline.id).collect(), + ); + for outline in &title.outlines { + index.outline(outline, title.id); + } + } + PageObject::Image(image) => { + index.add(image.id, PAGE, Own::Image(image.clone()), Vec::new()) + } + PageObject::Ink(ink) => index.add(ink.id, PAGE, Own::Ink(ink.clone()), Vec::new()), + PageObject::Unsupported(object) => index.add( + object.id, + PAGE, + Own::Unsupported(object.clone()), + Vec::new(), + ), + } + } + index + } + + fn children(&self, id: ExGuid) -> &[ExGuid] { + if id == PAGE { + &self.page } else { - j += 1; + self.nodes.get(&id).map_or(&[], |node| &node.children) } } - b.into_iter().filter(|id| !kept.contains(id)).collect() -} -fn merge_paragraph( - base: &PageParagraph, - ours: &PageParagraph, - theirs: &PageParagraph, -) -> Option { - let lists = pick(&base.lists, &ours.lists, &theirs.lists)?; - let tags = pick(&base.tags, &ours.tags, &theirs.tags)?; - let style = pick(&base.style, &ours.style, &theirs.style)?; - let format = pick(&base.format, &ours.format, &theirs.format)?; - let (parent, level) = pick( - &(base.parent, base.level), - &(ours.parent, ours.level), - &(theirs.parent, theirs.level), - )?; - let collapsed = pick(&base.collapsed, &ours.collapsed, &theirs.collapsed)?; - let content = match pick(&base.content, &ours.content, &theirs.content) { - Some(picked) => picked, - None => match (&base.content, &ours.content, &theirs.content) { - (ParagraphContent::Text(x), ParagraphContent::Text(y), ParagraphContent::Text(z)) - if x.id == y.id && x.id == z.id => - { - let mut text = z.clone(); - text.date_field = pick(&x.date_field, &y.date_field, &z.date_field)?; - text.tags = pick(&x.tags, &y.tags, &z.tags)?; - text.text = merge_text(&x.text, &y.text, &z.text)?; - ParagraphContent::Text(text) - } - _ => return None, - }, - }; - Some(PageParagraph { - id: theirs.id, - parent, - level, - style, - format, - content, - lists, - tags, - media: pick(&base.media, &ours.media, &theirs.media)?, - collapsed, - }) -} - -/// Re-applies our replacement inside the remote text when its range maps unambiguously. -fn merge_text( - base: &onestore::page::Paragraph, - ours: &onestore::page::Paragraph, - theirs: &onestore::page::Paragraph, -) -> Option { - if let Some(picked) = pick(base, ours, theirs) { - return Some(picked); - } - if base.text() == ours.text() { - // A local formatting-only change cannot be attributed to a range of the remote text. - return None; + fn add(&mut self, id: ExGuid, parent: ExGuid, own: Own, children: Vec) { + self.nodes.insert( + id, + Node { + own, + parent, + children, + }, + ); } - let (b, o) = (base.text(), ours.text()); - let prefix = b - .char_indices() - .zip(o.chars()) - .take_while(|((_, x), y)| x == y) - .map(|((i, x), _)| i + x.len_utf8()) - .last() - .unwrap_or(0); - let suffix = b[prefix..] - .chars() - .rev() - .zip(o[prefix..].chars().rev()) - .take_while(|(x, y)| x == y) - .map(|(x, _)| x.len_utf8()) - .sum::(); - let start = base.utf16_offset(prefix).ok()?; - let end = base.utf16_offset(b.len() - suffix).ok()?; - let inserted = ours.utf16_offset(o.len() - suffix).ok()?; - let replacement = ours.slice(start..inserted).ok()?; - let our_end = ours.utf16_offset(o.len()).ok()?; - let base_end = base.utf16_offset(b.len()).ok()?; - if (start > 0 && ours.slice(0..start).ok()? != base.slice(0..start).ok()?) - || (end < base_end - && ours.slice(inserted..our_end).ok()? != base.slice(end..base_end).ok()?) - { - // Formatting changed outside the replaced range; the remote text would hide it. - return None; + + fn outline(&mut self, outline: &Outline, parent: ExGuid) { + let children = self.paragraphs(&outline.paragraphs, outline.id); + self.add( + outline.id, + parent, + Own::Outline { + title: outline.title, + min_width: outline.min_width, + layout: outline.layout.clone(), + indents: outline.indents.clone(), + unsupported: outline.unsupported.clone(), + }, + children, + ); } - let mapped = crate::rebase::rebase(b, theirs.text(), start..end)?; - // Text they reformatted is not ours to replace, nor to move as a split would. - if start < end && theirs.slice(mapped.clone()).ok()? != base.slice(start..end).ok()? { - return None; + + /// Indexes a flat paragraph list; returns the container's direct children. + fn paragraphs(&mut self, list: &[PageParagraph], container: ExGuid) -> Vec { + let mut children: BTreeMap> = BTreeMap::new(); + for paragraph in list { + children + .entry(paragraph.parent.unwrap_or(container)) + .or_default() + .push(paragraph.id); + } + for paragraph in list { + let content = match ¶graph.content { + ParagraphContent::Text(text) => { + self.add(text.id, paragraph.id, Own::Text(text.clone()), Vec::new()); + text.id + } + ParagraphContent::Table(table) => { + let rows = table.rows.iter().map(|row| row.id).collect(); + for row in &table.rows { + let cells = row.cells.iter().map(|cell| cell.id).collect(); + for cell in &row.cells { + let children = self.paragraphs(&cell.paragraphs, cell.id); + self.add( + cell.id, + row.id, + Own::Cell { + layout: cell.layout.clone(), + indents: cell.indents.clone(), + shading: cell.shading, + unsupported: cell.unsupported.clone(), + }, + children, + ); + } + self.add(row.id, table.id, Own::Row, cells); + } + self.add( + table.id, + paragraph.id, + Own::Table { + columns: table.columns.clone(), + borders: table.borders, + layout: table.layout.clone(), + tags: table.tags.clone(), + }, + rows, + ); + table.id + } + ParagraphContent::Image(image) => { + self.add( + image.id, + paragraph.id, + Own::Image(image.clone()), + Vec::new(), + ); + image.id + } + ParagraphContent::Attachment(attachment) => { + self.add( + attachment.id, + paragraph.id, + Own::Attachment(attachment.clone()), + Vec::new(), + ); + attachment.id + } + ParagraphContent::Ink(ink) => { + self.add(ink.id, paragraph.id, Own::Ink(ink.clone()), Vec::new()); + ink.id + } + ParagraphContent::Unsupported(object) => { + self.add( + object.id, + paragraph.id, + Own::Unsupported(object.clone()), + Vec::new(), + ); + object.id + } + }; + self.add( + paragraph.id, + paragraph.parent.unwrap_or(container), + Own::Paragraph { + level: paragraph.level, + style: paragraph.style, + format: paragraph.format.clone(), + lists: paragraph.lists.clone(), + tags: paragraph.tags.clone(), + media: paragraph.media.clone(), + collapsed: paragraph.collapsed, + content, + }, + children.remove(¶graph.id).unwrap_or_default(), + ); + } + children.remove(&container).unwrap_or_default() } - let mut merged = theirs.clone(); - merged - .apply(Edit { - range: mapped, - replacement, - }) - .ok()?; - Some(merged) } #[cfg(test)] mod tests { use super::*; - use onestore::page::Paragraph; fn text(s: &str) -> Paragraph { - Paragraph::new(s.into(), Default::default()) + Paragraph::new(s.into(), Format::default()) } #[test] - fn same_paragraph_edits_merge_when_their_ranges_do_not_overlap() { - let base = text("one two three"); - let ours = text("one two three four"); - assert_eq!( - merge_text(&base, &ours, &text("zero one two three")) - .unwrap() - .text(), - "zero one two three four" - ); - assert_eq!( - merge_text(&base, &ours, &text("one 2 three")) - .unwrap() - .text(), - "one 2 three four" - ); - assert_eq!( - merge_text(&base, &text("one two THREE"), &text("one two 3")), - None - ); - assert_eq!( - merge_text(&base, &text("one two three"), &text("x")) - .unwrap() - .text(), - "x" - ); - assert_eq!( - merge_text(&base, &text("y"), &text("one two three")) - .unwrap() - .text(), - "y" - ); - let bold = onestore::document::Format { + fn ranges_clear_of_the_remote_changes_shift_past_them() { + let region = |start, end, delta| Region { start, end, delta }; + let changes = Changes::between(&text("one two three"), &text("one 2 three")); + assert_eq!(changes.0, [region(4, 7, -2)]); + assert_eq!(changes.map(&(0..3)), Some(0..3)); + assert_eq!(changes.map(&(0..4)), Some(0..4)); + assert_eq!(changes.map(&(8..13)), Some(6..11)); + assert_eq!(changes.map(&(7..8)), Some(5..6)); + assert_eq!(changes.map(&(13..13)), Some(11..11)); + assert_eq!(changes.map(&(4..4)), None); + assert_eq!(changes.map(&(7..7)), None); + assert_eq!(changes.map(&(3..5)), None); + let mut moved = changes.clone(); + moved.replaced(&(0..0), 5); + assert_eq!(moved.0, [region(9, 12, -2)]); + let both = Changes::between(&text("ab🦀cd"), &text("Xab🦀cYd")); + assert_eq!(both.0, [region(0, 0, 1), region(5, 5, 1)]); + assert_eq!(both.map(&(2..4)), Some(3..5)); + assert_eq!(both.map(&(1..1)), Some(2..2)); + assert_eq!(both.map(&(0..0)), None); + assert_eq!(both.map(&(3..6)), None); + let bold = Format { bold: Some(true), ..Default::default() }; - let remote = Paragraph::new("one two three".into(), bold); - let merged = merge_text(&base, &text("one two three four"), &remote).unwrap(); - assert_eq!(merged.text(), "one two three four"); - assert_eq!(merged.format_at(0).unwrap(), remote.format_at(0).unwrap()); - assert_eq!( - merge_text(&base, &remote, &text("one two three four")), - None - ); - assert_eq!( - merge_text(&base, &text("one two"), &text("X one two three")) - .unwrap() - .text(), - "X one two" + let formatted = Changes::between( + &text("abc"), + &Paragraph::from_runs([ + ("a".to_owned(), Format::default()), + ("b".to_owned(), bold), + ("c".to_owned(), Format::default()), + ]), ); + assert_eq!(formatted.0, [region(1, 2, 0)]); + let emoji = Changes::between(&text("🦀a"), &text("🦀ab")); + assert_eq!(emoji.0, [region(3, 3, 1)]); + let mut cut = Changes::between(&text("abcdef"), &text("aXbcdeYf")); + assert_eq!(cut.map(&(3..4)), Some(4..5)); + let right = cut.split(3); assert_eq!( - merge_text(&base, &text("two three"), &text("one two three!")) - .unwrap() - .text(), - "two three!" + (cut.0, right.0), + (vec![region(1, 1, 1)], vec![region(2, 2, 1)]) ); + let mut typed = Changes::between(&text("ab🦀cd"), &text("abX🦀cd")); + assert_eq!(typed.map(&(2..3)), Some(3..4)); + assert!(typed.split(2).0.is_empty()); + assert_eq!(typed.0, [region(2, 2, 1)]); } } diff --git a/crates/notebook/src/migrate.rs b/crates/notebook/src/migrate.rs new file mode 100644 index 0000000000000000000000000000000000000000..86fd430ad49763b4647863ad309e110a0d511387 --- /dev/null +++ b/crates/notebook/src/migrate.rs @@ -0,0 +1,303 @@ +//! Converts a schema-14 cache, whose queue holds whole-page edits over a working image, +//! to this schema's op queue. The cache is exported first; each queued page is lowered to +//! ops against the page the conversion has so far, and every converted page must equal +//! the page in the old working image, or nothing changes. + +use crate::{Result, base, queue, schema}; +use onestore::{ + Arena, ExGuid, PageCreation, PageEdit, RevisionIndex, Section, Store, + document::Document, + op::{Edit, Op, SectionOp}, + page::Page, +}; +use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params}; +use std::{collections::BTreeMap, io, path::Path}; + +pub(crate) const VERSION: u32 = 14; + +/// Schema 14's queued intents, as it serialized them. +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +enum Operation { + Page(PageIntent), + CreatePage(PageCreation), + Pages(PageEdits), + DeletePages(Vec), +} + +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct PageIntent { + #[allow(dead_code)] + before: Page, + after: Page, + author: String, +} + +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct PageEdits { + edits: Vec, + #[allow(dead_code)] + observed: Vec<(ExGuid, u32)>, +} + +fn invalid(message: String) -> crate::Error { + io::Error::new(io::ErrorKind::InvalidData, message).into() +} + +/// Schema 14 stored the working image as its length and the runs where it differs from +/// the base. +fn working(mut base: Vec, patch: &[u8]) -> Result> { + let damaged = || invalid("Damaged working image".into()); + let Some((length, mut runs)) = patch.split_first_chunk::<8>() else { + return if patch.is_empty() { + Ok(base) + } else { + Err(damaged()) + }; + }; + let length = usize::try_from(u64::from_le_bytes(*length)) + .ok() + .filter(|length| *length <= base.len() + patch.len()) + .ok_or_else(damaged)?; + base.resize(length, 0); + while let Some((header, rest)) = runs.split_first_chunk::<16>() { + let offset = usize::try_from(u64::from_le_bytes(header[..8].try_into().unwrap())) + .map_err(|_| damaged())?; + let size = usize::try_from(u64::from_le_bytes(header[8..].try_into().unwrap())) + .map_err(|_| damaged())?; + let (bytes, rest) = rest.split_at_checked(size).ok_or_else(damaged)?; + base.get_mut(offset..) + .and_then(|target| target.get_mut(..size)) + .ok_or_else(damaged)? + .copy_from_slice(bytes); + runs = rest; + } + if runs.is_empty() { + Ok(base) + } else { + Err(damaged()) + } +} + +struct Converted { + id: i64, + author: String, + edit: Edit, + /// Schema 14's publication evidence, when this edit was attempted. + attempted: Option, + conflict: Option<(i64, ExGuid)>, +} + +pub(crate) fn migrate(connection: &mut Connection, path: &Path) -> Result<()> { + let mut archive = path.as_os_str().to_owned(); + archive.push(".v14-recovery"); + let archive = std::path::PathBuf::from(archive); + crate::recovery::export(connection, &archive, true)?; + let failed = |error: crate::Error| { + invalid(format!( + "The schema-14 cache could not be converted ({error}); it is unchanged and archived at {}", + archive.display() + )) + }; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Exclusive)?; + convert(&transaction).map_err(failed)?; + transaction.commit()?; + Ok(()) +} + +fn convert(transaction: &rusqlite::Transaction<'_>) -> Result<()> { + let (base, patch): (Vec, Vec) = + transaction.query_row("SELECT base, working FROM replica WHERE id=1", [], |row| { + Ok((row.get(0)?, row.get(1)?)) + })?; + let expected = working(base.clone(), &patch)?; + let attempt: Option<(i64, String)> = transaction + .query_row("SELECT edit_id, revisions FROM attempt", [], |row| { + Ok((row.get(0)?, row.get(1)?)) + }) + .optional()?; + let conflicts: BTreeMap = transaction + .prepare("SELECT edit_id, kind FROM conflicts")? + .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))? + .collect::>()?; + let queued: Vec<(i64, String, String)> = transaction + .prepare("SELECT id, space, operation FROM edits ORDER BY id")? + .query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))? + .collect::>()?; + let sequence: i64 = transaction.query_row( + "SELECT max(coalesce((SELECT seq FROM sqlite_sequence WHERE name='edits'), 0), + coalesce((SELECT max(edit_id) FROM receipts), 0), + coalesce((SELECT max(edit_id) FROM archived), 0))", + [], + |row| row.get(0), + )?; + + let arena = Arena::default(); + let mut section = Section::open(&arena, base.clone())?; + let at = crate::now(); + let mut converted = Vec::new(); + let mut sealed = None; + let mut edited = BTreeMap::new(); + let (mut created, mut deleted) = (Vec::new(), Vec::new()); + for (id, space, operation) in queued { + let space: ExGuid = space.parse()?; + let operation: Operation = serde_json::from_str(&operation) + .map_err(|error| invalid(format!("Queued edit {id} is unreadable: {error}")))?; + let (author, ops) = match operation { + Operation::Page(intent) => { + let current = section.page(space)?; + let ops = onestore::op::lower_page(¤t, &intent.after)?; + edited.insert(space, id); + ( + intent.author, + ops.into_iter().map(|op| Op::Page { space, op }).collect(), + ) + } + Operation::CreatePage(creation) => { + created.push(creation.space()); + ( + String::new(), + vec![Op::Section(SectionOp::Create(creation))], + ) + } + Operation::Pages(batch) => ( + String::new(), + vec![Op::Section(SectionOp::Pages(batch.edits))], + ), + Operation::DeletePages(pages) => { + deleted.extend(&pages); + (String::new(), vec![Op::Section(SectionOp::Delete(pages))]) + } + }; + let edit = Edit { at, ops }; + section.apply(&author, &edit)?; + let attempted = attempt + .as_ref() + .filter(|(edit, _)| *edit == id) + .map(|(_, revisions)| revisions.clone()); + if attempted.is_some() { + if !converted.is_empty() { + return Err(invalid( + "Only the oldest queued edit can hold an attempt".into(), + )); + } + sealed = Some(section.seal()?); + } + // A conflicting page-list edit is the section's conflict, whichever page it names. + let conflicted = match edit.ops.first() { + Some(Op::Section(_)) => section.root(), + _ => space, + }; + converted.push(Converted { + id, + author, + edit, + attempted, + conflict: conflicts.get(&id).map(|kind| (*kind, conflicted)), + }); + } + let listed = |image: &[u8]| -> Result> { + let store = Store::parse(image)?; + let index = RevisionIndex::parse(&store)?; + Ok(Document::parse(&index)? + .pages()? + .into_iter() + .map(|(space, _)| space) + .collect()) + }; + let based = listed(&base)?; + let listing: Vec = section + .pages()? + .into_iter() + .map(|(space, ..)| space) + .collect(); + let cached = listed(&expected)?; + // A page the remote added or removed while edits waited is in one list and not the + // other; one the queue created or deleted must agree. + let agree = cached + .iter() + .filter(|space| !listing.contains(space)) + .all(|space| !based.contains(space) && !created.contains(space)) + && listing + .iter() + .filter(|space| !cached.contains(space)) + .all(|space| based.contains(space) && !deleted.contains(space)); + if !agree { + return Err(invalid( + "The converted page list differs from the cached one".into(), + )); + } + verify(&mut section, &expected, &edited)?; + + transaction.execute_batch( + "DROP TABLE replica; DROP TABLE attempt; DROP TABLE conflicts; DROP TABLE edits;", + )?; + transaction.execute_batch(schema::QUEUE)?; + base::write(transaction, base::Image::Base, &base)?; + let mut batch = None; + for edit in converted { + let current = match (batch, &edit.attempted) { + (Some(current), None) => current, + _ => { + transaction.execute("INSERT INTO batches DEFAULT VALUES", [])?; + transaction.last_insert_rowid() + } + }; + if let Some(evidence) = &edit.attempted { + transaction.execute( + "UPDATE batches SET sealed=?1, revisions=?2, attempted=1 WHERE id=?3", + params![ + serde_json::to_string(sealed.as_ref().unwrap()).map_err(io::Error::other)?, + evidence, + current + ], + )?; + batch = None; + } else { + batch = Some(current); + } + if let Some((kind, space)) = edit.conflict { + transaction.execute( + "UPDATE batches SET conflict=?1, space=?2 WHERE id=?3", + params![kind, space.to_string(), current], + )?; + } + queue::insert( + transaction, + Some(crate::unsigned(edit.id)?), + current, + &edit.author, + &edit.edit, + )?; + } + transaction.execute("DELETE FROM sqlite_sequence WHERE name='edits'", [])?; + transaction.execute( + "INSERT INTO sqlite_sequence(name, seq) VALUES ('edits', ?1)", + [sequence], + )?; + transaction.pragma_update(None, "user_version", schema::VERSION)?; + Ok(()) +} + +/// Requires each page a queued page edit changed to read as the old working image has it. +fn verify( + section: &mut Section<'_>, + expected: &[u8], + edited: &BTreeMap, +) -> Result<()> { + let store = Store::parse(expected)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + for (space, id) in edited { + let converted = section.page(*space).ok(); + let stored = Page::from_space(&document, *space).ok(); + if converted != stored { + return Err(invalid(format!( + "Queued edit {id} converts to a page that differs from the cached page" + ))); + } + } + Ok(()) +} diff --git a/crates/notebook/src/pages.rs b/crates/notebook/src/pages.rs deleted file mode 100644 index f77536addde20c68f8e8f0c76a2c315d7f883cbb..0000000000000000000000000000000000000000 --- a/crates/notebook/src/pages.rs +++ /dev/null @@ -1,168 +0,0 @@ -use super::*; -use onestore::{PageEdit, PagePosition}; -use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; - -type PageOrder = Vec<(ExGuid, u32)>; - -/// An atomic page batch with its observed order and indentation. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PageEdits { - pub edits: Vec, - observed: PageOrder, -} - -fn observe(source: &[u8]) -> Result> { - let store = Store::parse(source)?; - let index = RevisionIndex::parse(&store)?; - let document = Document::parse(&index)?; - let mut pages = Vec::new(); - for (sid, _) in document.pages()? { - let view = document.active(sid)?; - let Some(metadata) = view.roots.get(&2).and_then(|id| view.nodes.get(id)) else { - return Ok(None); - }; - let Kind::Metadata { level, .. } = metadata.kind else { - return Ok(None); - }; - pages.push((sid, level.unwrap_or(1))); - } - Ok(Some((document.root, pages))) -} - -fn positions(pages: &[(ExGuid, u32)]) -> Result> { - let mut positions = BTreeMap::new(); - for (at, (sid, level)) in pages.iter().enumerate() { - if sid.guid == [0; 16] - || !(1..=3).contains(level) - || positions.insert(*sid, (at, *level)).is_some() - { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Page observations need unique identities and valid indentation", - ) - .into()); - } - } - Ok(positions) -} - -impl Replica { - /// Durably queues the complete page batch using the supplied local snapshot. - pub fn pages(&self, source: &[u8], edits: &[PageEdit]) -> Result> { - let (space, batch, prepared) = PageEdits::capture(source, edits)?; - self.record(source, space, Operation::Pages(batch), &prepared) - } -} - -impl PageEdits { - fn capture<'a>( - source: &'a [u8], - edits: &[PageEdit], - ) -> Result<(ExGuid, Self, PreparedEdit<'a>)> { - let prepared = PreparedEdit::pages(source, edits)?; - let (space, observed) = observe(source)?.ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "Prepared page edits have no page metadata", - ) - })?; - positions(&observed)?; - Ok(( - space, - Self { - edits: edits.to_vec(), - observed, - }, - prepared, - )) - } - - pub(crate) fn prepare<'a>( - &self, - source: &'a [u8], - space: ExGuid, - ) -> Result, ConflictKind>> { - let Some((root, current)) = observe(source)? else { - return Ok(Err(ConflictKind::TargetUnavailable)); - }; - if root != space { - return Ok(Err(ConflictKind::TargetUnavailable)); - } - let before = positions(&self.observed)?; - let current = positions(¤t)?; - for edit in &self.edits { - let Some((_, original_level)) = before.get(&edit.space()) else { - return Ok(Err(ConflictKind::TargetUnavailable)); - }; - let Some((_, current_level)) = current.get(&edit.space()) else { - return Ok(Err(ConflictKind::TargetUnavailable)); - }; - if current_level != original_level && *current_level != edit.level() { - return Ok(Err(ConflictKind::StructureChanged)); - } - } - let prepared = match PreparedEdit::pages(source, &self.edits) { - Ok(prepared) => prepared, - Err(_) => return Ok(Err(ConflictKind::StructureChanged)), - }; - let (_, wanted) = observe(prepared.as_bytes())?.ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "Prepared page edits lost page metadata", - ) - })?; - let wanted = positions(&wanted)?; - for edit in &self.edits { - if edit.position() == PagePosition::Keep { - continue; - } - let sid = edit.space(); - let peers = before - .keys() - .filter(|id| **id != sid && current.contains_key(id)); - let mut unchanged = true; - let mut satisfied = true; - for id in peers { - let observed = current[id].0 < current[&sid].0; - unchanged &= observed == (before[id].0 < before[&sid].0); - satisfied &= observed == (wanted[id].0 < wanted[&sid].0); - } - if !unchanged && !satisfied { - return Ok(Err(ConflictKind::StructureChanged)); - } - } - Ok(Ok(prepared)) - } - - pub(crate) fn review(&self, source: &[u8], space: ExGuid, edits: &[PageEdit]) -> Result { - let identities_match = edits.len() == self.edits.len() - && edits.iter().all(|edit| { - self.edits - .iter() - .find(|original| original.space() == edit.space()) - .is_some_and(|original| { - original - .reposition(edit.position(), edit.level()) - .is_ok_and(|revised| revised == *edit) - }) - }); - if !identities_match { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Retain the original page and series identities when reviewing a page batch", - ) - .into()); - } - let (root, reviewed, _) = Self::capture(source, edits)?; - if root != space { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Review the original section for page edits", - ) - .into()); - } - Ok(reviewed) - } -} diff --git a/crates/notebook/src/queue.rs b/crates/notebook/src/queue.rs new file mode 100644 index 0000000000000000000000000000000000000000..66bf8399eca6aa71097a5da93af31e98521d8d66 --- /dev/null +++ b/crates/notebook/src/queue.rs @@ -0,0 +1,324 @@ +//! Edit rows. An edit is stored as its serialized ops with picture and attachment bytes +//! moved to the `payloads` table, named by SHA-256 in the order `slots` visits them. + +use crate::{PendingEdit, Result, signed, unsigned}; +use onestore::{ + op::{Edit, Op, PageOp, SectionOp, TableEdit}, + page::{PageObject, PageParagraph, ParagraphContent, TableCell}, +}; +use rusqlite::{Connection, OptionalExtension, params}; +use sha2::{Digest, Sha256}; +use std::{io, sync::Arc}; + +type Payload = Option>; + +fn paragraphs(list: &mut [PageParagraph], visit: &mut impl FnMut(&mut Payload)) { + for paragraph in list { + match &mut paragraph.content { + ParagraphContent::Image(image) => visit(&mut image.bytes), + ParagraphContent::Attachment(attachment) => { + visit(&mut attachment.bytes); + visit(&mut attachment.preview); + } + ParagraphContent::Table(table) => { + for row in &mut table.rows { + cells(&mut row.cells, visit); + } + } + ParagraphContent::Text(_) + | ParagraphContent::Ink(_) + | ParagraphContent::Unsupported(_) => {} + } + } +} + +fn cells(list: &mut [TableCell], visit: &mut impl FnMut(&mut Payload)) { + for cell in list { + paragraphs(&mut cell.paragraphs, visit); + } +} + +fn object(object: &mut PageObject, visit: &mut impl FnMut(&mut Payload)) { + match object { + PageObject::Outline(outline) => paragraphs(&mut outline.paragraphs, visit), + PageObject::Title(title) => { + for outline in &mut title.outlines { + paragraphs(&mut outline.paragraphs, visit); + } + } + PageObject::Image(image) => visit(&mut image.bytes), + PageObject::Ink(_) | PageObject::Unsupported(_) => {} + } +} + +/// Visits every payload an edit carries, in a fixed order. +fn slots(edit: &mut Edit, mut visit: impl FnMut(&mut Payload)) { + for op in &mut edit.ops { + match op { + Op::Page { op, .. } => match op { + PageOp::Insert { + paragraphs: list, .. + } => paragraphs(list, &mut visit), + PageOp::Add { object: added, .. } => object(added, &mut visit), + PageOp::Table { + edit: TableEdit::Rows { rows, .. }, + .. + } => { + for row in rows { + cells(&mut row.cells, &mut visit); + } + } + PageOp::Table { + edit: TableEdit::Column { cells: list, .. }, + .. + } => cells(list, &mut visit), + _ => {} + }, + Op::Section(SectionOp::Import { page, .. }) => { + for added in &mut page.objects { + object(added, &mut visit); + } + } + Op::Section(_) => {} + } + } +} + +fn damaged(message: &'static str) -> crate::Error { + io::Error::new(io::ErrorKind::InvalidData, message).into() +} + +/// Stores `edit` in `batch` under `id`, or the next id, its payloads once each by hash; +/// returns the edit's id. +pub(crate) fn insert( + connection: &Connection, + id: Option, + batch: i64, + author: &str, + edit: &Edit, +) -> Result { + let (text, names) = encode(connection, edit)?; + connection.execute( + "INSERT INTO edits(id, batch, author, edit, payloads) VALUES (?1, ?2, ?3, ?4, ?5)", + params![id.map(signed).transpose()?, batch, author, text, names], + )?; + unsigned(connection.last_insert_rowid()) +} + +fn hex(digest: &[u8]) -> String { + digest.iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// Replaces a stored edit's ops. +pub(crate) fn rewrite(connection: &Connection, id: u64, edit: &Edit) -> Result<()> { + let (text, names) = encode(connection, edit)?; + connection.execute( + "UPDATE edits SET edit=?1, payloads=?2 WHERE id=?3", + params![text, names, signed(id)?], + )?; + Ok(()) +} + +/// The serialized edit without payload bytes and the payload names, storing the payloads. +fn encode(connection: &Connection, edit: &Edit) -> Result<(String, Option)> { + let mut edit = edit.clone(); + let mut names: Vec> = Vec::new(); + let mut failure = None; + slots(&mut edit, |payload| { + let name = payload.take().map(|bytes| { + let digest = Sha256::digest(&bytes); + if let Err(error) = connection.execute( + "INSERT INTO payloads(sha256, bytes) VALUES (?1, ?2) ON CONFLICT DO NOTHING", + params![&digest[..], &bytes[..]], + ) { + failure.get_or_insert(error); + } + hex(&digest) + }); + names.push(name); + }); + if let Some(error) = failure { + return Err(error.into()); + } + let names = names + .iter() + .any(Option::is_some) + .then(|| serde_json::to_string(&names)) + .transpose() + .map_err(io::Error::other)?; + Ok(( + serde_json::to_string(&edit).map_err(io::Error::other)?, + names, + )) +} + +fn decode(connection: &Connection, text: &str, names: Option) -> Result { + let mut edit: Edit = serde_json::from_str(text) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + let Some(names) = names else { + return Ok(edit); + }; + let names: Vec> = serde_json::from_str(&names) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + let mut names = names.into_iter(); + let mut failure: Option = None; + let mut query = + connection.prepare_cached("SELECT bytes FROM payloads WHERE sha256=unhex(?1)")?; + slots(&mut edit, |payload| { + let Some(name) = names.next() else { + failure.get_or_insert(damaged("A queued edit names fewer payloads than it holds")); + return; + }; + let Some(name) = name else { + return; + }; + match query + .query_row([&name], |row| row.get::<_, Vec>(0)) + .optional() + { + Ok(Some(bytes)) if hex(&Sha256::digest(&bytes)) == name => { + *payload = Some(Arc::from(bytes)); + } + Ok(_) => { + failure.get_or_insert(damaged("A queued payload is missing or damaged")); + } + Err(error) => { + failure.get_or_insert(error.into()); + } + } + }); + if names.next().is_some() { + failure.get_or_insert(damaged("A queued edit names more payloads than it holds")); + } + match failure { + Some(error) => Err(error), + None => Ok(edit), + } +} + +/// Queued edits, oldest first; of one batch when `batch` is given. +pub(crate) fn load(connection: &Connection, batch: Option) -> Result> { + let mut query = connection.prepare_cached( + "SELECT id, author, edit, payloads FROM edits WHERE ?1 IS NULL OR batch=?1 ORDER BY id", + )?; + let mut rows = query.query([batch])?; + let mut edits = Vec::new(); + while let Some(row) = rows.next()? { + edits.push(PendingEdit { + id: unsigned(row.get(0)?)?, + author: row.get(1)?, + edit: decode(connection, &row.get::<_, String>(2)?, row.get(3)?)?, + }); + } + Ok(edits) +} + +/// Drops payloads no queued edit names. +pub(crate) fn collect(connection: &Connection) -> Result<()> { + connection.execute( + "DELETE FROM payloads WHERE NOT EXISTS (SELECT 1 FROM edits, json_each(edits.payloads) AS name + WHERE edits.payloads IS NOT NULL AND name.value=lower(hex(payloads.sha256)))", + [], + )?; + Ok(()) +} + +/// The object spaces an edit changes; section ops change the root space as well. +pub(crate) fn spaces(edit: &Edit, root: onestore::ExGuid) -> Vec { + let mut spaces = Vec::new(); + for op in &edit.ops { + match op { + Op::Page { space, .. } => spaces.push(*space), + Op::Section(op) => { + spaces.push(root); + match op { + SectionOp::Create(creation) | SectionOp::Import { creation, .. } => { + spaces.push(creation.space()); + } + SectionOp::Pages(edits) => spaces.extend(edits.iter().map(|edit| edit.space())), + SectionOp::Delete(pages) => spaces.extend(pages), + } + } + } + } + spaces.sort(); + spaces.dedup(); + spaces +} + +#[cfg(test)] +mod tests { + use super::*; + use onestore::{ExGuid, page::Image}; + + #[test] + fn payloads_are_stored_once_by_hash_and_restored_in_place() { + let connection = Connection::open_in_memory().unwrap(); + connection.execute_batch(crate::schema::QUEUE).unwrap(); + connection + .execute("INSERT INTO batches DEFAULT VALUES", []) + .unwrap(); + let bytes: Arc<[u8]> = Arc::from(vec![7_u8; 100_000]); + let image = |id: u32| { + PageObject::Image(Image { + id: ExGuid { + guid: [1; 16], + n: id, + }, + layout: Default::default(), + size: None, + bytes: Some(Arc::clone(&bytes)), + alt: None, + background: false, + }) + }; + let space = ExGuid { + guid: [2; 16], + n: 1, + }; + let edit = Edit { + at: 1, + ops: [image(1), image(2)] + .into_iter() + .map(|object| Op::Page { + space, + op: PageOp::Add { + object, + before: None, + }, + }) + .collect(), + }; + let id = insert(&connection, None, 1, "Author", &edit).unwrap(); + let stored: String = connection + .query_row("SELECT edit FROM edits WHERE id=?1", [id as i64], |row| { + row.get(0) + }) + .unwrap(); + assert!(stored.len() < 1000, "{}", stored.len()); + let payloads: i64 = connection + .query_row("SELECT count(*) FROM payloads", [], |row| row.get(0)) + .unwrap(); + assert_eq!(payloads, 1); + let loaded = load(&connection, None).unwrap(); + assert_eq!(loaded[0].edit, edit); + let Op::Page { + op: + PageOp::Add { + object: PageObject::Image(image), + .. + }, + .. + } = &loaded[0].edit.ops[1] + else { + panic!() + }; + assert_eq!(image.bytes.as_deref(), Some(&bytes[..])); + connection.execute("DELETE FROM edits", []).unwrap(); + collect(&connection).unwrap(); + let payloads: i64 = connection + .query_row("SELECT count(*) FROM payloads", [], |row| row.get(0)) + .unwrap(); + assert_eq!(payloads, 0); + } +} diff --git a/crates/notebook/src/rebase.rs b/crates/notebook/src/rebase.rs deleted file mode 100644 index 9a6dce0a8b16201774a6482925ad04659b6ab1b6..0000000000000000000000000000000000000000 --- a/crates/notebook/src/rebase.rs +++ /dev/null @@ -1,297 +0,0 @@ -use std::ops::Range; - -const INFINITY: u32 = 1 << 30; -const MAX_CELLS: usize = 4_000_000; - -struct Matrix { - band: usize, - values: Vec, -} - -impl Matrix { - fn get(&self, row: usize, column: usize) -> u32 { - let Some(column) = column - .checked_add(self.band) - .and_then(|at| at.checked_sub(row)) - else { - return INFINITY; - }; - let width = self.band * 2 + 1; - if column >= width { - return INFINITY; - } - self.values - .get(row * width + column) - .copied() - .unwrap_or(INFINITY) - } - - fn build(before: &[char], after: &[char], band: usize) -> Self { - let width = band * 2 + 1; - let mut matrix = Self { - band, - values: vec![INFINITY; (before.len() + 1) * width], - }; - for row in 0..=before.len() { - for column in row.saturating_sub(band)..=after.len().min(row + band) { - let mut cost = if row == 0 && column == 0 { 0 } else { INFINITY }; - if row > 0 { - cost = cost.min(matrix.get(row - 1, column) + 1); - } - if column > 0 { - cost = cost.min(matrix.get(row, column - 1) + 1); - } - if row > 0 && column > 0 && before[row - 1] == after[column - 1] { - cost = cost.min(matrix.get(row - 1, column - 1)); - } - matrix.values[row * width + column + band - row] = cost; - } - } - matrix - } -} - -/// Requires the same mapping in every minimum insertion/deletion alignment. -pub(crate) fn rebase(before: &str, after: &str, range: Range) -> Option> { - if range.start > range.end { - return None; - } - let mut a: Vec<_> = before.chars().collect(); - let offsets: Vec<_> = std::iter::once(0) - .chain(a.iter().scan(0_u32, |at, character| { - *at = at.checked_add(character.len_utf16() as u32)?; - Some(*at) - })) - .collect(); - let local = - offsets.binary_search(&range.start).ok()?..offsets.binary_search(&range.end).ok()?; - if before == after { - return Some(range); - } - let mut b: Vec<_> = after.chars().collect(); - let (n, m) = (a.len(), b.len()); - let mut band = n.abs_diff(m).max(1); - let forward = loop { - let width = band.checked_mul(2)?.checked_add(1)?; - if (n + 1).checked_mul(width)? > MAX_CELLS { - return None; - } - let matrix = Matrix::build(&a, &b, band); - if matrix.get(n, m) <= band as u32 { - break matrix; - } - band *= 2; - }; - let distance = forward.get(n, m); - a.reverse(); - b.reverse(); - let backward = Matrix::build(&a, &b, band); - a.reverse(); - b.reverse(); - let position = |index: usize| { - let mut matched = None; - for column in index.saturating_sub(band)..=m.min(index + band) { - let cost = forward.get(index, column); - if cost + 1 + backward.get(n - index - 1, m - column) == distance { - return None; - } - if b.get(column) == Some(&a[index]) - && cost + backward.get(n - index - 1, m - column - 1) == distance - { - if matched.is_some() { - return None; - } - matched = Some(column); - } - } - matched - }; - let mapped = if local.is_empty() { - if local.start > 0 { - position(local.start - 1)?; - } - if local.start < n { - position(local.start)?; - } - let mut boundary = None; - for column in local.start.saturating_sub(band)..=m.min(local.start + band) { - if forward.get(local.start, column) + backward.get(n - local.start, m - column) - == distance - { - if boundary.is_some() { - return None; - } - boundary = Some(column); - } - } - let at = boundary?; - at..at - } else { - let start = position(local.start)?; - for index in local.start + 1..local.end { - if position(index)? != start + index - local.start { - return None; - } - } - start..start + local.len() - }; - let start = b[..mapped.start] - .iter() - .map(|character| character.len_utf16()) - .sum::(); - let end = start - + b[mapped] - .iter() - .map(|character| character.len_utf16()) - .sum::(); - Some(u32::try_from(start).ok()?..u32::try_from(end).ok()?) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn optimal_paths(a: &[char], b: &[char]) -> Vec> { - fn visit( - a: &[char], - b: &[char], - path: &mut Vec<(usize, usize)>, - cost: usize, - best: &mut usize, - found: &mut Vec>, - ) { - if cost > *best { - return; - } - let (i, j) = *path.last().unwrap(); - if (i, j) == (a.len(), b.len()) { - if cost < *best { - found.clear(); - *best = cost; - } - found.push(path.clone()); - return; - } - for (next_i, next_j, charge) in [(i + 1, j + 1, 0), (i + 1, j, 1), (i, j + 1, 1)] { - if next_i > a.len() || next_j > b.len() || (charge == 0 && a[i] != b[j]) { - continue; - } - path.push((next_i, next_j)); - visit(a, b, path, cost + charge, best, found); - path.pop(); - } - } - let mut found = Vec::new(); - let mut best = usize::MAX; - visit(a, b, &mut vec![(0, 0)], 0, &mut best, &mut found); - found - } - - #[test] - fn bounded_alignment_agrees_with_exhaustive_paths_for_every_small_unicode_edit() { - let mut words = vec![String::new()]; - for length in 1..=4 { - for bits in 0..1 << length { - words.push( - (0..length) - .map(|bit| if bits & (1 << bit) == 0 { 'a' } else { '🦀' }) - .collect(), - ); - } - } - let mut cases = 0; - for before in &words { - let a: Vec<_> = before.chars().collect(); - for after in &words { - let b: Vec<_> = after.chars().collect(); - let paths = optimal_paths(&a, &b); - for start in 0..=a.len() { - for end in start..=a.len() { - let mut expected = None; - let mut valid = true; - for path in &paths { - let mapping: Vec<_> = (0..a.len()) - .map(|i| { - path.windows(2).find_map(|edge| { - (edge[0].0 == i && edge[1] == (i + 1, edge[0].1 + 1)) - .then_some(edge[0].1) - }) - }) - .collect(); - let candidate = if start == end { - let vertices: Vec<_> = path - .iter() - .filter(|(i, _)| *i == start) - .map(|(_, j)| *j) - .collect(); - if (start > 0 && mapping[start - 1].is_none()) - || (start < a.len() && mapping[start].is_none()) - || vertices.len() != 1 - { - None - } else { - Some(vertices[0]..vertices[0]) - } - } else if let Some(first) = mapping[start] { - (start..end) - .all(|i| mapping[i] == Some(first + i - start)) - .then_some(first..first + end - start) - } else { - None - }; - let Some(candidate) = candidate else { - valid = false; - break; - }; - if expected.as_ref().is_some_and(|old| *old != candidate) { - valid = false; - break; - } - expected = Some(candidate); - } - let expected = if valid { - expected.map(|range| { - b[..range.start].iter().map(|c| c.len_utf16() as u32).sum() - ..b[..range.end].iter().map(|c| c.len_utf16() as u32).sum() - }) - } else { - None - }; - let range = a[..start].iter().map(|c| c.len_utf16() as u32).sum() - ..a[..end].iter().map(|c| c.len_utf16() as u32).sum(); - assert_eq!( - rebase(before, after, range), - expected, - "{before:?} -> {after:?}, characters {start}..{end}" - ); - cases += 1; - } - } - } - } - assert_eq!(cases, 10881); - } - - #[test] - fn maps_disjoint_unicode_edits_and_rejects_ambiguous_or_overlapping_changes() { - assert_eq!(rebase("ab🦀cd", "Xab🦀cYd", 2..4), Some(3..5)); - assert_eq!(rebase("abc", "XabcY", 1..2), Some(2..3)); - assert_eq!(rebase("abc", "XabcY", 1..1), Some(2..2)); - assert_eq!(rebase("abc", "abcX", 3..3), None); - assert_eq!(rebase("abc", "ac", 1..2), None); - assert_eq!(rebase("abc", "", 1..1), None); - assert_eq!(rebase("aaaa", "aaaaa", 1..2), None); - assert_eq!(rebase("ab🦀cd", "ab🦀cd", 3..4), None); - assert_eq!(rebase("abc", "abc", 4..4), None); - } - - #[test] - fn long_paragraphs_with_small_remote_changes_use_a_narrow_band() { - let text = format!("{}🦀{}", "a".repeat(8192), "b".repeat(8192)); - assert_eq!( - rebase(&text, &format!("X{text}Y"), 8192..8194), - Some(8193..8195) - ); - assert_eq!(rebase(&"a".repeat(8192), &"b".repeat(8192), 1..2), None); - } -} diff --git a/crates/notebook/src/recovery.rs b/crates/notebook/src/recovery.rs index 8d1b28c808528026e99ff249bde794e92feaa7be..1999ad76d2bfd6f289551327e43762fcf59c86cd 100644 --- a/crates/notebook/src/recovery.rs +++ b/crates/notebook/src/recovery.rs @@ -11,7 +11,9 @@ pub struct RecoverySummary { pub conflicts: u64, pub uncertain_edits: u64, pub published_receipts: u64, - pub working_bytes: u64, + /// Bytes of the image the queued edits apply to. + pub base_bytes: u64, + /// Bytes of the observed remote image, while it is not the base. pub remote_bytes: u64, pub cached_assets: u64, pub cached_asset_bytes: u64, @@ -36,33 +38,48 @@ impl Recovery { io::Error::new(io::ErrorKind::InvalidData, "Not a recovery archive").into(), ); } - if version != SCHEMA_VERSION { + if version != schema::VERSION { return Err(io::Error::new( io::ErrorKind::InvalidData, format!( - "Archive schema version {version} is not the supported version {SCHEMA_VERSION}" + "Archive schema version {version} is not the supported version {}", + schema::VERSION ), ) .into()); } - validate_images(&connection)?; - for edit in pending(&connection)? { - sync::status(&connection, edit.id)?; + let recovery = Self { connection }; + recovery.snapshot()?; + for edit in pending(&recovery.connection)? { + sync::status(&recovery.connection, edit.id)?; } - receipts(&connection)?; - Ok(Self { connection }) + receipts(&recovery.connection)?; + Ok(recovery) } pub fn summary(&self) -> Result { summary(&self.connection) } + /// The image the archived queue leaves, as `Replica::snapshot` gives it. pub fn snapshot(&self) -> Result> { - crate::images::working(&self.connection) + working::image(&self.connection) } + fn base(&self) -> Result> { + Ok( + base::read(&self.connection, base::Image::Base)?.ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "The archive has no base image") + })?, + ) + } + + /// The last observed remote image. pub fn remote_snapshot(&self) -> Result> { - crate::images::base(&self.connection) + match base::read(&self.connection, base::Image::Remote)? { + Some(image) => Ok(image), + None => self.base(), + } } pub fn pending(&self) -> Result> { @@ -86,83 +103,88 @@ impl Recovery { impl Replica { /// Summarizes durable state without exposing notebook content. pub fn recovery_summary(&self) -> Result { - let connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - summary(&connection) + summary(&*self.lock()?) } /// Exports a consistent archive to a new local path without changing the live queue. /// Archives contain notebook content and cannot be opened as writable replicas. /// An error after publication can leave a complete archive at the destination. pub fn export_recovery(&self, path: impl AsRef) -> Result<()> { - let path = path.as_ref(); - if path.file_name().is_none() { + export(&*self.lock()?, path.as_ref(), false) + } +} + +/// Copies the cache to `path` as a recovery archive; `replace` overwrites an existing file. +pub(crate) fn export(source: &Connection, path: &Path, replace: bool) -> Result<()> { + if path.file_name().is_none() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Provide a new recovery archive filename", + ) + .into()); + } + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let temporary = tempfile::Builder::new() + .prefix(".onestore-recovery-") + .tempfile_in(parent)?; + let mut destination = cache_connection(temporary.path())?; + { + let backup = Backup::new(source, &mut destination)?; + if backup.step(-1)? != StepResult::Done { return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Provide a new recovery archive filename", + io::ErrorKind::WouldBlock, + "Recovery export could not acquire the database snapshot", ) .into()); } - let parent = path - .parent() - .filter(|parent| !parent.as_os_str().is_empty()) - .unwrap_or_else(|| Path::new(".")); - let temporary = tempfile::Builder::new() - .prefix(".onestore-recovery-") - .tempfile_in(parent)?; - let mut destination = cache_connection(temporary.path())?; - { - let source = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - let backup = Backup::new(&source, &mut destination)?; - if backup.step(-1)? != StepResult::Done { - return Err(io::Error::new( - io::ErrorKind::WouldBlock, - "Recovery export could not acquire the database snapshot", - ) - .into()); - } - } - destination.pragma_update(None, "application_id", RECOVERY_ID)?; - destination.close().map_err(|(_, error)| error)?; - temporary.as_file().sync_all()?; + } + // One self-contained file: the copied header says WAL, as the cache does. + destination.pragma_update(None, "journal_mode", "DELETE")?; + destination.pragma_update(None, "application_id", RECOVERY_ID)?; + destination.close().map_err(|(_, error)| error)?; + temporary.as_file().sync_all()?; + if replace { + temporary.persist(path).map_err(|error| error.error)?; + } else { temporary .persist_noclobber(path) .map_err(|error| error.error)?; - File::open(parent)?.sync_all()?; - Ok(()) } + File::open(parent)?.sync_all()?; + Ok(()) } fn summary(connection: &Connection) -> Result { - let (cached_assets, cached_asset_bytes) = connection.query_row( + let (cached_assets, cached_asset_bytes): (i64, i64) = connection.query_row( "SELECT count(*),coalesce(sum(length(data)),0) FROM assets", [], - |row| Ok((unsigned(row, 0)?, unsigned(row, 1)?)), + |row| Ok((row.get(0)?, row.get(1)?)), )?; - let working_bytes = crate::images::working_length(connection)?; - Ok(connection.query_row( - "SELECT (SELECT count(*) FROM edits), (SELECT count(*) FROM conflicts), - (SELECT count(*) FROM attempt), (SELECT count(*) FROM receipts), - length(base) FROM replica WHERE id=1", - [], - |row| { - Ok(RecoverySummary { - queued_edits: unsigned(row, 0)?, - conflicts: unsigned(row, 1)?, - uncertain_edits: unsigned(row, 2)?, - published_receipts: unsigned(row, 3)?, - working_bytes, - remote_bytes: unsigned(row, 4)?, - cached_assets, - cached_asset_bytes, - }) - }, - )?) + let length = |image| -> Result { + Ok(base::stamp(connection, image)?.map_or(0, |stamp| stamp.length)) + }; + let (queued_edits, conflicts, uncertain_edits, published_receipts): (i64, i64, i64, i64) = + connection.query_row( + "SELECT (SELECT count(*) FROM edits), + (SELECT count(*) FROM edits JOIN batches ON batches.id=edits.batch WHERE conflict IS NOT NULL), + (SELECT count(*) FROM edits JOIN batches ON batches.id=edits.batch WHERE attempted=1), + (SELECT count(*) FROM receipts)", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + )?; + Ok(RecoverySummary { + queued_edits: unsigned(queued_edits)?, + conflicts: unsigned(conflicts)?, + uncertain_edits: unsigned(uncertain_edits)?, + published_receipts: unsigned(published_receipts)?, + base_bytes: length(base::Image::Base)?, + remote_bytes: length(base::Image::Remote)?, + cached_assets: unsigned(cached_assets)?, + cached_asset_bytes: unsigned(cached_asset_bytes)?, + }) } fn receipts(connection: &Connection) -> Result> { @@ -171,17 +193,7 @@ fn receipts(connection: &Connection) -> Result> { let mut rows = statement.query([])?; let mut receipts = BTreeMap::new(); while let Some(row) = rows.next()? { - receipts.insert(unsigned(row, 0)?, row.get::<_, String>(1)?.parse()?); + receipts.insert(unsigned(row.get(0)?)?, row.get::<_, String>(1)?.parse()?); } Ok(receipts) } - -fn unsigned(row: &rusqlite::Row<'_>, column: usize) -> rusqlite::Result { - u64::try_from(row.get::<_, i64>(column)?).map_err(|error| { - rusqlite::Error::FromSqlConversionFailure( - column, - rusqlite::types::Type::Integer, - Box::new(error), - ) - }) -} diff --git a/crates/notebook/src/schema.rs b/crates/notebook/src/schema.rs index 4c163f268b3b3164cc6083a714c6729213884641..cb9a5628939f7337a71909d976fac544e94dae94 100644 --- a/crates/notebook/src/schema.rs +++ b/crates/notebook/src/schema.rs @@ -1,39 +1,53 @@ use super::*; use rusqlite::Transaction; -const CONFLICTS: &str = "CREATE TABLE conflicts ( - edit_id INTEGER PRIMARY KEY REFERENCES edits(id) ON DELETE CASCADE, - kind INTEGER NOT NULL CHECK(kind BETWEEN 0 AND 3) -) STRICT;"; +pub(crate) const VERSION: u32 = 15; -const ASSETS: &str = "CREATE TABLE assets ( - name TEXT PRIMARY KEY NOT NULL, - data BLOB NOT NULL, - sha256 BLOB NOT NULL CHECK(length(sha256)=32) -) STRICT;"; +/// Tables a schema-14 cache shares with this one. +pub(crate) const KEPT: &str = " + CREATE TABLE receipts ( + edit_id INTEGER PRIMARY KEY CHECK(edit_id>0), + revision TEXT NOT NULL + ) STRICT; + CREATE TABLE archived ( + edit_id INTEGER PRIMARY KEY CHECK(edit_id>0), + archive TEXT NOT NULL + ) STRICT; + CREATE TABLE assets ( + name TEXT PRIMARY KEY NOT NULL, + data BLOB NOT NULL, + sha256 BLOB NOT NULL CHECK(length(sha256)=32) + ) STRICT;"; + +/// The queue: the image its edits apply to, in chunks, and the edits in publication batches. +pub(crate) const QUEUE: &str = " + CREATE TABLE base (chunk INTEGER PRIMARY KEY CHECK(chunk>=0), bytes BLOB NOT NULL) STRICT; + CREATE TABLE remote (chunk INTEGER PRIMARY KEY CHECK(chunk>=0), bytes BLOB NOT NULL) STRICT; + CREATE TABLE batches ( + id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(id>0), + sealed TEXT, + revisions TEXT, + attempted INTEGER NOT NULL DEFAULT 0 CHECK(attempted IN (0,1)), + conflict INTEGER CHECK(conflict BETWEEN 0 AND 3), + space TEXT, + CHECK((sealed IS NULL) = (revisions IS NULL)), + CHECK(attempted=0 OR sealed IS NOT NULL), + CHECK((conflict IS NULL) = (space IS NULL)) + ) STRICT; + CREATE TABLE edits ( + id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(id>0), + batch INTEGER NOT NULL REFERENCES batches(id) ON DELETE CASCADE, + author TEXT NOT NULL, + edit TEXT NOT NULL, + payloads TEXT + ) STRICT; + CREATE TABLE payloads ( + sha256 BLOB PRIMARY KEY CHECK(length(sha256)=32), + bytes BLOB NOT NULL + ) STRICT;"; pub(crate) fn create(transaction: &Transaction<'_>) -> Result<()> { - transaction.execute_batch( - "CREATE TABLE edits ( - id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(id>0), - space TEXT NOT NULL, - operation TEXT NOT NULL - ) STRICT; - CREATE TABLE attempt ( - id INTEGER PRIMARY KEY CHECK(id=1), - edit_id INTEGER NOT NULL UNIQUE REFERENCES edits(id) ON DELETE CASCADE, - revisions TEXT NOT NULL - ) STRICT; - CREATE TABLE receipts ( - edit_id INTEGER PRIMARY KEY CHECK(edit_id>0), - revision TEXT NOT NULL - ) STRICT; - CREATE TABLE archived ( - edit_id INTEGER PRIMARY KEY CHECK(edit_id>0), - archive TEXT NOT NULL - ) STRICT;", - )?; - transaction.execute_batch(CONFLICTS)?; - transaction.execute_batch(ASSETS)?; + transaction.execute_batch(QUEUE)?; + transaction.execute_batch(KEPT)?; Ok(()) } diff --git a/crates/notebook/src/session.rs b/crates/notebook/src/session.rs index 9f379bb54b9651433e54ac842255b4369e427ed7..3ae33747104978bd0a241b0c381789626d3ebb3f 100644 --- a/crates/notebook/src/session.rs +++ b/crates/notebook/src/session.rs @@ -1,15 +1,18 @@ //! The application's view of a notebook: sections opened through a local replica that -//! publishes page saves to the section file in the background. +//! publishes their edits to the section file in the background. use crate::{ - ConflictKind, EditStatus, Error, PendingEdit, Remote, Replica, Result, SyncWorker, discover, + Conflict, ConflictKind, EditStatus, Error, PendingEdit, Remote, Replica, Resolution, Result, + SyncWorker, discover, }; use onestore::{ CommitError, ExGuid, PageCreation, PreparedEdit, RevisionIndex, Stamp, Store, Transaction, - document::Document, page::Page, + document::Document, + op::{Edit, Op, SectionOp}, + page::Page, }; use std::{ - collections::{BTreeMap, VecDeque}, + collections::BTreeMap, io, io::Write, path::{Path, PathBuf}, @@ -18,7 +21,6 @@ use std::{ atomic::{AtomicUsize, Ordering}, mpsc::{self, Receiver, Sender}, }, - thread::{self, JoinHandle}, time::Duration, }; @@ -723,10 +725,15 @@ fn catalog_path(folder: &str, name: &str) -> String { } } -/// What happened on the synchronization thread since the last poll. +/// What happened to the section since the last poll. #[derive(Debug)] pub enum Event { - /// The working image was refreshed from the section file with no local edit pending. + /// A remote change reached these pages; reload them where they are open. + Changed(Vec), + /// The section refused an edit `apply` handed it; the pages it names are as they were + /// before it, so an editor showing them should reload them. + Rejected { spaces: Vec, error: String }, + /// Old session API, until the application reloads on `Changed`: pages changed remotely. Refreshed, /// A publication attempt finished with this durable state. Attempt { id: u64, status: EditStatus }, @@ -736,7 +743,7 @@ pub enum Event { Failed(String), } -/// The outcome of saving an edited page. +/// The outcome of saving an edited page (old session API). #[derive(Debug, PartialEq, Eq)] pub enum Save { /// The model equals the stored page. @@ -755,31 +762,27 @@ pub struct QueuedEdit { pub status: EditStatus, } -/// A section file with its replica, background saves and background publication. +type Notify = Arc; + +/// A section file with its replica and background publication. `Send + Sync`: edits +/// arrive from any thread without waiting, and `notify` wakes the host when events wait. pub struct Section { file: PathBuf, replica: Arc, worker: Option, - events: Receiver, - saves: Option>, - saved: Receiver<(ExGuid, std::result::Result)>, + events: Mutex>, + sender: Sender, + notify: Notify, + saved: Mutex)>>, + saves: Sender<(ExGuid, std::result::Result)>, /// Saves queued and not yet reported. unsaved: Arc, - saver: Option>, -} - -/// An edited page `Section::queue_save` hands to the save thread. -struct PageSave { - space: ExGuid, - before: Page, - after: Page, - author: String, } impl Section { /// Opens the section file through a replica in `cache`, creating the replica from the - /// file on first use. `notify` runs on the synchronization thread whenever an event is - /// available. + /// file on first use and converting an older one. `notify` runs on a background thread + /// whenever an event is available. pub fn open( file: impl AsRef, cache: impl AsRef, @@ -847,58 +850,58 @@ impl Section { ) -> Result { let replica = Arc::new(replica); let (sender, events) = mpsc::channel(); - let (saved_sender, saved) = mpsc::channel(); - let notify = Arc::new(Mutex::new(notify)); - let signal = move || { + let (saves, saved) = mpsc::channel(); + let notify = Mutex::new(notify); + let notify: Notify = Arc::new(move || { if let Ok(notify) = notify.lock() { notify(); } - }; - let (saves, requests) = mpsc::channel(); - let unsaved = Arc::new(AtomicUsize::new(0)); - let saver = { - let (replica, signal) = (Arc::clone(&replica), signal.clone()); - let unsaved = Arc::clone(&unsaved); - thread::Builder::new() - .name("onestore-save".into()) - .spawn(move || { - save_pages(&replica, &requests, |outcome, count| { - unsaved.fetch_sub(count, Ordering::Release); - if saved_sender.send(outcome).is_ok() { - signal(); + }); + let worker = { + let (sender, notify) = (sender.clone(), Arc::clone(¬ify)); + replica.start_sync(Duration::from_secs(2), connect, move |result| { + let mut events = Vec::new(); + match result { + Ok(synced) => { + if !synced.changed.is_empty() { + events.push(Event::Changed(synced.changed.clone())); + events.push(Event::Refreshed); } - }); - })? - }; - let worker = replica.start_sync(Duration::from_secs(2), connect, move |result| { - let event = match result { - Ok(None) => Event::Refreshed, - Ok(Some((id, status))) => Event::Attempt { - id: *id, - status: status.clone(), - }, - Err(Error::RemoteIo(error)) => Event::Unreachable(match error.raw_os_error() { - Some(code) => io::Error::from_raw_os_error(code), - None => io::Error::new(error.kind(), error.to_string()), - }), - Err(Error::Remote(error)) => { - Event::Unreachable(io::Error::new(error.error.kind(), error.to_string())) + if let Some((id, status)) = &synced.edit { + events.push(Event::Attempt { + id: *id, + status: status.clone(), + }); + } + } + Err(Error::RemoteIo(error)) => { + events.push(Event::Unreachable(match error.raw_os_error() { + Some(code) => io::Error::from_raw_os_error(code), + None => io::Error::new(error.kind(), error.to_string()), + })) + } + Err(Error::Remote(error)) => events.push(Event::Unreachable(io::Error::new( + error.error.kind(), + error.to_string(), + ))), + Err(error) => events.push(Event::Failed(error.to_string())), } - Err(error) => Event::Failed(error.to_string()), - }; - if sender.send(event).is_ok() { - signal(); - } - })?; + if !events.is_empty() && events.into_iter().all(|event| sender.send(event).is_ok()) + { + notify(); + } + })? + }; Ok(Self { file, replica, worker: Some(worker), - events, - saves: Some(saves), - saved, - unsaved, - saver: Some(saver), + events: Mutex::new(events), + sender, + notify, + saved: Mutex::new(saved), + saves, + unsaved: Arc::new(AtomicUsize::new(0)), }) } @@ -910,111 +913,74 @@ impl Section { /// The section file identity, which internal links name as `section-id` /// (`onestore::page::link::internal_link`). pub fn identity(&self) -> Result<[u8; 16]> { - Ok(Store::parse(&self.replica.snapshot()?)?.header.file_id) + self.replica.identity() } - /// Page spaces, titles and outline levels (1 at the top) in section order, from the - /// local working image. + /// Applies an edit without waiting: it becomes durable on the section thread, which + /// reports a refusal as `Event::Rejected`. Edits apply in the order they arrive. + pub fn apply(&self, author: &str, edit: Edit) -> Result<()> { + let (sender, notify) = (self.sender.clone(), Arc::clone(&self.notify)); + let root = self.replica.root; + let spaces = crate::queue::spaces(&edit, root); + self.replica.submit( + author, + edit, + Box::new(move |result| { + let event = match result { + Ok(_) => return, + Err(Error::Rejected(error)) => Event::Rejected { + spaces, + error: error.to_string(), + }, + Err(error) => Event::Failed(error.to_string()), + }; + if sender.send(event).is_ok() { + notify(); + } + }), + ) + } + + /// Page spaces, titles and outline levels (1 at the top) in section order, as the + /// local edits leave them. pub fn pages(&self) -> Result> { - let snapshot = self.replica.snapshot()?; - let store = Store::parse(&snapshot)?; - let index = RevisionIndex::parse(&store)?; - let document = Document::parse(&index)?; - let pages = document.pages()?; - let mut spaces = std::collections::BTreeSet::new(); - pages - .into_iter() - .map(|(space, page)| { - if !spaces.insert(space) { - return Err(onestore::Error { - offset: 0, - message: "Choose an object space containing one active page", - } - .into()); - } - let (title, level) = Page::heading(document.active(space)?, page); - Ok((space, title, level)) - }) - .collect() + self.replica.pages() } + /// The page to show or edit; O(page), for opening and reloading. pub fn page(&self, space: ExGuid) -> Result { self.replica.page(space) } - /// Copies a page (usually from another section) to the end of this section as a - /// creation and a save queued like the user's own edits, under fresh identities. - /// Returns the new page's space. Content outside the model refuses to copy. + /// Copies a page (usually from another section) to the end of this section under + /// fresh identities, queued like the user's own edits. Returns the new page's space. + /// Content outside the model refuses to copy. pub fn import_page(&self, page: &Page, author: &str) -> Result { - let copy = page.copy()?; let creation = PageCreation::new(None, Some(&page.title), author)?; - self.replica - .create_page(&self.replica.snapshot()?, &creation)?; let space = creation.space(); - loop { - // The worker may publish the creation, and so replace the working image, - // between reading it and saving against it. - let source = self.replica.snapshot()?; - let mut after = Page::from_space( - &Document::parse(&RevisionIndex::parse(&Store::parse(&source)?)?)?, - space, - )?; - after - .objects - .retain(|object| matches!(object, onestore::page::PageObject::Title(_))); - after.objects.extend( - copy.objects - .iter() - .filter(|object| !matches!(object, onestore::page::PageObject::Title(_))) - .cloned(), - ); - after.definitions = copy.definitions.clone(); - match self.replica.save(&source, space, &after, author) { - Err(Error::Io(error)) if error.kind() == io::ErrorKind::ResourceBusy => continue, - result => result?, - }; - break; - } - self.wake(); + self.replica.apply( + author, + Edit { + at: crate::now(), + ops: vec![Op::Section(SectionOp::Import { + creation, + page: page.copy()?, + })], + }, + )?; Ok(space) } /// Removes pages permanently, queued like the user's own edits (a move across /// sections is `import_page` there, then this here). - pub fn delete_pages(&self, pages: &[ExGuid]) -> Result> { - let id = self - .replica - .delete_pages(&self.replica.snapshot()?, pages)?; - self.wake(); - Ok(id) - } - - /// Saves an edited page. `before` is the model the edit started from; a stored page - /// that differs from it means the section changed underneath the editor. - pub fn save(&self, space: ExGuid, before: &Page, after: &Page, author: &str) -> Result { - Ok(save(&self.replica, space, before, None, after, author)?.0) - } - - /// `save` on the section's save thread, returning at once; `saved` reports the outcome. - /// Saves queue in order, and one whose `before` is the previous save's `after` continues - /// it: pass each save's `after` as the next one's `before`. - pub fn queue_save(&self, space: ExGuid, before: Page, after: Page, author: &str) -> Result<()> { - let save = PageSave { - space, - before, - after, - author: author.to_owned(), - }; - self.unsaved.fetch_add(1, Ordering::Relaxed); - self.saves - .as_ref() - .and_then(|saves| saves.send(save).ok()) - .ok_or_else(|| io::Error::other("The save thread stopped").into()) - } - - /// Whether a queued save has yet to report, so the stored page may trail the editor's. - pub fn saving(&self) -> bool { - self.unsaved.load(Ordering::Acquire) > 0 + pub fn delete_pages(&self, pages: &[ExGuid]) -> Result { + self.replica.apply( + "", + Edit { + at: crate::now(), + ops: vec![Op::Section(SectionOp::Delete(pages.to_vec()))], + }, + ) } pub fn status(&self, id: u64) -> Result> { @@ -1025,78 +991,48 @@ impl Section { self.replica.pending() } - /// Every queued edit with its state: pending, awaiting confirmation of a retained - /// attempt, or a conflict awaiting review. - pub fn queue(&self) -> Result> { - self.replica - .queued()? - .into_iter() - .map(|(id, space)| { - Ok(QueuedEdit { - id, - space, - status: self.replica.status(id)?.unwrap_or(EditStatus::Pending), - }) - }) - .collect() - } - - /// The queued edits whose publication conflicted with a remote change. - pub fn conflicts(&self) -> Result> { - Ok(self - .queue()? - .into_iter() - .filter_map(|edit| match edit.status { - EditStatus::Conflict(kind) => Some((edit, kind)), - _ => None, - }) - .collect()) + /// The unpublished batch the remote no longer accepts, if any. Show `remote_page` + /// against `page` for its space, then `resolve`. + pub fn conflict(&self) -> Result> { + self.replica.conflict() } /// The page as last observed in the section file, for reviewing a conflict. pub fn remote_page(&self, space: ExGuid) -> Result { let snapshot = self.replica.remote_snapshot()?; - let store = Store::parse(&snapshot)?; - let index = RevisionIndex::parse(&store)?; - Ok(Page::from_space(&Document::parse(&index)?, space)?) + let arena = onestore::Arena::default(); + Ok(onestore::Section::open(&arena, snapshot)?.page(space)?) } - /// Resolves the oldest conflict with a page reviewed against `remote_page`; the - /// reviewed model publishes as a whole, keeping the edit's id. - pub fn review(&self, id: u64, after: &Page) -> Result<()> { - let local = self.replica.snapshot()?; - let remote = self.replica.remote_snapshot()?; - self.replica.review_page(id, &local, &remote, after)?; - self.wake(); - Ok(()) + /// Ends a conflict: `Mine` publishes the local page over the remote change, `Theirs` + /// drops the local edits to the conflicted page (and its later ones). + pub fn resolve(&self, id: u64, resolution: Resolution) -> Result<()> { + self.replica.resolve(id, resolution) } - /// Retires an uncertain attempt after review, exporting the branch to `archive` - /// first: `Some(page)` continues from the reviewed page against `remote_page`, - /// `None` abandons the local branch. Neither claims the attempt was acknowledged. - pub fn release(&self, id: u64, archive: impl AsRef, after: Option<&Page>) -> Result<()> { - let local = self.replica.snapshot()?; - let remote = self.replica.remote_snapshot()?; - self.replica - .release_attempt(id, &local, &remote, archive.as_ref(), after)?; - self.wake(); - Ok(()) + /// Retires an uncertain attempt after review, exporting the queue to `archive` first: + /// `Mine` publishes the local edits again, `Theirs` abandons them. Neither claims the + /// attempt was acknowledged. + pub fn release( + &self, + id: u64, + archive: impl AsRef, + resolution: Resolution, + ) -> Result<()> { + self.replica.release(id, archive.as_ref(), resolution) } - /// Captures both images, the queue and its states in a read-only archive. + /// Captures the queue, its images and its states in a read-only archive. pub fn export_recovery(&self, path: impl AsRef) -> Result<()> { self.replica.export_recovery(path) } /// Events since the last poll, oldest first. pub fn events(&self) -> Vec { - self.events.try_iter().collect() - } - - /// Outcomes of queued saves since the last poll, oldest first; `notify` runs for each. - /// Consecutive saves continuing one another finish as one, reported once. - pub fn saved(&self) -> Vec<(ExGuid, std::result::Result)> { - self.saved.try_iter().collect() + self.events + .lock() + .map(|events| events.try_iter().collect()) + .unwrap_or_default() } /// Requests a synchronization attempt now. @@ -1115,107 +1051,118 @@ impl Section { /// Dropping instead requests cancellation without waiting; the worker retains /// cache ownership until that operation finishes. Remote calls must be bounded. pub fn close(mut self) -> Result<()> { - drop(self.saves.take()); - if let Some(saver) = self.saver.take() { - saver - .join() - .map_err(|_| io::Error::other("The save thread panicked"))?; - } match self.worker.take() { Some(worker) => worker.stop(), None => Ok(()), } } -} -/// Saves `after` unless the stored page no longer matches `before`, returning the working -/// image the save leaves. `known` is the image a previous save of `before` left: while the -/// working image equals it the page is current unread, and otherwise the page `known` -/// stores stands in for `before`. -fn save( - replica: &Replica, - space: ExGuid, - before: &Page, - known: Option<&[u8]>, - after: &Page, - author: &str, -) -> Result<(Save, Vec)> { - let stored = |image: &[u8]| -> Result { - let store = Store::parse(image)?; - Ok(Page::from_space( - &Document::parse(&RevisionIndex::parse(&store)?)?, + // Old whole-page session API, kept until the application emits ops. + + /// Saves an edited page. `before` is the model the edit started from; a stored page + /// that differs from it means the section changed underneath the editor. + pub fn save(&self, space: ExGuid, before: &Page, after: &Page, author: &str) -> Result { + let outcome = self.replica.ask(|reply| crate::working::Request::Save { space, - )?) - }; - loop { - let snapshot = replica.snapshot()?; - if known != Some(snapshot.as_slice()) { - let current = stored(&snapshot)?; - let expected = match known { - Some(image) => current == stored(image)?, - None => current == *before, - }; - if !expected { - return Ok((Save::Stale, snapshot)); - } - } - match replica.save_image(&snapshot, space, after, author) { - Ok((Some(id), image)) => return Ok((Save::Queued(id), image)), - Ok((None, image)) => return Ok((Save::Unchanged, image)), - Err(Error::Io(error)) if error.kind() == io::ErrorKind::ResourceBusy => {} - Err(error) => return Err(error), - } + before: before.clone(), + after: after.clone(), + author: author.to_owned(), + reply, + })?; + Ok(outcome.unwrap_or(Save::Unchanged)) } -} -/// Runs queued saves until the section drops its sender, folding each run of saves that -/// continue one another into one. -fn save_pages( - replica: &Replica, - requests: &Receiver, - report: impl Fn((ExGuid, std::result::Result), usize), -) { - let mut queue = VecDeque::new(); - // The last save's page and the working image it left. - let mut last: Option<(ExGuid, Page, Vec)> = None; - loop { - if queue.is_empty() { - match requests.recv() { - Ok(request) => queue.push_back(request), - Err(_) => return, - } - } - queue.extend(requests.try_iter()); - let mut request = queue.pop_front().unwrap(); - let mut count = 1; - while let Some(next) = queue.front() - && (next.space, &next.before, &next.author) - == (request.space, &request.after, &request.author) - { - request.after = queue.pop_front().unwrap().after; - count += 1; - } - let known = last - .take() - .filter(|(space, after, _)| *space == request.space && *after == request.before); - let result = save( - replica, - request.space, - &request.before, - known.as_ref().map(|(_, _, image)| image.as_slice()), - &request.after, - &request.author, + /// `save` on the section thread, returning at once; `saved` reports the outcome. + /// Saves queue in order, and one whose `before` is the previous save's `after` + /// continues it: pass each save's `after` as the next one's `before`. + pub fn queue_save(&self, space: ExGuid, before: Page, after: Page, author: &str) -> Result<()> { + let (saves, notify, unsaved) = ( + self.saves.clone(), + Arc::clone(&self.notify), + Arc::clone(&self.unsaved), ); - let save = match result { - Ok((save, image)) => { - if save != Save::Stale { - last = Some((request.space, request.after, image)); + self.unsaved.fetch_add(1, Ordering::Relaxed); + self.replica.send(crate::working::Request::Save { + space, + before, + after, + author: author.to_owned(), + reply: Box::new(move |outcome| { + unsaved.fetch_sub(1, Ordering::Release); + let outcome = match outcome { + Ok(None) => return, + Ok(Some(save)) => Ok(save), + Err(error) => Err(error.to_string()), + }; + if saves.send((space, outcome)).is_ok() { + notify(); } - Ok(save) - } - Err(error) => Err(error.to_string()), - }; - report((request.space, save), count); + }), + }) + } + + /// Whether a queued save has yet to report, so the stored page may trail the editor's. + pub fn saving(&self) -> bool { + self.unsaved.load(Ordering::Acquire) > 0 + } + + /// Outcomes of queued saves since the last poll, oldest first; `notify` runs for each. + /// Consecutive saves continuing one another finish as one, reported once. + pub fn saved(&self) -> Vec<(ExGuid, std::result::Result)> { + self.saved + .lock() + .map(|saved| saved.try_iter().collect()) + .unwrap_or_default() + } + + /// Every queued edit with its state: pending, awaiting confirmation of a retained + /// attempt, or a conflict awaiting review. `space` is the first page it edits. + pub fn queue(&self) -> Result> { + self.replica + .pending()? + .into_iter() + .map(|edit| { + Ok(QueuedEdit { + id: edit.id, + space: edit + .edit + .ops + .iter() + .find_map(|op| match op { + Op::Page { space, .. } => Some(*space), + Op::Section(_) => None, + }) + .unwrap_or(self.replica.root), + status: self.replica.status(edit.id)?.unwrap_or(EditStatus::Pending), + }) + }) + .collect() + } + + /// The conflict as a queue entry, for the review dialog. + pub fn conflicts(&self) -> Result> { + Ok(self + .replica + .conflict()? + .map(|conflict| { + ( + QueuedEdit { + id: conflict.id, + space: conflict.space, + status: EditStatus::Conflict(conflict.kind), + }, + conflict.kind, + ) + }) + .into_iter() + .collect()) + } + + /// Resolves the conflict holding `id` with a page reviewed against `remote_page`: the + /// remote page is rewritten to `after`. + pub fn review(&self, id: u64, after: &Page) -> Result<()> { + self.replica + .resolve_with(id, Resolution::Mine, Some(after.clone())) } } diff --git a/crates/notebook/src/sync.rs b/crates/notebook/src/sync.rs index 40c35ad7d757852d1cf416d91be163fd2a96cbe9..f972d937376b28dcce55dedbbcac584e1c792d5c 100644 --- a/crates/notebook/src/sync.rs +++ b/crates/notebook/src/sync.rs @@ -1,9 +1,10 @@ use super::*; -use onestore::{CommitError, CommitState, Stamp, Transaction}; +use crate::working::{Rebased, Request, Resolve, Sealed}; +use onestore::{CommitError, CommitState, RevisionIndex, Stamp, Store, Transaction}; use rusqlite::OptionalExtension; use std::{ collections::BTreeMap, - sync::{MutexGuard, TryLockError, atomic::Ordering}, + sync::{MutexGuard, TryLockError}, }; /// A single remote file with fresh reads and native-compatible guarded publication. @@ -25,14 +26,32 @@ pub trait Remote { pub enum ConflictKind { /// The edited page or a selected page no longer exists remotely. TargetUnavailable = 0, - /// The remote image no longer accepts the intent's publication. + /// The remote section no longer accepts the edit. UnsupportedEdit = 1, /// Page order or indentation changed remotely in a way the batch cannot absorb. StructureChanged = 2, - /// The remote page changed where the local model changed too; review is required. + /// The remote page changed where the local edits changed it too; review is required. ContentChanged = 3, } +impl ConflictKind { + fn from_stored(kind: i64) -> Result { + Ok(match kind { + 0 => Self::TargetUnavailable, + 1 => Self::UnsupportedEdit, + 2 => Self::StructureChanged, + 3 => Self::ContentChanged, + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Unknown cached conflict kind", + ) + .into()); + } + }) + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub enum EditStatus { Pending, @@ -46,499 +65,534 @@ pub enum EditStatus { revision: ExGuid, }, /// Retired unpublished by a reviewed release; the archive at this path holds the - /// intent, its attempt evidence and both images. + /// edit, its attempt evidence and both images. Archived { archive: String, }, } +/// What one synchronization step did: the state its batch reached, named by the batch's +/// newest edit, and the pages a remote change replaced. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Synced { + pub edit: Option<(u64, EditStatus)>, + pub changed: Vec, +} + +/// The queue as a synchronization step starts it. +struct State { + base: Stamp, + /// The observed remote image's stamp while it is not the base. + remote: Option, + /// The first batch waiting on a remote answer: sealed, attempted, or in conflict. + blocked: Option, + queued: bool, +} + +struct Blocked { + batch: i64, + attempted: bool, + conflict: Option, + revisions: Option>, +} + +fn state(connection: &Connection) -> Result { + let base = base::stamp(connection, base::Image::Base)? + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "The cache has no base image"))?; + let blocked = connection + .query_row( + "SELECT id, attempted, conflict, revisions FROM batches + WHERE attempted=1 OR conflict IS NOT NULL ORDER BY id LIMIT 1", + [], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, bool>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + )) + }, + ) + .optional()? + .map(|(batch, attempted, conflict, revisions)| { + Ok::<_, Error>(Blocked { + batch, + attempted, + conflict: conflict.map(ConflictKind::from_stored).transpose()?, + revisions: revisions + .map(|revisions| decode_revisions(&revisions)) + .transpose()?, + }) + }) + .transpose()?; + Ok(State { + base, + remote: base::stamp(connection, base::Image::Remote)?, + blocked, + queued: connection + .query_row("SELECT EXISTS(SELECT 1 FROM batches)", [], |row| row.get(0))?, + }) +} + +fn decode_revisions(encoded: &str) -> Result> { + let revisions: BTreeMap = serde_json::from_str(encoded) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + if revisions + .iter() + .any(|(space, revision)| space.guid == [0; 16] || revision.guid == [0; 16]) + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Cached publication evidence is incomplete", + ) + .into()); + } + Ok(revisions) +} + +/// The revision a batch's seal stored for the first page an edit changes. +fn revision_of(edit: &onestore::op::Edit, revisions: &BTreeMap) -> Result { + edit.ops + .iter() + .find_map(|op| match op { + onestore::op::Op::Page { space, .. } => revisions.get(space), + onestore::op::Op::Section(_) => None, + }) + .or_else(|| revisions.values().next()) + .copied() + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "Cached publication evidence is incomplete", + ) + .into() + }) +} + +/// The newest edit of a batch. +fn newest(connection: &Connection, batch: i64) -> Result { + unsigned( + connection.query_row("SELECT max(id) FROM edits WHERE batch=?1", [batch], |row| { + row.get(0) + })?, + ) +} + impl Replica { /// Returns a durable receipt or the persisted state of a locally acknowledged edit. pub fn status(&self, id: u64) -> Result> { - let connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - status(&connection, id) + status(&*self.lock()?, id) } - /// The last observed remote image, retained alongside the complete local working image. - /// Observation alone does not acknowledge any pending edit's remote durability. + /// The last observed remote image. Observation alone does not acknowledge any pending + /// edit's remote durability. pub fn remote_snapshot(&self) -> Result> { - let connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - images::base(&connection) + let connection = self.lock()?; + let image = match base::read(&connection, base::Image::Remote)? { + Some(image) => Some(image), + None => base::read(&connection, base::Image::Base)?, + }; + Ok(image.ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "The cache has no base image") + })?) } - /// Reconciles one pending edit, or refreshes the working image when the queue is empty. - /// Network I/O holds synchronization ownership without holding the cache mutex. - /// Uncertain edits are never replayed; retired formatting may confirm its complete observed effect. - pub fn sync_once(&self, remote: &mut impl Remote) -> Result> { + /// The unpublished batch the remote no longer accepts, if any. + pub fn conflict(&self) -> Result> { + let connection = self.lock()?; + let row: Option<(i64, i64, String)> = connection + .query_row( + "SELECT id, conflict, space FROM batches WHERE conflict IS NOT NULL ORDER BY id LIMIT 1", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .optional()?; + row.map(|(batch, kind, space)| { + Ok(Conflict { + id: newest(&connection, batch)?, + space: space.parse()?, + kind: ConflictKind::from_stored(kind)?, + }) + }) + .transpose() + } + + /// Publishes the oldest unpublished batch, rebasing the queue first when the remote + /// changed. Reads the remote image only when its stamp moved; network I/O holds + /// synchronization ownership without holding the cache mutex. Uncertain attempts are + /// never replayed. + pub fn sync_once(&self, remote: &mut impl Remote) -> Result { let _owner = self.sync_owner()?; - struct Step<'a>(&'a Replica); - impl Drop for Step<'_> { - fn drop(&mut self) { - self.0.in_flight.store(0, Ordering::Release); - } + let state = state(&*self.lock()?)?; + let observed = remote.stamp().map_err(Error::RemoteIo)?; + if let (Some(observed), Some(blocked)) = (&observed, &state.blocked) + && Some(observed) == state.remote.as_ref().or(Some(&state.base)) + { + // Nothing new to decide: the remote is as it was when the batch blocked. + let id = newest(&*self.lock()?, blocked.batch)?; + return Ok(Synced { + edit: Some(( + id, + status(&*self.lock()?, id)?.unwrap_or(EditStatus::Pending), + )), + changed: Vec::new(), + }); } - let _step = Step(self); - let base = self.remote_snapshot()?; - let changed = match remote.stamp().map_err(Error::RemoteIo)? { - Some(stamp) if stamp == Stamp::of(&base)? => None, - _ => Some(remote.read().map_err(Error::RemoteIo)?).filter(|read| *read != base), + let image = match observed { + Some(observed) if observed == state.base => None, + _ => { + let image = remote.read().map_err(Error::RemoteIo)?; + // A read image is compared whole: a stamp stands for it only when read alone. + let base = base::read(&*self.lock()?, base::Image::Base)?; + (base.as_deref() != Some(&image[..])).then_some(image) + } }; - // An unchanged remote is the base image, validated when it was stored. - let identity = changed.as_deref().map(validate).transpose()?; - let snapshot = changed.unwrap_or(base); - let (intent, attempted) = { - let mut connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - let transaction = - connection.transaction_with_behavior(TransactionBehavior::Immediate)?; - if identity.is_none() && images::settled(&transaction)? { - return Ok(None); - } - let (base, working) = images::both(&transaction)?; - if let Some(identity) = identity - && RevisionIndex::parse(&Store::parse(&base)?)?.root != identity + let mut changed = Vec::new(); + if let Some(image) = image { + if let Some(Blocked { + batch, + attempted: true, + revisions, + .. + }) = &state.blocked { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Remote snapshot belongs to another document", - ) - .into()); - } - let intent = { - let mut query = transaction - .prepare("SELECT id, space, operation FROM edits ORDER BY id LIMIT 1")?; - let mut rows = query.query([])?; - rows.next()?.map(pending_edit).transpose()? - }; - let Some(intent) = intent else { - images::set_base(&transaction, &base, &snapshot, &snapshot)?; - transaction.commit()?; - return Ok(None); - }; - self.in_flight.store( - i64::try_from(intent.id).map_err(io::Error::other)?, - Ordering::Release, - ); - let attempted = transaction - .query_row( - "SELECT revisions FROM attempt WHERE edit_id=?1", - [i64::try_from(intent.id).map_err(io::Error::other)?], - |row| row.get::<_, String>(0), - ) - .optional()?; - images::set_base(&transaction, &base, &snapshot, &working)?; - transaction.commit()?; - (intent, attempted) - }; - if let Some(encoded) = attempted { - let revisions = attempted_revisions(&encoded, intent.space)?; - let mut revision = revisions[&intent.space]; - let store = Store::parse(&snapshot)?; - let index = RevisionIndex::parse(&store)?; - if !revisions.iter().all(|(space, revision)| { - index - .spaces - .get(space) - .is_some_and(|space| space.revisions.contains_key(revision)) - }) { - // A retired attempt confirms only when its complete effect is observed. - let satisfied = match &intent.operation { - Operation::Page(edit) if revisions.len() == 1 => { - page_of(&snapshot, intent.space)?.is_some_and(|page| page == edit.after) - } - _ => false, - }; - if !satisfied { - return Ok(Some(( - intent.id, - EditStatus::AwaitingConfirmation { revision }, - ))); + let id = newest(&*self.lock()?, *batch)?; + let revisions = revisions.clone().unwrap_or_default(); + let store = Store::parse(&image)?; + if !store.checksum_mismatches.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Notebook transaction checksum damage", + ) + .into()); } - revision = index.active(intent.space)?; - } - if let Err(error) = remote.confirm(&snapshot) { - if error.state == CommitState::Committed { - self.acknowledge(intent.id, revision, &snapshot)?; - } - return Err(error.into()); - } - self.acknowledge(intent.id, revision, &snapshot)?; - return Ok(Some((intent.id, EditStatus::Published { revision }))); - } - let candidate = match &intent.operation { - Operation::Page(edit) => page_of(&snapshot, intent.space)? - .ok_or(ConflictKind::TargetUnavailable) - .and_then(|current| { - let target = if current == edit.before { - edit.after.clone() - } else { - crate::merge::merge(&edit.before, &edit.after, ¤t) - .ok_or(ConflictKind::ContentChanged)? - }; - PreparedEdit::page(&snapshot, intent.space, &target, &edit.author) - .map_err(|_| ConflictKind::UnsupportedEdit) - }), - Operation::CreatePage(page) => PreparedEdit::create_page(&snapshot, page) - .map_err(|_| ConflictKind::StructureChanged), - Operation::Pages(edit) => edit.prepare(&snapshot, intent.space)?, - Operation::DeletePages(pages) => { - let store = Store::parse(&snapshot)?; let index = RevisionIndex::parse(&store)?; - if pages.iter().any(|page| { + index.validate_current()?; + if index.root != self.root { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Remote snapshot belongs to another document", + ) + .into()); + } + let observed = revisions.iter().all(|(space, revision)| { index .spaces - .get(page) - .is_none_or(|space| space.labels.is_empty()) - }) || Document::parse(&index)? - .pages()? - .iter() - .filter(|(sid, _)| pages.contains(sid)) - .count() - != pages.len() - { - Err(ConflictKind::TargetUnavailable) + .get(space) + .is_some_and(|space| space.revisions.contains_key(revision)) + }); + let sealed = working::sealed(&*self.lock()?)? + .filter(|sealed| sealed.batch == *batch) + .and_then(|sealed| sealed.transaction); + // Without its revisions the attempt still counts once the remote holds + // every page it changed as it changed them; its receipts then name the + // remote's revisions. + let receipts = if observed { + None } else { - PreparedEdit::delete_pages_permanently(&snapshot, pages) - .map_err(|_| ConflictKind::UnsupportedEdit) + let equal = self.holds(&image, sealed.as_ref(), &revisions)?; + if !equal { + base::write(&*self.lock()?, base::Image::Remote, &image)?; + return Ok(Synced { + edit: Some(( + id, + status(&*self.lock()?, id)?.unwrap_or(EditStatus::Pending), + )), + changed, + }); + } + Some( + revisions + .keys() + .filter_map(|space| Some((*space, index.active(*space).ok()?))) + .collect(), + ) + }; + if let Err(error) = remote.confirm(&image) { + if error.state == CommitState::Committed { + self.acknowledge(*batch, sealed.as_ref(), receipts.as_ref())?; + } + return Err(error.into()); } + self.acknowledge(*batch, sealed.as_ref(), receipts.as_ref())?; + let revision = self.receipt(id)?; + changed = self.rebase(Some(image), None)?.unwrap_or_default(); + return Ok(Synced { + edit: Some((id, EditStatus::Published { revision })), + changed, + }); } - }; - let prepared = match candidate { - Ok(prepared) => prepared, - Err(kind) => { - let connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - connection.execute("INSERT INTO conflicts(edit_id, kind) VALUES (?1, ?2) ON CONFLICT(edit_id) DO UPDATE SET kind=excluded.kind", params![i64::try_from(intent.id).map_err(io::Error::other)?, kind as i64])?; - return Ok(Some((intent.id, EditStatus::Conflict(kind)))); + match self.rebase(Some(image), None)? { + Ok(spaces) => changed = spaces, + Err(conflict) => { + return Ok(Synced { + edit: Some((conflict.id, EditStatus::Conflict(conflict.kind))), + changed, + }); + } + } + } else if let Some(blocked) = &state.blocked { + if state.remote.is_some() { + // The remote is back at the base: what blocked the queue is gone. + let connection = self.lock()?; + base::clear(&connection, base::Image::Remote)?; + if blocked.conflict.is_some() { + connection.execute("UPDATE batches SET conflict=NULL, space=NULL", [])?; + } + } + if blocked.attempted { + let id = newest(&*self.lock()?, blocked.batch)?; + return Ok(Synced { + edit: Some(( + id, + status(&*self.lock()?, id)?.unwrap_or(EditStatus::Pending), + )), + changed, + }); } + if let Some(kind) = blocked.conflict + && state.remote.is_none() + { + let id = newest(&*self.lock()?, blocked.batch)?; + return Ok(Synced { + edit: Some((id, EditStatus::Conflict(kind))), + changed, + }); + } + } else if !state.queued { + return Ok(Synced::default()); + } + // The cache lock is released before the section thread, which takes it, seals. + let waiting = working::sealed(&*self.lock()?)?; + let sealed = match waiting { + Some(sealed) => { + if sealed.transaction.is_some() { + self.lock()? + .execute("UPDATE batches SET attempted=1 WHERE id=?1", [sealed.batch])?; + } + Some(sealed) + } + None => self.ask(|reply| Request::Seal { reply })?, + }; + let Some(Sealed { batch, transaction }) = sealed else { + return Ok(Synced { + edit: None, + changed, + }); }; - if prepared.as_bytes() == snapshot { - let store = Store::parse(&snapshot)?; - let index = RevisionIndex::parse(&store)?; - let revision = index.active(intent.space)?; - if let Err(error) = remote.confirm(&snapshot) { + let id = newest(&*self.lock()?, batch)?; + let Some(transaction) = transaction else { + // Edits that changed nothing are published once the remote's image is durable. + let base = base::read(&*self.lock()?, base::Image::Base)?.unwrap_or_default(); + if let Err(error) = remote.confirm(&base) { if error.state == CommitState::Committed { - self.acknowledge(intent.id, revision, &snapshot)?; + self.acknowledge(batch, None, None)?; } return Err(error.into()); } - self.acknowledge(intent.id, revision, &snapshot)?; - return Ok(Some((intent.id, EditStatus::Published { revision }))); - } - // Once acknowledged this image is the base, which polls trust without validating. - validate(prepared.as_bytes())?; - let store = Store::parse(prepared.as_bytes())?; - let index = RevisionIndex::parse(&store)?; - let revision = index.active(intent.space)?; - let before_store = Store::parse(&snapshot)?; - let before = RevisionIndex::parse(&before_store)?; - let mut revisions: BTreeMap<_, _> = index - .spaces - .iter() - .filter_map(|(sid, space)| { - let revision = *space.labels.get(&(ExGuid::default(), 1))?; - (before - .spaces - .get(sid) - .and_then(|s| s.labels.get(&(ExGuid::default(), 1))) - != Some(&revision)) - .then_some((*sid, revision)) - }) - .collect(); - // The receipt's space can be unchanged in a multi-space edit. - revisions.insert(intent.space, revision); - { - let mut connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - let transaction = - connection.transaction_with_behavior(TransactionBehavior::Immediate)?; - transaction.execute( - "DELETE FROM conflicts WHERE edit_id=?1", - [i64::try_from(intent.id).map_err(io::Error::other)?], - )?; - transaction.execute( - "INSERT INTO attempt(id, edit_id, revisions) VALUES (1, ?1, ?2)", - params![ - i64::try_from(intent.id).map_err(io::Error::other)?, - serde_json::to_string(&revisions).map_err(io::Error::other)? - ], - )?; - transaction.commit()?; - } - match remote.publish(&prepared.transaction()) { + self.acknowledge(batch, None, None)?; + let revision = self.receipt(id)?; + return Ok(Synced { + edit: Some((id, EditStatus::Published { revision })), + changed, + }); + }; + match remote.publish(&transaction) { Ok(()) => {} Err(error) if error.state == CommitState::NotCommitted => { - let connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - connection.execute( - "DELETE FROM attempt WHERE edit_id=?1", - [i64::try_from(intent.id).map_err(io::Error::other)?], - )?; + self.lock()? + .execute("UPDATE batches SET attempted=0 WHERE id=?1", [batch])?; return Err(error.into()); } Err(error) if error.state == CommitState::Committed => { - self.acknowledge(intent.id, revision, prepared.as_bytes())?; + self.acknowledge(batch, Some(&transaction), None)?; return Err(error.into()); } Err(error) => return Err(error.into()), } - self.acknowledge(intent.id, revision, prepared.as_bytes())?; - Ok(Some((intent.id, EditStatus::Published { revision }))) + self.acknowledge(batch, Some(&transaction), None)?; + let revision = self.receipt(id)?; + Ok(Synced { + edit: Some((id, EditStatus::Published { revision })), + changed, + }) } - /// Repositions an unattempted page-creation conflict, retaining dependent object identities. - pub fn rebase_page_creation_conflict( + /// Whether `image` holds every page in `revisions` as the base with `sealed` has it. + fn holds( &self, - id: u64, - local: &[u8], - remote: &[u8], - before: Option, - ) -> Result<()> { - self.resolve_conflict(id, local, remote, |intent| { - let Operation::CreatePage(page) = intent.operation else { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Select a page-creation conflict", - ) - .into()); - }; - let page = page.reposition(before)?; - PreparedEdit::create_page(remote, &page)?; - Ok(Operation::CreatePage(page)) - }) + image: &[u8], + sealed: Option<&Transaction>, + revisions: &BTreeMap, + ) -> Result { + let base = base::read(&*self.lock()?, base::Image::Base)?.ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "The cache has no base image") + })?; + let (local, remote) = (onestore::Arena::default(), onestore::Arena::default()); + let mut local = onestore::Section::open(&local, base)?; + if let Some(sealed) = sealed { + local.replay(sealed)?; + } + let remote = onestore::Section::open(&remote, image.to_vec())?; + Ok(revisions.keys().all( + |space| matches!((local.page(*space), remote.page(*space)), (Ok(a), Ok(b)) if a == b), + )) } - /// Reviews a page batch against both cache images, retaining its page and series identities. - pub fn rebase_pages_conflict( + /// Asks the section thread to replay the queue on `image`; `Err` is the conflict found. + fn rebase( &self, - id: u64, - local: &[u8], - remote: &[u8], - edits: &[onestore::PageEdit], - ) -> Result<()> { - self.resolve_conflict(id, local, remote, |intent| { - let Operation::Pages(batch) = intent.operation else { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Select a page movement or indentation conflict", - ) - .into()); - }; - Ok(Operation::Pages(batch.review( - remote, - intent.space, - edits, - )?)) - }) + image: Option>, + resolve: Option, + ) -> Result, Conflict>> { + Ok( + match self.ask(|reply| Request::Rebase { + image, + resolve, + reply, + })? { + Rebased::Applied { changed } => Ok(changed), + Rebased::Conflict { id, space, kind } => Err(Conflict { id, space, kind }), + }, + ) } - /// Replaces a conflicting page save with a model reviewed against the remote page. - /// Both supplied images must match `snapshot` and `remote_snapshot`; the reviewed - /// model must publish against the remote image, and its author is retained. - pub fn review_page(&self, id: u64, local: &[u8], remote: &[u8], after: &Page) -> Result<()> { - self.resolve_conflict(id, local, remote, |intent| { - let Operation::Page(edit) = intent.operation else { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Select a page save conflict", - ) - .into()); - }; - let before = page_of(remote, intent.space)?.ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "The page is no longer in the remote section", - ) - })?; - PreparedEdit::page(remote, intent.space, after, &edit.author)?; - Ok(Operation::Page(PageIntent { - before, - after: after.clone(), - author: edit.author, - })) - }) + fn receipt(&self, id: u64) -> Result { + match status(&*self.lock()?, id)? { + Some(EditStatus::Published { revision }) => Ok(revision), + _ => Err(io::Error::other("A published edit has no receipt").into()), + } } - /// Retires the oldest edit's uncertain attempt after review. The branch is first - /// exported to `archive` (a new file), which is the record: no receipt is written. - /// `Some(after)` continues from the reviewed page as a fresh intent against the - /// current remote page, keeping the edit's id and its dependents; `None` abandons - /// the whole local branch, whose edits become `Archived`, and the working image - /// returns to the remote image. Both need the reviewed images to be current. - pub fn release_attempt( + /// Records a published batch: the base becomes the image it left, each of its edits + /// gets a receipt naming the revisions its seal stored, or `revisions`, and it leaves + /// the queue. + fn acknowledge( &self, - id: u64, - local: &[u8], - remote: &[u8], - archive: &Path, - after: Option<&Page>, + batch: i64, + transaction: Option<&Transaction>, + revisions: Option<&BTreeMap>, ) -> Result<()> { + let mut connection = self.lock()?; + let database = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + if let Some(transaction) = transaction { + base::publish(&database, transaction)?; + } + let revisions = match revisions { + Some(revisions) => revisions.clone(), + None => decode_revisions(&database.query_row( + "SELECT revisions FROM batches WHERE id=?1", + [batch], + |row| row.get::<_, String>(0), + )?)?, + }; + for queued in queue::load(&database, Some(batch))? { + database.execute( + "INSERT INTO receipts(edit_id, revision) VALUES (?1, ?2)", + params![ + signed(queued.id)?, + revision_of(&queued.edit, &revisions)?.to_string() + ], + )?; + } + database.execute("DELETE FROM batches WHERE id=?1", [batch])?; + queue::collect(&database)?; + database.commit()?; + Ok(()) + } + + /// Resolves the conflict holding edit `id`: the batch and everything queued after it + /// drop their ops on the conflicted page; `Mine` rewrites the remote page to the local + /// one first. A conflict on another page may follow. + pub fn resolve(&self, id: u64, resolution: Resolution) -> Result<()> { + self.resolve_with(id, resolution, None) + } + + pub(crate) fn resolve_with(&self, id: u64, keep: Resolution, page: Option) -> Result<()> { let owner = self.sync_owner()?; - let intent = self - .pending()? - .into_iter() - .next() - .filter(|intent| intent.id == id) - .ok_or_else(|| { - io::Error::new( + let (first, space) = { + let connection = self.lock()?; + let row: Option<(i64, String)> = connection + .query_row( + "SELECT (SELECT min(id) FROM edits WHERE batch=batches.id), space FROM batches + WHERE conflict IS NOT NULL AND id=(SELECT batch FROM edits WHERE id=?1)", + [signed(id)?], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional()?; + let Some((first, space)) = row else { + return Err(io::Error::new( io::ErrorKind::InvalidInput, - "Only the oldest edit can hold an attempt", + "The edit is not in conflict", ) - })?; - let continued = match after { - Some(after) => { - let Operation::Page(edit) = intent.operation else { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "Select a page save to continue from", - ) - .into()); - }; - let before = page_of(remote, intent.space)?.ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "The page is no longer in the remote section", - ) - })?; - PreparedEdit::page(remote, intent.space, after, &edit.author)?; - Some(Operation::Page(PageIntent { - before, - after: after.clone(), - author: edit.author, - })) - } - None => None, + .into()); + }; + (unsigned(first)?, space.parse()?) }; - let mut connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; - let (base, working) = images::both(&transaction)?; - if working != local || base != remote { - return Err(io::Error::new( - io::ErrorKind::ResourceBusy, - "The reviewed cache images changed", - ) - .into()); - } - let id = i64::try_from(id).map_err(io::Error::other)?; - let attempted: bool = transaction.query_row( - "SELECT EXISTS(SELECT 1 FROM attempt WHERE edit_id=?1)", - [id], - |row| row.get(0), + // A conflict on another page is recorded like any other. + let _ = self.rebase( + None, + Some(Resolve { + first, + space, + keep, + page, + }), )?; - if !attempted { + drop(owner); + self.wake_sync(); + Ok(()) + } + + /// Retires the uncertain attempt of the batch holding edit `id` after review. The + /// queue is first exported to `archive` (a new file), which is the record: no receipt + /// is written. `Mine` publishes the batch again against the current remote; `Theirs` + /// abandons every unpublished edit, which become `Archived`, and the local pages return + /// to the remote's. + pub fn release(&self, id: u64, archive: &Path, resolution: Resolution) -> Result<()> { + let owner = self.sync_owner()?; + let batch: Option = self + .lock()? + .query_row( + "SELECT id FROM batches WHERE attempted=1 AND id=(SELECT batch FROM edits WHERE id=?1)", + [signed(id)?], + |row| row.get(0), + ) + .optional()?; + let Some(batch) = batch else { return Err(io::Error::new( io::ErrorKind::InvalidInput, "The edit has no uncertain attempt", ) .into()); - } - drop(transaction); - drop(connection); + }; self.export_recovery(archive)?; - let mut connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; - transaction.execute("DELETE FROM attempt WHERE edit_id=?1", [id])?; - match continued { - Some(operation) => { - transaction.execute( - "UPDATE edits SET operation=?1 WHERE id=?2", - params![ - serde_json::to_string(&operation).map_err(io::Error::other)?, - id - ], - )?; + { + let mut connection = self.lock()?; + let transaction = + connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + match resolution { + Resolution::Mine => { + transaction.execute("UPDATE batches SET attempted=0 WHERE id=?1", [batch])?; + } + Resolution::Theirs => { + transaction.execute( + "INSERT INTO archived(edit_id, archive) SELECT id, ?1 FROM edits", + [archive.to_string_lossy().into_owned()], + )?; + transaction.execute("DELETE FROM batches", [])?; + if let Some(remote) = base::read(&transaction, base::Image::Remote)? { + base::write(&transaction, base::Image::Base, &remote)?; + base::clear(&transaction, base::Image::Remote)?; + } + queue::collect(&transaction)?; + } } - None => { - transaction.execute( - "INSERT INTO archived(edit_id, archive) SELECT id, ?1 FROM edits", - [archive.to_string_lossy().into_owned()], - )?; - transaction.execute("DELETE FROM edits", [])?; - transaction.execute("UPDATE replica SET working=x'' WHERE id=1", [])?; - } - } - transaction.commit()?; - drop(connection); - drop(owner); - self.wake_sync(); - Ok(()) - } - - fn resolve_conflict( - &self, - id: u64, - local: &[u8], - remote: &[u8], - update: impl FnOnce(PendingEdit) -> Result, - ) -> Result<()> { - let owner = self.sync_owner()?; - let intent = self - .pending()? - .into_iter() - .next() - .filter(|intent| intent.id == id) - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidInput, - "Only the oldest conflict can be rebased", - ) - })?; - let operation = update(intent)?; - let mut connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; - let (base, working) = images::both(&transaction)?; - if working != local || base != remote { - return Err(io::Error::new( - io::ErrorKind::ResourceBusy, - "The reviewed cache images changed", - ) - .into()); + transaction.commit()?; } - let id = i64::try_from(id).map_err(io::Error::other)?; - let eligible: bool = transaction.query_row( - "SELECT EXISTS(SELECT 1 FROM conflicts WHERE edit_id=?1) AND NOT EXISTS(SELECT 1 FROM attempt)", - [id], |row| row.get(0), - )?; - if !eligible { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "The edit is not an unattempted conflict", - ) - .into()); + if resolution == Resolution::Theirs { + self.ask(|reply| Request::Reopen { reply })?; } - transaction.execute( - "UPDATE edits SET operation=?1 WHERE id=?2", - params![ - serde_json::to_string(&operation).map_err(io::Error::other)?, - id - ], - )?; - transaction.execute("DELETE FROM conflicts WHERE edit_id=?1", [id])?; - transaction.commit()?; - drop(connection); drop(owner); self.wake_sync(); Ok(()) @@ -556,34 +610,25 @@ impl Replica { })?) } - fn acknowledge(&self, id: u64, revision: ExGuid, snapshot: &[u8]) -> Result<()> { - let id = i64::try_from(id).map_err(io::Error::other)?; - let mut connection = self - .connection - .lock() - .map_err(|_| io::Error::other("Cache owner panicked"))?; - let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; - transaction.execute( - "INSERT INTO receipts(edit_id, revision) VALUES (?1, ?2)", - params![id, revision.to_string()], - )?; - transaction.execute("DELETE FROM edits WHERE id=?1", [id])?; - let (base, working) = images::both(&transaction)?; - let pending: bool = - transaction.query_row("SELECT EXISTS(SELECT 1 FROM edits)", [], |row| row.get(0))?; - images::set_base( - &transaction, - &base, - snapshot, - if pending { &working } else { snapshot }, - )?; - transaction.commit()?; - Ok(()) + /// Whether nothing is queued and the remote still has the base's stamp, or the queue + /// is blocked on a remote that has not changed since; reads neither image and does not + /// lock the cache during remote I/O. + pub(crate) fn settled(&self, remote: &mut impl Remote) -> Result { + let state = state(&*self.lock()?)?; + let expected = match &state.blocked { + Some(_) => state.remote.unwrap_or(state.base), + None if !state.queued => state.base, + None => return Ok(false), + }; + let Some(stamp) = remote.stamp().map_err(Error::RemoteIo)? else { + return Ok(false); + }; + Ok(stamp == expected) } } pub(crate) fn status(connection: &Connection, id: u64) -> Result> { - let id = i64::try_from(id).map_err(io::Error::other)?; + let id = signed(id)?; if let Some(revision) = connection .query_row( "SELECT revision FROM receipts WHERE edit_id=?1", @@ -606,46 +651,26 @@ pub(crate) fn status(connection: &Connection, id: u64) -> Result, Option, String)> = connection.query_row( - "SELECT attempt.revisions, conflicts.kind, edits.space FROM edits LEFT JOIN attempt ON attempt.edit_id=edits.id LEFT JOIN conflicts ON conflicts.edit_id=edits.id WHERE edits.id=?1", [id], |row| Ok((row.get(0)?,row.get(1)?, row.get(2)?))).optional()?; + let record: Option<(bool, Option, Option, String)> = connection + .query_row( + "SELECT batches.attempted, batches.conflict, batches.revisions, edits.edit + FROM edits JOIN batches ON batches.id=edits.batch WHERE edits.id=?1", + [id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .optional()?; Ok(match record { None => None, - Some((Some(revision), _, space)) => Some(EditStatus::AwaitingConfirmation { - revision: { - let space = space.parse()?; - attempted_revisions(&revision, space)?[&space] - }, - }), - Some((None, Some(kind), _)) => Some(EditStatus::Conflict(match kind { - 0 => ConflictKind::TargetUnavailable, - 1 => ConflictKind::UnsupportedEdit, - 2 => ConflictKind::StructureChanged, - 3 => ConflictKind::ContentChanged, - _ => { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Unknown cached conflict kind", - ) - .into()); - } - })), - Some((None, None, _)) => Some(EditStatus::Pending), + Some((true, _, revisions, edit)) => { + let revisions = decode_revisions(revisions.as_deref().unwrap_or("{}"))?; + let edit: onestore::op::Edit = serde_json::from_str(&edit) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + let revision = revision_of(&edit, &revisions)?; + Some(EditStatus::AwaitingConfirmation { revision }) + } + Some((false, Some(kind), ..)) => { + Some(EditStatus::Conflict(ConflictKind::from_stored(kind)?)) + } + Some((false, None, ..)) => Some(EditStatus::Pending), }) } - -fn attempted_revisions(encoded: &str, space: ExGuid) -> Result> { - let revisions: BTreeMap = serde_json::from_str(encoded) - .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; - if !revisions.contains_key(&space) - || revisions - .iter() - .any(|(sid, rid)| sid.guid == [0; 16] || rid.guid == [0; 16]) - { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "Cached publication evidence is incomplete", - ) - .into()); - } - Ok(revisions) -} diff --git a/crates/notebook/src/worker.rs b/crates/notebook/src/worker.rs index 6e3d73d209efb9e219eeabd227aee3cd61da4c44..8f17ad73ac1236a192454bae4d01e0eb87b46945 100644 --- a/crates/notebook/src/worker.rs +++ b/crates/notebook/src/worker.rs @@ -61,7 +61,8 @@ impl Drop for SyncWorker { impl Replica { /// Starts one worker, reconnecting through `connect` after transport failures. /// Local edits wake it; `interval` controls idle polling and transport retries. While - /// nothing is queued, a remote whose `stamp` holds is not read again. + /// nothing is queued, or the queue waits on a remote that has not changed since, a + /// remote whose `stamp` holds is not read again. /// Contended operations returning `NotCommitted` also back off by up to one second. /// `observe` runs on the worker after each attempt, including connection errors. /// Cache/document errors stop the worker; inspect them through `observe` or `stop`. @@ -75,7 +76,7 @@ impl Replica { where R: Remote + 'static, F: FnMut() -> io::Result + Send + 'static, - O: FnMut(&Result>) + Send + 'static, + O: FnMut(&Result) + Send + 'static, { if interval.is_zero() || Instant::now().checked_add(interval).is_none() { return Err(io::Error::new( @@ -103,18 +104,19 @@ impl Replica { let mut remote: Option = None; let jitter = RandomState::new(); let mut contention = 0_u32; - // The first attempt always runs, so `observe` hears once that the remote is - // reachable. - let mut attempted = false; + // The first step, and the first after a failure, always runs, so `observe` + // hears that the remote is reachable and what state the queue is in. + let mut reported = false; while !worker_signal.stopped.load(Ordering::Acquire) { let result = match remote.as_mut() { Some(remote) => { - if attempted && replica.settled(remote).unwrap_or(false) { + if reported && replica.settled(remote).unwrap_or(false) { let _ = receiver.recv_timeout(interval); continue; } - attempted = true; - replica.sync_once(remote) + let result = replica.sync_once(remote); + reported = result.is_ok(); + result } None => match connect() { Ok(connected) => { @@ -126,11 +128,14 @@ impl Replica { }; observe(&result); match result { - Ok(Some((_, EditStatus::Published { .. }))) => { + Ok(Synced { + edit: Some((_, EditStatus::Published { .. })), + .. + }) => { contention = 0; continue; } - Ok(None) => contention = 0, + Ok(Synced { edit: None, .. }) => contention = 0, Ok(_) => {} Err(Error::Remote(onestore::CommitError { state: onestore::CommitState::NotCommitted, diff --git a/crates/notebook/src/working.rs b/crates/notebook/src/working.rs new file mode 100644 index 0000000000000000000000000000000000000000..6a2411809aec020e3182584f7a1697f89079641e --- /dev/null +++ b/crates/notebook/src/working.rs @@ -0,0 +1,776 @@ +//! The section thread: the only owner of the parsed section, which is the cached base +//! image with each sealed batch replayed and the open batch's edits applied. Edits apply +//! here and are written in one SQLite transaction per burst; the sync thread asks it to seal +//! and, when the remote changed, to rebase the queue. + +use crate::{ + ConflictKind, Resolution, Result, base, lock, merge, queue, session::Save, signed, + worker::Signal, +}; +use onestore::{ + Arena, ExGuid, Section, Transaction, + op::{Edit, Op, OpError}, + page::Page, +}; +use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params}; +use std::{ + collections::{BTreeMap, BTreeSet, VecDeque}, + io, + sync::{Arc, Mutex, Weak, mpsc}, + thread, +}; + +pub(crate) type Reply = Box) + Send>; + +pub(crate) enum Request { + Apply { + author: String, + edit: Edit, + reply: Reply, + }, + /// A whole-page save of the old session API: `None` answers a save a later one + /// continuing it absorbed. + Save { + space: ExGuid, + before: Page, + after: Page, + author: String, + reply: Reply>, + }, + Page { + space: ExGuid, + reply: Reply, + }, + Pages { + reply: Reply>, + }, + Image { + reply: Reply>, + }, + /// Seals the open batch, when no sealed batch waits for publication, recording the + /// attempt to publish it. + Seal { + reply: Reply>, + }, + /// Replays the queue on `image` (the stored remote image, or the base, when `None`). + Rebase { + image: Option>, + resolve: Option, + reply: Reply, + }, + /// Rereads the queue after the sync thread replaced it. + Reopen { + reply: Reply<()>, + }, +} + +/// A sealed batch: the transaction publishing it, or none when its edits changed nothing. +pub(crate) struct Sealed { + pub batch: i64, + pub transaction: Option, +} + +/// A conflict's resolution: the batch it holds and every later edit drop their ops on +/// `space`; `Mine` first rewrites the rebased page to the local one (or to `page`). +pub(crate) struct Resolve { + pub first: u64, + pub space: ExGuid, + pub keep: Resolution, + pub page: Option, +} + +pub(crate) enum Rebased { + /// The queue now applies to the image, which is the base; these pages changed. + Applied { changed: Vec }, + /// The batch holding edit `id` no longer applies; nothing changed but the record. + Conflict { + id: u64, + space: ExGuid, + kind: ConflictKind, + }, +} + +impl Request { + fn fail(self, message: &str) { + let error = || io::Error::other(message.to_owned()).into(); + match self { + Self::Apply { reply, .. } => reply(Err(error())), + Self::Save { reply, .. } => reply(Err(error())), + Self::Page { reply, .. } => reply(Err(error())), + Self::Pages { reply } => reply(Err(error())), + Self::Image { reply } => reply(Err(error())), + Self::Seal { reply } => reply(Err(error())), + Self::Rebase { reply, .. } => reply(Err(error())), + Self::Reopen { reply } => reply(Err(error())), + } + } +} + +type Worker = Arc>>; + +/// Starts the section thread once the queue opens. +pub(crate) fn spawn( + connection: Arc>, + worker: Worker, +) -> Result<(mpsc::Sender, thread::JoinHandle<()>, ExGuid)> { + let (sender, receiver) = mpsc::channel(); + let (ready, opened) = mpsc::sync_channel(1); + let thread = thread::Builder::new() + .name("onestore-section".into()) + .spawn(move || run(&connection, &worker, &receiver, ready))?; + match opened.recv() { + Ok(Ok(root)) => Ok((sender, thread, root)), + Ok(Err(error)) => { + let _ = thread.join(); + Err(error) + } + Err(_) => { + let _ = thread.join(); + Err(io::Error::other("The section thread panicked").into()) + } + } +} + +enum Next { + Stop, + Reopen, +} + +fn run( + connection: &Mutex, + worker: &Worker, + requests: &mpsc::Receiver, + ready: mpsc::SyncSender>, +) { + let mut ready = Some(ready); + let mut backlog = VecDeque::new(); + loop { + let arena = Arena::default(); + let working = match Working::open(&arena, connection) { + Ok(working) => working, + Err(error) => { + let message = error.to_string(); + if let Some(ready) = ready.take() { + let _ = ready.send(Err(error)); + return; + } + for request in backlog.drain(..).chain(requests.iter()) { + request.fail(&message); + } + return; + } + }; + if let Some(ready) = ready.take() { + let _ = ready.send(Ok(working.section.root())); + } + match working.serve(connection, worker, requests, &mut backlog) { + Next::Stop => return, + Next::Reopen => {} + } + } +} + +/// An edit applied to the section and not yet written, with who waits for it. +struct Accepted { + author: String, + edit: Edit, + done: Done, +} + +enum Done { + Apply(Reply), + Save(Reply>), +} + +impl Done { + fn reply(self, written: Result) { + match self { + Self::Apply(reply) => reply(written), + Self::Save(reply) => reply(written.map(|id| Some(Save::Queued(id)))), + } + } +} + +struct Working<'a> { + section: Section<'a>, + /// The batch collecting edits; sealed batches before it are replayed. + open: Option, + /// Spaces the open batch's edits change. + touched: BTreeSet, + /// The last whole-page save: its space, the page it was given, and the page as stored. + saved: Option<(ExGuid, Page, Page)>, +} + +impl<'a> Working<'a> { + fn open(arena: &'a Arena, connection: &Mutex) -> Result { + let (section, open, touched) = replay(arena, &*lock(connection)?)?; + Ok(Self { + section, + open, + touched, + saved: None, + }) + } + + fn serve( + mut self, + connection: &Mutex, + worker: &Worker, + requests: &mpsc::Receiver, + backlog: &mut VecDeque, + ) -> Next { + loop { + let first = match backlog.pop_front() { + Some(request) => request, + None => match requests.recv() { + Ok(request) => request, + Err(_) => return Next::Stop, + }, + }; + let mut burst: VecDeque = VecDeque::from([first]); + burst.extend(backlog.drain(..)); + burst.extend(requests.try_iter()); + let mut accepted = Vec::new(); + while let Some(request) = burst.pop_front() { + let reopen = match request { + Request::Apply { + author, + edit, + reply, + } => self + .accept( + author, + edit, + Done::Apply(reply), + &mut accepted, + connection, + worker, + ) + .err() + .unwrap_or(false), + Request::Save { + space, + before, + mut after, + author, + reply, + } => { + // Saves that continue one another are written once. + let mut replies = vec![reply]; + while let Some(Request::Save { + space: next, + before: continued, + author: by, + .. + }) = burst.front() + && (*next, continued, by) == (space, &after, &author) + { + let Some(Request::Save { + after: later, + reply, + .. + }) = burst.pop_front() + else { + unreachable!() + }; + after = later; + replies.push(reply); + } + let last = replies.pop().unwrap(); + for folded in replies { + folded(Ok(None)); + } + match self.save(space, &before, &after) { + Ok(Some(edit)) => match self.accept( + author, + edit, + Done::Save(last), + &mut accepted, + connection, + worker, + ) { + Ok(()) => { + self.saved = self + .section + .page(space) + .ok() + .map(|stored| (space, after, stored)); + false + } + Err(broken) => broken, + }, + Ok(None) => { + self.saved = self + .section + .page(space) + .ok() + .map(|stored| (space, after, stored)); + last(Ok(Some(Save::Unchanged))); + false + } + Err(Stale::Stale) => { + last(Ok(Some(Save::Stale))); + false + } + Err(Stale::Error(error)) => { + last(Err(error)); + false + } + } + } + Request::Page { space, reply } => { + reply(self.section.page(space).map_err(Into::into)); + false + } + Request::Pages { reply } => { + reply(self.section.pages().map_err(Into::into)); + false + } + Request::Image { reply } => { + if self.flush(connection, worker, &mut accepted) { + reply(lock(connection).and_then(|connection| image(&connection))); + false + } else { + reply(Err( + io::Error::other("The queue could not be written").into() + )); + true + } + } + Request::Seal { reply } => { + if !self.flush(connection, worker, &mut accepted) { + reply(Err( + io::Error::other("The queue could not be written").into() + )); + true + } else { + let sealed = self.seal(connection); + let failed = sealed.is_err(); + reply(sealed); + failed + } + } + Request::Rebase { + image, + resolve, + reply, + } => { + if !self.flush(connection, worker, &mut accepted) { + reply(Err( + io::Error::other("The queue could not be written").into() + )); + true + } else { + let rebased = self.rebase(connection, image, resolve); + let reopen = !matches!(rebased, Ok(Rebased::Conflict { .. })); + reply(rebased); + reopen + } + } + Request::Reopen { reply } => { + self.flush(connection, worker, &mut accepted); + reply(Ok(())); + true + } + }; + if reopen { + backlog.extend(burst); + return Next::Reopen; + } + } + if !self.flush(connection, worker, &mut accepted) { + return Next::Reopen; + } + } + } + + /// Applies an edit, keeping it to be written with the burst. A refused edit is answered + /// here; `Err(true)` when it failed part way and the section must be reread. + fn accept( + &mut self, + author: String, + edit: Edit, + done: Done, + accepted: &mut Vec, + connection: &Mutex, + worker: &Worker, + ) -> std::result::Result<(), bool> { + match self.section.apply(&author, &edit) { + Ok(()) => { + self.touched + .extend(queue::spaces(&edit, self.section.root())); + accepted.push(Accepted { author, edit, done }); + Ok(()) + } + Err(error) => { + let broken = matches!(error, OpError::Failed(_)); + if broken { + self.flush(connection, worker, accepted); + } + done.reply(Err(error.into())); + Err(broken) + } + } + } + + /// The edit that takes the stored page from `before` to `after`. A save continuing + /// the previous one finds the page as that one stored it, which may read back + /// normalized. + fn save( + &mut self, + space: ExGuid, + before: &Page, + after: &Page, + ) -> std::result::Result, Stale> { + let current = self + .section + .page(space) + .map_err(|error| Stale::Error(error.into()))?; + let expected = match &self.saved { + Some((saved, given, stored)) if *saved == space && given == before => stored, + _ => before, + }; + if current != *expected { + return Err(Stale::Stale); + } + let ops = onestore::op::lower_page(¤t, after) + .map_err(|error| Stale::Error(OpError::Unsupported(error.message).into()))?; + Ok((!ops.is_empty()).then(|| Edit { + at: crate::now(), + ops: ops.into_iter().map(|op| Op::Page { space, op }).collect(), + })) + } + + /// Writes the accepted edits in one transaction, then answers their senders; false + /// when the write failed and the section holds edits the queue lacks. + fn flush( + &mut self, + connection: &Mutex, + worker: &Worker, + accepted: &mut Vec, + ) -> bool { + if accepted.is_empty() { + return true; + } + let written = (|| -> Result> { + let mut connection = lock(connection)?; + let transaction = + connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let batch = match self.open { + Some(batch) => batch, + None => { + transaction.execute("INSERT INTO batches DEFAULT VALUES", [])?; + transaction.last_insert_rowid() + } + }; + let mut ids = Vec::new(); + for edit in accepted.iter() { + ids.push(queue::insert( + &transaction, + None, + batch, + &edit.author, + &edit.edit, + )?); + } + transaction.commit()?; + self.open = Some(batch); + Ok(ids) + })(); + let ok = written.is_ok(); + match written { + Ok(ids) => { + for (edit, id) in accepted.drain(..).zip(ids) { + edit.done.reply(Ok(id)); + } + crate::wake(worker); + } + Err(error) => { + let message = error.to_string(); + for edit in accepted.drain(..) { + edit.done + .reply(Err(io::Error::other(message.clone()).into())); + } + } + } + ok + } + + /// Seals the open batch unless a sealed batch still waits for publication. + fn seal(&mut self, connection: &Mutex) -> Result> { + let Some(batch) = self.open else { + return Ok(None); + }; + { + let connection = lock(connection)?; + let waiting: bool = connection.query_row( + "SELECT EXISTS(SELECT 1 FROM batches WHERE sealed IS NOT NULL OR conflict IS NOT NULL)", + [], + |row| row.get(0), + )?; + if waiting { + return Ok(None); + } + } + let transaction = self.section.seal()?; + // A batch whose edits change nothing is recorded against the section's root. + let root = self.section.root(); + let revisions: BTreeMap = self + .section + .revisions() + .filter(|(space, _)| { + self.touched.contains(space) || (self.touched.is_empty() && *space == root) + }) + .collect(); + // The sync thread publishes what it seals at once: the attempt is recorded with it. + lock(connection)?.execute( + "UPDATE batches SET sealed=?1, revisions=?2, attempted=?3 WHERE id=?4", + params![ + serde_json::to_string(&transaction).map_err(io::Error::other)?, + serde_json::to_string(&revisions).map_err(io::Error::other)?, + transaction.is_some(), + batch + ], + )?; + self.open = None; + self.touched.clear(); + Ok(Some(Sealed { batch, transaction })) + } + + /// Replays every queued edit on `image`, keeping it as the base when all apply. + fn rebase( + &mut self, + connection: &Mutex, + image: Option>, + resolve: Option, + ) -> Result { + let (base_image, image, edits) = { + let connection = lock(connection)?; + if connection.query_row( + "SELECT EXISTS(SELECT 1 FROM batches WHERE attempted=1)", + [], + |row| row.get::<_, bool>(0), + )? { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "An uncertain attempt is never rebased", + ) + .into()); + } + let base_image = base::read(&connection, base::Image::Base)?.ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "The cache has no base image") + })?; + let image = match image { + Some(image) => image, + None => base::read(&connection, base::Image::Remote)? + .unwrap_or_else(|| base_image.clone()), + }; + (base_image, image, queue::load(&connection, None)?) + }; + // The page the local edits leave, which `Mine` rewrites the remote page to; the + // root space lists pages and holds none. + let target = match &resolve { + Some(Resolve { + keep: Resolution::Mine, + page: None, + space, + .. + }) if *space != self.section.root() => Some(self.section.page(*space)?), + Some(Resolve { page, .. }) => page.clone(), + None => None, + }; + let old_arena = Arena::default(); + let mut old = Section::open(&old_arena, base_image.clone())?; + let before: BTreeMap = old.revisions().collect(); + let mut after: BTreeMap; + // Pages the remote already holds as the local edits leave them: their ops are done. + let mut converged = BTreeSet::new(); + let outcome = loop { + let arena = Arena::default(); + let mut new = Section::open(&arena, image.clone())?; + if old.root() != new.root() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Remote snapshot belongs to another document", + ) + .into()); + } + after = new.revisions().collect(); + let outcome = merge::rebase( + &mut old, + &mut new, + &edits, + &converged, + resolve.as_ref().map(|resolve| merge::Resolving { + first: resolve.first, + space: resolve.space, + mine: resolve.keep == Resolution::Mine, + target: target.as_ref(), + }), + )?; + if let merge::Outcome::Conflict { space, .. } = &outcome + && !converged.contains(space) + && let Ok(local) = self.section.page(*space) + && Section::open(&Arena::default(), image.clone())? + .page(*space) + .is_ok_and(|remote| remote == local) + { + converged.insert(*space); + continue; + } + break outcome; + }; + let mut connection = lock(connection)?; + let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let rebased = match outcome { + merge::Outcome::Conflict { id, space, kind } => { + transaction.execute("UPDATE batches SET conflict=NULL, space=NULL", [])?; + transaction.execute( + "UPDATE batches SET conflict=?1, space=?2 WHERE id=(SELECT batch FROM edits WHERE id=?3)", + params![kind as i64, space.to_string(), signed(id)?], + )?; + if image == base_image { + base::clear(&transaction, base::Image::Remote)?; + } else { + base::write(&transaction, base::Image::Remote, &image)?; + } + // A conflict is reported under its batch's newest edit. + let id = crate::unsigned(transaction.query_row( + "SELECT max(id) FROM edits WHERE batch=(SELECT batch FROM edits WHERE id=?1)", + [signed(id)?], + |row| row.get(0), + )?)?; + Rebased::Conflict { id, space, kind } + } + merge::Outcome::Applied(rewritten) => { + base::write(&transaction, base::Image::Base, &image)?; + base::clear(&transaction, base::Image::Remote)?; + transaction.execute("INSERT INTO batches DEFAULT VALUES", [])?; + let batch = transaction.last_insert_rowid(); + transaction.execute("UPDATE edits SET batch=?1", [batch])?; + transaction.execute("DELETE FROM batches WHERE id<>?1", [batch])?; + for (id, edit) in rewritten { + queue::rewrite(&transaction, id, &edit)?; + } + if transaction + .query_row("SELECT count(*) FROM edits", [], |row| row.get::<_, i64>(0))? + == 0 + { + transaction.execute("DELETE FROM batches", [])?; + } + queue::collect(&transaction)?; + let mut changed: BTreeSet = before + .iter() + .filter(|(space, rid)| after.get(space) != Some(rid)) + .map(|(space, _)| *space) + .chain( + after + .keys() + .filter(|space| !before.contains_key(space)) + .copied(), + ) + .collect(); + changed.extend(resolve.as_ref().map(|resolve| resolve.space)); + changed.extend(converged); + Rebased::Applied { + changed: changed.into_iter().collect(), + } + } + }; + transaction.commit()?; + Ok(rebased) + } +} + +enum Stale { + Stale, + Error(crate::Error), +} + +/// Batches in order with their sealed transactions: `None` while open, `Some(None)` when +/// sealing stored nothing. +fn batches(connection: &Connection) -> Result>)>> { + let mut query = connection.prepare("SELECT id, sealed FROM batches ORDER BY id")?; + let mut rows = query.query([])?; + let mut batches = Vec::new(); + while let Some(row) = rows.next()? { + let sealed: Option = row.get(1)?; + batches.push(( + row.get(0)?, + sealed + .map(|sealed| serde_json::from_str(&sealed)) + .transpose() + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?, + )); + } + Ok(batches) +} + +/// The cached base with each sealed batch replayed and the open batch's edits applied, +/// with the open batch and the spaces its edits change. +fn replay<'a>( + arena: &'a Arena, + connection: &Connection, +) -> Result<(Section<'a>, Option, BTreeSet)> { + let image = base::read(connection, base::Image::Base)? + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "The cache has no base image"))?; + let mut section = Section::open(arena, image)?; + let mut open = None; + let mut touched = BTreeSet::new(); + for (batch, sealed) in batches(connection)? { + if open.is_some() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "A cached batch follows the open batch", + ) + .into()); + } + match sealed { + Some(Some(transaction)) => section.replay(&transaction)?, + Some(None) => {} + None => { + for queued in queue::load(connection, Some(batch))? { + section + .apply(&queued.author, &queued.edit) + .map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("Queued edit {} no longer applies: {error}", queued.id), + ) + })?; + touched.extend(queue::spaces(&queued.edit, section.root())); + } + open = Some(batch); + } + } + } + Ok((section, open, touched)) +} + +/// The image the queue leaves, its unsealed edits sealed as one more revision; that +/// revision's identities are fresh on every call. O(section), for tests and recovery. +pub(crate) fn image(connection: &Connection) -> Result> { + let arena = Arena::default(); + let (mut section, ..) = replay(&arena, connection)?; + section.seal()?; + Ok(section.image()) +} + +/// The sealed batch waiting for publication, if any. +pub(crate) fn sealed(connection: &Connection) -> Result> { + let row: Option<(i64, String)> = connection + .query_row( + "SELECT id, sealed FROM batches WHERE sealed IS NOT NULL ORDER BY id LIMIT 1", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional()?; + row.map(|(batch, sealed)| { + Ok(Sealed { + batch, + transaction: serde_json::from_str(&sealed) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?, + }) + }) + .transpose() +} diff --git a/crates/notebook/tests/assets.rs b/crates/notebook/tests/assets.rs index 1eb7baae18e498c29c115b026c8156b9f922db79..16d1f0826eae82e50aa8ab2a995941efe14ed2ee 100644 --- a/crates/notebook/tests/assets.rs +++ b/crates/notebook/tests/assets.rs @@ -1,4 +1,4 @@ -use notebook::{EditStatus, Error, Operation, Recovery, Replica}; +use notebook::{EditStatus, Error, Recovery, Replica}; use onestore::{ ExGuid, RevisionIndex, Store, document::Document, @@ -13,6 +13,8 @@ use std::{ #[path = "support/model_ops.rs"] mod model_ops; +#[path = "support/server.rs"] +mod server; const FIXTURE: &str = "../../corpus/native-external-assets/notebook"; @@ -54,10 +56,7 @@ fn queued_cache(root: &Path) -> Replica { .unwrap() .unwrap(); let page = onestore::PageCreation::new(None, Some("Queued page"), "Author").unwrap(); - cache - .create_page(&cache.snapshot().unwrap(), &page) - .unwrap() - .unwrap(); + server::section_op(&cache, onestore::op::SectionOp::Create(page)); cache } @@ -118,7 +117,10 @@ fn downloaded_media_survives_reopen_and_recovery_with_the_queue_intact() { (summary.cached_assets, summary.cached_asset_bytes), (2, 1024) ); - assert_eq!(cache.snapshot().unwrap(), working); + assert_eq!( + server::pages(&cache.snapshot().unwrap()), + server::pages(&working) + ); assert_eq!(cache.remote_snapshot().unwrap(), remote); assert_eq!(cache.pending().unwrap(), pending); assert_eq!(cache.status(1).unwrap(), uncertain); @@ -234,10 +236,11 @@ fn download_network_wait_does_not_block_local_edits() { release.send(()).unwrap(); assert_eq!(download.join().unwrap(), bytes); assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); - let Operation::Page(intent) = &cache.pending().unwrap()[2].operation else { - panic!() - }; - assert_eq!(intent.text_change().unwrap().3, "during download "); + assert!(matches!( + &cache.pending().unwrap()[2].edit.ops[..], + [onestore::op::Op::Page { op: onestore::op::PageOp::Text { with, .. }, .. }] + if with == "during download " + )); }); } @@ -284,7 +287,7 @@ fn a_native_refresh_removing_the_reference_rejects_an_inflight_download() { panic!("Unexpected enumeration") } fn read(&mut self, _: &str, _: usize) -> io::Result> { - assert_eq!(self.0.sync_once(&mut Native).unwrap(), None); + assert_eq!(self.0.sync_once(&mut Native).unwrap().edit, None); Ok(payload(1024).1) } } diff --git a/crates/notebook/tests/cache.rs b/crates/notebook/tests/cache.rs index e5be94eb430795a1821518bb0bc23b751c03a3ca..54bdc5d9b4e90da0e5462e9d7694f9ed729b0012 100644 --- a/crates/notebook/tests/cache.rs +++ b/crates/notebook/tests/cache.rs @@ -1,7 +1,8 @@ -use notebook::{Error, Operation, Replica}; +use notebook::{Error, Replica}; use onestore::{ ExGuid, RevisionIndex, Store, document::{Document, Kind}, + op::{Edit, Op, PageOp, SectionOp}, page::{Outline, PageObject}, }; use std::{ @@ -14,6 +15,8 @@ use std::{ #[path = "support/model_ops.rs"] mod model_ops; +#[path = "support/server.rs"] +mod server; fn target(source: &[u8]) -> (ExGuid, ExGuid, String) { let store = Store::parse(source).unwrap(); @@ -36,25 +39,23 @@ fn target(source: &[u8]) -> (ExGuid, ExGuid, String) { .unwrap() } -fn text_of(page: &onestore::page::Page, text: ExGuid) -> String { - model_ops::paragraph_with(page, text) - .unwrap() - .text() - .unwrap() - .text - .text() - .to_owned() -} - -fn intent(replica: &Replica, at: usize) -> notebook::PageIntent { - match &replica.pending().unwrap()[at].operation { - Operation::Page(intent) => intent.clone(), - other => panic!("{other:?}"), +/// An edit typing `with` at the start of a text. +fn typed(space: ExGuid, text: ExGuid, with: &str) -> Edit { + Edit { + at: model_ops::now(), + ops: vec![Op::Page { + space, + op: PageOp::Text { + text, + range: 0..0, + with: with.into(), + }, + }], } } #[test] -fn cache_reopen_preserves_exact_images_and_intents() { +fn cache_reopen_preserves_the_base_and_queued_edits() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("section.sqlite"); let source = onestore::create_section("section.one", "café 🦀", "Fixture").unwrap(); @@ -72,77 +73,102 @@ fn cache_reopen_preserves_exact_images_and_intents() { }) .unwrap() .unwrap(); - let edited = replica.snapshot().unwrap(); - assert_eq!(target(&edited).2, "café 🐈 日本語"); - let intents = replica.pending().unwrap(); - assert_eq!(intents.len(), 1); - assert_eq!((intents[0].id, intents[0].space), (first, sid)); - let first_intent = intent(&replica, 0); + assert_eq!(target(&replica.snapshot().unwrap()).2, "café 🐈 日本語"); + let edits = replica.pending().unwrap(); + assert_eq!(edits.len(), 1); assert_eq!( - first_intent.text_change(), - Some((oid, "café 🦀".into(), 5..7, "🐈 日本語".into())) + (edits[0].id, edits[0].author.as_str()), + (first, model_ops::AUTHOR) ); - assert_eq!(first_intent.author, model_ops::AUTHOR); + assert!(matches!( + &edits[0].edit.ops[..], + [Op::Page { space, op: PageOp::Text { text, range, with } }] + if (*space, *text, range.clone(), with.as_str()) == (sid, oid, 5..7, "🐈 日本語") + )); + let beside = |suffix: &str| { + let mut file = path.as_os_str().to_owned(); + file.push(suffix); + std::path::PathBuf::from(file) + }; + // Commits go to the write-ahead log; its index stays in memory. + assert!(beside("-wal").exists()); + assert!(!beside("-shm").exists()); drop(replica); + assert!( + !beside("-wal").exists(), + "closing checkpoints and removes the log" + ); let replica = Replica::open(&path).unwrap(); - assert_eq!(replica.snapshot().unwrap(), edited); - assert_eq!(replica.pending().unwrap(), intents); - let second = model_ops::save(&replica, oid, |page| { - model_ops::replace_text(page, oid, 0..0, "Recovered ") - }) - .unwrap() - .unwrap(); - assert_eq!(second, first, "an unattempted save is replaced in place"); - let coalesced = intent(&replica, 0); - assert_eq!(coalesced.before, first_intent.before); - assert_eq!(text_of(&coalesced.after, oid), "Recovered café 🐈 日本語"); + assert_eq!(target(&replica.snapshot().unwrap()).2, "café 🐈 日本語"); + assert_eq!(replica.pending().unwrap(), edits); + let second = replica + .apply(model_ops::AUTHOR, typed(sid, oid, "Recovered ")) + .unwrap(); + assert!(second > first); assert_eq!( target(&replica.snapshot().unwrap()).2, "Recovered café 🐈 日本語" ); + assert_eq!(replica.pending().unwrap().len(), 2); } #[test] -fn failed_saves_preserve_both_intent_queue_and_working_image() { +fn refused_edits_and_failed_writes_leave_the_queue_as_it_was() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("section.sqlite"); let source = onestore::create_section("section.one", "café 🦀", "Fixture").unwrap(); let replica = Replica::create(&path, &source).unwrap(); let (sid, oid, _) = target(&source); - let (_, mut page) = model_ops::locate(&source, oid); - let mut empty = Outline { + let empty = Outline { id: onestore::page::text::new_id().unwrap(), title: false, min_width: None, - layout: Default::default(), + layout: onestore::document::Layout { + x: Some(72.0), + y: Some(400.0), + ..Default::default() + }, indents: Vec::new(), paragraphs: Vec::new(), unsupported: Vec::new(), }; - empty.layout.x = Some(72.0); - empty.layout.y = Some(400.0); - page.objects.push(PageObject::Outline(empty)); - assert!(replica.save(&source, sid, &page, "Author").is_err()); - let (_, mut page) = model_ops::locate(&source, oid); - model_ops::replace_text(&mut page, oid, 0..0, "Elsewhere "); - let foreign = ExGuid::default(); - assert!(replica.save(&source, foreign, &page, "Author").is_err()); + let refused = Edit { + at: model_ops::now(), + ops: vec![ + Op::Page { + space: sid, + op: PageOp::Text { + text: oid, + range: 0..0, + with: "Undone ".into(), + }, + }, + Op::Page { + space: sid, + op: PageOp::Add { + object: PageObject::Outline(empty), + before: None, + }, + }, + ], + }; + assert!(matches!( + replica.apply("Author", refused), + Err(Error::Rejected(_)) + )); + assert!(matches!( + replica.apply("Author", typed(ExGuid::default(), oid, "Elsewhere ")), + Err(Error::Rejected(_)) + )); assert_eq!(replica.snapshot().unwrap(), source); assert!(replica.pending().unwrap().is_empty()); drop(replica); let connection = rusqlite::Connection::open(&path).unwrap(); - connection.execute_batch("CREATE TRIGGER fail_image BEFORE UPDATE ON replica BEGIN SELECT RAISE(ABORT, 'Injected image update failure'); END;").unwrap(); + connection.execute_batch("CREATE TRIGGER fail_edit BEFORE INSERT ON edits BEGIN SELECT RAISE(ABORT, 'Injected queue failure'); END;").unwrap(); drop(connection); let replica = Replica::open(&path).unwrap(); - assert!(matches!( - model_ops::save(&replica, oid, |page| model_ops::replace_text( - page, - oid, - 0..0, - "lost? " - )), - Err(Error::Database(_)) - )); + assert!(replica.apply("Author", typed(sid, oid, "lost? ")).is_err()); + assert_eq!(replica.snapshot().unwrap(), source); drop(replica); let replica = Replica::open(&path).unwrap(); assert_eq!(replica.snapshot().unwrap(), source); @@ -155,7 +181,7 @@ fn concurrent_recovery_exports_capture_one_complete_acknowledged_queue() { let directory = tempfile::tempdir().unwrap(); let source = onestore::create_section("recovery.one", "base", "Fixture").unwrap(); - let (_, object, _) = target(&source); + let (space, object, _) = target(&source); let replica = Replica::create(directory.path().join("live.sqlite"), &source).unwrap(); let start = Barrier::new(4); std::thread::scope(|scope| { @@ -164,18 +190,12 @@ fn concurrent_recovery_exports_capture_one_complete_acknowledged_queue() { scope.spawn(move || { start.wait(); for edit in 0..20 { - loop { - let marker = format!("[{writer}:{edit}] "); - match model_ops::save(replica, object, |page| { - model_ops::replace_text(page, object, 0..0, &marker) - }) { - Ok(Some(_)) => break, - Err(Error::Io(error)) if error.kind() == ErrorKind::ResourceBusy => { - continue; - } - other => panic!("Unexpected local edit: {other:?}"), - } - } + replica + .apply( + "Fixture", + typed(space, object, &format!("[{writer}:{edit}] ")), + ) + .unwrap(); } }); } @@ -185,20 +205,22 @@ fn concurrent_recovery_exports_capture_one_complete_acknowledged_queue() { replica.export_recovery(&path).unwrap(); let archive = Recovery::open(path).unwrap(); let pending = archive.pending().unwrap(); - let snapshot = target(&archive.snapshot().unwrap()).2; - match pending.as_slice() { - [] => assert_eq!(snapshot, "base"), - [single] => { - let Operation::Page(intent) = &single.operation else { - panic!() - }; - let (id, before, range, replacement) = intent.text_change().unwrap(); - assert_eq!((id, before.as_str(), range), (object, "base", 0..0)); - assert_eq!(snapshot, format!("{replacement}base")); - assert_eq!(text_of(&intent.after, object), snapshot); - } - more => panic!("saves to one page coalesce: {more:?}"), - } + // Each edit types at the start, so the text is the queue read backwards. + let expected: String = pending + .iter() + .rev() + .map(|edit| match &edit.edit.ops[..] { + [ + Op::Page { + op: PageOp::Text { with, .. }, + .. + }, + ] => with.as_str(), + other => panic!("{other:?}"), + }) + .chain(["base"]) + .collect(); + assert_eq!(target(&archive.snapshot().unwrap()).2, expected); assert_eq!(archive.remote_snapshot().unwrap(), source); assert_eq!( archive.summary().unwrap().queued_edits, @@ -207,7 +229,7 @@ fn concurrent_recovery_exports_capture_one_complete_acknowledged_queue() { assert!(archive.receipts().unwrap().is_empty()); } }); - assert_eq!(replica.pending().unwrap().len(), 1); + assert_eq!(replica.pending().unwrap().len(), 60); let content = target(&replica.snapshot().unwrap()).2; for writer in 0..3 { for edit in 0..20 { @@ -242,9 +264,10 @@ fn ownership_and_foreign_file_rejection_preserve_existing_data() { .unwrap(); for sql in [ "PRAGMA application_id=0".to_owned(), + // Schema 14 converts; older caches do not. format!( "PRAGMA application_id=1330529615; PRAGMA user_version={}", - current - 1 + current - 2 ), format!("PRAGMA user_version={}", current + 1), ] { @@ -289,52 +312,26 @@ fn ownership_and_foreign_file_rejection_preserve_existing_data() { } #[test] -fn twelve_local_editors_reject_stale_images_and_coalesce_into_one_save() { +fn twelve_local_editors_queue_every_edit_once_in_order() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("section.sqlite"); let source = onestore::create_section("section.one", "Shared café 🦀", "Fixture").unwrap(); let replica = Replica::create(&path, &source).unwrap(); let (sid, oid, _) = target(&source); let barrier = Barrier::new(12); - let deadline = Instant::now() + Duration::from_secs(60); let outcomes = std::thread::scope(|scope| { let handles: Vec<_> = (0..12) .map(|writer| { - let (replica, source, barrier) = (&replica, &source, &barrier); + let (replica, barrier) = (&replica, &barrier); scope.spawn(move || { - let (_, mut page) = model_ops::locate(source, oid); - model_ops::replace_text(&mut page, oid, 0..0, &format!("[initial-{writer}] ")); barrier.wait(); - let mut ids = Vec::new(); - let first_won = match replica.save(source, sid, &page, model_ops::AUTHOR) { - Ok(Some(id)) => { - ids.push(id); - true - } - Err(Error::Io(error)) if error.kind() == ErrorKind::ResourceBusy => false, - other => panic!("Unexpected first edit: {other:?}"), - }; - for edit in 0..20 { - loop { - assert!( - Instant::now() < deadline, - "Writer {writer} stopped progressing at {edit}" - ); - let marker = format!("[{writer}-{edit}] "); - match model_ops::save(replica, oid, |page| { - model_ops::replace_text(page, oid, 0..0, &marker) - }) { - Ok(Some(id)) => { - ids.push(id); - break; - } - Err(Error::Io(error)) - if error.kind() == ErrorKind::ResourceBusy => {} - other => panic!("Unexpected edit: {other:?}"), - } - } - } - (first_won, ids) + (0..20) + .map(|edit| { + replica + .apply("Fixture", typed(sid, oid, &format!("[{writer}-{edit}] "))) + .unwrap() + }) + .collect::>() }) }) .collect(); @@ -343,31 +340,29 @@ fn twelve_local_editors_reject_stale_images_and_coalesce_into_one_save() { .map(|handle| handle.join().unwrap()) .collect::>() }); - assert_eq!(outcomes.iter().filter(|(won, _)| *won).count(), 1); - let ids: BTreeSet<_> = outcomes.into_iter().flat_map(|(_, ids)| ids).collect(); - assert_eq!(ids.len(), 1, "every save replaced the unattempted head"); - let final_bytes = replica.snapshot().unwrap(); - let content = target(&final_bytes).2; - let pending = replica.pending().unwrap(); - assert_eq!(pending.len(), 1); - assert_eq!(pending[0].id, *ids.first().unwrap()); - let (id, before, range, replacement) = intent(&replica, 0).text_change().unwrap(); - assert_eq!((id, before.as_str(), range), (oid, "Shared café 🦀", 0..0)); - assert_eq!(content, format!("{replacement}Shared café 🦀")); - assert_eq!(content.matches("[initial-").count(), 1); + for ids in &outcomes { + assert!( + ids.windows(2).all(|pair| pair[0] < pair[1]), + "a writer's edits keep its order" + ); + } + let ids: BTreeSet<_> = outcomes.into_iter().flatten().collect(); + assert_eq!(ids.len(), 240); + let content = target(&replica.snapshot().unwrap()).2; + assert!(content.ends_with("Shared café 🦀")); for writer in 0..12 { for edit in 0..20 { assert_eq!(content.matches(&format!("[{writer}-{edit}] ")).count(), 1); } } - let (_, mut stale) = model_ops::locate(&source, oid); - model_ops::replace_text(&mut stale, oid, 0..0, "stale "); - assert!( - matches!(replica.save(&source, sid, &stale, "Author"), Err(Error::Io(error)) if error.kind() == ErrorKind::ResourceBusy) + let pending = replica.pending().unwrap(); + assert_eq!( + pending.iter().map(|edit| edit.id).collect::>(), + ids ); drop(replica); let reopened = Replica::open(&path).unwrap(); - assert_eq!(reopened.snapshot().unwrap(), final_bytes); + assert_eq!(target(&reopened.snapshot().unwrap()).2, content); assert_eq!(reopened.pending().unwrap(), pending); } @@ -375,11 +370,10 @@ fn twelve_local_editors_reject_stale_images_and_coalesce_into_one_save() { fn seeded_unicode_edits_and_restarts_match_an_independent_text_model() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("model.sqlite"); - let original = "ab🚀ab🦀 é repeated repeated".to_owned(); - let mut text = original.clone(); + let mut text = "ab🚀ab🦀 é repeated repeated".to_owned(); let source = onestore::create_section("model.one", &text, "Fixture").unwrap(); let mut replica = Replica::create(&path, &source).unwrap(); - let (space, object, _) = target(&source); + let (_, object, _) = target(&source); let mut random = 911_u64; let mut next = || { random ^= random << 13; @@ -387,8 +381,7 @@ fn seeded_unicode_edits_and_restarts_match_an_independent_text_model() { random ^= random << 17; random }; - let mut acknowledged = 0; - let mut pending = Vec::new(); + let mut acknowledged = Vec::new(); for step in 0..512 { let boundaries: Vec<_> = text .char_indices() @@ -403,47 +396,45 @@ fn seeded_unicode_edits_and_restarts_match_an_independent_text_model() { let replacement = ["", "🐈", "日本語", "repeated", "é", "ab🦀ab"][next() as usize % 6]; let mut expected = text.clone(); expected.replace_range(bytes, replacement); - let before = replica.snapshot().unwrap(); let acknowledgement = model_ops::save(&replica, object, |page| { model_ops::replace_text(page, object, range.clone(), replacement) }) .unwrap(); - if let Some(id) = acknowledgement { - assert_ne!(text, expected); - acknowledged += 1; - pending = replica.pending().unwrap(); - assert_eq!(pending.len(), 1); - assert_eq!((pending[0].id, pending[0].space), (1, space)); - let saved = intent(&replica, 0); - assert_eq!(id, 1); - assert_eq!(text_of(&saved.before, object), original); - assert_eq!(text_of(&saved.after, object), expected); - if step % 37 == 0 { - let (_, mut stale) = model_ops::locate(&before, object); - model_ops::replace_text(&mut stale, object, 0..0, "stale"); - assert!( - matches!(replica.save(&before, space, &stale, "Author"), Err(Error::Io(error)) if error.kind() == ErrorKind::ResourceBusy) - ); + match acknowledgement { + Some(id) => { + assert_ne!(text, expected); + assert!(acknowledged.last().is_none_or(|last| *last < id)); + acknowledged.push(id); } - } else { - assert_eq!(text, expected); + None => assert_eq!(text, expected), } text = expected; - let source = replica.snapshot().unwrap(); - assert_eq!(target(&source).2, text, "seed 911, step {step}"); + assert_eq!( + target(&replica.snapshot().unwrap()).2, + text, + "seed 911, step {step}" + ); if step % 37 == 0 { + let pending = replica.pending().unwrap(); drop(replica); replica = Replica::open(&path).unwrap(); - assert_eq!(replica.snapshot().unwrap(), source); + assert_eq!(target(&replica.snapshot().unwrap()).2, text); assert_eq!(replica.pending().unwrap(), pending); } } - assert!(acknowledged > 256, "Most random edits change the text"); - let source = replica.snapshot().unwrap(); - drop(replica); - let replica = Replica::open(&path).unwrap(); - assert_eq!(replica.pending().unwrap(), pending); - assert_eq!(replica.snapshot().unwrap(), source); + assert!( + acknowledged.len() > 256, + "Most random edits change the text" + ); + assert_eq!( + replica + .pending() + .unwrap() + .iter() + .map(|edit| edit.id) + .collect::>(), + acknowledged + ); } #[test] @@ -465,71 +456,43 @@ fn twelve_local_clients_preserve_inserted_identities_and_dependent_edits() { let mut objects = Vec::new(); barrier.wait(); for sequence in 0..4 { + assert!(Instant::now() < deadline, "Client {client} stopped"); let content = if sequence == 0 { format!("Client {client}") } else { format!("Paragraph {client}:{sequence}") }; let mut inserted = None; - loop { - assert!( - Instant::now() < deadline, - "Client {client} stopped at insertion {sequence}" - ); - let result = model_ops::save(cache, anchor, |page| { - let text = if sequence == 0 { - model_ops::insert_outline( - page, - 72.0, - 144.0 + client as f32 * 72.0, - &content, - ) - .2 - } else { - model_ops::insert_after( - page, - *objects.last().unwrap(), - &content, - ) - .1 - }; - let end = content.encode_utf16().count() as u32; - model_ops::restyle(page, text, 0..end, |format| { - format.italic = None - }); - model_ops::restyle(page, text, 0..6, italic); - inserted = Some(text); - }); - match result { - Ok(Some(id)) => { - ids.push(id); - objects.push(inserted.unwrap()); - break; - } - Err(Error::Io(e)) if e.kind() == ErrorKind::ResourceBusy => {} - other => panic!("{other:?}"), - } - } + let id = model_ops::save(cache, anchor, |page| { + let text = if sequence == 0 { + model_ops::insert_outline( + page, + 72.0, + 144.0 + client as f32 * 72.0, + &content, + ) + .2 + } else { + model_ops::insert_after(page, *objects.last().unwrap(), &content).1 + }; + let end = content.encode_utf16().count() as u32; + model_ops::restyle(page, text, 0..end, |format| format.italic = None); + model_ops::restyle(page, text, 0..6, italic); + inserted = Some(text); + }) + .unwrap() + .unwrap(); + ids.push(id); + objects.push(inserted.unwrap()); if sequence == 0 { continue; } let text = *objects.last().unwrap(); - loop { - assert!( - Instant::now() < deadline, - "Client {client} stopped at text {sequence}" - ); - match model_ops::save(cache, text, |page| { - model_ops::replace_text(page, text, 0..0, "Edited ") - }) { - Ok(Some(id)) => { - ids.push(id); - break; - } - Err(Error::Io(e)) if e.kind() == ErrorKind::ResourceBusy => {} - other => panic!("{other:?}"), - } - } + ids.push( + cache + .apply(model_ops::AUTHOR, typed(sid, text, "Edited ")) + .unwrap(), + ); } (ids, objects) }) @@ -544,12 +507,14 @@ fn twelve_local_clients_preserve_inserted_identities_and_dependent_edits() { .iter() .flat_map(|(ids, _)| ids.iter().copied()) .collect(); - assert_eq!(all.iter().map(|(ids, _)| ids.len()).sum::(), 84); - assert_eq!(ids.len(), 1); + assert_eq!(ids.len(), 84); let snapshot = cache.snapshot().unwrap(); drop(cache); let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.snapshot().unwrap(), snapshot); + assert_eq!( + server::pages(&cache.snapshot().unwrap()), + server::pages(&snapshot) + ); assert_eq!( cache .pending() @@ -588,8 +553,8 @@ fn twelve_local_clients_preserve_inserted_identities_and_dependent_edits() { } #[test] -fn unrecognized_persisted_operations_are_rejected_without_dropping_fields() { - for operation in ["Page", "CreatePage", "Pages", "DeletePages"] { +fn unrecognized_persisted_ops_are_rejected_without_dropping_fields() { + for kind in ["Text", "Create", "Pages", "Delete"] { let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("unknown.sqlite"); let first = onestore::create_section("unknown.one", "Original", "Author").unwrap(); @@ -598,52 +563,47 @@ fn unrecognized_persisted_operations_are_rejected_without_dropping_fields() { .unwrap() .as_bytes() .to_vec(); - let (_, oid, _) = target(&source); + let (space, oid, _) = target(&source); let store = Store::parse(&source).unwrap(); let index = RevisionIndex::parse(&store).unwrap(); let sid = Document::parse(&index).unwrap().pages().unwrap()[1].0; let cache = Replica::create(&path, &source).unwrap(); - match operation { - "Page" => { - model_ops::save(&cache, oid, |page| { - model_ops::replace_text(page, oid, 0..0, "New ") - }) + let section = |op| { + server::section_op(&cache, op); + }; + match kind { + "Text" => { + cache.apply("Author", typed(space, oid, "New ")).unwrap(); + } + "Create" => section(SectionOp::Create( + onestore::PageCreation::new(None, Some("Created"), "Author").unwrap(), + )), + "Pages" => section(SectionOp::Pages(vec![ + onestore::PageEdit::set_level(sid, 2).unwrap(), + ])), + _ => section(SectionOp::Delete(vec![sid])), + } + drop(cache); + let encoded: String = rusqlite::Connection::open(&path) + .unwrap() + .query_row("SELECT edit FROM edits", [], |r| r.get(0)) + .unwrap(); + for target in ["", "/ops/0", "/ops/0/Page/op/Text", "/ops/0/Section"] { + let mut value: serde_json::Value = serde_json::from_str(&encoded).unwrap(); + let Some(object) = value.pointer_mut(target).and_then(|v| v.as_object_mut()) else { + continue; + }; + object.insert("future_option".into(), true.into()); + rusqlite::Connection::open(&path) .unwrap() + .execute("UPDATE edits SET edit=?1", [value.to_string()]) .unwrap(); - } - "CreatePage" => { - let page = onestore::PageCreation::new(None, Some("Created"), "Author").unwrap(); - cache.create_page(&source, &page).unwrap().unwrap(); - } - "Pages" => { - cache - .pages(&source, &[onestore::PageEdit::set_level(sid, 2).unwrap()]) - .unwrap() - .unwrap(); - } - _ => { - cache.delete_pages(&source, &[sid]).unwrap().unwrap(); - } + let before = fs::read(&path).unwrap(); + assert!( + matches!(Replica::open(&path),Err(Error::Io(error))if error.kind()==ErrorKind::InvalidData), + "{kind} {target}" + ); + assert_eq!(fs::read(&path).unwrap(), before); } - drop(cache); - let db = rusqlite::Connection::open(&path).unwrap(); - let encoded: String = db - .query_row("SELECT operation FROM edits", [], |r| r.get(0)) - .unwrap(); - let mut value: serde_json::Value = serde_json::from_str(&encoded).unwrap(); - if value[operation].is_object() { - value[operation]["future_option"] = true.into(); - } else { - value["future_option"] = true.into(); - } - db.execute("UPDATE edits SET operation=?1", [value.to_string()]) - .unwrap(); - drop(db); - let before = fs::read(&path).unwrap(); - assert!( - matches!(Replica::open(&path),Err(Error::Io(error))if error.kind()==ErrorKind::InvalidData), - "{operation}" - ); - assert_eq!(fs::read(&path).unwrap(), before); } } diff --git a/crates/notebook/tests/migrate.rs b/crates/notebook/tests/migrate.rs new file mode 100644 index 0000000000000000000000000000000000000000..4f929c388c58e8b9c04aa9c3246fbb1b1a762697 --- /dev/null +++ b/crates/notebook/tests/migrate.rs @@ -0,0 +1,384 @@ +//! Converting schema-14 caches: whole-page edits become ops, verified page by page against +//! the old working image, with a recovery archive first and nothing changed on a mismatch. + +use notebook::{ConflictKind, EditStatus, Replica, Resolution}; +use onestore::{ExGuid, PageCreation, PageEdit, PreparedEdit, page::Page}; +use rusqlite::{Connection, params}; +use serde_json::json; +use std::path::Path; + +#[path = "support/server.rs"] +mod server; +use server::*; +#[path = "support/model_ops.rs"] +mod model_ops; + +const SOURCE: &[u8] = include_bytes!("../../../corpus/outline-edit/before/notebook/synthetic.one"); + +/// The body pages of the fixture with their first body text. +fn body_pages(source: &[u8]) -> Vec<(ExGuid, ExGuid)> { + pages(source) + .into_iter() + .filter_map(|(space, page)| { + let text = page.objects.iter().find_map(|object| match object { + onestore::page::PageObject::Outline(outline) => { + outline.paragraphs.first()?.text().map(|text| text.id) + } + _ => None, + })?; + Some((space, text)) + }) + .collect() +} + +/// A schema-14 queue entry. +enum Queued { + Page(ExGuid, Page), + Create(PageCreation), + Pages(Vec), + Delete(Vec), +} + +/// Writes a schema-14 cache holding `queue` on `base`, as that schema stored it; returns +/// the working image. +fn v14(path: &Path, base: &[u8], queue: &[Queued], working: Option<&[u8]>) -> Vec { + let mut image = base.to_vec(); + let mut edits = Vec::new(); + for entry in queue { + let (space, operation, next) = match entry { + Queued::Page(space, after) => { + let before = model_ops::page_of(&image, *space); + let next = PreparedEdit::page(&image, *space, after, "Author").unwrap(); + ( + *space, + json!({"Page": {"before": before, "after": after, "author": "Author"}}), + next.as_bytes().to_vec(), + ) + } + Queued::Create(creation) => ( + creation.space(), + json!({"CreatePage": creation}), + PreparedEdit::create_page(&image, creation) + .unwrap() + .as_bytes() + .to_vec(), + ), + Queued::Pages(batch) => { + let observed: Vec<_> = pages(&image) + .into_iter() + .map(|(space, _)| (space, 1)) + .collect(); + ( + root(&image), + json!({"Pages": {"edits": batch, "observed": observed}}), + PreparedEdit::pages(&image, batch) + .unwrap() + .as_bytes() + .to_vec(), + ) + } + Queued::Delete(spaces) => ( + root(&image), + json!({"DeletePages": spaces}), + PreparedEdit::delete_pages_permanently(&image, spaces) + .unwrap() + .as_bytes() + .to_vec(), + ), + }; + edits.push((space, operation)); + image = next; + } + let working = working.map_or(image, <[u8]>::to_vec); + std::fs::File::create_new(path).unwrap(); + let connection = Connection::open(path).unwrap(); + connection + .execute_batch( + "PRAGMA application_id=1330529615; PRAGMA user_version=14; + CREATE TABLE replica (id INTEGER PRIMARY KEY CHECK(id=1), base BLOB NOT NULL, working BLOB NOT NULL) STRICT; + CREATE TABLE edits (id INTEGER PRIMARY KEY AUTOINCREMENT CHECK(id>0), space TEXT NOT NULL, operation TEXT NOT NULL) STRICT; + CREATE TABLE attempt (id INTEGER PRIMARY KEY CHECK(id=1), edit_id INTEGER NOT NULL UNIQUE REFERENCES edits(id) ON DELETE CASCADE, revisions TEXT NOT NULL) STRICT; + CREATE TABLE receipts (edit_id INTEGER PRIMARY KEY CHECK(edit_id>0), revision TEXT NOT NULL) STRICT; + CREATE TABLE archived (edit_id INTEGER PRIMARY KEY CHECK(edit_id>0), archive TEXT NOT NULL) STRICT; + CREATE TABLE conflicts (edit_id INTEGER PRIMARY KEY REFERENCES edits(id) ON DELETE CASCADE, kind INTEGER NOT NULL CHECK(kind BETWEEN 0 AND 3)) STRICT; + CREATE TABLE assets (name TEXT PRIMARY KEY NOT NULL, data BLOB NOT NULL, sha256 BLOB NOT NULL CHECK(length(sha256)=32)) STRICT;", + ) + .unwrap(); + // Schema 14 kept the working image as its length and the runs differing from the base. + let mut patch = (working.len() as u64).to_le_bytes().to_vec(); + patch.extend_from_slice(&0_u64.to_le_bytes()); + patch.extend_from_slice(&(working.len() as u64).to_le_bytes()); + patch.extend_from_slice(&working); + connection + .execute( + "INSERT INTO replica VALUES (1, ?1, ?2)", + params![base, patch], + ) + .unwrap(); + connection + .execute( + "INSERT INTO receipts VALUES (1, ?1)", + [ExGuid { + guid: [7; 16], + n: 1, + } + .to_string()], + ) + .unwrap(); + connection + .execute( + "INSERT INTO sqlite_sequence(name, seq) VALUES ('edits', 1)", + [], + ) + .unwrap(); + for (space, operation) in edits { + connection + .execute( + "INSERT INTO edits(space, operation) VALUES (?1, ?2)", + params![space.to_string(), operation.to_string()], + ) + .unwrap(); + } + working +} + +fn root(image: &[u8]) -> ExGuid { + onestore::RevisionIndex::parse(&onestore::Store::parse(image).unwrap()) + .unwrap() + .root +} + +fn appended(source: &[u8], space: ExGuid, text: ExGuid, suffix: &str) -> Page { + let mut page = model_ops::page_of(source, space); + let end = model_ops::paragraph_with(&page, text) + .unwrap() + .text() + .unwrap() + .text + .text() + .encode_utf16() + .count() as u32; + model_ops::replace_text(&mut page, text, end..end, suffix); + page +} + +fn archive(path: &Path) -> std::path::PathBuf { + let mut archive = path.as_os_str().to_owned(); + archive.push(".v14-recovery"); + archive.into() +} + +#[test] +fn a_schema_14_queue_converts_to_ops_that_reach_its_working_pages_and_publish() { + let body = body_pages(SOURCE); + let ((a, a_text), (b, b_text)) = (body[0], body[1]); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("cache.sqlite"); + let creation = PageCreation::new(None, Some("Created offline"), "Author").unwrap(); + let first = appended(SOURCE, a, a_text, " one"); + let working = v14( + &path, + SOURCE, + &[ + Queued::Page(a, first.clone()), + Queued::Page(b, appended(SOURCE, b, b_text, " other")), + Queued::Create(creation.clone()), + Queued::Pages(vec![PageEdit::move_to(b, Some(a), 1).unwrap()]), + Queued::Page( + a, + appended( + PreparedEdit::page(SOURCE, a, &first, "Author") + .unwrap() + .as_bytes(), + a, + a_text, + " two", + ), + ), + Queued::Delete(vec![body[2].0]), + ], + None, + ); + let cache = Replica::open(&path).unwrap(); + assert!(archive(&path).exists()); + let pending = cache.pending().unwrap(); + assert_eq!( + pending.iter().map(|edit| edit.id).collect::>(), + [2, 3, 4, 5, 6, 7] + ); + assert!(pending.iter().all(|edit| !edit.edit.ops.is_empty())); + assert_eq!( + cache.status(1).unwrap(), + Some(EditStatus::Published { + revision: ExGuid { + guid: [7; 16], + n: 1 + } + }) + ); + let local = pages(&cache.snapshot().unwrap()); + assert_eq!(local, pages(&working)); + drop(cache); + // A converted cache opens as it is. + let cache = Replica::open(&path).unwrap(); + assert_eq!(pages(&cache.snapshot().unwrap()), local); + let next = cache + .apply( + "Author", + onestore::op::Edit { + at: 133_000_000_000_000_000, + ops: vec![onestore::op::Op::Page { + space: b, + op: onestore::op::PageOp::Text { + text: b_text, + range: 0..0, + with: " three".into(), + }, + }], + }, + ) + .unwrap(); + assert!(next > 7); + let mut server = Server::new(SOURCE); + assert!(matches!( + cache.sync_once(&mut server).unwrap().edit, + Some((id, EditStatus::Published { .. })) if id == next + )); + let published = pages(&server.durable); + assert_eq!(published, pages(&cache.snapshot().unwrap())); + assert!( + published + .iter() + .any(|(space, _)| *space == creation.space()) + ); + assert!(published.iter().all(|(space, _)| *space != body[2].0)); +} + +#[test] +fn an_uncertain_attempt_and_a_conflict_keep_their_states() { + let body = body_pages(SOURCE); + let (a, a_text) = body[0]; + let directory = tempfile::tempdir().unwrap(); + + // The oldest edit was attempted: its schema-14 evidence names the revision it published. + let path = directory.path().join("attempted.sqlite"); + let after = appended(SOURCE, a, a_text, " attempted"); + let published = PreparedEdit::page(SOURCE, a, &after, "Author").unwrap(); + let revision = + onestore::RevisionIndex::parse(&onestore::Store::parse(published.as_bytes()).unwrap()) + .unwrap() + .active(a) + .unwrap(); + v14(&path, SOURCE, &[Queued::Page(a, after.clone())], None); + Connection::open(&path) + .unwrap() + .execute( + "INSERT INTO attempt VALUES (1, 2, ?1)", + [json!({a.to_string(): revision.to_string()}).to_string()], + ) + .unwrap(); + let cache = Replica::open(&path).unwrap(); + let awaiting = EditStatus::AwaitingConfirmation { revision }; + assert_eq!(cache.status(2).unwrap(), Some(awaiting.clone())); + let mut unchanged = Server::new(SOURCE); + assert_eq!( + cache.sync_once(&mut unchanged).unwrap().edit, + Some((2, awaiting)) + ); + assert_eq!(unchanged.publications, 0); + let mut landed = Server::new(published.as_bytes()); + assert_eq!( + cache.sync_once(&mut landed).unwrap().edit, + Some((2, EditStatus::Published { revision })) + ); + assert_eq!(landed.publications, 0); + drop(cache); + + // The oldest edit was in conflict: the conversion lowered it onto the remote page the + // conflict was recorded against, and keeping mine publishes it. + let path = directory.path().join("conflicted.sqlite"); + let remote = PreparedEdit::page(SOURCE, a, &appended(SOURCE, a, a_text, " remote"), "Native") + .unwrap() + .as_bytes() + .to_vec(); + let local = appended(SOURCE, a, a_text, " local"); + let working = PreparedEdit::page(SOURCE, a, &local, "Author") + .unwrap() + .as_bytes() + .to_vec(); + { + // Schema 14 stored the remote as the base and kept the local working image. + v14(&path, &remote, &[], Some(&working)); + let connection = Connection::open(&path).unwrap(); + connection + .execute( + "INSERT INTO edits(space, operation) VALUES (?1, ?2)", + params![ + a.to_string(), + json!({"Page": {"before": model_ops::page_of(SOURCE, a), "after": local, "author": "Author"}}).to_string() + ], + ) + .unwrap(); + connection + .execute("INSERT INTO conflicts VALUES (2, 3)", []) + .unwrap(); + } + let cache = Replica::open(&path).unwrap(); + assert_eq!( + cache.status(2).unwrap(), + Some(EditStatus::Conflict(ConflictKind::ContentChanged)) + ); + assert_eq!( + model_ops::page_of(&cache.snapshot().unwrap(), a), + model_ops::page_of(&working, a) + ); + let mut server = Server::new(&remote); + assert_eq!( + cache.sync_once(&mut server).unwrap().edit, + Some((2, EditStatus::Conflict(ConflictKind::ContentChanged))) + ); + cache.resolve(2, Resolution::Mine).unwrap(); + assert!(matches!( + cache.sync_once(&mut server).unwrap().edit, + Some((2, EditStatus::Published { .. })) + )); + assert_eq!( + model_ops::page_of(&server.durable, a), + model_ops::page_of(&working, a) + ); +} + +#[test] +fn a_page_the_conversion_cannot_reproduce_leaves_the_cache_untouched() { + let body = body_pages(SOURCE); + let (a, a_text) = body[0]; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("cache.sqlite"); + // The working image holds a page the queued edit does not reach. + let stray = PreparedEdit::page(SOURCE, a, &appended(SOURCE, a, a_text, " stray"), "Author") + .unwrap() + .as_bytes() + .to_vec(); + v14( + &path, + SOURCE, + &[Queued::Page(a, appended(SOURCE, a, a_text, " queued"))], + Some(&stray), + ); + let before = std::fs::read(&path).unwrap(); + let Err(notebook::Error::Io(error)) = Replica::open(&path) else { + panic!("the conversion must fail") + }; + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); + assert!(error.to_string().contains(".v14-recovery"), "{error}"); + assert_eq!(std::fs::read(&path).unwrap(), before); + assert!(archive(&path).exists()); + let version: u32 = Connection::open(archive(&path)) + .unwrap() + .pragma_query_value(None, "user_version", |row| row.get(0)) + .unwrap(); + assert_eq!(version, 14); + // A second open tries again from the same untouched cache. + assert!(Replica::open(&path).is_err()); + assert_eq!(std::fs::read(&path).unwrap(), before); +} diff --git a/crates/notebook/tests/session.rs b/crates/notebook/tests/session.rs index 65e040dc66bff1cae582bed73105d5fe9a24fb19..0592efe480ceee679ec5bdf9b4918d37e6cc7e73 100644 --- a/crates/notebook/tests/session.rs +++ b/crates/notebook/tests/session.rs @@ -1,5 +1,5 @@ use notebook::{ - ConflictKind, EditStatus, Recovery, + ConflictKind, EditStatus, Recovery, Resolution, session::{Event, Notebook, QueuedEdit, Save, Section}, }; use onestore::{ExGuid, PreparedEdit, page::Page}; @@ -62,10 +62,14 @@ fn wait(section: &Section, mut accept: impl FnMut(&Event) -> bool) { } fn published(section: &Section, id: u64) { - wait( - section, - |event| matches!(event, Event::Attempt { id: n, status: EditStatus::Published { .. } } if *n == id), - ); + let deadline = Instant::now() + Duration::from_secs(20); + while !matches!( + section.status(id).unwrap(), + Some(EditStatus::Published { .. }) + ) { + assert!(Instant::now() < deadline, "the edit was not published"); + std::thread::sleep(Duration::from_millis(20)); + } } fn stored_page(file: &Path, space: ExGuid) -> Page { @@ -79,6 +83,13 @@ fn assert_same(actual: Page, expected: &Page) { assert_eq!(actual, expected); } +#[test] +fn a_section_is_shared_between_threads() { + fn shared() {} + shared::
(); + shared::(); +} + #[test] fn a_section_opens_through_its_replica_and_a_save_reaches_the_file_and_survives_relaunch() { let directory = tempfile::tempdir().unwrap(); @@ -765,7 +776,7 @@ fn a_conflicting_save_is_reviewed_against_the_remote_page_and_archived_for_recov mod server; /// An uncertain attempt survives a restart as `AwaitingConfirmation`; after exporting the -/// archive the user either continues from a reviewed page or abandons the branch. Neither +/// archive the user either publishes the local edits again or abandons the branch. Neither /// path records a receipt for the uncertain attempt. #[test] fn an_uncertain_attempt_is_released_after_restart_by_review() { @@ -778,10 +789,8 @@ fn an_uncertain_attempt_is_released_after_restart_by_review() { let replica = notebook::Replica::create(&cache, &source).unwrap(); let space = space_of(&source); let page = model_ops::page_of(&source, space); - let text = first_text(&page); let local = edited(&page, "Uncertain "); - let id = replica - .save(&source, space, &local, "Author") + let id = model_ops::save_as(&replica, space, &local, "Author") .unwrap() .unwrap(); let mut faulty = server::Server::new(&source); @@ -815,14 +824,12 @@ fn an_uncertain_attempt_is_released_after_restart_by_review() { assert_same(section.page(space).unwrap(), &local); let archive = directory.path().join("review.sqlite"); if continued { - let mut reviewed = section.remote_page(space).unwrap(); - model_ops::replace_text(&mut reviewed, text, 0..0, "Reviewed "); - section.release(id, &archive, Some(&reviewed)).unwrap(); + section.release(id, &archive, Resolution::Mine).unwrap(); assert_eq!(section.status(id).unwrap(), Some(EditStatus::Pending)); published(§ion, id); - assert_same(stored_page(&file, space), &reviewed); + assert_same(stored_page(&file, space), &local); } else { - section.release(id, &archive, None).unwrap(); + section.release(id, &archive, Resolution::Theirs).unwrap(); assert_eq!( section.status(id).unwrap(), Some(EditStatus::Archived { @@ -841,7 +848,11 @@ fn an_uncertain_attempt_is_released_after_restart_by_review() { assert!(section.queue().unwrap().is_empty()); assert!( section - .release(id, directory.path().join("again.sqlite"), None) + .release( + id, + directory.path().join("again.sqlite"), + Resolution::Theirs + ) .is_err() ); section.close().unwrap(); diff --git a/crates/notebook/tests/smb_queue.rs b/crates/notebook/tests/smb_queue.rs new file mode 100644 index 0000000000000000000000000000000000000000..c817c1a575b6689295ee1876334c876ad1d97995 --- /dev/null +++ b/crates/notebook/tests/smb_queue.rs @@ -0,0 +1,249 @@ +//! The op queue against a disposable Samba lab (`ONESTORE_SMB_LAB=127.0.0.1:PORT`, share +//! `agent`): publications read only the file's header, a native change is read once and +//! merged, and a session edits through the embedded client. +#![cfg(feature = "smb")] + +use notebook::{ + EditStatus, Remote, Replica, SmbRemote, + session::{Event, Section}, + smb::{Client, Credentials}, +}; +use onestore::{ + CommitError, ExGuid, Stamp, Transaction, + op::{Edit, Op, PageOp}, +}; +use std::{ + io, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +fn client() -> Client { + Client::connect( + &std::env::var("ONESTORE_SMB_LAB").unwrap(), + "agent", + Credentials::default(), + Duration::from_secs(5), + ) + .unwrap() +} + +fn unique(name: &str) -> String { + format!( + "{name}-{}.one", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + ) +} + +/// The remote, counting whole-file reads. +struct Counted { + remote: SmbRemote, + reads: usize, + stamps: usize, +} + +impl Remote for Counted { + fn read(&mut self) -> io::Result> { + self.reads += 1; + self.remote.read() + } + fn stamp(&mut self) -> io::Result> { + self.stamps += 1; + self.remote.stamp() + } + fn publish(&mut self, transaction: &Transaction) -> Result<(), CommitError> { + self.remote.publish(transaction) + } + fn confirm(&mut self, snapshot: &[u8]) -> Result<(), CommitError> { + self.remote.confirm(snapshot) + } +} + +/// The first page's space and first body text. +fn target(replica: &Replica) -> (ExGuid, ExGuid) { + let space = replica.pages().unwrap()[0].0; + let text = replica + .page(space) + .unwrap() + .objects + .iter() + .find_map(|object| match object { + onestore::page::PageObject::Outline(outline) => outline + .paragraphs + .iter() + .find_map(|p| p.text().map(|t| t.id)), + _ => None, + }) + .unwrap(); + (space, text) +} + +fn typed(space: ExGuid, text: ExGuid, at: u32, with: &str) -> Edit { + Edit { + at: 133_000_000_000_000_000, + ops: vec![Op::Page { + space, + op: PageOp::Text { + text, + range: at..at, + with: with.into(), + }, + }], + } +} + +fn text_of(replica: &Replica, space: ExGuid, text: ExGuid) -> String { + let page = replica.page(space).unwrap(); + page.objects + .iter() + .find_map(|object| match object { + onestore::page::PageObject::Outline(outline) => outline + .paragraphs + .iter() + .find_map(|p| p.text().filter(|t| t.id == text)), + _ => None, + }) + .unwrap() + .text + .text() + .to_owned() +} + +#[test] +#[ignore = "requires ONESTORE_SMB_LAB pointing to disposable Samba"] +fn live_keystrokes_publish_reading_only_the_header_and_a_native_change_merges() { + let lab = client(); + let path = unique("queue"); + let source = onestore::create_section(&path, "Body", "Author").unwrap(); + lab.create(&path, &source).unwrap(); + let directory = tempfile::tempdir().unwrap(); + let replica = Replica::create(directory.path().join("cache.sqlite"), &source).unwrap(); + let (space, text) = target(&replica); + let mut remote = Counted { + remote: SmbRemote::new(client(), path.clone(), 1 << 24), + reads: 0, + stamps: 0, + }; + for n in 0..20 { + let at = 4 + n; + let id = replica + .apply("Author", typed(space, text, at, "x")) + .unwrap(); + assert!(matches!( + replica.sync_once(&mut remote).unwrap().edit, + Some((published, EditStatus::Published { .. })) if published == id + )); + } + assert_eq!(remote.reads, 0, "publications read only the header"); + assert_eq!(remote.stamps, 20); + assert_eq!( + lab.read(&path, 1 << 24).unwrap(), + replica.snapshot().unwrap() + ); + for _ in 0..5 { + assert_eq!(replica.sync_once(&mut remote).unwrap().edit, None); + } + assert_eq!(remote.reads, 0, "idle polls read only the header"); + + // Another writer types at the start of the same text; a local edit at its end merges. + let native = { + let arena = onestore::Arena::default(); + let mut section = + onestore::Section::open(&arena, lab.read(&path, 1 << 24).unwrap()).unwrap(); + section + .apply("Native", &typed(space, text, 0, "Native ")) + .unwrap(); + section.seal().unwrap().unwrap() + }; + lab.commit_transaction(&path, &native).unwrap(); + let end = text_of(&replica, space, text).encode_utf16().count() as u32; + let id = replica + .apply("Author", typed(space, text, end, " local")) + .unwrap(); + let synced = replica.sync_once(&mut remote).unwrap(); + assert_eq!(synced.changed, [space]); + assert!( + matches!(synced.edit, Some((published, EditStatus::Published { .. })) if published == id) + ); + assert_eq!(remote.reads, 1, "a changed remote is read once"); + let merged = text_of(&replica, space, text); + assert!(merged.starts_with("Native Body"), "{merged}"); + assert!(merged.ends_with(" local"), "{merged}"); + assert_eq!( + lab.read(&path, 1 << 24).unwrap(), + replica.snapshot().unwrap() + ); + lab.delete(&path).unwrap(); +} + +#[test] +#[ignore = "requires ONESTORE_SMB_LAB pointing to disposable Samba"] +fn live_session_applies_edits_through_the_embedded_client_and_reopens() { + let lab = client(); + let path = unique("session"); + let source = onestore::create_section(&path, "Session", "Author").unwrap(); + lab.create(&path, &source).unwrap(); + let directory = tempfile::tempdir().unwrap(); + let cache = directory.path().join("cache.sqlite"); + let section = Section::resume_smb( + path.clone(), + Replica::create(&cache, &source).unwrap(), + 1 << 24, + || { + Client::connect( + &std::env::var("ONESTORE_SMB_LAB").unwrap(), + "agent", + Credentials::default(), + Duration::from_secs(5), + ) + }, + || {}, + ) + .unwrap(); + let (space, text) = target(section.replica()); + for n in 0..10 { + section + .apply("Author", typed(space, text, n, &n.to_string())) + .unwrap(); + } + // Applying does not wait: the edits are done when the file holds them. + let remote_text = |bytes: Vec| { + let arena = onestore::Arena::default(); + let section = onestore::Section::open(&arena, bytes).unwrap(); + let page = section.page(space).unwrap(); + page.objects + .iter() + .find_map(|object| match object { + onestore::page::PageObject::Outline(outline) => { + outline.paragraphs.iter().find_map(|p| { + p.text() + .filter(|t| t.id == text) + .map(|t| t.text.text().to_owned()) + }) + } + _ => None, + }) + .unwrap() + }; + let deadline = Instant::now() + Duration::from_secs(30); + while remote_text(lab.read(&path, 1 << 24).unwrap()) != "0123456789Session" + || !section.pending().unwrap().is_empty() + { + assert!(Instant::now() < deadline, "the edits were not published"); + for event in section.events() { + assert!( + !matches!(event, Event::Rejected { .. } | Event::Failed(_)), + "{event:?}" + ); + } + std::thread::sleep(Duration::from_millis(50)); + } + let published = lab.read(&path, 1 << 24).unwrap(); + section.close().unwrap(); + let replica = Replica::open(&cache).unwrap(); + assert_eq!(text_of(&replica, space, text), "0123456789Session"); + assert_eq!(replica.snapshot().unwrap(), published); + lab.delete(&path).unwrap(); +} diff --git a/crates/notebook/tests/support/model_ops.rs b/crates/notebook/tests/support/model_ops.rs index 62fce890a5c9e3ada246348d6f57d53207befa82..8e49e7c98c7fee9df4bc671a8be943a38214af54 100644 --- a/crates/notebook/tests/support/model_ops.rs +++ b/crates/notebook/tests/support/model_ops.rs @@ -225,28 +225,79 @@ pub fn delete_paragraph(page: &mut Page, text: ExGuid) { panic!("text is on the page"); } -/// Saves `edit` applied to the page containing `text`, using the replica's current image. +/// FILETIME now. +pub fn now() -> u64 { + let unix = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap(); + (unix.as_secs() + 11_644_473_600) * 10_000_000 +} + +/// Queues the ops taking the page in `space` to `after`, or nothing when they are equal. +pub fn save_as( + cache: &Replica, + space: ExGuid, + after: &Page, + author: &str, +) -> Result, notebook::Error> { + save_from(cache, space, &cache.page(space)?, after, author) +} + +/// Queues the ops taking `before`, the page in `space` as it was read, to `after`. +pub fn save_from( + cache: &Replica, + space: ExGuid, + before: &Page, + after: &Page, + author: &str, +) -> Result, notebook::Error> { + let ops = onestore::op::lower_page(before, after)?; + if ops.is_empty() { + return Ok(None); + } + cache + .apply( + author, + onestore::op::Edit { + at: now(), + ops: ops + .into_iter() + .map(|op| onestore::op::Op::Page { space, op }) + .collect(), + }, + ) + .map(Some) +} + +/// Queues `edit` of the page in `space`. +pub fn save_page( + cache: &Replica, + space: ExGuid, + edit: impl FnOnce(&mut Page), +) -> Result, notebook::Error> { + let before = cache.page(space)?; + let mut after = before.clone(); + edit(&mut after); + save_from(cache, space, &before, &after, AUTHOR) +} + +/// Queues `edit` of the page containing `text`. pub fn save( cache: &Replica, text: ExGuid, edit: impl FnOnce(&mut Page), ) -> Result, notebook::Error> { - let source = cache.snapshot()?; - let (space, mut page) = locate(&source, text); - edit(&mut page); - cache.save(&source, space, &page, AUTHOR) + let (space, _) = locate(&cache.snapshot()?, text); + save_page(cache, space, edit) } -/// Reviews a conflicting save by applying `edit` to the current remote page. +/// Resolves a conflict by taking the remote page, then queues `edit` of it. pub fn review( cache: &Replica, id: u64, text: ExGuid, edit: impl FnOnce(&mut Page), -) -> Result<(), notebook::Error> { - let local = cache.snapshot()?; - let remote = cache.remote_snapshot()?; - let (_, mut page) = locate(&remote, text); - edit(&mut page); - cache.review_page(id, &local, &remote, &page) +) -> Result, notebook::Error> { + cache.resolve(id, notebook::Resolution::Theirs)?; + save(cache, text, edit) } diff --git a/crates/notebook/tests/support/model_schedule.rs b/crates/notebook/tests/support/model_schedule.rs index fa871299fc39192e63946d5791f5270d47cc94f2..b877ea01705cee00e27cdf0528ed010bb8ad0fa7 100644 --- a/crates/notebook/tests/support/model_schedule.rs +++ b/crates/notebook/tests/support/model_schedule.rs @@ -3,7 +3,7 @@ use crate::model_ops::{self, AUTHOR}; use crate::server::{Fault, Server}; -use notebook::{EditStatus, Operation, Remote, Replica}; +use notebook::{EditStatus, Remote, Replica, Resolution}; use onestore::{ CommitError, ExGuid, PreparedEdit, RevisionIndex, Store, Transaction, document::Document, @@ -126,11 +126,10 @@ pub(crate) fn move_subtree( } } -/// Asserts the publication invariants that survive the page model, then delegates. +/// Asserts the publication invariants, then delegates. struct Session<'a> { server: &'a mut Server, - intent: Option<(u64, Operation)>, - /// The intent was already attempted, so its publication must never be repeated. + /// The head batch was already attempted, so its publication must never be repeated. retired: bool, publications: usize, } @@ -144,18 +143,6 @@ impl Remote for Session<'_> { assert!(!self.retired, "a retired attempt was replayed"); self.publications += 1; assert_eq!(self.publications, 1); - let Some((_, Operation::Page(intent))) = &self.intent else { - panic!("the schedule queues page saves only") - }; - let mut image = self.server.visible.clone(); - if transaction.apply(&mut image).is_ok() - && model_ops::page_of(&self.server.visible, SOURCE.1) == intent.before - { - assert_eq!( - shape(&model_ops::page_of(&image, SOURCE.1)), - shape(&intent.after) - ); - } self.server.publish(transaction) } @@ -164,6 +151,11 @@ impl Remote for Session<'_> { } } +/// The page's shape as the replica's queue leaves it. +fn local(cache: &Replica) -> Vec { + shape(&cache.page(SOURCE.1).unwrap()) +} + fn statuses(cache: &Replica) -> Vec<(u64, Option)> { cache .pending() @@ -190,12 +182,13 @@ pub fn run(input: &[u8]) { let path = directory.path().join(format!("{actor}.sqlite")); let cache = replicas[actor].get_or_insert_with(|| Replica::create(&path, source).unwrap()); let snapshot = cache.snapshot().unwrap(); + let before = local(cache); let pending = cache.pending().unwrap(); - let local = rows(&snapshot); + let rows_now = rows(&snapshot); match step[1] % 8 { 0..=2 => { - let row = local[usize::from(step[2]) % local.len()].clone(); - let other = local[usize::from(step[4]) % local.len()].clone(); + let row = rows_now[usize::from(step[2]) % rows_now.len()].clone(); + let other = rows_now[usize::from(step[4]) % rows_now.len()].clone(); let change: Box = match step[3] % 6 { 0 if !row.2.is_empty() => Box::new(move |page| { model_ops::replace_text( @@ -229,24 +222,20 @@ pub fn run(input: &[u8]) { }), _ => Box::new(|_| {}), }; - let mut after = model_ops::page_of(&snapshot, *space); + let mut after = cache.page(*space).unwrap(); change(&mut after); - match cache.save(&snapshot, *space, &after, AUTHOR) { + match model_ops::save_as(cache, *space, &after, AUTHOR) { Ok(id) => { let next = cache.pending().unwrap(); let ids = |edits: &[notebook::PendingEdit]| -> Vec { edits.iter().map(|edit| edit.id).collect() }; assert_eq!(ids(&next)[..pending.len()], ids(&pending)[..]); - assert!(next.len() <= pending.len() + 1); - assert_eq!(id.is_some(), cache.snapshot().unwrap() != snapshot); - assert_eq!( - shape(&model_ops::page_of(&cache.snapshot().unwrap(), *space)), - shape(&after) - ); + assert_eq!(next.len(), pending.len() + usize::from(id.is_some())); + assert_eq!(local(cache), shape(&after)); } Err(_) => { - assert_eq!(cache.snapshot().unwrap(), snapshot); + assert_eq!(local(cache), before); assert_eq!(cache.pending().unwrap(), pending); } } @@ -270,9 +259,6 @@ pub fn run(input: &[u8]) { Some(EditStatus::AwaitingConfirmation { .. }) ) }), - intent: pending - .first() - .map(|edit| (edit.id, edit.operation.clone())), server: &mut server, publications: 0, }; @@ -280,22 +266,27 @@ pub fn run(input: &[u8]) { }; server.fault = Fault::None; let next = cache.pending().unwrap(); - if next.len() == pending.len() { - assert_eq!(next, pending); - if !next.is_empty() { - assert_eq!(cache.snapshot().unwrap(), snapshot); - } - } else { - assert_eq!(next, pending[1..]); - let Some(EditStatus::Published { revision }) = - cache.status(head.unwrap()).unwrap() + // Edits leave the queue oldest first, each with a durable receipt. + let left = pending.len() - next.len(); + assert!( + next.iter() + .all(|edit| pending[left..].iter().any(|kept| kept.id == edit.id)) + ); + for edit in &pending[..left] { + let Some(EditStatus::Published { revision }) = cache.status(edit.id).unwrap() else { - panic!("an intent leaves the queue only with its receipt") + panic!("an edit leaves the queue only with its receipt") }; durable(&server, revision); } if next.is_empty() && result.is_ok() { - assert_eq!(cache.snapshot().unwrap(), cache.remote_snapshot().unwrap()); + assert_eq!( + local(cache), + shape(&model_ops::page_of( + &cache.remote_snapshot().unwrap(), + *space + )) + ); } } 5 => { @@ -317,44 +308,40 @@ pub fn run(input: &[u8]) { } } 6 => { - if let Some(head) = pending.first() - && matches!( - cache.status(head.id).unwrap(), - Some(EditStatus::Conflict(_)) - ) - { - let remote = cache.remote_snapshot().unwrap(); - let published = rows(&remote); - let row = published[usize::from(step[2]) % published.len()].clone(); - let mut after = model_ops::page_of(&remote, *space); - if !row.2.is_empty() { - model_ops::replace_text(&mut after, row.1, 0..1, "R"); - } - if cache - .review_page(head.id, &snapshot, &remote, &after) - .is_ok() - { - assert_eq!(cache.status(head.id).unwrap(), Some(EditStatus::Pending)); - let (result, publications) = { - let mut session = Session { - intent: Some(( - head.id, - cache.pending().unwrap()[0].operation.clone(), - )), - retired: false, - server: &mut server, - publications: 0, - }; - (cache.sync_once(&mut session), session.publications) - }; - assert!(publications <= 1); - if let Ok(Some((id, EditStatus::Published { revision }))) = result { - assert_eq!(id, head.id); - durable(&server, revision); - assert!(cache.pending().unwrap().iter().all(|e| e.id != id)); - } + if let Some(conflict) = cache.conflict().unwrap() { + let keep = if step[2] & 1 == 0 { + Resolution::Mine } else { - assert_eq!(cache.pending().unwrap(), pending); + Resolution::Theirs + }; + let remote = model_ops::page_of(&cache.remote_snapshot().unwrap(), *space); + cache.resolve(conflict.id, keep).unwrap(); + match cache.conflict().unwrap() { + Some(next) => assert_ne!(next.space, conflict.space), + None => assert_eq!( + local(cache), + match keep { + Resolution::Mine => before.clone(), + Resolution::Theirs => shape(&remote), + } + ), + } + let (result, publications) = { + let mut session = Session { + retired: false, + server: &mut server, + publications: 0, + }; + (cache.sync_once(&mut session), session.publications) + }; + assert!(publications <= 1); + if let Ok(notebook::Synced { + edit: Some((id, EditStatus::Published { revision })), + .. + }) = result + { + durable(&server, revision); + assert!(cache.pending().unwrap().iter().all(|e| e.id != id)); } } } @@ -362,7 +349,7 @@ pub fn run(input: &[u8]) { let recorded = statuses(cache); replicas[actor] = None; let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.snapshot().unwrap(), snapshot); + assert_eq!(local(&cache), before); assert_eq!(cache.pending().unwrap(), pending); assert_eq!(statuses(&cache), recorded); replicas[actor] = Some(cache); diff --git a/crates/notebook/tests/support/page_schedule.rs b/crates/notebook/tests/support/page_schedule.rs index afaacb032c3d2c2bbc3e5817fdb71a2d02c04bc5..76f6ac49a40e1b22c5e5a63696e05718da051c53 100644 --- a/crates/notebook/tests/support/page_schedule.rs +++ b/crates/notebook/tests/support/page_schedule.rs @@ -1,7 +1,7 @@ -use crate::disk; -use notebook::{EditStatus, Operation, Remote, Replica}; +use crate::{disk, model_ops}; +use notebook::{EditStatus, Remote, Replica, Resolution}; use onestore::{ - CommitError, ExGuid, PageEdit, PagePosition, PreparedEdit, RevisionIndex, Store, Transaction, + CommitError, ExGuid, PageEdit, PreparedEdit, RevisionIndex, Store, Transaction, document::{Document, Format, Kind, Layout}, page::{ Outline, Page, PageObject, PageParagraph, Paragraph, ParagraphContent, TextObject, @@ -97,27 +97,9 @@ pub fn body_outline(page: &mut Page, text: &str) -> ExGuid { id } -/// The text identity of the first body outline's first paragraph. -fn body_text(page: &Page) -> ExGuid { - page.objects - .iter() - .find_map(|object| match object { - PageObject::Outline(outline) => outline.paragraphs[0].text().map(|text| text.id), - _ => None, - }) - .unwrap() -} - -fn page_of(source: &[u8], space: ExGuid) -> Page { - let store = Store::parse(source).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - Page::from_space(&document, space).unwrap() -} - +/// Publishes through the crash-injecting disk, asserting each commit is atomic. struct Session<'a> { disk: &'a mut disk::Disk, - operation: Option<&'a Operation>, } impl Remote for Session<'_> { @@ -128,41 +110,6 @@ impl Remote for Session<'_> { fn publish(&mut self, transaction: &Transaction) -> Result<(), CommitError> { let mut image = self.disk.visible.clone(); transaction.apply(&mut image).unwrap(); - let mut expected = pages(&self.disk.visible); - match self.operation.unwrap() { - Operation::CreatePage(page) => expected.push((page.space(), page.object(), 1)), - Operation::Pages(batch) => { - for change in &batch.edits { - let at = expected.iter().position(|p| p.0 == change.space()).unwrap(); - expected[at].2 = change.level(); - if let PagePosition::Before(before) = change.position() { - let page = expected.remove(at); - let at = before.map_or(expected.len(), |before| { - expected.iter().position(|p| p.0 == before).unwrap() - }); - expected.insert(at, page); - } - } - } - Operation::Page(intent) => { - let body = body_text(&intent.after); - let store = Store::parse(&image).unwrap(); - let index = RevisionIndex::parse(&store).unwrap(); - let document = Document::parse(&index).unwrap(); - let matching: Vec<_> = document - .spaces - .values() - .filter_map(|space| { - let view = &space.revisions[&space.contexts[&ExGuid::default()]]; - view.nodes.get(&body) - }) - .collect(); - assert_eq!(matching.len(), 1); - assert!(matches!(&matching[0].kind, Kind::RichText { text, .. } if text == BODY)); - } - _ => panic!(), - } - assert_eq!(pages(&image), expected); let old = current::current(&self.disk.durable); let new = current::current(&image); let result = transaction.commit(self.disk); @@ -179,6 +126,16 @@ impl Remote for Session<'_> { } } +fn section_op(cache: &Replica, op: onestore::op::SectionOp) -> Result { + cache.apply( + "Author", + onestore::op::Edit { + at: 133_000_000_000_000_000, + ops: vec![onestore::op::Op::Section(op)], + }, + ) +} + pub fn run(input: &[u8]) { let directory = tempfile::tempdir().unwrap(); let mut replicas: [Option; 12] = std::array::from_fn(|_| None); @@ -196,21 +153,27 @@ pub fn run(input: &[u8]) { let path = directory.path().join(format!("{actor}.sqlite")); let cache = replicas[actor].get_or_insert_with(|| Replica::create(&path, &SOURCE).unwrap()); let snapshot = cache.snapshot().unwrap(); + let listed = pages(&snapshot); let pending = cache.pending().unwrap(); match step[1] % 8 { 0 | 1 => { - let title = (step[2] & 1 != 0).then_some("Same 🦋 é"); + let title = (step[2] & 1 != 0).then_some("Same 🦋 é"); let page = onestore::PageCreation::new(None, title, "Author").unwrap(); - cache.create_page(&snapshot, &page).unwrap().unwrap(); + section_op(cache, onestore::op::SectionOp::Create(page.clone())).unwrap(); owned[actor].push((page.space(), page.object())); if step[1] % 8 == 1 { - let source = cache.snapshot().unwrap(); - let mut model = page_of(&source, page.space()); + let mut model = cache.page(page.space()).unwrap(); body_outline(&mut model, BODY); - cache - .save(&source, page.space(), &model, "Author") + model_ops::save_as(cache, page.space(), &model, "Author") .unwrap() .unwrap(); + let local = cache.page(page.space()).unwrap(); + assert!( + local + .objects + .iter() + .any(|object| matches!(object, PageObject::Outline(o) if o.paragraphs[0].text().is_some_and(|t| t.text.text() == BODY))) + ); } } 2 => { @@ -220,7 +183,7 @@ pub fn run(input: &[u8]) { .collect(); replicas[actor] = None; let cache = Replica::open(&path).unwrap(); - assert!(cache.snapshot().unwrap() == snapshot); + assert_eq!(pages(&cache.snapshot().unwrap()), listed); assert_eq!(cache.pending().unwrap(), pending); assert_eq!( pending @@ -240,15 +203,20 @@ pub fn run(input: &[u8]) { let prior = pending .first() .and_then(|edit| cache.status(edit.id).unwrap()); - let mut session = Session { - disk: &mut disk, - operation: pending.first().map(|edit| &edit.operation), - }; - let result = cache.sync_once(&mut session); + let result = cache.sync_once(&mut Session { disk: &mut disk }); if matches!(prior, Some(EditStatus::AwaitingConfirmation { .. })) { assert!( result.is_err() - || !matches!(result.unwrap(), Some((_, EditStatus::Conflict(_)))) + || !matches!( + result.as_ref().unwrap().edit, + Some((_, EditStatus::Conflict(_))) + ) + ); + } + if result.is_ok() && cache.pending().unwrap().is_empty() { + assert_eq!( + pages(&cache.snapshot().unwrap()), + pages(&cache.remote_snapshot().unwrap()) ); } } @@ -257,13 +225,12 @@ pub fn run(input: &[u8]) { disk.fail_at = None; } 6 => { - let order = pages(&snapshot); let count = 1 + usize::from(step[3] & 1 != 0); let selected: Vec<_> = (0..count) - .map(|i| order[(usize::from(step[2]) + i) % order.len()].0) + .map(|i| listed[(usize::from(step[2]) + i) % listed.len()].0) .collect(); let anchor = (step[4] & 1 != 0) - .then_some(order[usize::from(step[5]) % order.len()].0) + .then_some(listed[usize::from(step[5]) % listed.len()].0) .filter(|sid| !selected.contains(sid)); let edits: Vec<_> = selected .iter() @@ -278,43 +245,28 @@ pub fn run(input: &[u8]) { }) .collect(); let expected = PreparedEdit::pages(&snapshot, &edits); - match cache.pages(&snapshot, &edits) { - Ok(id) => { - let expected = expected.unwrap(); - assert_eq!( - current::current(&cache.snapshot().unwrap()), - current::current(expected.as_bytes()) - ); - assert_eq!(id.is_some(), expected.as_bytes() != snapshot); - } + match section_op(cache, onestore::op::SectionOp::Pages(edits)) { + Ok(_) => assert_eq!( + pages(&cache.snapshot().unwrap()), + pages(expected.unwrap().as_bytes()) + ), Err(_) => { assert!(expected.is_err()); - assert_eq!(cache.snapshot().unwrap(), snapshot); + assert_eq!(pages(&cache.snapshot().unwrap()), listed); assert_eq!(cache.pending().unwrap(), pending); } } } 7 => { - if let Some(edit) = pending.first() - && matches!( - cache.status(edit.id).unwrap(), - Some(EditStatus::Conflict(_)) - ) - && let Operation::Pages(batch) = &edit.operation - { - let remote = cache.remote_snapshot().unwrap(); - let order = pages(&remote); - let anchor = (step[2] & 1 != 0) - .then_some(order[usize::from(step[3]) % order.len()].0) - .filter(|sid| batch.edits.iter().all(|e| e.space() != *sid)); - let edits: Vec<_> = batch - .edits - .iter() - .map(|edit| edit.reposition(PagePosition::Before(anchor), 1).unwrap()) - .collect(); - let result = cache.rebase_pages_conflict(edit.id, &snapshot, &remote, &edits); - assert_eq!(cache.snapshot().unwrap(), snapshot); - if result.is_err() { + if let Some(conflict) = cache.conflict().unwrap() { + // Taking theirs for the page list would drop the pages this actor made. + let root = listed.iter().all(|(sid, ..)| *sid != conflict.space); + let keep = if root || step[2] & 1 == 0 { + Resolution::Mine + } else { + Resolution::Theirs + }; + if cache.resolve(conflict.id, keep).is_err() { assert_eq!(cache.pending().unwrap(), pending); } } diff --git a/crates/notebook/tests/support/server.rs b/crates/notebook/tests/support/server.rs index 504f8b4fdc74f51199a05e29713c7c8004c20b15..199421ab5489b423a522c6ab13d7ebf61608df59 100644 --- a/crates/notebook/tests/support/server.rs +++ b/crates/notebook/tests/support/server.rs @@ -133,3 +133,35 @@ pub fn text(source: &[u8]) -> (ExGuid, ExGuid, String) { }) .unwrap() } + +/// Every page of an image, in section order: what two images holding the same edits under +/// different revision identities share. +pub fn pages(source: &[u8]) -> Vec<(ExGuid, onestore::page::Page)> { + let store = Store::parse(source).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + document + .pages() + .unwrap() + .into_iter() + .map(|(space, _)| { + ( + space, + onestore::page::Page::from_space(&document, space).unwrap(), + ) + }) + .collect() +} + +/// Queues a section op as its own edit. +pub fn section_op(cache: ¬ebook::Replica, op: onestore::op::SectionOp) -> u64 { + cache + .apply( + "Fixture", + onestore::op::Edit { + at: 133_000_000_000_000_000, + ops: vec![onestore::op::Op::Section(op)], + }, + ) + .unwrap() +} diff --git a/crates/notebook/tests/sync.rs b/crates/notebook/tests/sync.rs index 0a7daccb1b546b4f9cfb023b8b9e7b91f9a9f5a9..581864cfcea245531af577d10627ad55a54e1988 100644 --- a/crates/notebook/tests/sync.rs +++ b/crates/notebook/tests/sync.rs @@ -1,7 +1,7 @@ -//! Replica protocol: durable receipts, retired attempts, conflict review and contention. +//! Replica protocol: durable receipts, retired attempts, conflict resolution and contention. -use notebook::{ConflictKind, EditStatus, Error, Remote, Replica}; -use onestore::{CommitError, CommitState, ExGuid, Transaction, page::Page}; +use notebook::{ConflictKind, EditStatus, Error, Remote, Replica, Resolution}; +use onestore::{CommitError, CommitState, ExGuid, Transaction}; use std::{io, ops::Range}; #[path = "support/server.rs"] @@ -10,13 +10,6 @@ use server::*; #[path = "support/model_ops.rs"] mod model_ops; -/// The page holding `text`, with a UTF-16 range of that text replaced. -fn edited(bytes: &[u8], text: ExGuid, range: Range, replacement: &str) -> Page { - let (_, mut page) = model_ops::locate(bytes, text); - model_ops::replace_text(&mut page, text, range, replacement); - page -} - /// Saves a replacement of `range` in the page holding `text`. fn save(cache: &Replica, text: ExGuid, range: Range, replacement: &str) -> Option { model_ops::save(cache, text, |page| { @@ -56,16 +49,16 @@ fn an_unchanged_stamp_publishes_and_settles_without_reading_the_remote() { reads: 0, }; assert!(matches!( - cache.sync_once(&mut remote).unwrap(), + cache.sync_once(&mut remote).unwrap().edit, Some((published, EditStatus::Published { .. })) if published == id )); - assert_eq!(cache.sync_once(&mut remote).unwrap(), None); + assert_eq!(cache.sync_once(&mut remote).unwrap().edit, None); assert_eq!(remote.reads, 0); // Another writer's commit moves the header, so the next step reads the file. let native = onestore::replace_text(&remote.server.visible, sid, object, 0..0, "Native ").unwrap(); remote.server.visible.clone_from(&native); - assert_eq!(cache.sync_once(&mut remote).unwrap(), None); + assert_eq!(cache.sync_once(&mut remote).unwrap().edit, None); assert_eq!(remote.reads, 1); assert_eq!(cache.snapshot().unwrap(), native); } @@ -82,7 +75,7 @@ fn recovery_archive_preserves_the_queue_uncertainty_and_receipts_without_becomin let cache = Replica::create(&path, &source).unwrap(); let published = save(&cache, object, 0..0, "Published ").unwrap(); let mut server = Server::new(&source); - let (_, receipt) = cache.sync_once(&mut server).unwrap().unwrap(); + let (_, receipt) = cache.sync_once(&mut server).unwrap().edit.unwrap(); let attempted = save(&cache, object, 0..0, "Uncertain ").unwrap(); server.fault = Fault::UnknownBefore; assert!(matches!( @@ -94,10 +87,7 @@ fn recovery_archive_preserves_the_queue_uncertainty_and_receipts_without_becomin )); save(&cache, object, 0..0, "Queued ").unwrap(); let created = onestore::PageCreation::new(None, Some("Recovery 🦀"), "Fixture").unwrap(); - cache - .create_page(&cache.snapshot().unwrap(), &created) - .unwrap() - .unwrap(); + section_op(&cache, onestore::op::SectionOp::Create(created)); save(&cache, object, 0..0, "Last ").unwrap(); let working = cache.snapshot().unwrap(); let remote = cache.remote_snapshot().unwrap(); @@ -109,8 +99,8 @@ fn recovery_archive_preserves_the_queue_uncertainty_and_receipts_without_becomin conflicts: 0, uncertain_edits: 1, published_receipts: 1, - working_bytes: working.len() as u64, - remote_bytes: remote.len() as u64, + base_bytes: remote.len() as u64, + remote_bytes: 0, cached_assets: 0, cached_asset_bytes: 0, }; @@ -123,7 +113,7 @@ fn recovery_archive_preserves_the_queue_uncertainty_and_receipts_without_becomin assert!(Recovery::open(&path).is_err()); let archive = Recovery::open(&archive_path).unwrap(); assert_eq!(archive.summary().unwrap(), summary); - assert_eq!(archive.snapshot().unwrap(), working); + assert_eq!(pages(&archive.snapshot().unwrap()), pages(&working)); assert_eq!(archive.remote_snapshot().unwrap(), remote); assert_eq!(archive.pending().unwrap(), pending); assert_eq!(archive.status(published).unwrap(), Some(receipt.clone())); @@ -150,7 +140,7 @@ fn recovery_archive_preserves_the_queue_uncertainty_and_receipts_without_becomin assert_eq!(std::fs::read(&path).unwrap(), source_file); assert_eq!(std::fs::read(&archive_path).unwrap(), archive_bytes); save(&cache, object, 0..0, "Later ").unwrap(); - assert_eq!(archive.snapshot().unwrap(), working); + assert_eq!(pages(&archive.snapshot().unwrap()), pages(&working)); assert_eq!(archive.pending().unwrap(), pending); drop(archive); assert_eq!( @@ -193,7 +183,7 @@ fn recovery_archive_retains_conflict_images_and_rejects_foreign_or_future_archiv let id = save(&cache, object, 0..8, "Local").unwrap(); let changed = onestore::replace_text(&source, space, object, 0..8, "Remote").unwrap(); let mut server = Server::new(&changed); - let outcome = cache.sync_once(&mut server).unwrap().unwrap(); + let outcome = cache.sync_once(&mut server).unwrap().edit.unwrap(); assert_eq!( outcome, (id, EditStatus::Conflict(ConflictKind::ContentChanged)) @@ -232,7 +222,7 @@ fn disjoint_remote_changes_merge_and_persist_the_remote_receipt() { let id = save(&cache, oid, 2..4, "🐈").unwrap(); let remote = onestore::replace_text(&source, sid, oid, 0..6, "Xab🦀cYd").unwrap(); let mut server = Server::new(&remote); - let outcome = cache.sync_once(&mut server).unwrap().unwrap(); + let outcome = cache.sync_once(&mut server).unwrap().edit.unwrap(); assert_eq!(outcome.0, id); assert!(matches!(outcome.1, EditStatus::Published { .. })); assert_eq!(text(&server.durable).2, "Xab🐈cYd"); @@ -241,7 +231,7 @@ fn disjoint_remote_changes_merge_and_persist_the_remote_receipt() { drop(cache); let cache = Replica::open(&path).unwrap(); assert_eq!(cache.status(id).unwrap(), Some(outcome.1.clone())); - assert_eq!(cache.sync_once(&mut server).unwrap(), None); + assert_eq!(cache.sync_once(&mut server).unwrap().edit, None); assert_eq!(server.publications, 1); assert_eq!(cache.status(id + 1).unwrap(), None); let next = save(&cache, oid, 0..0, "Later ").unwrap(); @@ -260,13 +250,13 @@ fn overlapping_changes_preserve_both_images_and_survive_restart() { let remote = onestore::replace_text(&source, sid, oid, 1..2, "R").unwrap(); let mut server = Server::new(&remote); assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))) ); assert_eq!(server.publications, 0); drop(cache); let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(pages(&cache.snapshot().unwrap()), pages(&local)); assert_eq!(cache.remote_snapshot().unwrap(), remote); assert_eq!(cache.pending().unwrap().len(), 1); assert_eq!( @@ -298,7 +288,7 @@ fn lost_replies_and_process_termination_never_blindly_replay_an_attempt() { assert!(matches!(attempted, EditStatus::AwaitingConfirmation { .. })); drop(cache); let cache = Replica::open(&path).unwrap(); - let result = cache.sync_once(&mut server).unwrap().unwrap(); + let result = cache.sync_once(&mut server).unwrap().edit.unwrap(); if matches!(fault, Fault::UnknownAfter | Fault::PanicAfter) { assert!(matches!(result.1, EditStatus::Published { .. })); assert_eq!(text(&server.durable).2, "Once abc"); @@ -333,7 +323,7 @@ fn failed_confirmation_does_not_promote_visible_bytes_to_a_receipt() { Some(EditStatus::AwaitingConfirmation { .. }) )); assert!(matches!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((_, EditStatus::Published { .. })) )); assert_eq!(server.publications, 1); @@ -383,7 +373,7 @@ fn proven_unpublished_attempts_retry_and_committed_cleanup_errors_keep_receipts( if matches!(fault, Fault::Before) { assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); assert!(matches!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((_, EditStatus::Published { .. })) )); assert_eq!(server.publications, 2); @@ -392,7 +382,7 @@ fn proven_unpublished_attempts_retry_and_committed_cleanup_errors_keep_receipts( cache.status(id).unwrap(), Some(EditStatus::Published { .. }) )); - assert_eq!(cache.sync_once(&mut server).unwrap(), None); + assert_eq!(cache.sync_once(&mut server).unwrap().edit, None); assert_eq!(server.publications, 1); } assert_eq!(text(&server.durable).2, "Once abc"); @@ -401,7 +391,10 @@ fn proven_unpublished_attempts_retry_and_committed_cleanup_errors_keep_receipts( #[test] fn database_failures_before_and_after_publication_preserve_recovery_state() { - for table in ["attempt", "receipts"] { + for (table, event) in [ + ("attempt", "UPDATE OF attempted ON batches"), + ("receipts", "INSERT ON receipts"), + ] { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("cache.sqlite"); let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); @@ -410,7 +403,7 @@ fn database_failures_before_and_after_publication_preserve_recovery_state() { let id = save(&cache, oid, 0..0, "Once ").unwrap(); drop(cache); let db = rusqlite::Connection::open(&path).unwrap(); - db.execute_batch(&format!("CREATE TRIGGER interrupted BEFORE INSERT ON {table} BEGIN SELECT RAISE(ABORT,'Injected cache failure'); END;")).unwrap(); + db.execute_batch(&format!("CREATE TRIGGER interrupted BEFORE {event} BEGIN SELECT RAISE(ABORT,'Injected cache failure'); END;")).unwrap(); drop(db); let cache = Replica::open(&path).unwrap(); let mut server = Server::new(&source); @@ -426,7 +419,7 @@ fn database_failures_before_and_after_publication_preserve_recovery_state() { drop(db); let cache = Replica::open(&path).unwrap(); assert!(matches!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((_, EditStatus::Published { .. })) )); assert!(matches!( @@ -474,7 +467,7 @@ fn twelve_local_editors_progress_during_remote_reads_publication_and_confirmatio for phase in ["read", "publish", "confirm"] { let dir = tempfile::tempdir().unwrap(); let source = onestore::create_section("sync.one", "abc", "Fixture").unwrap(); - let (_, oid, _) = text(&source); + let (sid, oid, _) = text(&source); let cache = Replica::create(dir.path().join("cache.sqlite"), &source).unwrap(); let first = save(&cache, oid, 0..0, "First ").unwrap(); let mut server = Server::new(&source); @@ -493,9 +486,17 @@ fn twelve_local_editors_progress_during_remote_reads_publication_and_confirmatio let mut server = std::thread::scope(|scope| { let running = scope.spawn(|| { let result = cache.sync_once(&mut paused); - assert!( - matches!(result, Ok(Some((id, EditStatus::Published { .. }))) if id == first) - ); + assert!(matches!( + result, + Ok(notebook::Synced { + edit: Some((_, EditStatus::Published { .. })), + .. + }) + )); + assert!(matches!( + cache.status(first).unwrap(), + Some(EditStatus::Published { .. }) + )); paused.server }); entered_rx @@ -504,26 +505,31 @@ fn twelve_local_editors_progress_during_remote_reads_publication_and_confirmatio assert!( matches!(cache.sync_once(&mut Server::new(&source)),Err(Error::Io(error)) if error.kind()==io::ErrorKind::WouldBlock) ); - let reviewed = edited(&source, oid, 0..0, "Reviewed "); assert!( - matches!(cache.review_page(first, &[], &[], &reviewed), Err(Error::Io(error)) if error.kind() == io::ErrorKind::WouldBlock) + matches!(cache.resolve(first, Resolution::Theirs), Err(Error::Io(error)) if error.kind() == io::ErrorKind::WouldBlock) ); let started = std::time::Instant::now(); let handles: Vec<_> = (0..12) .map(|writer| { let cache = &cache; scope.spawn(move || { - loop { - let marker = format!("[{writer}] "); - match model_ops::save(cache, oid, |page| { - model_ops::replace_text(page, oid, 0..0, &marker) - }) { - Ok(Some(id)) => break id, - Err(Error::Io(error)) - if error.kind() == io::ErrorKind::ResourceBusy => {} - other => panic!("Unexpected local outcome {other:?}"), - } - } + // Ops, unlike models, commute with the other writers' edits. + cache + .apply( + model_ops::AUTHOR, + onestore::op::Edit { + at: model_ops::now(), + ops: vec![onestore::op::Op::Page { + space: sid, + op: onestore::op::PageOp::Text { + text: oid, + range: 0..0, + with: format!("[{writer}] "), + }, + }], + }, + ) + .unwrap() }) }) .collect(); @@ -531,12 +537,7 @@ fn twelve_local_editors_progress_during_remote_reads_publication_and_confirmatio .into_iter() .map(|handle| handle.join().unwrap()) .collect(); - assert_eq!(ids.len(), 1, "saves to one page coalesce"); - assert_eq!( - ids.contains(&first), - phase == "read", - "saves coalesce into the head only until a step selects it" - ); + assert_eq!(ids.len(), 12, "every local edit is its own"); assert!( started.elapsed() < std::time::Duration::from_secs(2), "Local edits waited for remote {phase}" @@ -544,14 +545,18 @@ fn twelve_local_editors_progress_during_remote_reads_publication_and_confirmatio resume_tx.send(()).unwrap(); running.join().unwrap() }); - assert_eq!(cache.pending().unwrap().len(), usize::from(phase != "read")); + // Edits arriving while a batch publishes form the next batch. + assert_eq!( + cache.pending().unwrap().len(), + if phase == "read" { 0 } else { 12 } + ); let expected = text(&cache.snapshot().unwrap()).2; for writer in 0..12 { assert_eq!(expected.matches(&format!("[{writer}] ")).count(), 1); } while !cache.pending().unwrap().is_empty() { assert!(matches!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((_, EditStatus::Published { .. })) )); } @@ -573,136 +578,155 @@ fn unrelated_remote_files_never_replace_a_local_cache() { if pending { save(&cache, oid, 0..0, "Local ").unwrap(); } - let before = cache.snapshot().unwrap(); + let before = pages(&cache.snapshot().unwrap()); assert!( matches!(cache.sync_once(&mut server),Err(Error::Io(error)) if error.kind()==io::ErrorKind::InvalidInput) ); - assert_eq!(cache.snapshot().unwrap(), before); + assert_eq!(pages(&cache.snapshot().unwrap()), before); assert_eq!(cache.remote_snapshot().unwrap(), source); assert_eq!(server.publications, 0); } } +/// The text object's content in an image. +fn content(bytes: &[u8], text: ExGuid) -> String { + let (_, page) = model_ops::locate(bytes, text); + model_ops::paragraph_with(&page, text) + .unwrap() + .text() + .unwrap() + .text + .text() + .to_owned() +} + #[test] -fn reviewing_a_conflict_preserves_the_dependent_save_and_survives_reopen() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("cache.sqlite"); - let source = onestore::create_section("resolve.one", "abc", "Fixture").unwrap(); - let (sid, oid, _) = text(&source); - let cache = Replica::create(&path, &source).unwrap(); - let first = save(&cache, oid, 1..2, "L").unwrap(); - let remote = onestore::replace_text(&source, sid, oid, 0..3, "aRcZ").unwrap(); - let mut server = Server::new(&remote); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((first, EditStatus::Conflict(ConflictKind::ContentChanged))) - ); - let mut dependent = None; - for n in 0..12 { - dependent = save(&cache, oid, 0..0, &format!("[{n}] ")); - } - let dependent = dependent.unwrap(); - assert_ne!(dependent, first, "a conflicted save is never rewritten"); - let local = cache.snapshot().unwrap(); - let pending = cache.pending().unwrap(); - assert_eq!(pending.len(), 2); - model_ops::review(&cache, first, oid, |page| { - model_ops::replace_text(page, oid, 1..2, "L") - }) - .unwrap(); - assert_eq!(cache.snapshot().unwrap(), local); - assert_eq!(cache.remote_snapshot().unwrap(), remote); - let reviewed = cache.pending().unwrap(); - assert_eq!(reviewed[1..], pending[1..]); - assert_eq!(reviewed[0].id, first); - let notebook::Operation::Page(intent) = &reviewed[0].operation else { - panic!() - }; - assert_eq!( - intent.text_change(), - Some((oid, "aRcZ".into(), 1..2, "L".into())) - ); - assert_eq!(intent.author, model_ops::AUTHOR); - assert_eq!(cache.status(first).unwrap(), Some(EditStatus::Pending)); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.pending().unwrap(), reviewed); - assert_eq!(cache.snapshot().unwrap(), local); - for intent in &pending { - assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((id, EditStatus::Published { .. })) if id == intent.id) +fn a_conflict_holds_later_edits_until_resolved_either_way_and_survives_reopen() { + for resolution in [Resolution::Mine, Resolution::Theirs] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cache.sqlite"); + let source = onestore::create_section("resolve.one", "abc", "Fixture").unwrap(); + let (sid, oid, _) = text(&source); + let cache = Replica::create(&path, &source).unwrap(); + let first = save(&cache, oid, 1..2, "L").unwrap(); + let remote = onestore::replace_text(&source, sid, oid, 0..3, "aRcZ").unwrap(); + let mut server = Server::new(&remote); + let conflict = EditStatus::Conflict(ConflictKind::ContentChanged); + assert_eq!( + cache.sync_once(&mut server).unwrap().edit, + Some((first, conflict.clone())) + ); + let mut dependent = None; + for n in 0..12 { + dependent = save(&cache, oid, 0..0, &format!("[{n}] ")); + } + let dependent = dependent.unwrap(); + assert_ne!(dependent, first); + assert_eq!(cache.status(dependent).unwrap(), Some(conflict.clone())); + let local = content(&cache.snapshot().unwrap(), oid); + assert!(local.ends_with("[0] aLc"), "{local}"); + let pending = cache.pending().unwrap(); + assert_eq!(pending.len(), 13); + assert_eq!( + cache.sync_once(&mut server).unwrap().edit, + Some((dependent, conflict.clone())), + "an unchanged remote leaves the conflict as it was" + ); + assert_eq!(server.publications, 0); + drop(cache); + let cache = Replica::open(&path).unwrap(); + assert_eq!(cache.pending().unwrap(), pending); + assert_eq!(content(&cache.snapshot().unwrap(), oid), local); + assert_eq!( + cache.conflict().unwrap(), + Some(notebook::Conflict { + id: dependent, + space: sid, + kind: ConflictKind::ContentChanged + }) ); + cache.resolve(first, resolution).unwrap(); + assert_eq!(cache.conflict().unwrap(), None); + assert_eq!(cache.remote_snapshot().unwrap(), remote); + let expected = match resolution { + Resolution::Mine => local.clone(), + Resolution::Theirs => "aRcZ".to_owned(), + }; + assert_eq!(content(&cache.snapshot().unwrap(), oid), expected); + drop(cache); + let cache = Replica::open(&path).unwrap(); + match resolution { + Resolution::Mine => { + assert!(matches!( + cache.sync_once(&mut server).unwrap().edit, + Some((_, EditStatus::Published { .. })) + )); + assert_eq!(server.publications, 1); + for edit in &pending { + assert!(matches!( + cache.status(edit.id).unwrap(), + None | Some(EditStatus::Published { .. }) + )); + } + assert!(matches!( + cache.status(first).unwrap(), + Some(EditStatus::Published { .. }) + )); + } + Resolution::Theirs => { + // The emptied edits are acknowledged once the remote image is flushed. + assert!(matches!( + cache.sync_once(&mut server).unwrap().edit, + Some((id, EditStatus::Published { .. })) if id == dependent + )); + assert_eq!((server.publications, server.confirmations), (0, 1)); + assert!(matches!( + cache.status(first).unwrap(), + Some(EditStatus::Published { .. }) + )); + } + } + assert_eq!(content(&server.durable, oid), expected); + assert_eq!(pages(&cache.snapshot().unwrap()), pages(&server.durable)); + assert!(cache.pending().unwrap().is_empty()); } - assert_eq!(server.publications, 2); - assert_eq!(text(&server.durable).2, text(&local).2 + "Z"); - assert_eq!(cache.snapshot().unwrap(), server.durable); - assert!(cache.pending().unwrap().is_empty()); } #[test] -fn conflict_review_rejects_stale_images_and_nonconflicting_states() { +fn resolution_refuses_edits_that_are_not_in_conflict() { let dir = tempfile::tempdir().unwrap(); let source = onestore::create_section("resolve.one", "abc", "Fixture").unwrap(); let (sid, oid, _) = text(&source); let cache = Replica::create(dir.path().join("cache.sqlite"), &source).unwrap(); let id = save(&cache, oid, 1..2, "L").unwrap(); - let local = cache.snapshot().unwrap(); - let pending_review = edited(&source, oid, 1..2, "L"); - assert!( - matches!(cache.review_page(id, &local, &source, &pending_review), Err(Error::Io(error)) if error.kind() == io::ErrorKind::InvalidInput) - ); + let refused = |cache: &Replica, id| matches!(cache.resolve(id, Resolution::Mine), Err(Error::Io(error)) if error.kind() == io::ErrorKind::InvalidInput); + assert!(refused(&cache, id)); let remote = onestore::replace_text(&source, sid, oid, 0..3, "🦀Rc").unwrap(); let mut server = Server::new(&remote); assert!(matches!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((_, EditStatus::Conflict(_))) )); let pending = cache.pending().unwrap(); - let reviewed = edited(&remote, oid, 2..3, "L"); - assert!( - matches!(cache.review_page(id + 1, &local, &remote, &reviewed), Err(Error::Io(error)) if error.kind() == io::ErrorKind::InvalidInput) - ); + assert!(refused(&cache, id + 1)); assert_eq!(cache.pending().unwrap(), pending); - save(&cache, oid, 0..0, "Later ").unwrap(); - let changed = cache.snapshot().unwrap(); - assert!( - matches!(cache.review_page(id, &local, &remote, &reviewed), Err(Error::Io(error)) if error.kind() == io::ErrorKind::ResourceBusy) - ); - let new_remote = onestore::replace_text(&remote, sid, oid, 2..3, "Q").unwrap(); - server.visible = new_remote.clone(); - server.durable = new_remote; - assert!(matches!( - cache.sync_once(&mut server).unwrap(), - Some((_, EditStatus::Conflict(_))) - )); - assert!( - matches!(cache.review_page(id, &changed, &remote, &reviewed), Err(Error::Io(error)) if error.kind() == io::ErrorKind::ResourceBusy) - ); - assert_eq!(cache.snapshot().unwrap(), changed); - assert_eq!(cache.pending().unwrap()[0], pending[0]); - assert_eq!(server.publications, 0); - - let current = cache.remote_snapshot().unwrap(); - let reviewed = edited(¤t, oid, 2..3, "L"); - cache - .review_page(id, &changed, ¤t, &reviewed) - .unwrap(); + cache.resolve(id, Resolution::Mine).unwrap(); server.fault = Fault::UnknownBefore; assert!( matches!(cache.sync_once(&mut server), Err(Error::Remote(error)) if error.state == CommitState::Unknown) ); let attempted = cache.status(id).unwrap(); - let pending = cache.pending().unwrap(); - assert!( - matches!(cache.review_page(id, &changed, ¤t, &reviewed), Err(Error::Io(error)) if error.kind() == io::ErrorKind::InvalidInput) - ); + assert!(matches!( + attempted, + Some(EditStatus::AwaitingConfirmation { .. }) + )); + assert!(refused(&cache, id)); assert_eq!(cache.status(id).unwrap(), attempted); - assert_eq!(cache.pending().unwrap(), pending); assert_eq!(server.publications, 1); } #[test] -fn failure_between_review_and_conflict_clear_rolls_back_the_entire_resolution() { +fn a_failed_resolution_leaves_the_conflict_as_it_was() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("cache.sqlite"); let source = onestore::create_section("resolve.one", "abc", "Fixture").unwrap(); @@ -712,30 +736,29 @@ fn failure_between_review_and_conflict_clear_rolls_back_the_entire_resolution() let remote = onestore::replace_text(&source, sid, oid, 0..3, "XaRc").unwrap(); let mut server = Server::new(&remote); cache.sync_once(&mut server).unwrap(); - let local = cache.snapshot().unwrap(); + let local = pages(&cache.snapshot().unwrap()); let pending = cache.pending().unwrap(); let status = cache.status(id).unwrap(); drop(cache); let connection = rusqlite::Connection::open(&path).unwrap(); - connection.execute_batch("CREATE TRIGGER fail_clear BEFORE DELETE ON conflicts BEGIN SELECT RAISE(ABORT, 'test conflict clear failure'); END;").unwrap(); + connection.execute_batch("CREATE TRIGGER fail_clear BEFORE DELETE ON batches BEGIN SELECT RAISE(ABORT, 'test resolution failure'); END;").unwrap(); drop(connection); let cache = Replica::open(&path).unwrap(); - let reviewed = edited(&remote, oid, 2..3, "L"); assert!(matches!( - cache.review_page(id, &local, &remote, &reviewed), + cache.resolve(id, Resolution::Mine), Err(Error::Database(_)) )); drop(cache); let cache = Replica::open(&path).unwrap(); assert_eq!(cache.pending().unwrap(), pending); assert_eq!(cache.status(id).unwrap(), status); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(pages(&cache.snapshot().unwrap()), local); assert_eq!(cache.remote_snapshot().unwrap(), remote); assert_eq!(server.publications, 0); } #[test] -fn seeded_reviews_preserve_unicode_around_the_reviewed_replacement() { +fn seeded_conflicts_resolve_to_the_remote_text_and_later_edits_publish_once() { let dir = tempfile::tempdir().unwrap(); let original: Vec<_> = "abcdefghij🦀klmnop".chars().collect(); let mut seed = 911_u64; @@ -757,8 +780,7 @@ fn seeded_reviews_preserve_unicode_around_the_reviewed_replacement() { let (sid, oid, _) = text(&source); let cache = Replica::create(dir.path().join(format!("{case}.sqlite")), &source).unwrap(); let id = save(&cache, oid, start..end, "λ🦊μ").unwrap(); - let dependent = save(&cache, oid, start + 1..start + 3, "🐕").unwrap(); - assert_eq!(dependent, id, "case {case}, seed {seed}"); + save(&cache, oid, start + 1..start + 3, "🐕").unwrap(); let mut remote_text = original.clone(); remote_text.splice(at..at + 1, "Ω🐈π".chars()); let remote_text = prefix.clone() + &remote_text.iter().collect::() + &suffix; @@ -771,18 +793,21 @@ fn seeded_reviews_preserve_unicode_around_the_reviewed_replacement() { ) .unwrap(); let mut server = Server::new(&remote); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))), + assert!( + matches!( + cache.sync_once(&mut server).unwrap().edit, + Some((_, EditStatus::Conflict(ConflictKind::ContentChanged))) + ), "case {case}, seed {seed}" ); let at_remote = start + prefix.len() as u32; - model_ops::review(&cache, id, oid, |page| { + let reviewed = model_ops::review(&cache, id, oid, |page| { model_ops::replace_text(page, oid, at_remote..at_remote + 4, "λ🐕μ") }) + .unwrap() .unwrap(); assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id), + matches!(cache.sync_once(&mut server).unwrap().edit, Some((actual, EditStatus::Published { .. })) if actual == reviewed), "case {case}, seed {seed}" ); let mut expected = original.clone(); @@ -799,7 +824,7 @@ fn seeded_reviews_preserve_unicode_around_the_reviewed_replacement() { } #[test] -fn remote_changes_after_review_cannot_be_overwritten_by_the_reviewed_page() { +fn remote_changes_after_resolution_are_never_overwritten() { struct ChangedAfterRead { server: Server, change: Option>, @@ -826,20 +851,22 @@ fn remote_changes_after_review_cannot_be_overwritten_by_the_reviewed_page() { let (sid, oid, _) = text(&source); let cache = Replica::create(dir.path().join("cache.sqlite"), &source).unwrap(); let id = save(&cache, oid, 1..2, "L").unwrap(); - let local = cache.snapshot().unwrap(); let remote = onestore::replace_text(&source, sid, oid, 1..2, "R").unwrap(); let mut remote = ChangedAfterRead { server: Server::new(&remote), change: None, }; assert!(matches!( - cache.sync_once(&mut remote).unwrap(), + cache.sync_once(&mut remote).unwrap().edit, Some((_, EditStatus::Conflict(_))) )); - model_ops::review(&cache, id, oid, |page| { + let id = model_ops::review(&cache, id, oid, |page| { model_ops::replace_text(page, oid, 1..2, "L") }) + .unwrap() .unwrap(); + let local = content(&cache.snapshot().unwrap(), oid); + assert_eq!(local, "aLc"); let changed = onestore::replace_text(&remote.server.visible, sid, oid, 1..2, "Q").unwrap(); if after_read { remote.change = Some(changed.clone()); @@ -854,20 +881,19 @@ fn remote_changes_after_review_cannot_be_overwritten_by_the_reviewed_page() { assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); } assert_eq!( - cache.sync_once(&mut remote).unwrap(), + cache.sync_once(&mut remote).unwrap().edit, Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))) ); assert_eq!(remote.server.durable, changed); assert_eq!(remote.server.visible, changed); assert_eq!(remote.server.publications, usize::from(after_read)); - assert_eq!(cache.snapshot().unwrap(), local); - let notebook::Operation::Page(intent) = &cache.pending().unwrap()[0].operation else { - panic!() - }; - assert_eq!( - intent.text_change(), - Some((oid, "aRc".into(), 1..2, "L".into())) - ); + assert_eq!(content(&cache.snapshot().unwrap(), oid), local); + assert!(cache.pending().unwrap()[0].edit.ops.is_empty()); + assert!(matches!( + &cache.pending().unwrap()[1].edit.ops[..], + [onestore::op::Op::Page { op: onestore::op::PageOp::Text { range, with, .. }, .. }] + if *range == (1..2) && with == "L" + )); } } @@ -880,7 +906,7 @@ fn remote_restore_retains_historical_receipts_without_replaying_them() { let cache = Replica::create(&path, &source).unwrap(); let mut server = Server::new(&source); let published = save(&cache, object, 0..0, "Published ").unwrap(); - let (_, receipt) = cache.sync_once(&mut server).unwrap().unwrap(); + let (_, receipt) = cache.sync_once(&mut server).unwrap().edit.unwrap(); assert!(matches!(receipt, EditStatus::Published { .. })); let published_image = cache.snapshot().unwrap(); cache @@ -891,7 +917,7 @@ fn remote_restore_retains_historical_receipts_without_replaying_them() { server.visible.clone_from(&source); server.durable.clone_from(&source); let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.sync_once(&mut server).unwrap(), None); + assert_eq!(cache.sync_once(&mut server).unwrap().edit, None); assert_eq!(cache.snapshot().unwrap(), source); assert_eq!(cache.remote_snapshot().unwrap(), source); assert_eq!(cache.status(published).unwrap(), Some(receipt.clone())); @@ -918,7 +944,7 @@ fn remote_restore_rebases_unsent_work_but_never_replays_an_uncertain_attempt() { let cache = Replica::create(&path, &source).unwrap(); let mut server = Server::new(&source); let published = save(&cache, object, 0..0, "Published ").unwrap(); - let (_, receipt) = cache.sync_once(&mut server).unwrap().unwrap(); + let (_, receipt) = cache.sync_once(&mut server).unwrap().edit.unwrap(); let end = u32::try_from(text(&cache.snapshot().unwrap()).2.encode_utf16().count()).unwrap(); let queued = save(&cache, object, end..end, " Local").unwrap(); if uncertain { @@ -936,14 +962,14 @@ fn remote_restore_rebases_unsent_work_but_never_replays_an_uncertain_attempt() { server.durable.clone_from(&source); let attempts = server.publications; let cache = Replica::open(&path).unwrap(); - let (_, status) = cache.sync_once(&mut server).unwrap().unwrap(); + let (_, status) = cache.sync_once(&mut server).unwrap().edit.unwrap(); assert_eq!(cache.status(published).unwrap(), Some(receipt.clone())); if uncertain { assert!(matches!(status, EditStatus::AwaitingConfirmation { .. })); assert_eq!(Some(status.clone()), prior_status); assert_eq!(server.publications, attempts); assert_eq!(server.visible, source); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(pages(&cache.snapshot().unwrap()), pages(&local)); assert_eq!(cache.pending().unwrap(), pending); } else { assert!(matches!(status, EditStatus::Published { .. })); @@ -953,7 +979,7 @@ fn remote_restore_rebases_unsent_work_but_never_replays_an_uncertain_attempt() { } let archive = notebook::Recovery::open(directory.path().join("before-restore.sqlite")).unwrap(); - assert_eq!(archive.snapshot().unwrap(), local); + assert_eq!(pages(&archive.snapshot().unwrap()), pages(&local)); assert_eq!(archive.pending().unwrap(), pending); assert_eq!(archive.status(queued).unwrap(), prior_status); drop(cache); @@ -964,7 +990,7 @@ fn remote_restore_rebases_unsent_work_but_never_replays_an_uncertain_attempt() { } #[test] -fn restore_conflicts_keep_dependent_work_and_reject_stale_review() { +fn restore_conflicts_keep_dependent_work_until_mine_publishes_it() { let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("cache.sqlite"); let source = onestore::create_section("restore.one", "Original", "Fixture").unwrap(); @@ -972,19 +998,20 @@ fn restore_conflicts_keep_dependent_work_and_reject_stale_review() { let cache = Replica::create(&path, &source).unwrap(); let mut server = Server::new(&source); let published = save(&cache, object, 0..0, "Published ").unwrap(); - let (_, receipt) = cache.sync_once(&mut server).unwrap().unwrap(); + let (_, receipt) = cache.sync_once(&mut server).unwrap().edit.unwrap(); let head = save(&cache, object, 0..9, "Revised").unwrap(); server.visible.clone_from(&source); server.durable.clone_from(&source); let conflict = EditStatus::Conflict(ConflictKind::ContentChanged); assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((head, conflict.clone())) ); let end = u32::try_from(text(&cache.snapshot().unwrap()).2.encode_utf16().count()).unwrap(); let dependent = save(&cache, object, end..end, " Later").unwrap(); assert_ne!(head, dependent); - let local = cache.snapshot().unwrap(); + let local = text(&cache.snapshot().unwrap()).2; + assert_eq!(local, "Revised Original Later"); let pending = cache.pending().unwrap(); assert_eq!(pending.len(), 2); cache @@ -993,60 +1020,39 @@ fn restore_conflicts_keep_dependent_work_and_reject_stale_review() { drop(cache); let cache = Replica::open(&path).unwrap(); assert_eq!(cache.status(head).unwrap(), Some(conflict.clone())); + assert_eq!(cache.status(dependent).unwrap(), Some(conflict.clone())); assert_eq!(cache.pending().unwrap(), pending); - assert_eq!(cache.snapshot().unwrap(), local); - let reviewed = edited(&source, object, 0..0, "Revised "); let changed = onestore::replace_text(&source, space, object, 8..8, " Remote").unwrap(); server.visible.clone_from(&changed); server.durable.clone_from(&changed); assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((head, conflict.clone())) - ); - assert!( - matches!(cache.review_page(head, &local, &source, &reviewed), - Err(Error::Io(error)) if error.kind() == io::ErrorKind::ResourceBusy) - ); - assert_eq!(cache.pending().unwrap(), pending); - assert_eq!(server.publications, 1); - let reviewed = edited(&changed, object, 0..0, "Revised "); - cache - .review_page(head, &local, &changed, &reviewed) - .unwrap(); - assert_eq!(cache.pending().unwrap()[1], pending[1]); - drop(cache); - let cache = Replica::open(&path).unwrap(); - assert!(matches!(cache.sync_once(&mut server).unwrap(), - Some((actual, EditStatus::Published { .. })) if actual == head)); - assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((dependent, conflict.clone())) ); - assert_eq!(server.publications, 2); - let dependent_local = cache.snapshot().unwrap(); - let current = cache.remote_snapshot().unwrap(); - let reviewed = edited(¤t, object, 16..16, " Later"); - cache - .review_page(dependent, &dependent_local, ¤t, &reviewed) - .unwrap(); + assert_eq!(cache.remote_snapshot().unwrap(), changed); + assert_eq!(server.publications, 1); + cache.resolve(head, Resolution::Mine).unwrap(); drop(cache); let cache = Replica::open(&path).unwrap(); - assert!(matches!(cache.sync_once(&mut server).unwrap(), - Some((actual, EditStatus::Published { .. })) if actual == dependent)); - assert_eq!(text(&server.durable).2, "Revised Original Later Remote"); + assert!(matches!( + cache.sync_once(&mut server).unwrap().edit, + Some((_, EditStatus::Published { .. })) + )); + assert_eq!(text(&server.durable).2, local); + assert!(matches!( + cache.status(head).unwrap(), + Some(EditStatus::Published { .. }) + )); assert_eq!(cache.status(published).unwrap(), Some(receipt.clone())); let archive = notebook::Recovery::open(directory.path().join("conflicted.sqlite")).unwrap(); - assert_eq!(archive.snapshot().unwrap(), local); + assert_eq!(text(&archive.snapshot().unwrap()).2, local); assert_eq!(archive.pending().unwrap(), pending); assert_eq!(archive.status(head).unwrap(), Some(conflict.clone())); } #[test] fn cache_open_validates_the_tail_before_head_only_synchronization() { - for damage in [ - "operation='{}'", - "space=(SELECT space FROM edits ORDER BY id LIMIT 1)", - ] { + for damage in ["edit='{}'", "edit=replace(edit, 'Create', 'Delete')"] { let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("cache.sqlite"); let source = onestore::create_section("tail.one", "Original", "Fixture").unwrap(); @@ -1054,10 +1060,7 @@ fn cache_open_validates_the_tail_before_head_only_synchronization() { let cache = Replica::create(&path, &source).unwrap(); save(&cache, object, 0..0, "Head ").unwrap(); let page = onestore::PageCreation::new(None, Some("Tail"), "Fixture").unwrap(); - let tail = cache - .create_page(&cache.snapshot().unwrap(), &page) - .unwrap() - .unwrap(); + let tail = section_op(&cache, onestore::op::SectionOp::Create(page)); assert_eq!(cache.pending().unwrap().len(), 2); drop(cache); let database = rusqlite::Connection::open(&path).unwrap(); diff --git a/crates/notebook/tests/sync/tree.rs b/crates/notebook/tests/sync/tree.rs index abd11b017dfc1385a44051dac995bd6976d97cde..41aac56061497585288e7811cef9426685ea691c 100644 --- a/crates/notebook/tests/sync/tree.rs +++ b/crates/notebook/tests/sync/tree.rs @@ -7,12 +7,7 @@ use onestore::page::{Paragraph, ParagraphContent, TableCell}; /// Saves an edited model of `space`, reaching content that outline helpers cannot. fn save_page(cache: &Replica, space: ExGuid, edit: impl FnOnce(&mut Page)) -> Option { - let source = cache.snapshot().unwrap(); - let mut page = page_of(&source, space); - edit(&mut page); - cache - .save(&source, space, &page, model_ops::AUTHOR) - .unwrap() + model_ops::save_page(cache, space, edit).unwrap() } #[test] @@ -97,7 +92,7 @@ fn move_preserves_remote_content_and_dependent_edits_through_reopen() { drop(cache); let cache = Replica::open(&path).unwrap(); assert_eq!(cache.pending().unwrap(), queue); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(pages(&cache.snapshot().unwrap()), pages(&local)); let mut server = Server::new(&remote_with(&source, space, |page| { insert_child(page, paragraphs[0], "New remote child"); restyle(page, texts[1], 0..7, |format| format.bold = Some(true)); @@ -169,7 +164,7 @@ fn native_empty_child_list_normalization_merges_with_a_local_deletion() { let cache = Replica::open(&path).unwrap(); let mut server = Server::new(native); assert!(matches!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((published, EditStatus::Published { .. })) if published == id )); assert_eq!(server.publications, 1); @@ -224,18 +219,12 @@ fn deletion_requires_review_of_remote_content_and_preserves_later_work() { let local = cache.snapshot().unwrap(); let mut server = Server::new(&remote_with(&source, space, remote)); assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))) ); assert_eq!(server.publications, 0); assert_eq!(cache.pending().unwrap(), queue); - assert_eq!(cache.snapshot().unwrap(), local); - let observed = cache.remote_snapshot().unwrap(); - assert!( - cache - .review_page(id, &source, &observed, &page_of(&observed, space)) - .is_err() - ); + assert_eq!(pages(&cache.snapshot().unwrap()), pages(&local)); drop(cache); let cache = Replica::open(&path).unwrap(); assert_eq!( @@ -270,9 +259,10 @@ fn sibling_and_ancestry_changes_have_explicit_merge_or_conflict() { Box::new(move |page| delete_paragraph(page, texts[1])), Some(&[2, 3, 0]), ), + // Both moved a paragraph last; the local move replays after the remote one. ( Box::new(move |page| move_subtree(page, paragraphs[1], None, None)), - Some(&[2, 3, 0, 1]), + Some(&[2, 3, 1, 0]), ), ( Box::new(move |page| move_subtree(page, paragraphs[0], Some(paragraphs[1]), None)), @@ -289,7 +279,7 @@ fn sibling_and_ancestry_changes_have_explicit_merge_or_conflict() { .unwrap() .unwrap(); let mut server = Server::new(&remote_with(&source, space, remote)); - let result = cache.sync_once(&mut server).unwrap().unwrap(); + let result = cache.sync_once(&mut server).unwrap().edit.unwrap(); assert_eq!(result.0, id); let Some(expected) = expected else { assert_eq!(result.1, EditStatus::Conflict(ConflictKind::ContentChanged)); @@ -361,9 +351,14 @@ fn cell_mut(page: &mut Page, id: ExGuid) -> &mut TableCell { .expect("the cell is on the page") } -/// Moves a cell's sole paragraph to the end of another cell, or deletes it. +/// Moves a cell's sole paragraph to the end of another cell, or deletes it, leaving the +/// cell an empty paragraph as the writers do. fn empty_cell(page: &mut Page, cell: ExGuid, destination: Option) { - let moved = cell_mut(page, cell).paragraphs.remove(0); + let emptied = cell_mut(page, cell); + let mut replacement = model_ops::fresh_paragraph(&emptied.paragraphs[0], ""); + replacement.level = emptied.paragraphs[0].level; + emptied.paragraphs.insert(0, replacement); + let moved = cell_mut(page, cell).paragraphs.remove(1); let Some(destination) = destination else { return; }; @@ -436,7 +431,7 @@ fn emptied_cell_replacement_is_durable_and_cannot_be_silently_omitted_on_replay( drop(cache); let cache = Replica::open(&path).unwrap(); assert_eq!(cache.pending().unwrap(), queue); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(pages(&cache.snapshot().unwrap()), pages(&local)); let mut server = Server::new(&if competing { remote_with(source, space, |page| { let target = cell_mut(page, cell); @@ -452,13 +447,21 @@ fn emptied_cell_replacement_is_durable_and_cannot_be_silently_omitted_on_replay( }) }); if competing { - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))) + // The remote's new paragraph and the replacement both stay; the target goes. + published(&cache, &mut server, id); + let durable = page_of(&server.durable, space); + let texts: Vec<_> = cell_of(&durable, cell) + .paragraphs + .iter() + .map(|paragraph| paragraph.text().unwrap()) + .collect(); + assert_eq!(texts.len(), 2); + assert!(texts.iter().all(|text| text.id != target)); + assert!( + texts + .iter() + .any(|text| text.text.text() == "Remote sibling") ); - assert_eq!(server.publications, 0); - assert_eq!(cache.pending().unwrap(), queue); - assert_eq!(cache.snapshot().unwrap(), local); continue; } published(&cache, &mut server, id); @@ -573,7 +576,7 @@ fn native_tree_and_layout_changes_reconcile_without_discarding_unreviewed_conten let id = save(&cache, dependent, save_both).unwrap().unwrap(); let local = cache.snapshot().unwrap(); let queue = cache.pending().unwrap(); - let merged = match cache.sync_once(&mut server).unwrap().unwrap() { + let merged = match cache.sync_once(&mut server).unwrap().edit.unwrap() { (actual, EditStatus::Published { .. }) => { assert_eq!(actual, id, "{name}"); true @@ -585,7 +588,7 @@ fn native_tree_and_layout_changes_reconcile_without_discarding_unreviewed_conten "{name}" ); assert_eq!(cache.pending().unwrap(), queue); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(pages(&cache.snapshot().unwrap()), pages(&local)); drop(cache); let cache = Replica::open(&path).unwrap(); review(&cache, id, dependent, save_both).unwrap(); diff --git a/crates/notebook/tests/sync_formatting.rs b/crates/notebook/tests/sync_formatting.rs index 4396f27daef2375976ad5cb673f4f125fc0c4271..dca00aa6e1511bafceb8e6c0f5b63620c83d7fbf 100644 --- a/crates/notebook/tests/sync_formatting.rs +++ b/crates/notebook/tests/sync_formatting.rs @@ -10,7 +10,7 @@ use onestore::{ #[path = "support/server.rs"] mod server; -use server::{Fault, Server}; +use server::{Fault, Server, pages}; #[path = "support/model_ops.rs"] mod model_ops; use model_ops::{ @@ -103,7 +103,7 @@ fn offline_formatting_merges_with_a_remote_edit_to_another_paragraph() { .unwrap(); let mut server = remote_with(space, |page| replace_text(page, ids[3], 0..0, "Remote ")); assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) + matches!(cache.sync_once(&mut server).unwrap().edit, Some((actual, EditStatus::Published { .. })) if actual == id) ); let published = page_of(&server.durable, space); assert!( @@ -116,7 +116,7 @@ fn offline_formatting_merges_with_a_remote_edit_to_another_paragraph() { } #[test] -fn offline_formatting_conflicts_with_any_remote_change_to_its_own_paragraph() { +fn offline_formatting_conflicts_only_with_remote_changes_it_overlaps() { let (space, page) = page_titled(OUTLINES, PAGE); let ids = plain(&page); for typed in [false, true] { @@ -135,13 +135,28 @@ fn offline_formatting_conflicts_with_any_remote_change_to_its_own_paragraph() { restyle(page, ids[0], 0..3, |format| format.italic = Some(true)); } }); - assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))), - "typed {typed}" - ); - assert_eq!(server.publications, 0); - assert_eq!(cache.snapshot().unwrap(), local); + let outcome = cache.sync_once(&mut server).unwrap().edit; + if typed { + // Text typed before the formatted range moves it; the ranges do not meet. + assert!(matches!(outcome, Some((n, EditStatus::Published { .. })) if n == id)); + let published = page_of(&server.durable, space); + assert!(text_of(&published, ids[0]).starts_with("Remote Anchor")); + let styles = formats(&published, ids[0]); + for (offset, format) in styles.iter().enumerate() { + assert_eq!( + format.bold == Some(true), + (7..13).contains(&offset), + "{offset}" + ); + } + } else { + assert_eq!( + outcome, + Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))) + ); + assert_eq!(server.publications, 0); + assert_eq!(pages(&cache.snapshot().unwrap()), pages(&local)); + } } } @@ -163,7 +178,7 @@ fn competing_formatting_of_one_paragraph_conflicts_until_reviewed() { }); for _ in 0..2 { assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))) ); } @@ -179,7 +194,7 @@ fn competing_formatting_of_one_paragraph_conflicts_until_reviewed() { }) .unwrap(); assert!(matches!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((_, EditStatus::Published { .. })) )); let published = formats(&page_of(&server.durable, space), ids[0]); @@ -223,7 +238,7 @@ fn every_visual_attribute_publishes_through_a_page_model_save() { .unwrap(); let mut server = Server::new(OUTLINES); assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id), + matches!(cache.sync_once(&mut server).unwrap().edit, Some((actual, EditStatus::Published { .. })) if actual == id), "{name}" ); let mut expected = formats(&page_of(OUTLINES, space), ids[0]); @@ -282,9 +297,21 @@ fn seeded_formatting_reconciles_exactly_when_the_remote_left_the_paragraph_alone } } }); - let outcome = cache.sync_once(&mut server).unwrap().unwrap(); + let remote_page = page_of(&server.durable, space); + let outcome = cache.sync_once(&mut server).unwrap().edit.unwrap(); assert_eq!(outcome.0, id, "seed {seed}"); - if remote.contains(&target) { + // Bullets the remote typed before the target's text, and whether it underlined a + // character the target already had. + let mut bullets = vec![0_u32; ids.len()]; + let mut underlined = vec![false; ids.len()]; + for (step, &at) in remote.iter().enumerate() { + if step % 2 == 0 { + bullets[at] += 1; + } else if bullets[at] == 0 { + underlined[at] = true; + } + } + if underlined[target] && start == 0 { conflicts += 1; assert_eq!( outcome.1, @@ -292,7 +319,11 @@ fn seeded_formatting_reconciles_exactly_when_the_remote_left_the_paragraph_alone "seed {seed}" ); assert_eq!(server.publications, 0, "seed {seed}"); - assert_eq!(cache.snapshot().unwrap(), local, "seed {seed}"); + assert_eq!( + pages(&cache.snapshot().unwrap()), + pages(&local), + "seed {seed}" + ); continue; } published += 1; @@ -301,8 +332,9 @@ fn seeded_formatting_reconciles_exactly_when_the_remote_left_the_paragraph_alone "seed {seed}: {outcome:?}" ); let page = page_of(&server.durable, space); - let mut expected: Vec = formats(&page_of(OUTLINES, space), ids[target]); - for format in &mut expected[start as usize..end as usize] { + let shift = bullets[target] as usize; + let mut expected: Vec = formats(&remote_page, ids[target]); + for format in &mut expected[start as usize + shift..end as usize + shift] { attribute(format); } assert_eq!(formats(&page, ids[target]), expected, "seed {seed}"); @@ -316,7 +348,7 @@ fn seeded_formatting_reconciles_exactly_when_the_remote_left_the_paragraph_alone assert_eq!(cache.snapshot().unwrap(), server.durable, "seed {seed}"); } assert!( - published >= 16 && conflicts >= 16, + published >= 16, "{published} published, {conflicts} conflicts" ); } @@ -338,11 +370,11 @@ fn offline_insertions_survive_reopen_and_carry_their_formatting() { let pending = cache.pending().unwrap(); drop(cache); cache = Replica::open(&path).unwrap(); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(pages(&cache.snapshot().unwrap()), pages(&local)); assert_eq!(cache.pending().unwrap(), pending); let mut server = remote_with(space, |page| replace_text(page, ids[3], 0..0, "Remote ")); assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) + matches!(cache.sync_once(&mut server).unwrap().edit, Some((actual, EditStatus::Published { .. })) if actual == id) ); let published = page_of(&server.durable, space); let inserted = plain(&published) @@ -361,7 +393,7 @@ fn offline_insertions_survive_reopen_and_carry_their_formatting() { .unwrap() .unwrap(); assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == styled) + matches!(cache.sync_once(&mut server).unwrap().edit, Some((actual, EditStatus::Published { .. })) if actual == styled) ); let styles = formats(&page_of(&server.durable, space), inserted); for (offset, format) in styles.iter().enumerate() { @@ -395,17 +427,17 @@ fn uncertain_formatting_attempts_keep_the_original_attempt_and_never_replay() { if matches!(fault, Fault::UnknownBefore) { for _ in 0..3 { assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((id, state.clone())) ); } assert_eq!(server.publications, 1); assert_eq!(server.durable, OUTLINES); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(pages(&cache.snapshot().unwrap()), pages(&local)); continue; } assert!(matches!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((_, EditStatus::Published { .. })) )); assert_eq!(server.publications, 1); @@ -438,7 +470,7 @@ fn a_format_the_remote_already_carries_is_confirmed_without_republishing() { assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); assert_eq!(server.durable, OUTLINES); assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) + matches!(cache.sync_once(&mut server).unwrap().edit, Some((actual, EditStatus::Published { .. })) if actual == id) ); assert_eq!((server.publications, server.confirmations), (0, 2)); assert!(cache.pending().unwrap().is_empty()); @@ -459,8 +491,7 @@ fn a_retired_attempt_confirms_only_once_the_remote_page_equals_it() { restyle(&mut after, ids[0], 0..6, |format| format.bold = Some(true)); let directory = tempfile::tempdir().unwrap(); let cache = cache(&directory); - let id = cache - .save(OUTLINES, space, &after, AUTHOR) + let id = model_ops::save_as(&cache, space, &after, AUTHOR) .unwrap() .unwrap(); let mut server = Server::new(OUTLINES); @@ -476,7 +507,7 @@ fn a_retired_attempt_confirms_only_once_the_remote_page_equals_it() { }); server.visible.clone_from(&partial.visible); assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, attempt.map(|state| (id, state)) ); assert_eq!(server.confirmations, 0); @@ -486,7 +517,7 @@ fn a_retired_attempt_confirms_only_once_the_remote_page_equals_it() { server.visible.clone_from(&complete.visible); assert_eq!(page_of(&server.visible, space), after); assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) + matches!(cache.sync_once(&mut server).unwrap().edit, Some((actual, EditStatus::Published { .. })) if actual == id) ); assert_eq!((server.publications, server.confirmations), (1, 1)); assert!(cache.pending().unwrap().is_empty()); @@ -497,7 +528,10 @@ fn a_formatted_save_of_an_unchanged_model_queues_nothing() { let (space, page) = page_titled(OUTLINES, PAGE); let directory = tempfile::tempdir().unwrap(); let cache = cache(&directory); - assert_eq!(cache.save(OUTLINES, space, &page, AUTHOR).unwrap(), None); + assert_eq!( + model_ops::save_as(&cache, space, &page, AUTHOR).unwrap(), + None + ); assert!(cache.pending().unwrap().is_empty()); } @@ -533,7 +567,7 @@ fn paragraph_formatting_survives_reopen_rebase_and_a_lost_publication_reply() { drop(cache); let cache = Replica::open(directory.path().join("cache.sqlite")).unwrap(); assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) + matches!(cache.sync_once(&mut server).unwrap().edit, Some((actual, EditStatus::Published { .. })) if actual == id) ); assert_eq!(server.publications, 1); let page = page_of(&server.durable, space); diff --git a/crates/notebook/tests/sync_model.rs b/crates/notebook/tests/sync_model.rs index b22176a9e37953ba0df47945b9557659951ac98d..5132e5f0cb78f260ebda6bb0a5091b5e7e9331ad 100644 --- a/crates/notebook/tests/sync_model.rs +++ b/crates/notebook/tests/sync_model.rs @@ -10,7 +10,6 @@ use server::*; #[path = "support/model_ops.rs"] mod model_ops; -use notebook::{Operation, PageIntent}; use onestore::page::{Page, PageObject}; const OUTLINES: &[u8] = @@ -77,70 +76,58 @@ fn a_saved_page_model_publishes_and_reads_back() { let directory = tempfile::tempdir().unwrap(); let cache = Replica::create(directory.path().join("cache.sqlite"), OUTLINES).unwrap(); append(&mut after, " saved 🦀"); - let id = cache - .save(OUTLINES, space, &after, "Model author") + let id = model_ops::save_as(&cache, space, &after, "Model author") .unwrap() .unwrap(); assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); assert_eq!(page_of(&cache.snapshot().unwrap(), space), after); let pending = cache.pending().unwrap(); assert_eq!(pending.len(), 1); - let Operation::Page(PageIntent { - before, - after: stored, - author, - }) = &pending[0].operation - else { - panic!() - }; - assert_eq!( - (before, stored, author.as_str()), - (&page_of(OUTLINES, space), &after, "Model author") + assert_eq!(pending[0].author, "Model author"); + assert!( + pending[0] + .edit + .ops + .iter() + .all(|op| matches!(op, onestore::op::Op::Page { space: s, .. } if *s == space)) ); let mut server = Server::new(OUTLINES); assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == id) + matches!(cache.sync_once(&mut server).unwrap().edit, Some((n, EditStatus::Published { .. })) if n == id) ); assert_eq!(page_of(&server.durable, space), after); assert_eq!(server.publications, 1); assert!(cache.pending().unwrap().is_empty()); assert_eq!( - cache - .save(&cache.snapshot().unwrap(), space, &after, "Model author") - .unwrap(), + model_ops::save_as(&cache, space, &after, "Model author").unwrap(), None ); } #[test] -fn saves_before_an_attempt_coalesce_into_one_publication() { +fn saves_before_an_attempt_publish_together() { let (space, mut after) = page_titled(OUTLINES, "Move leaf down"); let directory = tempfile::tempdir().unwrap(); let cache = Replica::create(directory.path().join("cache.sqlite"), OUTLINES).unwrap(); append(&mut after, " one"); - let id = cache - .save(OUTLINES, space, &after, "Model author") + let id = model_ops::save_as(&cache, space, &after, "Model author") .unwrap() .unwrap(); let mut later = page_of(&cache.snapshot().unwrap(), space); append(&mut later, " two"); - assert_eq!( - cache - .save(&cache.snapshot().unwrap(), space, &later, "Model author") - .unwrap(), - Some(id) - ); - let pending = cache.pending().unwrap(); - assert_eq!(pending.len(), 1); - let Operation::Page(intent) = &pending[0].operation else { - panic!() - }; - assert_eq!(intent.before, page_of(OUTLINES, space)); - assert_eq!(intent.after, later); + let second = model_ops::save_as(&cache, space, &later, "Model author") + .unwrap() + .unwrap(); + assert!(second > id); + assert_eq!(cache.pending().unwrap().len(), 2); let mut server = Server::new(OUTLINES); assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == id) + matches!(cache.sync_once(&mut server).unwrap().edit, Some((n, EditStatus::Published { .. })) if n == second) ); + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::Published { .. }) + )); assert_eq!(server.publications, 1); assert_eq!( first_text(&page_of(&server.durable, space)), @@ -148,11 +135,10 @@ fn saves_before_an_attempt_coalesce_into_one_publication() { ); let mut third = page_of(&cache.snapshot().unwrap(), space); append(&mut third, " three"); - let next = cache - .save(&cache.snapshot().unwrap(), space, &third, "Model author") + let next = model_ops::save_as(&cache, space, &third, "Model author") .unwrap() .unwrap(); - assert!(next > id); + assert!(next > second); } #[test] @@ -161,8 +147,7 @@ fn an_attempted_save_is_never_rewritten() { let directory = tempfile::tempdir().unwrap(); let cache = Replica::create(directory.path().join("cache.sqlite"), OUTLINES).unwrap(); append(&mut after, " uncertain"); - let id = cache - .save(OUTLINES, space, &after, "Model author") + let id = model_ops::save_as(&cache, space, &after, "Model author") .unwrap() .unwrap(); let mut server = Server::new(OUTLINES); @@ -174,21 +159,18 @@ fn an_attempted_save_is_never_rewritten() { )); let mut more = page_of(&cache.snapshot().unwrap(), space); append(&mut more, " more"); - let next = cache - .save(&cache.snapshot().unwrap(), space, &more, "Model author") + let next = model_ops::save_as(&cache, space, &more, "Model author") .unwrap() .unwrap(); assert!(next > id); let pending = cache.pending().unwrap(); assert_eq!(pending.iter().map(|p| p.id).collect::>(), [id, next]); - let Operation::Page(first) = &pending[0].operation else { - panic!() - }; - assert_eq!(first.after, after); - let Operation::Page(second) = &pending[1].operation else { - panic!() - }; - assert_eq!((&second.before, &second.after), (&after, &more)); + assert_eq!(page_of(&cache.snapshot().unwrap(), space), more); + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::AwaitingConfirmation { .. }) + )); + assert_eq!(cache.status(next).unwrap(), Some(EditStatus::Pending)); } #[test] @@ -197,8 +179,7 @@ fn a_remote_change_to_the_saved_page_conflicts_until_reviewed() { let directory = tempfile::tempdir().unwrap(); let cache = Replica::create(directory.path().join("cache.sqlite"), OUTLINES).unwrap(); append(&mut after, " local"); - let id = cache - .save(OUTLINES, space, &after, "Model author") + let id = model_ops::save_as(&cache, space, &after, "Model author") .unwrap() .unwrap(); let mut remote_page = page_of(OUTLINES, space); @@ -206,26 +187,42 @@ fn a_remote_change_to_the_saved_page_conflicts_until_reviewed() { let remote = PreparedEdit::page(OUTLINES, space, &remote_page, "Native author").unwrap(); let mut server = Server::new(remote.as_bytes()); assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))) ); assert_eq!(server.publications, 0); assert!(cache.pending().unwrap().len() == 1); - let local = cache.snapshot().unwrap(); let observed = cache.remote_snapshot().unwrap(); assert_eq!(observed, remote.as_bytes()); let mut reviewed = page_of(&observed, space); append(&mut reviewed, " local"); - assert!( - cache - .review_page(id + 1, &local, &observed, &reviewed) - .is_err() - ); - assert!(cache.review_page(id, &local, OUTLINES, &reviewed).is_err()); - cache.review_page(id, &local, &observed, &reviewed).unwrap(); + assert!(cache.resolve(id + 1, notebook::Resolution::Theirs).is_err()); + let text = model_ops::paragraph_with(&reviewed, { + let outline = reviewed + .objects + .iter() + .find_map(|object| match object { + PageObject::Outline(outline) => Some(outline), + _ => None, + }) + .unwrap(); + outline.paragraphs[0].text().unwrap().id + }) + .unwrap() + .text() + .unwrap() + .id; + let reviewed_id = model_ops::review(&cache, id, text, |page| append(page, " local")) + .unwrap() + .unwrap(); + // The conflicted edit stays, emptied, and publishes with the reviewed one. assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); + assert_eq!( + cache.status(reviewed_id).unwrap(), + Some(EditStatus::Pending) + ); assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == id) + matches!(cache.sync_once(&mut server).unwrap().edit, Some((n, EditStatus::Published { .. })) if n == reviewed_id) ); assert_eq!( first_text(&page_of(&server.durable, space)), @@ -241,15 +238,14 @@ fn remote_changes_to_other_pages_do_not_conflict() { let directory = tempfile::tempdir().unwrap(); let cache = Replica::create(directory.path().join("cache.sqlite"), OUTLINES).unwrap(); append(&mut after, " local"); - let id = cache - .save(OUTLINES, space, &after, "Model author") + let id = model_ops::save_as(&cache, space, &after, "Model author") .unwrap() .unwrap(); append(&mut other_page, " elsewhere"); let remote = PreparedEdit::page(OUTLINES, other, &other_page, "Native author").unwrap(); let mut server = Server::new(remote.as_bytes()); assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == id) + matches!(cache.sync_once(&mut server).unwrap().edit, Some((n, EditStatus::Published { .. })) if n == id) ); assert_eq!(page_of(&server.durable, space), after); assert_eq!(page_of(&server.durable, other), other_page); @@ -263,8 +259,7 @@ fn pending_saves_survive_reopening_the_cache() { let path = directory.path().join("cache.sqlite"); let cache = Replica::create(&path, OUTLINES).unwrap(); append(&mut after, " durable"); - let id = cache - .save(OUTLINES, space, &after, "Model author") + let id = model_ops::save_as(&cache, space, &after, "Model author") .unwrap() .unwrap(); let pending = cache.pending().unwrap(); @@ -327,13 +322,12 @@ fn concurrent_edits_to_different_paragraphs_merge() { let directory = tempfile::tempdir().unwrap(); let cache = Replica::create(directory.path().join("cache.sqlite"), OUTLINES).unwrap(); edit_paragraph(&mut after, 0, 0..0, "Local "); - let id = cache - .save(OUTLINES, space, &after, "Model author") + let id = model_ops::save_as(&cache, space, &after, "Model author") .unwrap() .unwrap(); let mut server = remote_with(space, |page| edit_paragraph(page, 2, 0..0, "Remote ")); assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == id) + matches!(cache.sync_once(&mut server).unwrap().edit, Some((n, EditStatus::Published { .. })) if n == id) ); let published = texts(&page_of(&server.durable, space)); assert!(published[0].starts_with("Local Anchor"), "{published:?}"); @@ -353,13 +347,12 @@ fn concurrent_edits_to_one_paragraph_merge_unless_their_ranges_overlap() { let directory = tempfile::tempdir().unwrap(); let cache = Replica::create(directory.path().join("cache.sqlite"), OUTLINES).unwrap(); append(&mut after, " local"); - let id = cache - .save(OUTLINES, space, &after, "Model author") + let id = model_ops::save_as(&cache, space, &after, "Model author") .unwrap() .unwrap(); let mut server = remote_with(space, |page| edit_paragraph(page, 0, 0..0, "Remote ")); assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == id) + matches!(cache.sync_once(&mut server).unwrap().edit, Some((n, EditStatus::Published { .. })) if n == id) ); assert_eq!( texts(&page_of(&server.durable, space))[0], @@ -369,13 +362,12 @@ fn concurrent_edits_to_one_paragraph_merge_unless_their_ranges_overlap() { let cache = Replica::create(directory.path().join("overlap.sqlite"), OUTLINES).unwrap(); let mut after = page_of(OUTLINES, space); edit_paragraph(&mut after, 0, 0..6, "Local"); - let id = cache - .save(OUTLINES, space, &after, "Model author") + let id = model_ops::save_as(&cache, space, &after, "Model author") .unwrap() .unwrap(); let mut server = remote_with(space, |page| edit_paragraph(page, 0, 0..6, "Remote")); assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))) ); assert_eq!(server.publications, 0); @@ -408,8 +400,7 @@ fn a_local_insertion_merges_with_a_remote_deletion_elsewhere() { ); outline.paragraphs.insert(1, fresh); } - let id = cache - .save(OUTLINES, space, &after, "Model author") + let id = model_ops::save_as(&cache, space, &after, "Model author") .unwrap() .unwrap(); let mut server = remote_with(space, |page| { @@ -424,7 +415,7 @@ fn a_local_insertion_merges_with_a_remote_deletion_elsewhere() { outline.paragraphs.pop(); }); assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == id) + matches!(cache.sync_once(&mut server).unwrap().edit, Some((n, EditStatus::Published { .. })) if n == id) ); assert_eq!( texts(&page_of(&server.durable, space)), @@ -442,8 +433,7 @@ fn a_remote_deletion_of_the_edited_paragraph_conflicts() { let directory = tempfile::tempdir().unwrap(); let cache = Replica::create(directory.path().join("cache.sqlite"), OUTLINES).unwrap(); edit_paragraph(&mut after, 2, 0..0, "Local "); - let id = cache - .save(OUTLINES, space, &after, "Model author") + let id = model_ops::save_as(&cache, space, &after, "Model author") .unwrap() .unwrap(); let mut server = remote_with(space, |page| { @@ -458,7 +448,7 @@ fn a_remote_deletion_of_the_edited_paragraph_conflicts() { outline.paragraphs.pop(); }); assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))) ); } @@ -481,13 +471,12 @@ fn outline_moves_merge_with_remote_text_edits_but_not_with_remote_moves() { outline.layout.y = Some(300.0); outline.id }; - let id = cache - .save(OUTLINES, space, &after, "Model author") + let id = model_ops::save_as(&cache, space, &after, "Model author") .unwrap() .unwrap(); let mut server = remote_with(space, |page| edit_paragraph(page, 0, 0..0, "Remote ")); assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == id) + matches!(cache.sync_once(&mut server).unwrap().edit, Some((n, EditStatus::Published { .. })) if n == id) ); let published = page_of(&server.durable, space); let PageObject::Outline(outline) = published @@ -512,8 +501,7 @@ fn outline_moves_merge_with_remote_text_edits_but_not_with_remote_moves() { ); let cache = Replica::create(directory.path().join("moves.sqlite"), OUTLINES).unwrap(); - let id = cache - .save(OUTLINES, space, &after, "Model author") + let id = model_ops::save_as(&cache, space, &after, "Model author") .unwrap() .unwrap(); let mut server = remote_with(space, |page| { @@ -528,7 +516,7 @@ fn outline_moves_merge_with_remote_text_edits_but_not_with_remote_moves() { outline.layout.x = Some(50.0); }); assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))) ); } @@ -565,13 +553,12 @@ fn a_local_bullet_merges_with_a_remote_text_edit_and_publishes_its_definition() }) .unwrap(); outline.paragraphs[0].lists = vec![bullet]; - let id = cache - .save(OUTLINES, space, &after, "Model author") + let id = model_ops::save_as(&cache, space, &after, "Model author") .unwrap() .unwrap(); let mut server = remote_with(space, |page| edit_paragraph(page, 1, 0..0, "Remote ")); assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((n, EditStatus::Published { .. })) if n == id) + matches!(cache.sync_once(&mut server).unwrap().edit, Some((n, EditStatus::Published { .. })) if n == id) ); let published = page_of(&server.durable, space); let outline = published diff --git a/crates/notebook/tests/sync_outline.rs b/crates/notebook/tests/sync_outline.rs index f83c7b95204621a9e4716416b987c6ce416a4354..f136c756bf1812076e9fc045271e37caacc0d554 100644 --- a/crates/notebook/tests/sync_outline.rs +++ b/crates/notebook/tests/sync_outline.rs @@ -11,7 +11,7 @@ use serde_json::json; #[path = "support/server.rs"] mod server; -use server::{Fault, Server}; +use server::{Fault, Server, pages}; #[path = "support/model_ops.rs"] mod model_ops; use model_ops::{ @@ -105,10 +105,21 @@ pub(crate) fn remote_with(source: &[u8], space: ExGuid, change: impl FnOnce(&mut .to_vec() } +/// Publishes the batch holding `id`, unless an earlier step already did. pub(crate) fn published(cache: &Replica, server: &mut Server, id: u64) { - assert!( - matches!(cache.sync_once(server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) - ); + if !matches!( + cache.status(id).unwrap(), + Some(EditStatus::Published { .. }) + ) { + assert!(matches!( + cache.sync_once(server).unwrap().edit, + Some((_, EditStatus::Published { .. })) + )); + } + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::Published { .. }) + )); } fn moved(x: f32, y: f32) -> impl Fn(&mut Outline) { @@ -144,7 +155,7 @@ fn layout_and_dependent_text_survive_reopen_and_a_remote_format_of_the_same_page drop(cache); let cache = Replica::open(&path).unwrap(); assert_eq!(cache.pending().unwrap(), pending); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(pages(&cache.snapshot().unwrap()), pages(&local)); let mut server = Server::new(&remote_with(&source, space, |page| { restyle(page, text_id, 0..8, |format| format.bold = Some(true)); })); @@ -232,7 +243,7 @@ fn an_uncertain_move_confirms_from_the_remote_revision_and_adopts_the_remote_tit } #[test] -fn competing_layout_requires_review_against_the_current_remote_page() { +fn competing_layout_conflicts_until_the_local_page_is_kept() { let (source, space, outline, text_id) = fixture(); let cases: [(Change, Change); 2] = [ (Box::new(moved(180.0, 216.0)), Box::new(moved(288.0, 216.0))), @@ -255,41 +266,25 @@ fn competing_layout_requires_review_against_the_current_remote_page() { remote_change(outline_mut(page, outline)); })); assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))) ); assert_eq!(server.publications, 0); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(pages(&cache.snapshot().unwrap()), pages(&local)); let dependent = save(&cache, text_id, |page| { replace_text(page, text_id, 0..0, "Local ") }) .unwrap() .unwrap(); - let queue = cache.pending().unwrap(); let local = cache.snapshot().unwrap(); - let observed = cache.remote_snapshot().unwrap(); - assert!( - cache - .review_page(id, &source, &observed, &page_of(&observed, space)) - .is_err() - ); - assert!( - cache - .review_page(id, &local, &source, &page_of(&source, space)) - .is_err() - ); drop(cache); let cache = Replica::open(&path).unwrap(); assert_eq!( cache.status(id).unwrap(), Some(EditStatus::Conflict(ConflictKind::ContentChanged)) ); - review(&cache, id, text_id, |page| { - local_change(outline_mut(page, outline)) - }) - .unwrap(); + cache.resolve(id, notebook::Resolution::Mine).unwrap(); assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); - assert_eq!(cache.pending().unwrap()[1], queue[1]); drop(cache); let cache = Replica::open(&path).unwrap(); for id in [id, dependent] { @@ -354,7 +349,7 @@ fn twelve_offline_writers_preserve_all_layout_intents_through_review_and_restart let expected = page_of(&cache.snapshot().unwrap(), space); drop(cache); let cache = Replica::open(&path).unwrap(); - if cache.sync_once(&mut server).unwrap() + if cache.sync_once(&mut server).unwrap().edit == Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))) { review(&cache, id, text_id, change).unwrap(); @@ -411,17 +406,17 @@ fn an_uncertain_layout_attempt_confirms_by_revision_or_by_an_equal_remote_page() continue; } assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((id, status.clone())) ); server.visible = remote_with(&source, space, |page| { resized(216.0, true)(outline_mut(page, outline)); }); assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((id, status.clone())) ); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(pages(&cache.snapshot().unwrap()), pages(&local)); server.visible = remote_with(&source, space, |page| { resized(144.0, true)(outline_mut(page, outline)); }); @@ -503,7 +498,7 @@ fn native_moves_deletions_and_layout_changes_merge_or_require_review() { let id = save(&cache, dependent, save_both).unwrap().unwrap(); let local = cache.snapshot().unwrap(); let queue = cache.pending().unwrap(); - let merged = match cache.sync_once(&mut server).unwrap().unwrap() { + let merged = match cache.sync_once(&mut server).unwrap().edit.unwrap() { (actual, EditStatus::Published { .. }) => { assert_eq!(actual, id, "{name}"); true @@ -515,7 +510,7 @@ fn native_moves_deletions_and_layout_changes_merge_or_require_review() { "{name}" ); assert_eq!(cache.pending().unwrap(), queue); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(pages(&cache.snapshot().unwrap()), pages(&local)); drop(cache); let cache = Replica::open(&path).unwrap(); review(&cache, id, dependent, save_both).unwrap(); @@ -590,7 +585,7 @@ fn a_new_native_wrap_reservation_requires_review_before_width_replacement() { .unwrap(); let mut server = Server::new(NATIVE); assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))) ); review(&cache, id, text_id, |page| { diff --git a/crates/notebook/tests/sync_page.rs b/crates/notebook/tests/sync_page.rs index 59fb372db5712cb4cdfbe2415e31fa0e92f528c0..2c434a6a1b628b0499c9ac12b530f23506aa542b 100644 --- a/crates/notebook/tests/sync_page.rs +++ b/crates/notebook/tests/sync_page.rs @@ -1,7 +1,8 @@ //! Page creation reconciliation: dependent body saves, uncertain publication, anchor review, //! and deterministic multi-actor schedules. -use notebook::{ConflictKind, EditStatus, Operation, Recovery, Replica}; +use notebook::{ConflictKind, EditStatus, Recovery, Replica, Resolution}; +use onestore::op::SectionOp; use onestore::{ ExGuid, PageCreation, RevisionIndex, Store, document::{Document, Kind}, @@ -49,47 +50,38 @@ fn offline_page_creation_rebases_with_dependent_edits_and_duplicate_titles() { let path = directory.path().join(format!("{actor}.sqlite")); let cache = Replica::create(&path, &source).unwrap(); let page = PageCreation::new(None, Some("Same 🦋 é"), "Offline author").unwrap(); - let id = cache.create_page(&source, &page).unwrap().unwrap(); - let mut created = model_ops::page_of(&cache.snapshot().unwrap(), page.space()); + let id = section_op(&cache, SectionOp::Create(page.clone())); + let mut created = cache.page(page.space()).unwrap(); let body = body_outline(&mut created, "Body"); - let save = cache - .save( - &cache.snapshot().unwrap(), - page.space(), - &created, - model_ops::AUTHOR, - ) + let save = model_ops::save_as(&cache, page.space(), &created, model_ops::AUTHOR) .unwrap() .unwrap(); - assert_eq!( - model_ops::save(&cache, body, |page| model_ops::replace_text( - page, - body, - 4..4, - &format!(" {actor}") - )) - .unwrap(), - Some(save), - "a dependent text edit coalesces into the unattempted body save" - ); + let dependent = model_ops::save(&cache, body, |page| { + model_ops::replace_text(page, body, 4..4, &format!(" {actor}")) + }) + .unwrap() + .unwrap(); + assert!(id < save && save < dependent); let original = cache.pending().unwrap(); - assert_eq!(original.len(), 2); + assert_eq!(original.len(), 3); assert!( - matches!(&original[0].operation, Operation::CreatePage(retained) if retained == &page) + matches!(&original[0].edit.ops[..], [onestore::op::Op::Section(SectionOp::Create(retained))] if retained == &page) ); let local = cache.snapshot().unwrap(); drop(cache); let cache = Replica::open(&path).unwrap(); assert_eq!(cache.pending().unwrap(), original); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(pages(&cache.snapshot().unwrap()), pages(&local)); + assert!(matches!( + cache.sync_once(&mut server).unwrap().edit, + Some((published, EditStatus::Published { .. })) if published == dependent + )); for edit in original { - assert!(matches!(cache.sync_once(&mut server).unwrap(), - Some((published, EditStatus::Published { .. })) if published == edit.id)); + assert!(matches!( + cache.status(edit.id).unwrap(), + Some(EditStatus::Published { .. }) + )); } - assert!(matches!( - cache.status(id).unwrap(), - Some(EditStatus::Published { .. }) - )); expected.push((page.space(), page.object(), body, format!("Body {actor}"))); } let store = Store::parse(&server.durable).unwrap(); @@ -103,7 +95,7 @@ fn offline_page_creation_rebases_with_dependent_edits_and_duplicate_titles() { let view = &space.revisions[&space.contexts[&ExGuid::default()]]; assert!(matches!(&view.nodes[text].kind, Kind::RichText { text, .. } if text == expected)); } - assert_eq!(server.publications, 24); + assert_eq!(server.publications, 12); if let Some(output) = std::env::var_os("ONESTORE_OFFLINE_PAGE_OUTPUT") { std::fs::create_dir(&output).unwrap(); std::fs::write( @@ -127,7 +119,7 @@ fn uncertain_page_publication_retains_both_revisions_and_never_replays() { let path = directory.path().join("pages.sqlite"); let cache = Replica::create(&path, &source).unwrap(); let page = PageCreation::new(None, Some("Created"), "Author").unwrap(); - let id = cache.create_page(&source, &page).unwrap().unwrap(); + let id = section_op(&cache, SectionOp::Create(page.clone())); let local = cache.snapshot().unwrap(); let mut server = Server::new(&source); server.fault = fault; @@ -145,7 +137,11 @@ fn uncertain_page_publication_retains_both_revisions_and_never_replays() { drop(cache); let db = rusqlite::Connection::open(&path).unwrap(); let encoded: String = db - .query_row("SELECT revisions FROM attempt", [], |row| row.get(0)) + .query_row( + "SELECT revisions FROM batches WHERE attempted=1", + [], + |row| row.get(0), + ) .unwrap(); let revisions: BTreeMap = serde_json::from_str(&encoded).unwrap(); assert_eq!(revisions.len(), 2); @@ -153,7 +149,7 @@ fn uncertain_page_publication_retains_both_revisions_and_never_replays() { drop(db); let cache = Replica::open(&path).unwrap(); let observed = server.visible.clone(); - let result = cache.sync_once(&mut server).unwrap().unwrap(); + let result = cache.sync_once(&mut server).unwrap().edit.unwrap(); let visible = matches!(fault, Fault::UnknownAfter | Fault::PanicAfter); assert_eq!(server.publications, 1); assert_eq!(server.confirmations, usize::from(visible)); @@ -164,7 +160,7 @@ fn uncertain_page_publication_retains_both_revisions_and_never_replays() { assert!(observed[252..] == server.durable[252..]); } else { assert_eq!(result, (id, attempted.clone())); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(pages(&cache.snapshot().unwrap()), pages(&local)); } } } @@ -176,7 +172,7 @@ fn surviving_page_revision_alone_does_not_confirm_section_publication() { let path = directory.path().join("pages.sqlite"); let cache = Replica::create(&path, &source).unwrap(); let page = PageCreation::new(None, None, "Author").unwrap(); - let id = cache.create_page(&source, &page).unwrap().unwrap(); + let id = section_op(&cache, SectionOp::Create(page.clone())); let mut server = Server::new(&source); server.fault = Fault::UnknownAfter; assert!(cache.sync_once(&mut server).is_err()); @@ -184,7 +180,11 @@ fn surviving_page_revision_alone_does_not_confirm_section_publication() { drop(cache); let db = rusqlite::Connection::open(&path).unwrap(); let encoded: String = db - .query_row("SELECT revisions FROM attempt", [], |row| row.get(0)) + .query_row( + "SELECT revisions FROM batches WHERE attempted=1", + [], + |row| row.get(0), + ) .unwrap(); let revisions: BTreeMap = serde_json::from_str(&encoded).unwrap(); drop(db); @@ -222,21 +222,21 @@ fn surviving_page_revision_alone_does_not_confirm_section_publication() { ); let cache = Replica::open(&path).unwrap(); assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((id, attempted.clone())) ); assert_eq!(server.confirmations, 0); assert_eq!(server.publications, 1); server.visible = complete; assert!(matches!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((_, EditStatus::Published { .. })) )); assert_eq!(server.publications, 1); } #[test] -fn changed_page_anchor_requires_review_without_regenerating_dependent_identities() { +fn changed_page_anchor_conflicts_until_kept_without_regenerating_dependent_identities() { let source = include_bytes!("../../../corpus/page-lifecycle/03-renamed/notebook/Lifecycle.one"); let remote = include_bytes!("../../../corpus/page-lifecycle/04-nested/notebook/Lifecycle.one"); let store = Store::parse(source).unwrap(); @@ -246,48 +246,36 @@ fn changed_page_anchor_requires_review_without_regenerating_dependent_identities let path = directory.path().join("pages.sqlite"); let cache = Replica::create(&path, source).unwrap(); let page = PageCreation::new(Some(pages[4].0), Some("Created"), "Author").unwrap(); - let id = cache.create_page(source, &page).unwrap().unwrap(); - let mut created = model_ops::page_of(&cache.snapshot().unwrap(), page.space()); + let id = section_op(&cache, SectionOp::Create(page.clone())); + let mut created = cache.page(page.space()).unwrap(); let body = body_outline(&mut created, "Retained body"); - cache - .save(&cache.snapshot().unwrap(), page.space(), &created, "Author") + let save = model_ops::save_as(&cache, page.space(), &created, "Author") .unwrap() .unwrap(); let local = cache.snapshot().unwrap(); let pending = cache.pending().unwrap(); let mut server = Server::new(remote); assert_eq!( - cache.sync_once(&mut server).unwrap(), - Some((id, EditStatus::Conflict(ConflictKind::StructureChanged))) + cache.sync_once(&mut server).unwrap().edit, + Some((save, EditStatus::Conflict(ConflictKind::StructureChanged))) ); - assert!( - cache - .rebase_page_creation_conflict(id, source, remote, None) - .is_err() - ); - assert!( - cache - .rebase_page_creation_conflict(id, &local, source, None) - .is_err() - ); - cache - .rebase_page_creation_conflict(id, &local, remote, None) - .unwrap(); - assert_eq!(cache.snapshot().unwrap(), local); + cache.resolve(id, Resolution::Mine).unwrap(); assert_eq!(cache.pending().unwrap()[1], pending[1]); - let Operation::CreatePage(reviewed) = &cache.pending().unwrap()[0].operation else { - panic!() - }; - assert_eq!(*reviewed, page.reposition(None).unwrap()); + assert!(matches!( + &cache.pending().unwrap()[0].edit.ops[..], + [onestore::op::Op::Section(SectionOp::Create(reviewed))] if *reviewed == page.reposition(None).unwrap() + )); + assert_eq!( + model_ops::page_of(&cache.snapshot().unwrap(), page.space()), + model_ops::page_of(&local, page.space()) + ); drop(cache); let cache = Replica::open(&path).unwrap(); - for _ in 0..2 { - assert!(matches!( - cache.sync_once(&mut server).unwrap(), - Some((_, EditStatus::Published { .. })) - )); - } - assert_eq!(server.publications, 2); + assert!(matches!( + cache.sync_once(&mut server).unwrap().edit, + Some((_, EditStatus::Published { .. })) + )); + assert_eq!(server.publications, 1); let store = Store::parse(&server.durable).unwrap(); let index = RevisionIndex::parse(&store).unwrap(); let document = Document::parse(&index).unwrap(); @@ -296,3 +284,110 @@ fn changed_page_anchor_requires_review_without_regenerating_dependent_identities assert!(matches!(&view.nodes[&body].kind, Kind::RichText { text, .. } if text == "Retained body")); } + +#[test] +fn keeping_mine_puts_back_a_page_the_remote_removed_where_it_stood() { + let source = include_bytes!("../../../corpus/page-lifecycle/04-nested/notebook/Lifecycle.one"); + let order = |image: &[u8]| -> Vec<(ExGuid, String, u32)> { + let arena = onestore::Arena::default(); + onestore::Section::open(&arena, image.to_vec()) + .unwrap() + .pages() + .unwrap() + }; + let pages = order(source); + // A subpage without subpages of its own and with a page after it, so it has a position + // and a level to return to. + let at = (0..pages.len() - 1) + .find(|at| { + pages[*at].2 > 1 + && pages[*at + 1].2 <= pages[*at].2 + && model_ops::page_of(source, pages[*at].0) + .objects + .iter() + .any(|object| matches!(object, onestore::page::PageObject::Outline(_))) + }) + .expect("a subpage with body text"); + let (space, _, level) = pages[at].clone(); + let text = model_ops::page_of(source, space) + .objects + .iter() + .find_map(|object| match object { + onestore::page::PageObject::Outline(outline) => { + outline.paragraphs[0].text().map(|t| t.id) + } + _ => None, + }) + .unwrap(); + for keep in [Resolution::Mine, Resolution::Theirs] { + let directory = tempfile::tempdir().unwrap(); + let cache = Replica::create(directory.path().join("pages.sqlite"), source).unwrap(); + let id = model_ops::save(&cache, text, |page| { + model_ops::replace_text(page, text, 0..0, "Kept ") + }) + .unwrap() + .unwrap(); + let local = cache.page(space).unwrap(); + let removed = onestore::PreparedEdit::delete_pages_permanently(source, &[space]) + .unwrap() + .as_bytes() + .to_vec(); + let mut server = Server::new(&removed); + assert_eq!( + cache.sync_once(&mut server).unwrap().edit, + Some((id, EditStatus::Conflict(ConflictKind::TargetUnavailable))) + ); + assert_eq!( + server.publications, 0, + "a removed page is not edited in place" + ); + cache.resolve(id, keep).unwrap(); + assert!(matches!( + cache.sync_once(&mut server).unwrap().edit, + Some((_, EditStatus::Published { .. })) + )); + let published = order(&server.durable); + assert!(published.iter().all(|(page, ..)| *page != space)); + match keep { + Resolution::Mine => { + assert_eq!(published.len(), pages.len()); + let (restored, title, restored_level) = published[at].clone(); + assert_eq!( + (title.as_str(), restored_level), + (pages[at].1.as_str(), level) + ); + let page = model_ops::page_of(&server.durable, restored); + let texts = |page: &onestore::page::Page| -> Vec { + page.objects + .iter() + .filter_map(|object| match object { + onestore::page::PageObject::Outline(outline) => Some(outline), + _ => None, + }) + .flat_map(|outline| &outline.paragraphs) + .filter_map(|p| p.text().map(|t| t.text.text().to_owned())) + .collect() + }; + assert_eq!(texts(&page), texts(&local)); + assert!(texts(&page)[0].starts_with("Kept ")); + let others: Vec<_> = published + .iter() + .map(|(page, ..)| *page) + .filter(|page| *page != restored) + .collect(); + let before: Vec<_> = pages + .iter() + .map(|(page, ..)| *page) + .filter(|page| *page != space) + .collect(); + assert_eq!(others, before); + } + Resolution::Theirs => assert_eq!(published, order(&removed)), + } + assert_eq!(pages_of(&cache), published); + } +} + +fn pages_of(cache: &Replica) -> Vec<(ExGuid, String, u32)> { + cache.pages().unwrap() +} diff --git a/crates/notebook/tests/sync_pages.rs b/crates/notebook/tests/sync_pages.rs index 14b91ecaa59634fae77dea998e68fe230d2b96aa..35d493870769e00b9ba018ab948e7150ad985f51 100644 --- a/crates/notebook/tests/sync_pages.rs +++ b/crates/notebook/tests/sync_pages.rs @@ -1,7 +1,8 @@ //! Page batch reconciliation: atomic batches with dependent body saves, competing moves and //! indentation, uncertain batches, and native page movement fixtures. -use notebook::{ConflictKind, EditStatus, Operation, Recovery, Replica}; +use notebook::{ConflictKind, EditStatus, Recovery, Replica, Resolution}; +use onestore::op::{Op, SectionOp}; use onestore::{ ExGuid, PageEdit, PagePosition, PreparedEdit, RevisionIndex, Store, document::{Document, Format, Kind, Layout}, @@ -117,8 +118,7 @@ fn atomic_page_batches_survive_reopen_with_dependent_text() { .iter() .map(|(sid, level)| PageEdit::move_to(*sid, None, *level).unwrap()) .collect(); - let id = cache.pages(SOURCE, &edits).unwrap().unwrap(); - assert!(cache.pages(SOURCE, &edits).is_err()); + let id = section_op(&cache, SectionOp::Pages(edits.to_vec())); let mut expected_text = texts(SOURCE); let (&(sid, oid), original) = expected_text .iter() @@ -132,17 +132,22 @@ fn atomic_page_batches_survive_reopen_with_dependent_text() { .unwrap() .unwrap(); let queue = cache.pending().unwrap(); - assert!(matches!(&queue[0].operation, Operation::Pages(batch) if batch.edits == edits)); + assert!( + matches!(&queue[0].edit.ops[..], [Op::Section(SectionOp::Pages(batch))] if *batch == edits) + ); let local = cache.snapshot().unwrap(); drop(cache); let cache = Replica::open(&path).unwrap(); assert_eq!(cache.pending().unwrap(), queue); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!( + server::pages(&cache.snapshot().unwrap()), + server::pages(&local) + ); let mut server = Server::new(SOURCE); - for edit in queue { - assert!(matches!(cache.sync_once(&mut server).unwrap(), - Some((published, EditStatus::Published { .. })) if published == edit.id)); - } + assert!(matches!( + cache.sync_once(&mut server).unwrap().edit, + Some((_, EditStatus::Published { .. })) + )); let expected = [ before[..3].to_vec(), before[6..].to_vec(), @@ -155,7 +160,7 @@ fn atomic_page_batches_survive_reopen_with_dependent_text() { cache.status(id).unwrap(), Some(EditStatus::Published { .. }) )); - assert_eq!(server.publications, 2); + assert_eq!(server.publications, 1); } #[test] @@ -164,10 +169,10 @@ fn indentation_only_edits_preserve_remote_page_movement() { let cache = Replica::create(directory.path().join("pages.sqlite"), SOURCE).unwrap(); let before = order(SOURCE); let sid = before[4].0; - let id = cache - .pages(SOURCE, &[PageEdit::set_level(sid, 3).unwrap()]) - .unwrap() - .unwrap(); + let id = section_op( + &cache, + SectionOp::Pages([PageEdit::set_level(sid, 3).unwrap()].to_vec()), + ); let mut server = Server::new(SOURCE); PreparedEdit::pages( SOURCE, @@ -178,22 +183,19 @@ fn indentation_only_edits_preserve_remote_page_movement() { .unwrap(); let mut expected = order(&server.durable); expected.iter_mut().find(|p| p.0 == sid).unwrap().1 = 3; - assert!(matches!(cache.sync_once(&mut server).unwrap(), + assert!(matches!(cache.sync_once(&mut server).unwrap().edit, Some((published, EditStatus::Published { .. })) if published == id)); assert_eq!(order(&server.durable), expected); } #[test] -fn competing_page_moves_require_current_review_and_retain_intent_identities() { +fn competing_page_moves_conflict_until_mine_is_kept_with_its_identities() { let directory = tempfile::tempdir().unwrap(); let cache = Replica::create(directory.path().join("pages.sqlite"), SOURCE).unwrap(); let pages = order(SOURCE); let sid = pages[6].0; let original = PageEdit::move_to(sid, None, 1).unwrap(); - let id = cache - .pages(SOURCE, std::slice::from_ref(&original)) - .unwrap() - .unwrap(); + let id = section_op(&cache, SectionOp::Pages(vec![original.clone()])); let mut server = Server::new(SOURCE); PreparedEdit::pages( SOURCE, @@ -205,37 +207,20 @@ fn competing_page_moves_require_current_review_and_retain_intent_identities() { let remote = server.visible.clone(); let local = cache.snapshot().unwrap(); assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((id, EditStatus::Conflict(ConflictKind::StructureChanged))) ); assert_eq!(server.publications, 0); - assert_eq!(cache.snapshot().unwrap(), local); - let revised = original - .reposition(PagePosition::Before(Some(pages[2].0)), 1) - .unwrap(); - let fresh = PageEdit::move_to(sid, Some(pages[2].0), 1).unwrap(); - assert!( - cache - .rebase_pages_conflict(id, &local, &remote, &[fresh]) - .is_err() + assert_eq!( + server::pages(&cache.snapshot().unwrap()), + server::pages(&local) ); - assert!( - cache - .rebase_pages_conflict(id, SOURCE, &remote, std::slice::from_ref(&revised)) - .is_err() - ); - assert!( - cache - .rebase_pages_conflict(id, &local, SOURCE, std::slice::from_ref(&revised)) - .is_err() - ); - cache - .rebase_pages_conflict(id, &local, &remote, std::slice::from_ref(&revised)) - .unwrap(); - assert!(matches!(&cache.pending().unwrap()[0].operation, - Operation::Pages(batch) if batch.edits == [revised.clone()])); - let expected = PreparedEdit::pages(&remote, &[revised]).unwrap(); - assert!(matches!(cache.sync_once(&mut server).unwrap(), + // Keeping mine moves the page last again, from where the remote put it. + cache.resolve(id, Resolution::Mine).unwrap(); + assert!(matches!(&cache.pending().unwrap()[0].edit.ops[..], + [Op::Section(SectionOp::Pages(batch))] if *batch == [original.clone()])); + let expected = PreparedEdit::pages(&remote, &[original]).unwrap(); + assert!(matches!(cache.sync_once(&mut server).unwrap().edit, Some((published, EditStatus::Published { .. })) if published == id)); assert_eq!(order(&server.durable), order(expected.as_bytes())); } @@ -245,16 +230,13 @@ fn one_competing_level_prevents_publication_of_the_whole_batch() { let directory = tempfile::tempdir().unwrap(); let cache = Replica::create(directory.path().join("pages.sqlite"), SOURCE).unwrap(); let pages = order(SOURCE); - let id = cache - .pages( - SOURCE, - &[ - PageEdit::set_level(pages[4].0, 3).unwrap(), - PageEdit::set_level(pages[5].0, 2).unwrap(), - ], - ) - .unwrap() - .unwrap(); + let id = section_op( + &cache, + SectionOp::Pages(vec![ + PageEdit::set_level(pages[4].0, 3).unwrap(), + PageEdit::set_level(pages[5].0, 2).unwrap(), + ]), + ); let local = cache.snapshot().unwrap(); let mut server = Server::new(SOURCE); PreparedEdit::pages(SOURCE, &[PageEdit::set_level(pages[5].0, 1).unwrap()]) @@ -263,12 +245,15 @@ fn one_competing_level_prevents_publication_of_the_whole_batch() { .unwrap(); let remote = server.durable.clone(); assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((id, EditStatus::Conflict(ConflictKind::StructureChanged))) ); assert_eq!(server.publications, 0); assert_eq!(server.durable, remote); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!( + server::pages(&cache.snapshot().unwrap()), + server::pages(&local) + ); } #[test] @@ -287,7 +272,7 @@ fn uncertain_page_batches_survive_recovery_and_do_not_replay() { PageEdit::set_level(pages[4].0, 3).unwrap(), PageEdit::set_level(pages[5].0, 2).unwrap(), ]; - let id = cache.pages(SOURCE, &edits).unwrap().unwrap(); + let id = section_op(&cache, SectionOp::Pages(edits.to_vec())); let local = cache.snapshot().unwrap(); let queue = cache.pending().unwrap(); let mut server = Server::new(SOURCE); @@ -304,13 +289,12 @@ fn uncertain_page_batches_survive_recovery_and_do_not_replay() { let recovery = Recovery::open(&archive).unwrap(); assert_eq!(recovery.pending().unwrap(), queue); assert_eq!(recovery.status(id).unwrap(), Some(attempted.clone())); - assert_eq!(recovery.snapshot().unwrap(), local); - assert!( - cache - .rebase_pages_conflict(id, &local, SOURCE, &edits) - .is_err() + assert_eq!( + server::pages(&recovery.snapshot().unwrap()), + server::pages(&local) ); - let result = cache.sync_once(&mut server).unwrap().unwrap(); + assert!(cache.resolve(id, Resolution::Mine).is_err()); + let result = cache.sync_once(&mut server).unwrap().edit.unwrap(); assert_eq!(server.publications, 1); if matches!(fault, Fault::UnknownAfter | Fault::PanicAfter) { assert!(matches!(result.1, EditStatus::Published { .. })); @@ -341,10 +325,10 @@ fn an_unchanged_section_revision_cannot_confirm_a_changed_page() { let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("pages.sqlite"); let cache = Replica::create(&path, &source).unwrap(); - let id = cache - .pages(&source, &[PageEdit::set_level(sid, 3).unwrap()]) - .unwrap() - .unwrap(); + let id = section_op( + &cache, + SectionOp::Pages([PageEdit::set_level(sid, 3).unwrap()].to_vec()), + ); let mut server = Server::new(&source); server.fault = Fault::UnknownAfter; assert!(cache.sync_once(&mut server).is_err()); @@ -352,7 +336,9 @@ fn an_unchanged_section_revision_cannot_confirm_a_changed_page() { drop(cache); let db = rusqlite::Connection::open(&path).unwrap(); let encoded: String = db - .query_row("SELECT revisions FROM attempt", [], |r| r.get(0)) + .query_row("SELECT revisions FROM batches WHERE attempted=1", [], |r| { + r.get(0) + }) .unwrap(); let proofs: BTreeMap = serde_json::from_str(&encoded).unwrap(); assert_eq!(proofs.len(), 2); @@ -368,13 +354,13 @@ fn an_unchanged_section_revision_cannot_confirm_a_changed_page() { let complete = server.visible.clone(); server.visible = source; assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((id, attempted.clone())) ); assert_eq!(server.confirmations, 0); server.visible = complete; assert!(matches!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((_, EditStatus::Published { .. })) )); assert_eq!(server.publications, 1); @@ -386,16 +372,13 @@ fn an_explicit_anchor_is_retained_even_when_originally_in_place() { let directory = tempfile::tempdir().unwrap(); let cache = Replica::create(directory.path().join("pages.sqlite"), SOURCE).unwrap(); let pages = order(SOURCE); - let id = cache - .pages( - SOURCE, - &[ - PageEdit::move_to(pages[6].0, Some(pages[7].0), 1).unwrap(), - PageEdit::set_level(pages[4].0, 3).unwrap(), - ], - ) - .unwrap() - .unwrap(); + let id = section_op( + &cache, + SectionOp::Pages(vec![ + PageEdit::move_to(pages[6].0, Some(pages[7].0), 1).unwrap(), + PageEdit::set_level(pages[4].0, 3).unwrap(), + ]), + ); let mut server = Server::new(SOURCE); PreparedEdit::pages( SOURCE, @@ -405,7 +388,7 @@ fn an_explicit_anchor_is_retained_even_when_originally_in_place() { .commit(&mut server) .unwrap(); assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((id, EditStatus::Conflict(ConflictKind::StructureChanged))) ); assert_eq!(server.publications, 0); @@ -416,10 +399,10 @@ fn convergent_page_indentation_requires_confirmation_without_republishing() { let directory = tempfile::tempdir().unwrap(); let cache = Replica::create(directory.path().join("pages.sqlite"), SOURCE).unwrap(); let sid = order(SOURCE)[4].0; - let id = cache - .pages(SOURCE, &[PageEdit::set_level(sid, 1).unwrap()]) - .unwrap() - .unwrap(); + let id = section_op( + &cache, + SectionOp::Pages([PageEdit::set_level(sid, 1).unwrap()].to_vec()), + ); let mut server = Server::new(SOURCE); PreparedEdit::pages(SOURCE, &[PageEdit::set_level(sid, 1).unwrap()]) .unwrap() @@ -429,7 +412,7 @@ fn convergent_page_indentation_requires_confirmation_without_republishing() { assert!(cache.sync_once(&mut server).is_err()); assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); assert_eq!(server.publications, 0); - assert!(matches!(cache.sync_once(&mut server).unwrap(), + assert!(matches!(cache.sync_once(&mut server).unwrap().edit, Some((published, EditStatus::Published { .. })) if published == id)); assert_eq!(server.publications, 0); assert_eq!(server.confirmations, 2); @@ -464,7 +447,7 @@ fn native_page_changes_reconcile_with_atomic_offline_batches_and_review() { .map(|(sid, level)| PageEdit::move_to(*sid, None, *level).unwrap()) .collect() }; - let id = cache.pages(SOURCE, &edits).unwrap().unwrap(); + let id = section_op(&cache, SectionOp::Pages(edits.to_vec())); let text_id = model_ops::save(&cache, text_object, |page| { model_ops::replace_text(page, text_object, 0..0, "Offline ") }) @@ -484,8 +467,11 @@ fn native_page_changes_reconcile_with_atomic_offline_batches_and_review() { "02-promoted-parent" | "03-selected-group-move" | "06-promoted-level-two" ) }; - let first = cache.sync_once(&mut server).unwrap().unwrap(); - assert_eq!(first.0, id); + let first = if expected_conflict { + cache.sync_once(&mut server).unwrap().edit.unwrap() + } else { + (text_id, EditStatus::Pending) + }; let mut reviewed = edits.clone(); if expected_conflict { assert_eq!( @@ -494,26 +480,25 @@ fn native_page_changes_reconcile_with_atomic_offline_batches_and_review() { "{mode}:{phase}" ); assert_eq!(server.publications, 0); - assert_eq!(cache.snapshot().unwrap(), local); - if mode == "indent" && phase == "08-collapsed-group-move" { - reviewed[0] = reviewed[0].reposition(PagePosition::Keep, 1).unwrap(); - } - cache - .rebase_pages_conflict(id, &local, &native, &reviewed) - .unwrap(); - assert!(matches!(cache.sync_once(&mut server).unwrap(), - Some((published, EditStatus::Published { .. })) if published == id)); - } else { - assert!( - matches!(first.1, EditStatus::Published { .. }), - "{mode}:{phase}" + assert_eq!( + server::pages(&cache.snapshot().unwrap()), + server::pages(&local) ); + // Keeping mine drops the moves that no longer apply. + cache.resolve(id, Resolution::Mine).unwrap(); + reviewed = match &cache.pending().unwrap()[0].edit.ops[..] { + [Op::Section(SectionOp::Pages(kept))] => kept.clone(), + [] => Vec::new(), + other => panic!("{other:?}"), + }; } + assert!( + matches!(cache.sync_once(&mut server).unwrap().edit, Some((published, EditStatus::Published { .. })) if published == text_id), + "{mode}:{phase}" + ); counts[usize::from(expected_conflict)] += 1; drop(cache); let cache = Replica::open(&path).unwrap(); - assert!(matches!(cache.sync_once(&mut server).unwrap(), - Some((published, EditStatus::Published { .. })) if published == text_id)); assert!(cache.pending().unwrap().is_empty()); let mut expected = order(&native); for edit in &reviewed { @@ -529,10 +514,7 @@ fn native_page_changes_reconcile_with_atomic_offline_batches_and_review() { } assert_eq!(order(&server.durable), expected, "{mode}:{phase}"); assert_eq!(texts(&server.durable), expected_text, "{mode}:{phase}"); - assert_eq!( - server.publications, - 1 + usize::from(expected != order(&native)) - ); + assert_eq!(server.publications, 1); if let Some(output) = std::env::var_os("ONESTORE_OFFLINE_PAGE_EDIT_OUTPUT") { let output = std::path::Path::new(&output); assert!(output.is_absolute()); @@ -580,13 +562,12 @@ fn twelve_disjoint_page_batches_merge_with_dependent_bodies_without_review() { PageEdit::move_to(group[1].space(), Some(group[0].space()), 1).unwrap(), PageEdit::set_level(group[2].space(), 2).unwrap(), ]; - cache.pages(&source, &edits).unwrap().unwrap(); + section_op(&cache, SectionOp::Pages(edits.to_vec())); let body = format!("Actor {actor} 🦀 e\u{301}"); let local = cache.snapshot().unwrap(); let mut model = model_ops::page_of(&local, group[1].space()); let text = body_outline(&mut model, &body); - cache - .save(&local, group[1].space(), &model, "Author") + model_ops::save_as(&cache, group[1].space(), &model, "Author") .unwrap() .unwrap(); expected.extend([ @@ -601,18 +582,18 @@ fn twelve_disjoint_page_batches_merge_with_dependent_bodies_without_review() { for (path, queue) in &queues { let cache = Replica::open(path).unwrap(); assert_eq!(cache.pending().unwrap(), *queue); - for edit in queue { - assert!(matches!(cache.sync_once(&mut server).unwrap(), - Some((published, EditStatus::Published { .. })) if published == edit.id)); - } + assert!(matches!( + cache.sync_once(&mut server).unwrap().edit, + Some((_, EditStatus::Published { .. })) + )); assert!(cache.pending().unwrap().is_empty()); } - assert_eq!(server.publications, 24); + assert_eq!(server.publications, 12); assert_eq!(order(&server.durable), expected); assert_eq!(texts(&server.durable), expected_text); for (path, queue) in &queues { let cache = Replica::open(path).unwrap(); - assert_eq!(cache.sync_once(&mut server).unwrap(), None); + assert_eq!(cache.sync_once(&mut server).unwrap().edit, None); assert_eq!(order(&cache.snapshot().unwrap()), expected); for edit in queue { assert!(matches!( @@ -631,10 +612,7 @@ fn twelve_disjoint_page_batches_merge_with_dependent_bodies_without_review() { .map(|(_, queue)| { queue .iter() - .map(|edit| { - serde_json::json!({"id": edit.id, "space": edit.space, - "operation": edit.operation}) - }) + .map(|edit| serde_json::json!({"id": edit.id, "edit": edit.edit})) .collect::>() }) .collect(); diff --git a/crates/notebook/tests/sync_paragraph.rs b/crates/notebook/tests/sync_paragraph.rs index 1dd492545de64fbbff2b0d414920ff7493ade744..787b98694edef420111e836f4c98496766537612 100644 --- a/crates/notebook/tests/sync_paragraph.rs +++ b/crates/notebook/tests/sync_paragraph.rs @@ -11,7 +11,7 @@ use std::path::Path; #[path = "support/server.rs"] mod server; -use server::{Fault, Server}; +use server::{Fault, Server, pages}; #[path = "support/model_ops.rs"] mod model_ops; use model_ops::{AUTHOR, outlines_mut, page_of, replace_text}; @@ -178,16 +178,13 @@ fn save_page( let source = cache.snapshot()?; let (space, mut page) = holder(&source, text); edit(&mut page); - cache.save(&source, space, &page, AUTHOR) + model_ops::save_as(cache, space, &page, AUTHOR) } -/// Reviews a conflicting save by applying `edit` to the page the remote now holds. +/// Resolves a conflict by taking the remote page, then saves `edit` of it. fn review_page(cache: &Replica, id: u64, text: ExGuid, edit: impl FnOnce(&mut Page)) { - let local = cache.snapshot().unwrap(); - let remote = cache.remote_snapshot().unwrap(); - let (_, mut page) = holder(&remote, text); - edit(&mut page); - cache.review_page(id, &local, &remote, &page).unwrap(); + cache.resolve(id, notebook::Resolution::Theirs).unwrap(); + save_page(cache, text, edit).unwrap().unwrap(); } /// The styled text of an object anywhere on a page. @@ -246,18 +243,20 @@ fn reconcile_controls() -> (Vec, Vec) { let mut cache = Replica::create(&path, BEFORE).unwrap(); let mut after = original.clone(); apply(&mut after); - let id = cache.save(BEFORE, space, &after, AUTHOR).unwrap().unwrap(); + let id = model_ops::save_as(&cache, space, &after, AUTHOR) + .unwrap() + .unwrap(); let pending = cache.pending().unwrap(); drop(cache); cache = Replica::open(&path).unwrap(); assert_eq!(cache.pending().unwrap(), pending); - let (actual, status) = cache.sync_once(&mut server).unwrap().unwrap(); + let (actual, status) = cache.sync_once(&mut server).unwrap().edit.unwrap(); assert_eq!(actual, id); let reviewed = matches!(status, EditStatus::Conflict(_)); let receipt = if reviewed { assert_eq!(cache.pending().unwrap(), pending); review_page(&cache, id, left, apply); - cache.sync_once(&mut server).unwrap().unwrap().1 + cache.sync_once(&mut server).unwrap().edit.unwrap().1 } else { automatic += 1; status @@ -293,7 +292,7 @@ fn reconcile_controls() -> (Vec, Vec) { let dependent = save_page(&cache, target, |page| dependent_edit(page, target)) .unwrap() .unwrap(); - let (actual, status) = cache.sync_once(&mut server).unwrap().unwrap(); + let (actual, status) = cache.sync_once(&mut server).unwrap().edit.unwrap(); assert_eq!(actual, dependent); let EditStatus::Published { revision: dependent_revision, @@ -309,7 +308,7 @@ fn reconcile_controls() -> (Vec, Vec) { "receipt": revision, "dependent_receipt": dependent_revision}), ); } - assert_eq!((recorded.len(), automatic), (16, 6)); + assert_eq!((recorded.len(), automatic), (16, 8)); assert_eq!(server.publications, 32); (server.durable, recorded) } @@ -321,19 +320,20 @@ fn native_keyboard_edits_merge_with_offline_splits_and_joins_or_survive_review() ("Split remote prefix", "automatic", vec!["Xab", "🦀Cd"]), ("Split remote boundary", "automatic", vec!["abX", "🦀Cd"]), ("Split remote child", "reviewed", vec!["ab", "🦀Cd"]), - ("Split remote list", "automatic", vec!["ab", "🦀Cd"]), + // A split names copies of its paragraph's list nodes; the remote's new list has none. + ("Split remote list", "reviewed", vec!["ab", "🦀Cd"]), ("Split remote tag", "automatic", vec!["ab", "🦀Cd"]), ("Split remote format", "reviewed", vec!["ab", "🦀Cd"]), ("Split remote sibling", "automatic", vec!["ab", "🦀Cd"]), ("Join remote prefix", "reviewed", vec!["Xab🦀CdRightY"]), - ("Join left boundary", "reviewed", vec!["ab🦀CdXRight"]), + ("Join left boundary", "automatic", vec!["ab🦀CdXRight"]), ("Join right boundary", "reviewed", vec!["ab🦀CdXRight"]), - ("Join remote child", "reviewed", vec!["ab🦀CdRight"]), + ("Join remote child", "automatic", vec!["ab🦀CdRight"]), ("Join remote list", "reviewed", vec!["ab🦀CdRight"]), ("Join remote tag", "reviewed", vec!["ab🦀CdRight"]), ("Join remote format", "reviewed", vec!["ab🦀CdRight"]), ("Join remote sibling", "automatic", vec!["ab🦀CdRight"]), - ("Join empty adoption", "reviewed", vec!["XIight"]), + ("Join empty adoption", "automatic", vec!["XIight"]), ]; for (case, (name, outcome, prefix)) in cases.iter().zip(expected) { assert_eq!(case["case"], name); @@ -363,17 +363,19 @@ fn a_native_text_object_replacement_conflicts_with_every_offline_split_and_join( } else { join_paragraphs(&mut after, left, body(&original)[1].text().unwrap().id); } - let id = cache.save(BEFORE, space, &after, AUTHOR).unwrap().unwrap(); + let id = model_ops::save_as(&cache, space, &after, AUTHOR) + .unwrap() + .unwrap(); let local = cache.snapshot().unwrap(); let pending = cache.pending().unwrap(); assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))), "{name}" ); drop(cache); let cache = Replica::open(&path).unwrap(); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(pages(&cache.snapshot().unwrap()), pages(&local)); assert_eq!(cache.pending().unwrap(), pending); assert_eq!( cache.status(id).unwrap(), @@ -448,6 +450,7 @@ fn offline_paragraphs(output: Option<&Path>) { let cases = if split { &manifest["cases"] } else { &manifest }; let mut entries = Vec::new(); let mut refused: Vec = Vec::new(); + let mut reviewed: Vec = Vec::new(); for case in cases .as_array() .unwrap() @@ -474,16 +477,10 @@ fn offline_paragraphs(output: Option<&Path>) { continue; }; let id = queued.unwrap(); - entries.push( - serde_json::json!({"case": case["case"], "id": id, "space": space, - "intent": intent, "remote_prefix": prefixed}), - ); + let mut entry = serde_json::json!({"case": case["case"], "id": id, "space": space, + "intent": intent, "remote_prefix": prefixed}); drop(cache); cache = Replica::open(&path).unwrap(); - } - let mut reviewed: Vec = Vec::new(); - for entry in &mut entries { - let id = entry["id"].as_u64().unwrap(); let case = entry["case"].as_str().unwrap().to_owned(); let intent = entry["intent"].clone(); let mut uncertain = id % 2 == 0; @@ -501,11 +498,20 @@ fn offline_paragraphs(output: Option<&Path>) { drop(cache); cache = Replica::open(&path).unwrap(); } - Ok(Some((actual, EditStatus::Published { revision }))) => { - assert_eq!(actual, id); + Ok(notebook::Synced { + edit: Some((_, EditStatus::Published { revision })), + .. + }) => { + assert!(matches!( + cache.status(id).unwrap(), + Some(EditStatus::Published { .. }) + )); break revision; } - Ok(Some((actual, EditStatus::Conflict(_)))) => { + Ok(notebook::Synced { + edit: Some((actual, EditStatus::Conflict(_))), + .. + }) => { assert_eq!(actual, id); server.fault = Fault::None; reviewed.push(case.clone()); @@ -518,16 +524,15 @@ fn offline_paragraphs(output: Option<&Path>) { replay(page, &placed, split) }); } - other => panic!("{name} {}: {other:?}", entry["case"]), + other => panic!("{name} {case}: {other:?}"), } }; entry["revision"] = serde_json::to_value(revision).unwrap(); + entries.push(entry); } - // A split inside a table cell the remote also changed overlaps at the cell. - let structural = ["Split cell".to_owned()]; let unrepresentable = ["Join inherited styles".to_owned()]; let expected: (usize, &[String], &[String]) = match name { - "splits" | "joins" => (12, &structural, &[]), + "splits" | "joins" => (12, &[], &[]), "inheritance" => (4, &[], &unrepresentable), _ => (3, &[], &[]), }; @@ -537,7 +542,7 @@ fn offline_paragraphs(output: Option<&Path>) { "{name}" ); assert_eq!(server.publications, entries.len()); - assert_eq!(cache.sync_once(&mut server).unwrap(), None); + assert_eq!(cache.sync_once(&mut server).unwrap().edit, None); assert_eq!(cache.snapshot().unwrap(), server.durable); if output.is_some() { cache @@ -585,7 +590,7 @@ fn a_split_publishes_two_paragraphs_and_a_join_puts_them_back() { .unwrap() .unwrap(); assert!(matches!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((_, EditStatus::Published { .. })) )); let published = page_of(&server.durable, space); @@ -598,7 +603,7 @@ fn a_split_publishes_two_paragraphs_and_a_join_puts_them_back() { .unwrap() .unwrap(); assert!(matches!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((_, EditStatus::Published { .. })) )); assert_eq!( @@ -623,7 +628,7 @@ fn a_split_merges_with_a_remote_edit_to_another_paragraph() { .unwrap(); let mut server = remote_with(space, |page| replace_text(page, other, 0..0, "Remote ")); assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == id) + matches!(cache.sync_once(&mut server).unwrap().edit, Some((actual, EditStatus::Published { .. })) if actual == id) ); assert_eq!( texts(&page_of(&server.durable, space)), @@ -645,13 +650,13 @@ fn a_join_conflicts_when_the_remote_changed_the_paragraph_it_removes() { .unwrap(); let mut server = remote_with(space, |page| replace_text(page, right, 0..0, "Remote ")); assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))) ); assert_eq!(server.publications, 0); review_page(&cache, id, left, |page| join_paragraphs(page, left, right)); assert!(matches!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((_, EditStatus::Published { .. })) )); assert_eq!( @@ -674,7 +679,7 @@ fn edits_dependent_on_a_published_split_survive_reopen() { save_page(&cache, left, |page| split_paragraph(page, left, 2)) .unwrap() .unwrap(); - cache.sync_once(&mut server).unwrap().unwrap(); + cache.sync_once(&mut server).unwrap().edit.unwrap(); let tail = body(&page_of(&cache.snapshot().unwrap(), space))[1] .text() .unwrap() @@ -688,7 +693,7 @@ fn edits_dependent_on_a_published_split_survive_reopen() { let pending = cache.pending().unwrap(); drop(cache); cache = Replica::open(&path).unwrap(); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(pages(&cache.snapshot().unwrap()), pages(&local)); assert_eq!(cache.pending().unwrap(), pending); let mut remote = page_of(&server.durable, space); replace_text(&mut remote, other, 0..0, "Remote "); @@ -698,7 +703,7 @@ fn edits_dependent_on_a_published_split_survive_reopen() { .to_vec(); server.durable.clone_from(&server.visible); assert!( - matches!(cache.sync_once(&mut server).unwrap(), Some((actual, EditStatus::Published { .. })) if actual == dependent) + matches!(cache.sync_once(&mut server).unwrap().edit, Some((actual, EditStatus::Published { .. })) if actual == dependent) ); assert_eq!( texts(&page_of(&server.durable, space)), @@ -751,7 +756,7 @@ fn uncertain_splits_and_joins_keep_the_original_attempt_across_reopen() { assert_eq!(state, EditStatus::Pending); assert_eq!(cache.pending().unwrap(), pending); assert!(matches!( - cache.sync_once(&mut server).unwrap().unwrap().1, + cache.sync_once(&mut server).unwrap().edit.unwrap().1, EditStatus::Published { .. } )); assert_eq!(server.publications, 2); @@ -759,10 +764,10 @@ fn uncertain_splits_and_joins_keep_the_original_attempt_across_reopen() { Fault::UnknownBefore => { assert!(matches!(state, EditStatus::AwaitingConfirmation { .. })); assert_eq!(cache.pending().unwrap(), pending); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(pages(&cache.snapshot().unwrap()), pages(&local)); for _ in 0..3 { assert_eq!( - cache.sync_once(&mut server).unwrap(), + cache.sync_once(&mut server).unwrap().edit, Some((id, state.clone())) ); } @@ -772,9 +777,9 @@ fn uncertain_splits_and_joins_keep_the_original_attempt_across_reopen() { } Fault::UnknownAfter => { assert!(matches!(state, EditStatus::AwaitingConfirmation { .. })); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(pages(&cache.snapshot().unwrap()), pages(&local)); assert!(matches!( - cache.sync_once(&mut server).unwrap().unwrap().1, + cache.sync_once(&mut server).unwrap().edit.unwrap().1, EditStatus::Published { .. } )); assert_eq!(server.publications, 1); @@ -791,7 +796,7 @@ fn uncertain_splits_and_joins_keep_the_original_attempt_across_reopen() { &["ab", "\u{1f980}cd", "Right", "Preserved sibling"] }; assert_eq!(texts(&page_of(&server.durable, space)), expected); - assert_eq!(cache.sync_once(&mut server).unwrap(), None); + assert_eq!(cache.sync_once(&mut server).unwrap().edit, None); assert_eq!(cache.snapshot().unwrap(), server.durable); } } @@ -818,17 +823,17 @@ fn a_competing_join_leaves_the_other_client_a_reviewable_split() { let pending = second.pending().unwrap(); let mut server = Server::new(BEFORE); assert!(matches!( - first.sync_once(&mut server).unwrap().unwrap().1, + first.sync_once(&mut server).unwrap().edit.unwrap().1, EditStatus::Published { .. } )); assert_eq!( - second.sync_once(&mut server).unwrap(), + second.sync_once(&mut server).unwrap().edit, Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))) ); assert_eq!(server.publications, 1); drop(second); let second = Replica::open(&path).unwrap(); - assert_eq!(second.snapshot().unwrap(), local); + assert_eq!(pages(&second.snapshot().unwrap()), pages(&local)); assert_eq!(second.pending().unwrap(), pending); assert_eq!( second.status(id).unwrap(), @@ -838,7 +843,7 @@ fn a_competing_join_leaves_the_other_client_a_reviewable_split() { split_paragraph(page, left, 2); }); assert!(matches!( - second.sync_once(&mut server).unwrap().unwrap().1, + second.sync_once(&mut server).unwrap().edit.unwrap().1, EditStatus::Published { .. } )); assert_eq!( diff --git a/crates/notebook/tests/sync_worker.rs b/crates/notebook/tests/sync_worker.rs index 9221beeadbb5296ff10b642b8e624a6d3496abc8..ecf4189e963aaed6629d73fc19acad8f23059d34 100644 --- a/crates/notebook/tests/sync_worker.rs +++ b/crates/notebook/tests/sync_worker.rs @@ -111,7 +111,12 @@ fn reconnects_after_connect_read_and_uncertain_publish_without_replaying() { }, move |result| { observed_tx - .send(result.as_ref().cloned().map_err(|error| error.to_string())) + .send( + result + .as_ref() + .map(|synced| synced.edit.clone()) + .map_err(|error| error.to_string()), + ) .unwrap(); }, ) @@ -166,7 +171,9 @@ fn local_saves_wake_an_idle_worker_and_publish_every_writer_marker() { Duration::from_secs(3600), move || Ok(shared.clone()), move |result| { - observed_tx.send(result.as_ref().unwrap().clone()).unwrap(); + observed_tx + .send(result.as_ref().unwrap().edit.clone()) + .unwrap(); if first { first = false; resume_rx.recv_timeout(Duration::from_secs(5)).unwrap(); @@ -185,16 +192,11 @@ fn local_saves_wake_an_idle_worker_and_publish_every_writer_marker() { let cache = &cache; scope.spawn(move || { let marker = format!("[{writer}] "); - loop { - match model_ops::save(cache, oid, |page| { - model_ops::replace_text(page, oid, 0..0, &marker) - }) { - Ok(Some(id)) => break id, - Err(Error::Io(error)) - if error.kind() == io::ErrorKind::ResourceBusy => {} - other => panic!("Unexpected local outcome: {other:?}"), - } - } + model_ops::save(cache, oid, |page| { + model_ops::replace_text(page, oid, 0..0, &marker) + }) + .unwrap() + .unwrap() }) }) .collect::>() @@ -203,11 +205,9 @@ fn local_saves_wake_an_idle_worker_and_publish_every_writer_marker() { .collect::>() }); resume_tx.send(()).unwrap(); - let mut published = std::collections::BTreeSet::new(); while !cache.pending().unwrap().is_empty() { - if let Some((id, status)) = observed_rx.recv_timeout(Duration::from_secs(5)).unwrap() { + if let Some((_, status)) = observed_rx.recv_timeout(Duration::from_secs(5)).unwrap() { assert!(matches!(status, EditStatus::Published { .. }), "{status:?}"); - published.insert(id); } } assert!( @@ -215,7 +215,12 @@ fn local_saves_wake_an_idle_worker_and_publish_every_writer_marker() { "Edits waited for the hourly poll" ); worker.stop().unwrap(); - assert_eq!(published, queued); + for id in &queued { + assert!(matches!( + cache.status(*id).unwrap(), + Some(EditStatus::Published { .. }) + )); + } let content = text(&cache.snapshot().unwrap()).2; for writer in 0..12 { assert_eq!(content.matches(&format!("[{writer}] ")).count(), 1); @@ -223,7 +228,8 @@ fn local_saves_wake_an_idle_worker_and_publish_every_writer_marker() { assert!(content.ends_with("abc")); let server = server.lock().unwrap(); assert_eq!(text(&server.durable).2, content); - assert_eq!(server.publications, queued.len()); + // Edits queued while the worker was busy publish together. + assert_eq!(server.publications, 1); assert_eq!(cache.snapshot().unwrap(), server.durable); } @@ -308,7 +314,7 @@ fn dropping_during_publication_is_nonblocking_and_retains_ownership_until_recove Duration::from_secs(3600), move || Ok(shared.clone()), move |result| { - tx.send(result.as_ref().unwrap().clone()).unwrap(); + tx.send(result.as_ref().unwrap().edit.clone()).unwrap(); }, ) .unwrap(); @@ -332,7 +338,7 @@ fn cache_failures_stop_retries_and_return_the_error_without_remote_publication() let id = save(&cache, oid, 0..0, "L ").unwrap(); drop(cache); let connection = rusqlite::Connection::open(&path).unwrap(); - connection.execute_batch("CREATE TRIGGER fail_attempt BEFORE INSERT ON attempt BEGIN SELECT RAISE(ABORT, 'test cache write failure'); END;").unwrap(); + connection.execute_batch("CREATE TRIGGER fail_attempt BEFORE UPDATE OF attempted ON batches BEGIN SELECT RAISE(ABORT, 'test cache write failure'); END;").unwrap(); drop(connection); let cache = Arc::new(Replica::open(&path).unwrap()); let server = Arc::new(Mutex::new(Server::new(&source))); @@ -356,14 +362,14 @@ fn cache_failures_stop_retries_and_return_the_error_without_remote_publication() } #[test] -fn polling_preserves_conflicts_and_absent_uncertain_revisions_without_replay() { +fn polling_reports_a_blocked_queue_once_per_remote_change_without_replay() { for uncertain in [false, true] { let dir = tempfile::tempdir().unwrap(); let source = onestore::create_section("worker.one", "abc", "Fixture").unwrap(); let (sid, oid, _) = text(&source); let cache = Arc::new(Replica::create(dir.path().join("cache.sqlite"), &source).unwrap()); let id = save(&cache, oid, 1..2, "L").unwrap(); - let local = cache.snapshot().unwrap(); + let local = text(&cache.snapshot().unwrap()).2; let mut server = if uncertain { Server::new(&source) } else { @@ -380,38 +386,48 @@ fn polling_preserves_conflicts_and_absent_uncertain_revisions_without_replay() { Duration::from_millis(10), move || Ok(shared.clone()), move |result| { - tx.send(result.as_ref().cloned().map_err(|_| ())).unwrap(); + tx.send( + result + .as_ref() + .map(|synced| synced.edit.clone()) + .map_err(|_| ()), + ) + .unwrap(); }, ) .unwrap(); if uncertain { assert!(rx.recv_timeout(Duration::from_secs(5)).unwrap().is_err()); } - let mut previous = None; - let started = Instant::now(); - for _ in 0..5 { + let expected = |status: &EditStatus| { + if uncertain { + matches!(status, EditStatus::AwaitingConfirmation { .. }) + } else { + *status == EditStatus::Conflict(ConflictKind::ContentChanged) + } + }; + for round in 0..3 { let (actual, status) = rx .recv_timeout(Duration::from_secs(5)) .unwrap() .unwrap() .unwrap(); assert_eq!(actual, id); - if uncertain { - assert!(matches!(status, EditStatus::AwaitingConfirmation { .. })); - } else { - assert_eq!(status, EditStatus::Conflict(ConflictKind::ContentChanged)); - } - if let Some(previous) = previous { - assert_eq!(previous, status); - } - previous = Some(status.clone()); + assert!(expected(&status), "{status:?}"); + // An unchanged remote is not read or reported again. + assert!(rx.recv_timeout(Duration::from_millis(100)).is_err()); + // Another writer's change elsewhere in the text is read and decided again. + let mut server = server.lock().unwrap(); + let changed = + onestore::replace_text(&server.visible, sid, oid, 0..0, &round.to_string()) + .unwrap(); + server.visible = changed.clone(); + server.durable = changed; + drop(server); + worker.wake(); } - assert!( - started.elapsed() >= Duration::from_millis(30), - "Worker spun instead of waiting between retries" - ); worker.stop().unwrap(); - assert_eq!(cache.snapshot().unwrap(), local); + assert_eq!(text(&cache.snapshot().unwrap()).2, local); assert_eq!(cache.pending().unwrap().len(), 1); let server = server.lock().unwrap(); assert_eq!(server.publications, usize::from(uncertain)); @@ -511,7 +527,7 @@ fn invalid_intervals_and_callback_panics_leave_worker_ownership_recoverable() { Duration::from_secs(3600), move || Ok(shared.clone()), move |result| { - tx.send(result.as_ref().unwrap().clone()).unwrap(); + tx.send(result.as_ref().unwrap().edit.clone()).unwrap(); }, ) .unwrap(); @@ -521,7 +537,7 @@ fn invalid_intervals_and_callback_panics_leave_worker_ownership_recoverable() { } #[test] -fn reviewed_conflict_wakes_the_worker_and_publishes_the_original_intent_once() { +fn a_resolved_conflict_wakes_the_worker_and_publishes_the_original_edit_once() { let dir = tempfile::tempdir().unwrap(); let source = onestore::create_section("resolve.one", "abc", "Fixture").unwrap(); let (sid, oid, _) = text(&source); @@ -538,7 +554,7 @@ fn reviewed_conflict_wakes_the_worker_and_publishes_the_original_intent_once() { Duration::from_secs(3600), move || Ok(shared.clone()), move |result| { - tx.send(result.as_ref().unwrap().clone()).unwrap(); + tx.send(result.as_ref().unwrap().edit.clone()).unwrap(); if first { first = false; resume_rx.recv_timeout(Duration::from_secs(5)).unwrap(); @@ -550,10 +566,7 @@ fn reviewed_conflict_wakes_the_worker_and_publishes_the_original_intent_once() { rx.recv_timeout(Duration::from_secs(5)).unwrap(), Some((id, EditStatus::Conflict(ConflictKind::ContentChanged))) ); - model_ops::review(&cache, id, oid, |page| { - model_ops::replace_text(page, oid, 1..2, "L") - }) - .unwrap(); + cache.resolve(id, notebook::Resolution::Mine).unwrap(); assert_eq!(cache.status(id).unwrap(), Some(EditStatus::Pending)); resume_tx.send(()).unwrap(); assert!( @@ -618,8 +631,13 @@ fn ordinary_read_and_unpublished_write_contention_reuse_the_connection() { Duration::from_millis(1), move || Ok(remote.take().expect("Contention caused a reconnect")), move |result| { - tx.send(result.as_ref().cloned().map_err(|error| error.to_string())) - .unwrap(); + tx.send( + result + .as_ref() + .map(|synced| synced.edit.clone()) + .map_err(|error| error.to_string()), + ) + .unwrap(); }, ) .unwrap(); @@ -675,14 +693,14 @@ fn publication_backoff_drains_local_wakes_without_waiting_for_the_idle_poll() { failed = true; let page = onestore::PageCreation::new(None, Some("Queued"), "Fixture").unwrap(); - let second = observed - .create_page(&observed.snapshot().unwrap(), &page) - .unwrap() - .unwrap(); + let second = section_op(&observed, onestore::op::SectionOp::Create(page)); tx.send((second, None)).unwrap(); } - Ok(Some((id, status))) => tx.send((*id, Some(status.clone()))).unwrap(), - Ok(None) => {} + Ok(notebook::Synced { + edit: Some((id, status)), + .. + }) => tx.send((*id, Some(status.clone()))).unwrap(), + Ok(_) => {} other => panic!("Unexpected worker result: {other:?}"), }, ) diff --git a/crates/onestore/src/commit.rs b/crates/onestore/src/commit.rs index 1ac0817662d22240f0ac7955e2b95262d37c61f2..87ecff00fd31b2f8ef33abfdce63dc3a1bac15b0 100644 --- a/crates/onestore/src/commit.rs +++ b/crates/onestore/src/commit.rs @@ -644,6 +644,19 @@ impl Transaction { } } + /// The image this transaction applies to. + pub fn base(&self) -> &Stamp { + &self.base + } + + /// The bytes a commit writes, by offset, in the order `apply` writes them; the header, + /// last, covers bytes 0..1024. + pub fn writes(&self) -> impl Iterator { + std::iter::once((self.base.length, self.append.as_slice())) + .chain(self.patches.iter().map(|(offset, bytes)| (*offset, bytes.as_slice()))) + .chain(std::iter::once((0, self.header.as_slice()))) + } + /// Writes this transaction into its base image, as a successful commit leaves the file. pub fn apply(&self, image: &mut Vec) -> Result<(), crate::Error> { if Stamp::of(image)? != self.base { diff --git a/crates/onestore/src/op/apply.rs b/crates/onestore/src/op/apply.rs index 6e624be1f405de5d128ca456c1879386d45aa296..dbbe31ef9e3761bc88b6fa968da0291194759b60 100644 --- a/crates/onestore/src/op/apply.rs +++ b/crates/onestore/src/op/apply.rs @@ -15,7 +15,7 @@ use crate::{ active::{ActivePage, Changes}, document::Kind, page::{ - Attachment, Image, Ink, Page, PageObject, PageParagraph, Paragraph, ParagraphContent, + Attachment, Image, Ink, PageObject, PageParagraph, Paragraph, ParagraphContent, TableCell, TableColumn, }, }; @@ -57,12 +57,13 @@ impl<'a> Section<'a> { /// Lowers `after` against the page `space` holds and applies the result as one edit; /// returns the ops, which reach `after` where the writers can store it. - pub fn apply_page( + #[cfg(test)] + pub(crate) fn apply_page( &mut self, author: &str, at: u64, space: ExGuid, - after: &Page, + after: &crate::page::Page, ) -> Result, OpError> { let before = self.page(space).map_err(|error| self.classify(Failure::Rejected(error)))?; let ops = super::lower_page(&before, after) diff --git a/crates/onestore/src/op/lower.rs b/crates/onestore/src/op/lower.rs index 207d77184efdcc5bb74d52be412be1e333f8cb9d..ccdb388c0eef145e51c55b07c561ea0334be69e9 100644 --- a/crates/onestore/src/op/lower.rs +++ b/crates/onestore/src/op/lower.rs @@ -20,7 +20,8 @@ fn invalid(message: &'static str) -> Error { Error { offset: 0, message } } -/// The ops that turn `before`, a page as stored, into `after`. +/// The ops that turn `before`, a page as stored, into `after`: O(page), for importing a +/// page, converting queued whole-page edits and resolving a conflict, never per keystroke. pub fn lower_page(before: &Page, after: &Page) -> Result, Error> { let old = View::new(before, false)?; let new = View::new(after, false)?; diff --git a/crates/onestore/src/op/mod.rs b/crates/onestore/src/op/mod.rs index fb9f498bcc3a1b9a5ad5e726702106a6d1763591..ce0ac5b4bfda3a172ad7e216be2c8c1c3b79dca2 100644 --- a/crates/onestore/src/op/mod.rs +++ b/crates/onestore/src/op/mod.rs @@ -28,6 +28,7 @@ pub(crate) type Values = Vec<(u32, Vec)>; /// One user action, applied whole or not at all. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Edit { /// FILETIME when the action happened; the modification time of what it changes. pub at: u64, @@ -35,6 +36,7 @@ pub struct Edit { } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub enum Op { Page { space: ExGuid, op: PageOp }, Section(SectionOp), @@ -43,6 +45,7 @@ pub enum Op { /// A change to the page an object space holds. Targets are stored identities; paragraph /// properties name the paragraph, text edits its text object. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub enum PageOp { /// Replaces a range; inserted text takes the format of the run it lands in, the /// following run at a boundary except at the end. @@ -159,6 +162,7 @@ pub enum PageOp { } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub enum TableEdit { /// New rows before a row, or last, each with one cell per column. Rows { @@ -185,6 +189,7 @@ pub enum TableEdit { } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub enum SectionOp { Create(PageCreation), /// A page created with `creation` holding `page`'s content under the identities it diff --git a/crates/onestore/src/section.rs b/crates/onestore/src/section.rs index 8f405310127a520ff747ad28f7487d6ebb43dfea..f0892fb238f7fc462d88948b5d5114e1d11da097 100644 --- a/crates/onestore/src/section.rs +++ b/crates/onestore/src/section.rs @@ -202,6 +202,13 @@ impl<'a> Section<'a> { image } + /// Each object space with the active revision the sealed image stores for it. + pub fn revisions(&self) -> impl Iterator + '_ { + self.spaces + .iter() + .filter_map(|(space, stored)| Some((*space, stored.rid?))) + } + /// The page an object space holds, with the changes applied since the last seal. pub fn page(&self, space: ExGuid) -> Result { let stored = self.spaces.get(&space).ok_or(Error { @@ -429,7 +436,7 @@ impl<'a> Section<'a> { } /// The root object space, which lists the pages. - pub(crate) fn root(&self) -> ExGuid { + pub fn root(&self) -> ExGuid { self.root } diff --git a/crates/onestore/src/store.rs b/crates/onestore/src/store.rs index 5c8f202448c47fc4af2954fef5cde4910a5756ec..b1799d85dff71b09908594d0805bda687a455ecc 100644 --- a/crates/onestore/src/store.rs +++ b/crates/onestore/src/store.rs @@ -86,7 +86,7 @@ pub struct Header { } impl Header { - pub(crate) fn parse(data: &[u8]) -> Result { + pub fn parse(data: &[u8]) -> Result { let mut c = Cursor { bytes: data, offset: 0, diff --git a/tools/cache_images.py b/tools/cache_images.py index 6b241891b9f8f36bc438d2b23151af21f4fcc179..9bd30f8fc880c76ee2c2f18dfef8a249baec590b 100644 --- a/tools/cache_images.py +++ b/tools/cache_images.py @@ -1,17 +1,37 @@ -"""The cache's working image, stored as the byte ranges where it differs from the base.""" +"""Reading a notebook cache (schema 15) from outside: its section images, stored as 16 KiB +chunk rows, and the state of its write-ahead log.""" import struct -def working(connection): - base, patch = connection.execute('SELECT base, working FROM replica WHERE id=1').fetchone() - if not patch: return base - length, = struct.unpack_from('-wal` since its last reset: frames whose header records the + database size after them, among the frames carrying the log's current salts. Frames + after the last commit belong to a transaction that has not committed.""" + try: + data = path.with_name(path.name + '-wal').read_bytes() + except FileNotFoundError: + return 0, 0 + if len(data) < 32: + return 0, 0 + magic, _, page_size, _, salt1, salt2 = struct.unpack_from('>IIIIII', data) + assert magic in (0x377f0682, 0x377f0683), 'Not a SQLite write-ahead log' + commits = pending = 0 + at = 32 + while at + 24 + page_size <= len(data): + _, committed, frame1, frame2 = struct.unpack_from('>IIII', data, at) + if (frame1, frame2) != (salt1, salt2): + break + if committed: + commits, pending = commits + 1, 0 + else: + pending += 1 + at += 24 + page_size + return commits, pending diff --git a/tools/offline_cache_crash.py b/tools/offline_cache_crash.py index 8d752f4a6e6dda79ad32102531fd7d2d2150dca7..598d91410532d1f234bf17d1e4b30637d1eb1e49 100644 --- a/tools/offline_cache_crash.py +++ b/tools/offline_cache_crash.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Interrupt owned offline-cache writers and verify every retained intent and image.""" import argparse +import cache_images import hashlib import json import os @@ -31,7 +32,7 @@ def run(binary, output): retained = [] retained_ids = [] results = [] - delays = [None, "ack", "unack", "journal", "database-write", 0, .001, .01, .02, .04, .08, .16, .32, .64, 1.28, 2.56, None, "ack"] + delays = [None, "ack", "unack", "wal", 0, .001, .01, .02, .04, .08, .16, .32, .64, 1.28, 2.56, None, "ack"] for operation, delay in enumerate(delays, 1): with (output / f"owner-{operation}.stderr").open("w") as stderr: owner = subprocess.Popen([binary, "edit", cache], stdin=subprocess.PIPE, @@ -48,34 +49,28 @@ def run(binary, output): contender = subprocess.run([binary, "read", cache], capture_output=True, text=True, timeout=15) (output / f"contender-{operation}.stderr").write_text(contender.stderr) assert contender.returncode != 0 and "DatabaseBusy" in contender.stderr, contender - before_write = cache.stat().st_mtime_ns owner.stdin.write(f"{operation} {'unack' if delay == 'unack' else 'ack'}\n") owner.stdin.flush() expect(f"editing {operation}") acknowledgement = None - journal_observation = None + log_observation = None if delay == "ack": acknowledgement = expect(f"ack {operation}") elif delay == "unack": expect(f"durable {operation}") - elif delay in ("journal", "database-write"): + elif delay == "wal": + # Cut the edit's transaction once its frames are in the log and its + # commit frame is not. deadline = time.monotonic() + 10 - journal = cache.with_name(cache.name + "-journal") + committed, _ = cache_images.wal_commits(cache) while time.monotonic() < deadline: - try: - with journal.open("rb") as active: - header = active.read(28) - size = journal.stat().st_size - database_changed = cache.stat().st_mtime_ns != before_write - if (header[:8] == bytes.fromhex("d9d505f920a163d7") and - (delay == "journal" or database_changed)): - journal_observation = {"header": header.hex(), "bytes": size, - "database_changed": database_changed} - break - except FileNotFoundError: - pass - time.sleep(.0001) - assert journal_observation, f"No active {delay} phase observed" + commits, pending = cache_images.wal_commits(cache) + if commits == committed and pending: + log_observation = {"wal_commits": commits, "uncommitted_frames": pending} + break + if commits != committed: + break + assert log_observation, "No uncommitted log frames observed" elif delay is not None: time.sleep(delay) owner.kill() @@ -88,9 +83,7 @@ def run(binary, output): actual = json.loads(read.stdout) operations = actual["operations"] ids = actual["ids"] - # A save replaces the newest pending save of its page under the same intent. - coalesced = retained[:-1] + [operation] if ids == retained_ids else None - assert operations in (retained, retained + [operation], coalesced), (retained, operation, actual) + assert operations in (retained, retained + [operation]), (retained, operation, actual) assert ids[:len(retained_ids)] == retained_ids, (retained_ids, actual) if acknowledgement: assert operations[-1] == operation @@ -105,7 +98,7 @@ def run(binary, output): "retained_operations": operations, "ids": ids, "section_bytes": actual["section_bytes"], "complete_payloads": True, "exclusive_owner": True, - "journal_observation": journal_observation}) + "log_observation": log_observation}) retained, retained_ids = operations, ids (output / "results.json").write_text(json.dumps(results, indent=2)) print(json.dumps(results[-1]), flush=True) diff --git a/tools/offline_publication_crash.py b/tools/offline_publication_crash.py index 733a2f7eb984bb76aefa23c6a27683fb9fbdb2c2..ae1fe408a34a1dfc368b7d3e76745f5b6ed51fe9 100644 --- a/tools/offline_publication_crash.py +++ b/tools/offline_publication_crash.py @@ -31,11 +31,6 @@ def run_command(binary, args, output): def kill_at(binary, args, phase, output, receipt_window=None): events = queue.Queue() database = Path(args[1]) / 'cache.sqlite' - journal = database.with_name(database.name + '-journal') - def header(): - try: - with journal.open('rb') as stream: return stream.read(8) - except FileNotFoundError: return b'' with output.with_suffix('.jsonl').open('w') as log, output.with_suffix('.stderr').open('w') as error: process = subprocess.Popen([str(binary), *map(str, args)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=error, text=True, env={**os.environ, 'ONESTORE_RECOVERY_PAUSE': phase}) @@ -57,24 +52,26 @@ def kill_at(binary, args, phase, output, receipt_window=None): if event.get('event') == 'phase' and event['name'] == phase: if receipt_window is not None: assert args[0] == 'sync' and phase == 'publish-after' - assert receipt_window in ('journal', 'database') - before_mtime = database.stat().st_mtime_ns + assert receipt_window == 'wal' + # The receipt transaction is cut once its frames are in the log and + # its commit frame is not. + committed, _ = cache_images.wal_commits(database) process.stdin.write('\n') process.stdin.flush() deadline = time.monotonic() + 5 while True: - if header() == bytes.fromhex('d9d505f920a163d7') and (receipt_window == 'journal' or database.stat().st_mtime_ns != before_mtime): break - assert process.poll() is None, 'Receipt transaction finished before the requested cut' + commits, pending = cache_images.wal_commits(database) + if commits == committed and pending: break + assert commits == committed and process.poll() is None, 'Receipt transaction finished before the requested cut' assert time.monotonic() < deadline, 'Receipt transaction window was not observed' - time.sleep(.0001) process.kill() break assert process.wait(timeout=10) == -signal.SIGKILL, 'Expected an actual process kill' proof = {'pid': process.pid, 'exit': process.returncode, 'phase': phase} if receipt_window is not None: - proof.update(journal_header_after_kill=header().hex(), database_changed=database.stat().st_mtime_ns != before_mtime, receipt_window=receipt_window) - assert proof['journal_header_after_kill'] == 'd9d505f920a163d7', 'Receipt transaction committed before the process died' - assert receipt_window != 'database' or proof['database_changed'] + commits, pending = cache_images.wal_commits(database) + proof.update(wal_commits_before=committed, wal_commits_after_kill=commits, uncommitted_frames=pending, receipt_window=receipt_window) + assert commits == committed, 'Receipt transaction committed before the process died' output.with_suffix('.cut.json').write_text(json.dumps(proof, indent=2)) finally: if process.poll() is None: process.kill() @@ -103,7 +100,7 @@ def state(rows, original): else: assert len(result['pending']) == 1, 'Unacknowledged intent was lost or duplicated' pending, = result['pending'] - assert pending['id'] == 1 and pending['before'] == original and pending['replacement'] == TOKEN + assert pending['id'] == 1 and pending['replacement'] == TOKEN at = len(original.encode('utf-16-le')) // 2 assert pending['range'] == [at, at], 'Durable intent range changed' return result @@ -187,11 +184,12 @@ def run(source, output): try: assert connection.execute('PRAGMA quick_check').fetchall() == [('ok',)] assert connection.execute('PRAGMA foreign_key_check').fetchall() == [] - local_image = save_image(output, cache_images.working(connection)) + # The image the queue applies to; the local text is the probe's, checked above. + base_image = save_image(output, cache_images.image(connection)) finally: connection.close() result = {'case': index, 'phase': phase, 'during_confirmation': confirmation, 'before_status': before['status'], 'after_status': after['status'], - 'visible_before_recovery': before['remote_text'] != original, 'remote_image': remote_image, 'local_image': local_image, + 'visible_before_recovery': before['remote_text'] != original, 'remote_image': remote_image, 'base_image': base_image, 'remote_text': after['remote_text'], 'local_text': after['local_text']} results.append(result) (output / 'results.json').write_text(json.dumps(results, indent=2)) @@ -206,14 +204,14 @@ def run(source, output): def receipt_windows(source, output): binary, source_hash = prepare_run(source, output) results = [] - for window in ('journal', 'database'): + for window in ('wal',): folder = output / window initialized = run_command(binary, ['init', folder, source], output / (window+'-init')) original = next(row['remote_text'] for row in initialized if row['event'] == 'state') state(initialized, original) kill_at(binary, ['sync', folder], 'publish-after', output / (window+'-kill'), receipt_window=window) before = state(run_command(binary, ['inspect', folder], output / (window+'-inspect')), original) - assert before['status'] == 'uncertain' and before['remote_text'] == original+TOKEN, 'Hot-journal recovery lost the pending confirmation' + assert before['status'] == 'uncertain' and before['remote_text'] == original+TOKEN, 'Log recovery lost the pending confirmation' before_bytes = (folder / 'remote.one').read_bytes() recovered = run_command(binary, ['sync', folder], output / (window+'-recover')) after = state(recovered, original) @@ -226,7 +224,7 @@ def receipt_windows(source, output): print(json.dumps({'receipt_window':window, 'status':after['status'], 'remote_image':digest}), flush=True) assert hashlib.sha256(source.read_bytes()).hexdigest() == source_hash (output/'results.json').write_text(json.dumps(results, indent=2)) - (output/'summary.json').write_text(json.dumps({'process_kills':2, 'recovered_receipts':2, 'republished_edits':0}, indent=2)) + (output/'summary.json').write_text(json.dumps({'process_kills':len(results), 'recovered_receipts':len(results), 'republished_edits':0}, indent=2)) if __name__ == '__main__': diff --git a/tools/test_offline_history.py b/tools/test_offline_history.py index fa80e5683259a08e21541b693c936595d4b08985..ed34b5507c6cbf622cc1359c39e3b350f2472cec 100644 --- a/tools/test_offline_history.py +++ b/tools/test_offline_history.py @@ -139,9 +139,10 @@ class OfflineLedgerTests(unittest.TestCase): events.extend({'event': 'reopened_receipt', 'id': op+1, 'revision': f'{actor}-{op}'} for op in range(2)) events.append({'event': 'done'}) connection = sqlite3.connect(output / 'rust' / f'{actor}.sqlite') - connection.executescript('CREATE TABLE receipts(edit_id INTEGER, revision TEXT); CREATE TABLE edits(id INTEGER); CREATE TABLE attempt(id INTEGER); CREATE TABLE conflicts(id INTEGER); CREATE TABLE replica(id INTEGER, base BLOB, working BLOB);') + # Schema 15 in WAL mode, as a drained cache leaves it. + connection.executescript('PRAGMA journal_mode=WAL; CREATE TABLE receipts(edit_id INTEGER, revision TEXT); CREATE TABLE edits(id INTEGER); CREATE TABLE batches(id INTEGER); CREATE TABLE payloads(sha256 BLOB); CREATE TABLE base(chunk INTEGER, bytes BLOB); CREATE TABLE remote(chunk INTEGER, bytes BLOB);') connection.executemany('INSERT INTO receipts VALUES (?,?)', [(op+1, f'{actor}-{op}') for op in range(2)]) - connection.execute('INSERT INTO replica VALUES (1, ?, ?)', (b'opaque image', b'')) + connection.execute('INSERT INTO base VALUES (0, ?)', (b'opaque image',)) connection.commit() connection.close() for i in range(4): @@ -165,8 +166,8 @@ class OfflineLedgerTests(unittest.TestCase): os.utime(output / 'rust/stop', ns=(0, 0)) connection = sqlite3.connect(output / 'rust/w0.sqlite') for sql, undo in [("UPDATE receipts SET revision='wrong' WHERE edit_id=1", "UPDATE receipts SET revision='w0-0' WHERE edit_id=1"), - ('INSERT INTO attempt VALUES (1)', 'DELETE FROM attempt'), - ("UPDATE replica SET working=X'00'", "UPDATE replica SET working=X''")]: + ('INSERT INTO batches VALUES (1)', 'DELETE FROM batches'), + ("INSERT INTO remote VALUES (0, X'00')", 'DELETE FROM remote')]: connection.execute(sql) connection.commit() with self.assertRaises(AssertionError): verify(output) diff --git a/tools/test_offline_publication_crash.py b/tools/test_offline_publication_crash.py index 0080ac05cb3158a7bc9b8c17f4ebcde393c9e045..7249307fcf3f575df5a589ba0affcda818ae0791 100644 --- a/tools/test_offline_publication_crash.py +++ b/tools/test_offline_publication_crash.py @@ -28,7 +28,7 @@ class RecoveryOracleTests(unittest.TestCase): at = len(original.encode('utf-16-le')) // 2 row = {'event': 'state', 'status': 'uncertain', 'revision': 'new', 'remote_revision': 'old', 'local_text': original+TOKEN, 'remote_text': original, - 'pending': [{'id': 1, 'before': original, 'replacement': TOKEN, 'range': [at, at]}]} + 'pending': [{'id': 1, 'replacement': TOKEN, 'range': [at, at]}]} self.assertEqual(state([row], original), row) for field, value in [('revision', None), ('revision', 'old'), ('local_text', original), ('remote_text', original+TOKEN+TOKEN), ('status', 'missing'), ('pending', [])]: @@ -36,7 +36,7 @@ class RecoveryOracleTests(unittest.TestCase): changed[field] = value with self.subTest(field=field): with self.assertRaises(AssertionError): state([changed], original) - for field, value in [('id', 2), ('before', 'other'), ('replacement', 'other'), ('range', [at-1, at-1])]: + for field, value in [('id', 2), ('replacement', 'other'), ('range', [at-1, at-1])]: changed = copy.deepcopy(row) changed['pending'][0][field] = value with self.assertRaises(AssertionError): state([changed], original) diff --git a/tools/verify_offline.py b/tools/verify_offline.py index 965ebc60e072d0796a6f42503648dd0d3f15a3b2..de251e8960ffc3f74152e042f0b1498767eb0f82 100644 --- a/tools/verify_offline.py +++ b/tools/verify_offline.py @@ -4,6 +4,7 @@ import argparse import hashlib import json from pathlib import Path +import cache_images import sqlite3 from native_stress import edit_history, native_history @@ -41,12 +42,11 @@ def verify(output, max_gap=120): try: assert connection.execute('PRAGMA quick_check').fetchall() == [('ok',)], 'Cache integrity failed' assert connection.execute('PRAGMA foreign_key_check').fetchall() == [], 'Cache foreign keys failed' - for table in ('edits', 'attempt', 'conflicts'): + for table in ('edits', 'batches', 'payloads', 'remote'): assert connection.execute(f'SELECT count(*) FROM {table}').fetchone() == (0,), 'Completed cache retains unresolved state' persisted = dict(connection.execute('SELECT edit_id, revision FROM receipts')) assert persisted == {id: event['revision'] for id, event in receipts.items()}, 'SQLite receipts differ from observed acknowledgements' - base, working = connection.execute('SELECT base, working FROM replica WHERE id=1').fetchone() - assert working == b'', 'A drained cache retained a divergent local branch' + base = cache_images.image(connection) caches[actor] = {'receipts': len(persisted), 'image_bytes': len(base), 'image_sha256': hashlib.sha256(base).hexdigest(), 'database_sha256': hashlib.sha256(path.read_bytes()).hexdigest()} finally: connection.close()