From 00efb7f2a201741ccea64df3120cf80bb4d0f6cd Mon Sep 17 00:00:00 2001 From: clover caruso Date: Mon, 19 Oct 2026 17:01:25 -0700 Subject: [PATCH] fix: keep each section's uploads beside its replica, once Review fixes for the lost-upload stack (yvvmzxws, rpmtnlzk, toxstspw, pswvkryr, ssnownqr). They share one change of design, so they land as one commit rather than folded into each. The images a section's publications keep now live beside its replica (".published"), as the session chose it, instead of in a folder guessed from whether a copy's folder had been labelled: the remotes the session and the background make are handed the replica's path. A copy found after the notebook opened kept its images under the original's identity and replaced the original's; now each keeps its own. They move with the replica (location::moved, adopt) and go with it (remove_replica), so a password set elsewhere, which supersedes the replica, takes the plaintext kept with it; closing a notebook, as the app does before renaming it, keeps an offline copy whose images are not yet merged, as a waiting edit keeps it, and only removing the notebook deletes them, and copies' folders are labelled as they are first named, so a rename carries them. Android's provider keeps its images the same way. An upload iCloud Drive keeps as a conflict version is no longer also merged from the store, which kept it beside the file twice: the store skips an image a conflict version is, lists none while a conflict version cannot be read yet, and drops it when that version is retired. A confirmation, which rewrites only the header, carries the kept image along rather than letting it list as lost and go. The store reads the file's stamp only where it keeps something. The Mac names its versions by SCDynamicStoreCopyComputerName instead of the deprecated NSHost, and its iCloud functions take no cache, which pswvkryr had added only on macOS. A share copy's replica is named as the session names a copy's (session::replica_location) instead of by a second rule. Tests: the store's own (a lost image lists once, after LAG, not beside a conflict version; a confirmation; nothing read when nothing is kept), and on the phone an upload undone then merged back after LAG, a phone-only notebook keeping nothing, a password set elsewhere, a copy found while open, and the launch test now asserting nothing kept beside the file and no stale image. The test coordinator holds files per test, so parallel tests no longer take each other's hold. Assisted-by: claude-opus-5.5 --- arc/sync.md | 3 +- crates/mobile/src/library.rs | 205 +++++++++-------- crates/mobile/src/tests.rs | 235 ++++++++++++++++++-- crates/notebook/src/background.rs | 42 ++-- crates/notebook/src/lib.rs | 22 ++ crates/notebook/src/location.rs | 82 +++---- crates/notebook/src/provider.rs | 54 +++-- crates/notebook/src/provider/tests.rs | 32 +-- crates/notebook/src/published.rs | 305 +++++++++++++++++++++----- crates/notebook/src/session.rs | 122 ++++++----- crates/notebook/src/session/tests.rs | 2 +- crates/notebook/tests/session.rs | 6 +- crates/snowbound/Cargo.toml | 2 +- crates/snowbound/src/icloud_linux.rs | 3 - crates/snowbound/src/icloud_macos.rs | 110 +++++----- crates/snowbound/src/library.rs | 19 +- crates/snowbound/src/undo.rs | 2 +- 17 files changed, 836 insertions(+), 410 deletions(-) diff --git a/arc/sync.md b/arc/sync.md index 06700958857d7bf65b897cb2b5b88fdff19430f5..9cbb7bbb202426d19866de67cb337f4e0f04bd57 100644 --- a/arc/sync.md +++ b/arc/sync.md @@ -265,7 +265,8 @@ mount reports none of another client's changes to inotify, so there each section read every 15 seconds. On iOS, a folder on the device (Snowbound's own, or On My iPhone) has no copies and learns of other apps' writes through file coordination (`NSFilePresenter`); one a file provider keeps elsewhere, as iCloud Drive does, keeps copies and is checked every 15 -seconds. Closing a notebook deletes its copies, except any with edits still waiting. +seconds. Closing a notebook deletes its copies, except any with edits still waiting or images +of uploads it keeps (`notebook::published`). ## iCloud Drive diff --git a/crates/mobile/src/library.rs b/crates/mobile/src/library.rs index a611290597e0843e3bea19fc5ec2d73622b32464..841569ef9c2acdc5132b35accb72698237abeacd 100644 --- a/crates/mobile/src/library.rs +++ b/crates/mobile/src/library.rs @@ -20,7 +20,7 @@ use onestore::{ protected::Key, }; use std::{ - collections::{BTreeMap, HashMap}, + collections::{BTreeMap, BTreeSet, HashMap}, ffi::{CString, c_char, c_void}, io, path::{Path, PathBuf}, @@ -123,44 +123,51 @@ pub unsafe extern "C" fn sb_set_versions( /// A local section file read and published under the host's file coordination. One a file /// provider keeps elsewhere, which syncs it whole with no lock, keeps each image this device -/// publishes until the file holds it, and merges one another device's upload replaced back -/// (`notebook::published`). +/// publishes beside its replica until the file holds it, and merges one another device's +/// upload replaced back (`notebook::published`). struct Coordinated { file: PathBuf, published: Option, } impl Coordinated { - /// The file `file` of the notebook folder, or lone section, at `notebook`. - fn new(file: &Path, notebook: &Path, cache: &Path, elsewhere: bool) -> Self { - let device = VERSIONS.get().map(|(.., device)| device.clone()); - let published = elsewhere - .then(|| Published::of(cache, notebook, file, device)) - .flatten(); + /// The section file `file`, its replica at `replica` where it has one. + fn new(file: &Path, replica: Option<&Path>, elsewhere: bool) -> Self { + let device = VERSIONS + .get() + .filter(|_| elsewhere) + .map(|(.., device)| device); Self { file: file.to_owned(), - published, + published: replica + .zip(device) + .map(|(replica, device)| Published::of(replica, device)), } } } +/// The header and length of `file`, as the file remote's uncoordinated change detector reads +/// them. +fn stamp(file: &Path) -> io::Result { + coordinated(file, false, || { + use std::io::Read; + let mut opened = std::fs::File::open(file)?; + let mut header = [0; 1024]; + opened.read_exact(&mut header)?; + Ok(Stamp { + header, + length: opened.metadata()?.len(), + }) + })? +} + impl Remote for Coordinated { fn read(&mut self) -> io::Result> { coordinated(&self.file, false, || onestore::read_file(&self.file))? } - /// The header and length, as the file remote's uncoordinated change detector reads them. fn stamp(&mut self) -> io::Result { - coordinated(&self.file, false, || { - use std::io::Read; - let mut file = std::fs::File::open(&self.file)?; - let mut header = [0; 1024]; - file.read_exact(&mut header)?; - Ok(Stamp { - header, - length: file.metadata()?.len(), - }) - })? + stamp(&self.file) } fn publish(&mut self, transaction: &Transaction) -> std::result::Result<(), CommitError> { @@ -178,7 +185,12 @@ impl Remote for Coordinated { fn confirm(&mut self, base: &Stamp) -> std::result::Result<(), CommitError> { coordinated(&self.file, true, || { - onestore::confirm_file(&self.file, base) + onestore::confirm_file(&self.file, base)?; + if let Some(published) = &self.published { + let _ = std::fs::File::open(&self.file) + .and_then(|mut file| published.confirmed(Written::InPlace(&mut file), base)); + } + Ok(()) }) .map_err(uncommitted)? } @@ -198,13 +210,9 @@ impl Remote for Coordinated { let path = CString::new(self.file.as_os_str().as_encoded_bytes())?; versions(path.as_ptr(), found, (&raw mut listed).cast()); } - if self.published.is_some() { - let stamp = self.stamp()?; - listed.extend( - self.published - .iter() - .flat_map(|published| published.versions(&stamp)), - ); + if let Some(published) = &self.published { + let lost = published.versions(|| stamp(&self.file), &listed)?; + listed.extend(lost); } Ok(listed) } @@ -221,12 +229,14 @@ impl Remote for Coordinated { && published.holds(id) { if keep { - let device = VERSIONS.get().map_or("this device", |(.., device)| device); - let beside = notebook::published::beside(&self.file, device); + let beside = notebook::published::beside(&self.file, published.device()); coordinated(&beside, true, || std::fs::copy(id, &beside))??; } return std::fs::remove_file(id); } + if let Some(published) = &self.published { + let _ = published.retired(Path::new(id)); + } let Some((_, retire, _)) = VERSIONS.get() else { return Ok(()); }; @@ -304,40 +314,38 @@ struct Joined { /// A share notebook's sections as last listed, so they open from their replicas while the /// server cannot be reached. -#[derive(serde::Serialize, serde::Deserialize)] -struct Listed { +#[derive(serde::Serialize, serde::Deserialize, Default)] +pub(crate) struct Listed { tabs: Vec, - /// The name of each readable section's replica (`files`), by catalog path. - files: BTreeMap, + /// Each readable section's file identity in hex, which names its replica, by catalog path. + pub(crate) files: BTreeMap, + /// The sections that copy another, holding its identity, whose replicas are their own + /// (`session::replica_location`). + #[serde(default)] + pub(crate) copies: BTreeSet, } fn hex(bytes: &[u8]) -> String { bytes.iter().map(|byte| format!("{byte:02x}")).collect() } -/// The name of the replica of every readable section in `folder` and its groups, by catalog -/// path, as `Notebook::replicas` lists a local notebook's: its file identity in hex, and for a -/// copy of another section, which holds the same identity, its path besides, so that each -/// keeps its own edits. `Client::read` answers a protected section as it does one mid-write -/// (WouldBlock), so syncing one would show it in use for good. -pub(crate) fn files(folder: &Folder, files: &mut BTreeMap) { +/// Every readable section's file identity in `folder` and its groups, by catalog path, and +/// the copies among them, as `Notebook::replicas` lists a local notebook's. `Client::read` +/// answers a protected section as it does one mid-write (WouldBlock), so syncing one would +/// show it in use for good. +pub(crate) fn files(folder: &Folder, listed: &mut Listed) { for section in &folder.sections { if matches!(section.state, SectionState::Readable { .. }) { - let mut replica = hex(§ion.file_id); + listed + .files + .insert(section.path.clone(), hex(§ion.file_id)); if section.copy { - replica.push('-'); - replica.extend( - section - .path - .chars() - .map(|char| if char.is_alphanumeric() { char } else { '_' }), - ); + listed.copies.insert(section.path.clone()); } - files.insert(section.path.clone(), replica); } } for group in &folder.groups { - self::files(group, files); + self::files(group, listed); } } @@ -473,11 +481,10 @@ impl Library { (None, Place::File(path.to_owned()), None) } else { let mut notebook = Notebook::open(path, cache)?; - let (root, kept) = (path.to_owned(), cache.to_owned()); let background = notebook.background_with( local, !local, - move |file| Coordinated::new(file, &root, &kept, !local), + move |file, replica| Coordinated::new(file, replica, !local), sync_woken, )?; background.watch(notebook.replicas()); @@ -523,7 +530,7 @@ impl Library { }; let reached = library.with_notebook(false, |_| Ok(())); match library.listed() { - Some(listed) => library.watch(&listed.files), + Some(listed) => library.watch(&listed), None => reached?, } Ok(library) @@ -619,17 +626,18 @@ impl Library { }) } - /// Has the background sync a share notebook's sections, `files` as `Listed` keeps them. - fn watch(&self, files: &BTreeMap) { - let (Some(background), Some(replicas)) = (&self.background, self.share_replicas()) else { + /// Has the background sync a share notebook's sections, as `listed` names them. + fn watch(&self, listed: &Listed) { + let Some(background) = &self.background else { return; }; background.watch( - files - .iter() - .map(|(path, replica)| Known { + listed + .files + .keys() + .map(|path| Known { path: path.clone(), - replica: Some(replicas.join(format!("{replica}.sqlite"))), + replica: self.share_replica(listed, path), found: None, image: None, }) @@ -637,13 +645,16 @@ impl Library { ); } - /// The cache folder of a share notebook's replicas. - fn share_replicas(&self) -> Option { + /// The replica of the share section at catalog `path`, as `listed` names it: by its file + /// identity in the notebook's folder, a copy's in a folder of its own. + fn share_replica(&self, listed: &Listed, path: &str) -> Option { let Place::Share { server, root, .. } = &self.place else { return None; }; - let location = notebook::location::smb(&server.address, &server.share, root); - Some(notebook::location::folder(&self.cache, &location)) + let notebook = notebook::location::smb(&server.address, &server.share, root); + let location = session::replica_location(¬ebook, path, listed.copies.contains(path)); + let folder = notebook::location::claimed(&self.cache, &location).ok()?; + Some(folder.join(format!("{}.sqlite", listed.files.get(path)?))) } /// Where a share notebook's last listing is kept. @@ -757,13 +768,10 @@ impl Library { } let listed = self.with_notebook(false, |notebook| { notebook.refresh()?; - let mut listed = Listed { - tabs: Vec::new(), - files: BTreeMap::new(), - }; + let mut listed = Listed::default(); tabs(notebook.catalog(), &self.unlocked(), &mut listed.tabs); if matches!(self.place, Place::Share { .. }) { - files(notebook.catalog(), &mut listed.files); + files(notebook.catalog(), &mut listed); } if let (Place::Folder(_) | Place::Hosted(_) | Place::Provider(_), Some(background)) = (&self.place, &self.background) @@ -779,7 +787,7 @@ impl Library { Ok(listed) => { std::fs::create_dir_all(listing.parent().ok_or("No cache")?)?; std::fs::write(&listing, serde_json::to_vec(&listed)?)?; - self.watch(&listed.files); + self.watch(&listed); Ok(listed.tabs) } Err(error) => Ok(self.listed().ok_or(error)?.tabs), @@ -874,21 +882,19 @@ impl Library { notify: impl Fn() + Send + Sync + 'static, ) -> Result { let offline = self.offline.load(Ordering::Relaxed); - let (cache, elsewhere) = (self.cache.clone(), self.elsewhere); - let connect = |notebook: &Path| { - let notebook = notebook.to_owned(); - move |file: &Path| Ok(Coordinated::new(file, ¬ebook, &cache, elsewhere)) + let elsewhere = self.elsewhere; + let connect = move |file: &Path, replica: Option<&Path>| { + Ok(Coordinated::new(file, replica, elsewhere)) }; match &self.place { Place::File(file) => Ok(session::Section::open_with( file, &self.cache, - connect(file), + connect, offline, notify, )?), - Place::Folder(root) => { - let connect = connect(root); + Place::Folder(_) => { let notebook = self.notebook(); let notebook = notebook.as_ref().ok_or("A lone section has no notebook")?; Ok(match self.unlocked().get(path) { @@ -908,14 +914,10 @@ impl Library { .as_ref() .ok_or("The notebook hasn’t been read")? .replica_path(path)?, - None => { - let replica = self - .listed() - .and_then(|mut listed| listed.files.remove(path)) - .ok_or("The notebook hasn’t listed this section")?; - let replicas = self.share_replicas().ok_or("Not a share")?; - replicas.join(format!("{replica}.sqlite")) - } + None => self + .listed() + .and_then(|listed| self.share_replica(&listed, path)) + .ok_or("The notebook hasn’t listed this section")?, }; std::fs::create_dir_all(cache.parent().ok_or("No cache")?)?; let replica = Replica::open_or_create(&cache, key.as_ref(), || { @@ -2753,12 +2755,9 @@ mod tests { onestore::create_section("Read.one", "Offline source content", "Fixture").unwrap(); std::fs::write(root.join("Read.one"), &bytes).unwrap(); let notebook = Notebook::open(&root, &cache).unwrap(); - let mut listed = Listed { - tabs: Vec::new(), - files: BTreeMap::new(), - }; + let mut listed = Listed::default(); tabs(notebook.catalog(), &HashMap::new(), &mut listed.tabs); - files(notebook.catalog(), &mut listed.files); + files(notebook.catalog(), &mut listed); let identity = listed.files["Read.one"].clone(); drop(notebook); @@ -2840,21 +2839,21 @@ mod tests { std::fs::write(root.join(name), &bytes).unwrap(); } let notebook = Notebook::open(&root, &cache).unwrap(); - let mut listed = Listed { - tabs: Vec::new(), - files: BTreeMap::new(), - }; + let mut listed = Listed::default(); tabs(notebook.catalog(), &HashMap::new(), &mut listed.tabs); - files(notebook.catalog(), &mut listed.files); + files(notebook.catalog(), &mut listed); drop(notebook); - let address = "127.0.0.1:0"; - let replicas = notebook::location::folder( - &cache, - ¬ebook::location::smb(address, "agent", "Notebook"), + assert_eq!( + listed.copies.iter().collect::>(), + ["Notes - Copy.one"] ); - std::fs::create_dir_all(&replicas).unwrap(); - for replica in listed.files.values() { - let _ = Replica::create(replicas.join(format!("{replica}.sqlite")), &bytes); + let address = "127.0.0.1:0"; + let notebook = notebook::location::smb(address, "agent", "Notebook"); + for (path, identity) in &listed.files { + let copy = listed.copies.contains(path); + let location = session::replica_location(¬ebook, path, copy); + let replicas = notebook::location::claimed(&cache, &location).unwrap(); + Replica::create(replicas.join(format!("{identity}.sqlite")), &bytes).unwrap(); } std::fs::create_dir_all(cache.join("smb")).unwrap(); std::fs::write( diff --git a/crates/mobile/src/tests.rs b/crates/mobile/src/tests.rs index e912bb725ea7727b7a487c3efcb6917487b9ce23..b907f3b7983668741caee1689a1136877691c8bf 100644 --- a/crates/mobile/src/tests.rs +++ b/crates/mobile/src/tests.rs @@ -1273,22 +1273,22 @@ fn a_share_notebook_syncs_only_its_readable_sections() { .unwrap(); let notebook = notebook::session::Notebook::open(&root, directory.path().join("cache")).unwrap(); - let mut files = std::collections::BTreeMap::new(); - library::files(notebook.catalog(), &mut files); - assert_eq!(files.keys().collect::>(), ["Empty.one"]); + let mut listed = library::Listed::default(); + library::files(notebook.catalog(), &mut listed); + assert_eq!(listed.files.keys().collect::>(), ["Empty.one"]); } /// Paths the test coordinator was asked for, with whether to write. static COORDINATED: Mutex> = Mutex::new(Vec::new()); -/// A file whose next coordinated access the test coordinator holds, as another app may: who -/// hears that it is held, and what lets it go. +/// Files whose next coordinated access the test coordinator holds, as another app may: who +/// hears that each is held, and what lets it go. Tests running at once hold files of their own. type Held = ( String, std::sync::mpsc::Sender<()>, std::sync::mpsc::Receiver<()>, ); -static HELD: Mutex> = Mutex::new(None); +static HELD: Mutex> = Mutex::new(Vec::new()); extern "C" fn coordinator( path: *const c_char, @@ -1300,10 +1300,8 @@ extern "C" fn coordinator( COORDINATED.lock().unwrap().push((path.clone(), write)); let held = { let mut held = HELD.lock().unwrap(); - match &*held { - Some((file, ..)) if *file == path => held.take(), - _ => None, - } + let at = held.iter().position(|(file, ..)| *file == path); + at.map(|at| held.remove(at)) }; if let Some((_, entered, released)) = held { let _ = entered.send(()); @@ -2043,7 +2041,9 @@ fn a_page_edited_while_it_moves_stays_with_its_edit() { let (entered, holding) = std::sync::mpsc::channel(); let (release, released) = std::sync::mpsc::channel(); let empty = root.canonicalize().unwrap().join("Empty.one"); - *HELD.lock().unwrap() = Some((empty.to_string_lossy().into_owned(), entered, released)); + HELD.lock() + .unwrap() + .push((empty.to_string_lossy().into_owned(), entered, released)); let moving = Arc::clone(&library); let date = ["Thursday, October 1, 2026", "9:00 AM"]; let move_page = std::thread::spawn(move || { @@ -4707,6 +4707,35 @@ fn every_page(root: &Path, cache: &Path) -> String { shown } +/// The images the cache keeps of this device's publications. +fn kept(cache: &Path) -> Vec> { + let folders = std::fs::read_dir(cache.join("replicas")) + .into_iter() + .flatten(); + folders + .filter_map(|folder| std::fs::read_dir(folder.ok()?.path()).ok()) + .flatten() + .filter_map(|entry| { + let path = entry.ok()?.path(); + (path.extension()? == "published").then_some(path) + }) + .filter_map(|store| std::fs::read_dir(store).ok()) + .flatten() + .map(|image| std::fs::read(image.unwrap().path()).unwrap()) + .collect() +} + +/// How many versions of a section are kept beside it in `root`, as "Name (iPhone).one". +fn beside(root: &Path) -> usize { + std::fs::read_dir(root) + .unwrap() + .filter(|entry| { + let name = entry.as_ref().unwrap().file_name(); + name.to_string_lossy().contains("(iPhone") + }) + .count() +} + /// The titles and text of every page and conflict page each section file in `root` stores. fn stored(root: &Path) -> String { let mut text = String::new(); @@ -4741,13 +4770,11 @@ fn stored(root: &Path) -> String { /// types on it, and a section the phone adds. The Mac's upload wins: iCloud Drive keeps the /// phone's file as a conflict version, or loses it. The phone's next launch opens, lists and /// merges it back through the C surface as iOS does, its pages open and typed on in the -/// Reading View all the while. Nothing either device typed goes missing, merged or set aside -/// whole. +/// Reading View all the while. Nothing either device typed goes missing, and nothing is kept +/// beside the file twice. #[test] fn an_icloud_notebook_merges_the_phones_upload_back_at_launch() { - unsafe { - library::sb_set_versions(cloud_versions, cloud_retire, c"Clover’s iPhone".as_ptr()) - }; + unsafe { library::sb_set_versions(cloud_versions, cloud_retire, c"iPhone".as_ptr()) }; // Whether the devices' copies already share an edit of the page both type on, and // whether iCloud Drive lost the phone's upload rather than keep it as a version. for (shared, lost) in [(false, false), (true, false), (false, true), (true, true)] { @@ -4825,7 +4852,7 @@ fn an_icloud_notebook_merges_the_phones_upload_back_at_launch() { std::fs::rename(&file, &version).unwrap(); std::fs::copy(mac.join(&path), &file).unwrap(); if !lost { - let device = "Clover’s iPhone".to_owned(); + let device = "iPhone".to_owned(); let file = std::fs::canonicalize(&file).unwrap(); CLOUD_VERSIONS.lock().unwrap().push((file, version, device)); } @@ -4864,9 +4891,25 @@ fn an_icloud_notebook_merges_the_phones_upload_back_at_launch() { unsafe { sb_insert(&mut view, c"After ".as_ptr()) }; sb_view_keep(&view); } + // What the phone kept of its uploads goes once its file moves on, keeping them; only + // what a file stands as stays kept. + assert!( + eventually(|| { + section.shared.section.wake(); + let files: Vec> = (std::fs::read_dir(&phone).unwrap()) + .map(|entry| std::fs::read(entry.unwrap().path()).unwrap_or_default()) + .collect(); + kept(&folder("phone-cache")) + .iter() + .all(|image| files.contains(image)) + }), + "the phone's uploads stay kept (shared {shared}, lost {lost})" + ); close_section(section); library.flush(Duration::from_secs(20)).unwrap(); drop(library); + // Every upload merged: none is kept beside the file, nor twice. + assert_eq!(beside(&phone), 0); let shown = every_page(&phone, &folder("phone-cache")); for typed in [ @@ -4881,13 +4924,158 @@ fn an_icloud_notebook_merges_the_phones_upload_back_at_launch() { } } +/// A notebook Snowbound made in a scratch folder standing in for iCloud Drive's, opened as +/// the phone opens one there, and its one section. +fn cloud_notebook(directory: &Path) -> (PathBuf, PathBuf, String) { + let (root, cache) = (directory.join("Notes"), directory.join("cache")); + let creation = onestore::PageCreation::new(None, Some("Shared page"), "Clover").unwrap(); + let notebook = + notebook::session::Notebook::create(&root, &cache, 0x00f0_a0c0, &creation).unwrap(); + let path = notebook.catalog().sections[0].path.clone(); + (root, cache, path) +} + +/// Ages the images the cache keeps past the time reads get to catch up with an upload. +fn aged(cache: &Path) { + let folders = std::fs::read_dir(cache.join("replicas")).unwrap(); + for folder in folders { + for entry in std::fs::read_dir(folder.unwrap().path()).unwrap() { + let store = entry.unwrap().path(); + if store + .extension() + .is_none_or(|extension| extension != "published") + { + continue; + } + for image in std::fs::read_dir(store).unwrap() { + let file = std::fs::File::options() + .write(true) + .open(image.unwrap().path()) + .unwrap(); + file.set_modified(std::time::SystemTime::now() - Duration::from_secs(61)) + .unwrap(); + } + } + } +} + +/// iCloud Drive puts back the file as it stood before the phone's upload, as when another +/// device's upload made on that state wins: the phone's edits merge back once reads have had a +/// minute to catch up with its upload, not before. +#[test] +fn an_upload_icloud_undid_merges_back_once_reads_have_caught_up() { + unsafe { library::sb_set_versions(cloud_versions, cloud_retire, c"iPhone".as_ptr()) }; + let directory = tempfile::tempdir().unwrap(); + let (root, cache, path) = cloud_notebook(directory.path()); + let library = cloud_library(&root, &cache); + let section = cloud_section(&library, &path); + let page = listed_ids(§ion)[0]; + type_on(§ion, page, "Before "); + assert!(section.flush(Duration::from_secs(20))); + let before = std::fs::read(root.join(&path)).unwrap(); + type_on(§ion, page, "Phone words "); + assert!(section.flush(Duration::from_secs(20))); + std::fs::write(root.join(&path), before).unwrap(); + for _ in 0..20 { + section.shared.section.wake(); + std::thread::sleep(Duration::from_millis(50)); + } + assert!(!stored(&root).contains("Phone words")); + aged(&cache); + assert!(eventually(|| { + section.shared.section.wake(); + stored(&root).contains("Phone words") + })); + close_section(section); +} + +/// A notebook on this device alone loses no upload, and keeps nothing of its publications. +#[test] +fn a_notebook_on_the_phone_alone_keeps_no_uploads() { + unsafe { library::sb_set_versions(cloud_versions, cloud_retire, c"iPhone".as_ptr()) }; + let directory = tempfile::tempdir().unwrap(); + let (root, cache, path) = cloud_notebook(directory.path()); + let library = Arc::new(Library::open(&root, &cache, true).unwrap()); + let section = cloud_section(&library, &path); + type_on(§ion, listed_ids(§ion)[0], "Phone words "); + assert!(section.flush(Duration::from_secs(20))); + assert!(stored(&root).contains("Phone words")); + assert!(kept(&cache).is_empty()); + close_section(section); +} + +/// Another device sets a password on the section: the file is written anew under a new +/// identity, and the plaintext the phone kept of its uploads goes with the replica it backed. +#[test] +fn a_password_set_elsewhere_takes_the_plaintext_kept() { + unsafe { library::sb_set_versions(cloud_versions, cloud_retire, c"iPhone".as_ptr()) }; + let directory = tempfile::tempdir().unwrap(); + let (root, cache, path) = cloud_notebook(directory.path()); + let library = cloud_library(&root, &cache); + let section = cloud_section(&library, &path); + type_on(§ion, listed_ids(§ion)[0], "Secret words "); + close_section(section); + assert!(!kept(&cache).is_empty()); + + let other = directory.path().join("other"); + let copied = std::process::Command::new("cp") + .arg("-R") + .arg(&root) + .arg(&other) + .status(); + assert!(copied.unwrap().success()); + let mut notebook = + notebook::session::Notebook::open(&other, directory.path().join("other-cache")).unwrap(); + notebook + .set_password(&path, None, Some("password")) + .unwrap(); + drop(notebook); + for entry in std::fs::read_dir(&other).unwrap() { + let entry = entry.unwrap(); + if entry.path().is_file() { + std::fs::copy(entry.path(), root.join(entry.file_name())).unwrap(); + } + } + let tabs = library.tabs().unwrap(); + assert!(tabs.iter().any(|tab| tab.path == path && tab.locked)); + assert!(kept(&cache).is_empty()); +} + +/// A copy of a section made while the notebook is open keeps what it publishes of its own, +/// beside its own replica, never in place of the original's. +#[test] +fn a_copy_found_while_open_keeps_its_own_uploads() { + unsafe { library::sb_set_versions(cloud_versions, cloud_retire, c"iPhone".as_ptr()) }; + let directory = tempfile::tempdir().unwrap(); + let (root, cache, path) = cloud_notebook(directory.path()); + let library = cloud_library(&root, &cache); + let section = cloud_section(&library, &path); + type_on(§ion, listed_ids(§ion)[0], "Original words "); + close_section(section); + let original = kept(&cache); + assert_eq!(original.len(), 1); + std::fs::copy(root.join(&path), root.join("Copy.one")).unwrap(); + assert!( + library + .tabs() + .unwrap() + .iter() + .any(|tab| tab.path == "Copy.one") + ); + let copy = cloud_section(&library, "Copy.one"); + type_on(©, listed_ids(©)[0], "Copy words "); + close_section(copy); + let images = kept(&cache); + assert_eq!(images.len(), 2); + assert!(images.contains(&original[0])); + assert!(images.contains(&std::fs::read(root.join("Copy.one")).unwrap())); +} + /// The phone's upload to an iCloud Drive notebook is lost after the phone renamed the /// notebook: the images its publications keep move with it, and the next launch merges it back. #[test] fn an_upload_lost_after_the_notebook_is_renamed_merges_back() { - unsafe { - library::sb_set_versions(cloud_versions, cloud_retire, c"Clover’s iPhone".as_ptr()) - }; + unsafe { library::sb_set_versions(cloud_versions, cloud_retire, c"iPhone".as_ptr()) }; let directory = tempfile::tempdir().unwrap(); let folder = |name: &str| directory.path().join(name); let (phone, mac, cache) = (folder("Notes"), folder("mac"), folder("phone-cache")); @@ -4916,7 +5104,8 @@ fn an_upload_lost_after_the_notebook_is_renamed_merges_back() { let section = cloud_section(&library, &path); type_on(§ion, page, typed); close_section(section); - library.flush(Duration::from_secs(20)).unwrap(); + // As the app closes a notebook before renaming it (Lists.swift). + library.close(Duration::from_secs(20)).unwrap(); } // Renamed in the app, as Arranging.swift renames a notebook's folder. @@ -5025,7 +5214,9 @@ fn main_thread_calls_return_while_icloud_holds_the_section_file() { let file = root.canonicalize().unwrap().join(&path); let (entered, held) = std::sync::mpsc::channel(); let (release, released) = std::sync::mpsc::channel(); - *HELD.lock().unwrap() = Some((file.to_string_lossy().into_owned(), entered, released)); + HELD.lock() + .unwrap() + .push((file.to_string_lossy().into_owned(), entered, released)); type_on(§ion, space, "Typed while iCloud downloads "); sb_library_sync_now(&library); held.recv_timeout(Duration::from_secs(10)).unwrap(); diff --git a/crates/notebook/src/background.rs b/crates/notebook/src/background.rs index bbed5fbcd1d33e19faecba1a3c5c148cd937e072..b7eac704e8b3ad15fd27cba885ce013d941bb8b1 100644 --- a/crates/notebook/src/background.rs +++ b/crates/notebook/src/background.rs @@ -144,7 +144,7 @@ impl Background { ) -> Result where R: Remote, - B: FnMut(&str) -> R, + B: FnMut(&str, Option<&Path>) -> R, L: FnMut(String) -> LF, LF: Future>>, { @@ -262,7 +262,7 @@ impl Background { continue; }; let (queued, outcome) = step_async( - &mut bind(&path), + &mut bind(&path, replica.as_deref()), replica.as_deref(), seen.as_deref(), current, @@ -360,7 +360,7 @@ impl Background { Err(error) => return Err(error), }; let (bound, folder) = (Arc::clone(&client), root.clone()); - let bind = move |path: &str| { + let bind = move |path: &str, _: Option<&Path>| { let file = match folder.as_str() { "" => path.to_owned(), root => format!("{root}/{path}"), @@ -394,7 +394,9 @@ impl Background { move |reports| { guest.watch(reports)?; let (bound, listing) = (Arc::clone(&guest), Arc::clone(&guest)); - let bind = move |path: &str| crate::live::share::HostedRemote::new(&bound, path); + let bind = move |path: &str, _: Option<&Path>| { + crate::live::share::HostedRemote::new(&bound, path) + }; let list = move |folder: String| { let listing = listing.clone(); async move { listing.entries_async(&folder).await } @@ -418,7 +420,9 @@ impl Background { true, move |_| { let (bound, listing) = (Arc::clone(&documents), Arc::clone(&documents)); - let bind = move |path: &str| crate::provider::ProviderRemote::new(&bound, path); + let bind = move |path: &str, replica: Option<&Path>| { + crate::provider::ProviderRemote::new(&bound, path, replica) + }; let list = move |folder: String| std::future::ready(listing.entries(&folder)); Ok(((bind, list), false)) }, @@ -909,17 +913,17 @@ async fn list>>>( listings } -/// Deletes the replica at `replica` if it holds nothing unpublished and no one holds it. +/// Deletes the replica at `replica` if it holds nothing unpublished and no one holds it. One +/// whose section keeps images of its publications stays as an edit waiting keeps it: an image +/// may be an upload the file lost, which merges back once the notebook opens again. fn discard(replica: &Path) { - if matches!( - crate::closed(replica).and_then(|held| crate::peek(&held)), - Ok((_, 0)) - ) { - for suffix in ["-wal", "-shm", ""] { - let mut file = replica.as_os_str().to_owned(); - file.push(suffix); - let _ = fs::remove_file(file); - } + if !crate::published::keeps(replica) + && matches!( + crate::closed(replica).and_then(|held| crate::peek(&held)), + Ok((_, 0)) + ) + { + let _ = crate::remove_replica(replica); } } @@ -1448,7 +1452,7 @@ mod tests { move |reports| { armed.send(reports).unwrap(); let (files, listed) = (remote.clone(), remote.clone()); - let bind = move |path: &str| File(files.clone(), path.to_owned()); + let bind = move |path: &str, _: Option<&Path>| File(files.clone(), path.to_owned()); let list = move |folder: String| std::future::ready(Ok(listed.list(&folder))); Ok(((bind, list), true)) }, @@ -1524,7 +1528,7 @@ mod tests { move |reports| { armed.send(reports).unwrap(); let (files, listed) = (remote.clone(), remote.clone()); - let bind = move |path: &str| File(files.clone(), path.to_owned()); + let bind = move |path: &str, _: Option<&Path>| File(files.clone(), path.to_owned()); let list = move |folder: String| std::future::ready(Ok(listed.list(&folder))); Ok(((bind, list), true)) }, @@ -1575,7 +1579,7 @@ mod tests { false, move |_| { let (files, listed) = (remote.clone(), remote.clone()); - let bind = move |path: &str| File(files.clone(), path.to_owned()); + let bind = move |path: &str, _: Option<&Path>| File(files.clone(), path.to_owned()); let list = move |folder: String| std::future::ready(Ok(listed.list(&folder))); Ok(((bind, list), true)) }, @@ -1674,7 +1678,7 @@ mod tests { move |reports| { armed.send(reports).unwrap(); let (files, listed) = (remote.clone(), remote.clone()); - let bind = move |path: &str| File(files.clone(), path.to_owned()); + let bind = move |path: &str, _: Option<&Path>| File(files.clone(), path.to_owned()); let list = move |folder: String| std::future::ready(Ok(listed.list(&folder))); Ok(((bind, list), true)) }, diff --git a/crates/notebook/src/lib.rs b/crates/notebook/src/lib.rs index f28cf69b8648d81ff2d9fc998ca55f6ddaf43a10..dff88249c9399bf035db785988655c645eeb4b83 100644 --- a/crates/notebook/src/lib.rs +++ b/crates/notebook/src/lib.rs @@ -153,6 +153,23 @@ impl Drop for Released { } } +/// Deletes the closed replica at `replica`, with the files SQLite keeps beside it and the +/// images its section's publications keep (`published`). +pub(crate) fn remove_replica(replica: &Path) -> io::Result<()> { + for suffix in ["", "-wal", "-shm"] { + let mut file = replica.as_os_str().to_owned(); + file.push(suffix); + match fs::remove_file(file) { + Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error), + _ => {} + } + } + match fs::remove_dir_all(published::folder(replica)) { + Err(error) if error.kind() != io::ErrorKind::NotFound => Err(error), + _ => Ok(()), + } +} + impl Replica { /// Seeds a new cache from a validated section image, refusing any existing path. /// An initialization error preserves the created file for inspection. @@ -286,6 +303,11 @@ impl Replica { lock(&self.section.cache) } + /// Where the cache is. + pub fn path(&self) -> Option { + self.lock().ok()?.path().map(PathBuf::from) + } + /// Hands `request` to the section thread. fn send(&self, request: working::Request) -> Result<()> { self.section.send(request) diff --git a/crates/notebook/src/location.rs b/crates/notebook/src/location.rs index f7eba5ec7ee008d1aed9d935beb21d42c28da9a4..2d1ddbf34728736772c5e525e45526ab13857244 100644 --- a/crates/notebook/src/location.rs +++ b/crates/notebook/src/location.rs @@ -41,29 +41,37 @@ pub fn smb(address: &str, share: &str, root: &str) -> String { format!("smb://{address}/{share}/{}", root.trim_matches('/')) } -/// Moves the replicas of the sections at `from` to `to`, and the images their publications +/// Moves the replicas of the sections at `from` to `to`, with the images their publications /// keep (`published`), as when the app renames or moves a notebook; with them those of every /// location within it, as a copy of a section keeps its own. Each replica `to` already holds /// for the same section stays. Every replica moved must be closed. pub fn moved(cache: &Path, from: &str, to: &str) -> io::Result<()> { - let within = format!("{from}/"); - let inner: Vec = fs::read_dir(cache.join("replicas")) - .into_iter() - .flatten() - .filter_map(|entry| fs::read_to_string(entry.ok()?.path().join(LOCATION)).ok()) - .filter(|location| location.starts_with(&within)) - .collect(); + let inner = within(cache, from); move_folder(cache, from, to)?; - for location in inner { + for (_, location) in inner { move_folder( cache, &location, - &format!("{to}/{}", &location[within.len()..]), + &format!("{to}{}", &location[from.len()..]), )?; } Ok(()) } +/// The folders of the locations within `location`, labelled, with their locations. +fn within(cache: &Path, location: &str) -> Vec<(PathBuf, String)> { + let within = format!("{location}/"); + fs::read_dir(cache.join("replicas")) + .into_iter() + .flatten() + .filter_map(|entry| { + let folder = entry.ok()?.path(); + let label = fs::read_to_string(folder.join(LOCATION)).ok()?; + label.starts_with(&within).then_some((folder, label)) + }) + .collect() +} + fn move_folder(cache: &Path, from: &str, to: &str) -> io::Result<()> { let old = folder(cache, from); if fs::metadata(&old).is_err() { @@ -78,50 +86,22 @@ fn move_folder(cache: &Path, from: &str, to: &str) -> io::Result<()> { adopt(&old.join(&name), &new.join(&name))?; } } - if let Ok(images) = fs::read_dir(old.join(PUBLISHED)) { - fs::create_dir_all(new.join(PUBLISHED))?; - for entry in images { - let name = entry?.file_name(); - let image = new.join(PUBLISHED).join(&name); - if fs::metadata(&image).is_err() { - fs::rename(old.join(PUBLISHED).join(&name), image)?; - } - } - let _ = fs::remove_dir(old.join(PUBLISHED)); - } tidy(&old); Ok(()) } -/// The folder of a replica folder holding the images its sections' publications keep. -pub const PUBLISHED: &str = "published"; - -/// The replica folder of the section file at location `file` in the notebook at location -/// `notebook`: the file's own where one was claimed, as a copy's and a lone section's are, -/// else the notebook's. -pub fn section_folder(cache: &Path, notebook: &str, file: &str) -> PathBuf { - let own = folder(cache, file); - if fs::metadata(own.join(LOCATION)).is_ok() { - own - } else { - folder(cache, notebook) - } -} - /// Deletes what the cache keeps of the notebook at `location`, as when the app deletes the /// notebook: its replicas, edits waiting in them included, its sections' copies' and its /// listing. Every replica must be closed. pub fn forget(cache: &Path, location: &str) -> io::Result<()> { - let within = format!("{location}/"); - if let Ok(entries) = fs::read_dir(cache.join("replicas")) { - for entry in entries { - let folder = entry?.path(); - let ours = fs::read_to_string(folder.join(LOCATION)) - .is_ok_and(|label| label == location || label.starts_with(&within)); - if ours { - fs::remove_dir_all(&folder)?; - } - } + let own = folder(cache, location); + let ours = fs::read_to_string(own.join(LOCATION)).is_ok_and(|label| label == location); + for folder in ours.then_some(own).into_iter().chain( + within(cache, location) + .into_iter() + .map(|(folder, _)| folder), + ) { + fs::remove_dir_all(&folder)?; } match fs::remove_file(crate::session::listing(cache, location)) { Err(error) if error.kind() != io::ErrorKind::NotFound => Err(error), @@ -130,7 +110,7 @@ pub fn forget(cache: &Path, location: &str) -> io::Result<()> { } /// The folder for `location`, made and labelled with it where it was not. -fn claimed(cache: &Path, location: &str) -> io::Result { +pub fn claimed(cache: &Path, location: &str) -> io::Result { let folder = folder(cache, location); let label = folder.join(LOCATION); if fs::metadata(&label).is_err() { @@ -223,7 +203,8 @@ fn follows(file: &Stamp, base: &Stamp) -> bool { ) } -/// Moves the replica at `from`, with the write-ahead log a crash may have left, to `to`. +/// Moves the replica at `from`, with the write-ahead log a crash may have left and the images +/// its publications keep, to `to`. fn adopt(from: &Path, to: &Path) -> io::Result<()> { for suffix in ["-wal", "-shm", ""] { let (mut source, mut target) = (from.as_os_str().to_owned(), to.as_os_str().to_owned()); @@ -234,7 +215,10 @@ fn adopt(from: &Path, to: &Path) -> io::Result<()> { moved => moved?, } } - Ok(()) + match fs::rename(crate::published::folder(from), crate::published::folder(to)) { + Err(error) if error.kind() != io::ErrorKind::NotFound => Err(error), + _ => Ok(()), + } } /// Removes a replica folder that holds no replica, with the owner lock a crash may have left diff --git a/crates/notebook/src/provider.rs b/crates/notebook/src/provider.rs index 2a9691e17b92bec8f85cc98b2515bcd0166c2669..a1c2e87d48484080ad1d0f377dd48ff6525e0f58 100644 --- a/crates/notebook/src/provider.rs +++ b/crates/notebook/src/provider.rs @@ -6,7 +6,7 @@ use crate::{ Error, Result, discover, - published::{Published, Written}, + published::{Published, Written, whole}, session::Storage, }; use onestore::{CommitError, CommitIo, CommitState, Stamp, Store, Transaction}; @@ -14,7 +14,7 @@ use std::{ collections::BTreeMap, fs::File, io::{self, Read, Seek, SeekFrom, Write}, - path::PathBuf, + path::{Path, PathBuf}, sync::{Arc, Mutex, MutexGuard, PoisonError}, }; @@ -78,12 +78,6 @@ impl Documents { format!("provider:{}", self.root) } - /// Where this device keeps the images it published until the documents are seen holding - /// them: a provider without compare-and-swap may let another device's upload replace one. - fn published(&self) -> PathBuf { - crate::location::folder(&self.cache, &self.location()).join(crate::location::PUBLISHED) - } - fn id(&self, path: &str) -> io::Result { if path.is_empty() { return Ok(self.root.clone()); @@ -288,15 +282,6 @@ fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { mutex.lock().unwrap_or_else(PoisonError::into_inner) } -fn whole(file: &mut File, limit: usize) -> io::Result> { - let mut bytes = Vec::new(); - file.take(limit as u64 + 1).read_to_end(&mut bytes)?; - if bytes.len() > limit { - return Err(io::ErrorKind::FileTooLarge.into()); - } - Ok(bytes) -} - /// A document as a commit writes it: its seekable descriptor, or its image to write whole. enum Io { File(File), @@ -349,21 +334,22 @@ impl CommitIo for Io { } /// A section a document provider keeps, as its replica publishes to it, each image it -/// publishes kept until the document holds it (`published.rs`). +/// publishes kept beside the replica until the document holds it (`published.rs`). pub(crate) struct ProviderRemote { documents: Arc, path: String, - published: Published, + /// None for a section with no replica yet, which publishes nothing. + published: Option, /// The image the step read whole for a stamp, which its read takes. read: Option>, } impl ProviderRemote { - pub(crate) fn new(documents: &Arc, path: &str) -> Self { + pub(crate) fn new(documents: &Arc, path: &str, replica: Option<&Path>) -> Self { Self { documents: Arc::clone(documents), path: path.to_owned(), - published: Published::new(documents.published(), Some(documents.device.clone())), + published: replica.map(|replica| Published::of(replica, &documents.device)), read: None, } } @@ -372,10 +358,8 @@ impl ProviderRemote { /// put back and read again: the newest image kept that the torn bytes begin. Any other /// document that does not parse is left as it is. fn repair(&self, torn: &[u8]) -> io::Result> { - let image = self - .published - .torn(torn)? - .ok_or(io::ErrorKind::WouldBlock)?; + let published = self.published.as_ref().ok_or(io::ErrorKind::WouldBlock)?; + let image = published.torn(torn)?.ok_or(io::ErrorKind::WouldBlock)?; let id = self.documents.id(&self.path)?; self.documents.writable(&id)?; self.documents @@ -420,14 +404,23 @@ impl crate::Remote for ProviderRemote { self.documents.write( &self.path, |io| transaction.commit(io), - |written| self.published.keep(written, transaction), + |written| match &self.published { + Some(published) => published.keep(written, transaction), + None => Ok(()), + }, ) } fn confirm(&mut self, base: &Stamp) -> std::result::Result<(), CommitError> { self.read = None; - self.documents - .write(&self.path, |io| onestore::confirm(io, base), |_| Ok(())) + self.documents.write( + &self.path, + |io| onestore::confirm(io, base), + |written| match &self.published { + Some(published) => published.confirmed(written, base), + None => Ok(()), + }, + ) } /// The images kept of this document other than the one it stands as @@ -441,7 +434,10 @@ impl crate::Remote for ProviderRemote { (stamp, image) = (repaired_stamp, Some(repaired)); } self.read = image; - Ok(self.published.versions(&stamp)) + match &self.published { + Some(published) => published.versions(|| Ok(stamp), &[]), + None => Ok(Vec::new()), + } } fn version(&mut self, id: &str) -> io::Result> { diff --git a/crates/notebook/src/provider/tests.rs b/crates/notebook/src/provider/tests.rs index 58fff58ad285f9c53b4e566c996c38d7a906885f..9324a3e840e92c65aa4fdb6e1a1340ee6332b4c5 100644 --- a/crates/notebook/src/provider/tests.rs +++ b/crates/notebook/src/provider/tests.rs @@ -899,8 +899,8 @@ fn a_provider_that_lists_but_cannot_open_opens_sections_from_their_replicas() { section.close().unwrap(); } -/// The images a device keeps of what it published, for every notebook in its cache. -fn kept_images(fake: &Fake) -> Vec { +/// Where a device keeps the images it published, beside each replica in its cache. +fn stores(fake: &Fake) -> Vec { let cache = fake .store .folder @@ -908,7 +908,21 @@ fn kept_images(fake: &Fake) -> Vec { .join(format!("cache-{}", fake.device)); let folders = fs::read_dir(cache.join("replicas")).into_iter().flatten(); folders - .filter_map(|folder| fs::read_dir(folder.ok()?.path().join("published")).ok()) + .filter_map(|folder| fs::read_dir(folder.ok()?.path()).ok()) + .flatten() + .map(|entry| entry.unwrap().path()) + .filter(|path| { + path.extension() + .is_some_and(|extension| extension == "published") + }) + .collect() +} + +/// The images a device keeps of what it published, for every notebook in its cache. +fn kept_images(fake: &Fake) -> Vec { + stores(fake) + .into_iter() + .filter_map(|store| fs::read_dir(store).ok()) .flatten() .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) .filter(|name| name.ends_with(".one")) @@ -962,16 +976,8 @@ fn a_publication_another_devices_upload_replaced_merges_back() { /// Ages the images a device keeps past `LAG`, as if reads had long had time to catch up. fn aged(fake: &Fake) { - let cache = fake - .store - .folder - .path() - .join(format!("cache-{}", fake.device)); - for folder in fs::read_dir(cache.join("replicas")).unwrap() { - for image in fs::read_dir(folder.unwrap().path().join("published")) - .into_iter() - .flatten() - { + for store in stores(fake) { + for image in fs::read_dir(store).into_iter().flatten() { let file = File::options() .write(true) .open(image.unwrap().path()) diff --git a/crates/notebook/src/published.rs b/crates/notebook/src/published.rs index 573852ab73a37358c42f41cdecd238020e819635..3475c0c0c966c7746feda60a18d862844ae57149 100644 --- a/crates/notebook/src/published.rs +++ b/crates/notebook/src/published.rs @@ -1,8 +1,9 @@ //! The images this device publishes to section files a provider syncs whole, with no lock or //! compare-and-swap, as iCloud Drive and Android's document providers do: another device's -//! upload can replace this one's. Each image is kept until the file is seen holding it, and -//! one the file lost is listed as a `Remote::versions` version, so that it merges back as a -//! conflict version does (`resolve.rs`). +//! upload can replace this one's. Each image is kept beside the section's replica until the +//! file is seen holding it, and one the file lost is listed as a `Remote::versions` version, +//! so that it merges back as a conflict version does (`resolve.rs`). They move and go with +//! the replica (`location::moved`, `remove_replica`). use crate::Version; use onestore::{Stamp, Transaction}; @@ -23,13 +24,13 @@ pub enum Written<'a> { Image(&'a [u8]), } -/// The images this device published to the files of one place, kept in `folder` and named -/// `--.one`: the file's identity, then the digests of the image's stamp -/// and of the stamp of the image it was published on. Version ids are their paths. +/// The images this device published to one section file, named `--.one`: +/// the file's identity, then the digests of the image's stamp and of the stamp of the image it +/// was published on. Version ids are their paths. pub struct Published { folder: PathBuf, /// This device, as the conflict pages a lost publication makes name it. - device: Option, + device: String, } struct Kept { @@ -40,21 +41,22 @@ struct Kept { at: SystemTime, } +/// The folder beside the replica at `replica` holding its section's images. +pub(crate) fn folder(replica: &Path) -> PathBuf { + replica.with_extension("published") +} + impl Published { - pub fn new(folder: impl Into, device: Option) -> Self { + /// The images kept of the section whose replica is at `replica`. + pub fn of(replica: &Path, device: impl Into) -> Self { Self { - folder: folder.into(), - device, + folder: folder(replica), + device: device.into(), } } - /// The images kept of the section file `file` of the notebook folder, or lone section, at - /// `notebook`, which `cache` holds the replicas of: beside the section's replica, so that - /// they move with the notebook (`location::moved`). - pub fn of(cache: &Path, notebook: &Path, file: &Path, device: Option) -> Option { - use crate::location::{PUBLISHED, local, section_folder}; - let folder = section_folder(cache, &local(notebook).ok()?, &local(file).ok()?); - Some(Self::new(folder.join(PUBLISHED), device)) + pub fn device(&self) -> &str { + &self.device } fn kept(&self) -> Vec { @@ -75,6 +77,19 @@ impl Published { .collect() } + fn put(&self, image: &[u8], name: String, replacing: Option) -> io::Result<()> { + crate::fs::create_dir_all(&self.folder)?; + let mut kept = tempfile::NamedTempFile::new_in(&self.folder)?; + kept.write_all(image)?; + kept.as_file().sync_all()?; + kept.persist(self.folder.join(name)) + .map_err(|error| error.error)?; + if let Some(kept) = replacing { + crate::fs::remove_file(kept.path)?; + } + Ok(()) + } + /// Keeps the image `transaction` published, in place of the one it was published on: /// that kept image with the transaction applied where this device kept it, else read back. pub fn keep(&self, written: Written<'_>, transaction: &Transaction) -> io::Result<()> { @@ -93,31 +108,46 @@ impl Published { } (Written::InPlace(file), None) => { file.seek(SeekFrom::Start(0))?; - let mut image = Vec::new(); - file.take(crate::MAX_FILE_BYTES as u64 + 1) - .read_to_end(&mut image)?; - if image.len() > crate::MAX_FILE_BYTES { - return Err(io::ErrorKind::FileTooLarge.into()); - } - image + whole(file, crate::MAX_FILE_BYTES)? } }; let name = format!("{document}-{}-{base}.one", digest(&transaction.stamp())); - crate::fs::create_dir_all(&self.folder)?; - let mut kept = tempfile::NamedTempFile::new_in(&self.folder)?; - kept.write_all(&image)?; - kept.as_file().sync_all()?; - kept.persist(self.folder.join(name)) - .map_err(|error| error.error)?; - if let Some(kept) = on { - crate::fs::remove_file(kept.path)?; + self.put(&image, name, on) + } + + /// Follows the image kept as `base` through a confirmation (`onestore::confirm`), which + /// rewrites only the header: the image `written` now begins with. + pub fn confirmed(&self, written: Written<'_>, base: &Stamp) -> io::Result<()> { + let image = digest(base); + let Some(kept) = (self.kept().into_iter()).find(|kept| kept.image == image) else { + return Ok(()); + }; + let mut header = [0; 1024]; + match written { + Written::InPlace(file) => { + file.seek(SeekFrom::Start(0))?; + file.read_exact(&mut header)?; + } + Written::Image(image) => { + header.copy_from_slice(image.get(..1024).ok_or(io::ErrorKind::UnexpectedEof)?) + } } - Ok(()) + let mut confirmed = crate::fs::read(&kept.path)?; + confirmed + .get_mut(..1024) + .ok_or(io::ErrorKind::InvalidData)? + .copy_from_slice(&header); + let stamp = Stamp { + header, + length: confirmed.len() as u64, + }; + let name = format!("{}-{}-{}.one", kept.document, digest(&stamp), kept.base); + self.put(&confirmed, name, Some(kept)) } /// The newest image kept that `torn` begins, as a provider writing in place leaves this /// device's upload cut off. - pub fn torn(&self, torn: &[u8]) -> io::Result>> { + pub(crate) fn torn(&self, torn: &[u8]) -> io::Result>> { let mut newest = None; for kept in self.kept() { let image = crate::fs::read(&kept.path)?; @@ -128,28 +158,60 @@ impl Published { Ok(newest.map(|(_, image)| image)) } - /// The images kept of the file `stamp` stamps, other than the one it stands as; one whose - /// upload the file still stands before is listed only once reads have had `LAG` to catch up. - pub fn versions(&self, stamp: &Stamp) -> Vec { + /// The images kept other than the one the file stands as, by `stamp`, which is read only + /// where any is kept; one whose upload the file still stands before is listed only once + /// reads have had `LAG` to catch up. One the provider keeps as a conflict version too, in + /// `conflicts`, is left to that version, so that it is not merged, or kept beside, twice; + /// none is listed while a conflict version cannot be read. + pub fn versions( + &self, + stamp: impl FnOnce() -> io::Result, + conflicts: &[Version], + ) -> io::Result> { + let kept = self.kept(); + if kept.is_empty() { + return Ok(Vec::new()); + } + let stamp = stamp()?; // A header mid-write names no file. - let Some(document) = document(stamp) else { - return Vec::new(); + let Some(document) = document(&stamp) else { + return Ok(Vec::new()); }; - let current = digest(stamp); + let current = digest(&stamp); let lagging = |kept: &Kept| { kept.base == current && kept.at.elapsed().is_ok_and(|elapsed| elapsed < LAG) }; - (self.kept().into_iter()) - .filter(|kept| kept.document == document && kept.image != current && !lagging(kept)) + // A conflict version not read yet, as one iCloud Drive has not brought down, may be + // one of these: they wait for it, rather than be merged or kept beside the file twice. + let Ok(conflicts) = (conflicts.iter()) + .map(|version| Ok(digest(&file_stamp(Path::new(&version.id))?))) + .collect::>>() + else { + return Ok(Vec::new()); + }; + Ok(kept + .into_iter() + .filter(|kept| { + kept.document == document + && kept.image != current + && !lagging(kept) + && !conflicts.contains(&kept.image) + }) .map(|kept| Version { id: kept.path.to_string_lossy().into_owned(), - device: self.device.clone(), + device: Some(self.device.clone()), }) - .collect() + .collect()) } - pub fn device(&self) -> Option<&str> { - self.device.as_deref() + /// The provider's conflict version at `version` is retired, merged or kept beside the + /// file: an image kept here that it is goes with it, so that it is not merged again. + pub fn retired(&self, version: &Path) -> io::Result<()> { + let image = digest(&file_stamp(version)?); + for kept in self.kept().into_iter().filter(|kept| kept.image == image) { + crate::fs::remove_file(kept.path)?; + } + Ok(()) } /// Whether version `id` is an image kept here. @@ -158,18 +220,48 @@ impl Published { } } +/// The stamp of the file at `path`: its header and length. +fn file_stamp(path: &Path) -> io::Result { + let mut file = File::open(path)?; + let mut header = [0; 1024]; + file.read_exact(&mut header)?; + Ok(Stamp { + header, + length: file.metadata()?.len(), + }) +} + +/// Whether the section whose replica is at `replica` keeps images of its publications. +pub(crate) fn keeps(replica: &Path) -> bool { + crate::fs::read_dir(folder(replica)).is_ok_and(|mut images| images.next().is_some()) +} + /// Where a version of `file` that cannot merge is kept beside it, as iCloud Drive keeps a /// conflict version: "Name (Device).one", numbered past any name taken. pub fn beside(file: &Path, device: &str) -> PathBuf { let stem = file.file_stem().unwrap_or_default().to_string_lossy(); - (1..) - .map(|n| match n { + let mut n = 1; + loop { + let name = match n { 1 => format!("{stem} ({device}).one"), n => format!("{stem} ({device} {n}).one"), - }) - .map(|name| file.with_file_name(name)) - .find(|beside| std::fs::symlink_metadata(beside).is_err()) - .unwrap_or_default() + }; + let beside = file.with_file_name(name); + if std::fs::symlink_metadata(&beside).is_err() { + return beside; + } + n += 1; + } +} + +/// All of `file`, at most `limit` bytes. +pub(crate) fn whole(file: &mut File, limit: usize) -> io::Result> { + let mut bytes = Vec::new(); + file.take(limit as u64 + 1).read_to_end(&mut bytes)?; + if bytes.len() > limit { + return Err(io::ErrorKind::FileTooLarge.into()); + } + Ok(bytes) } /// The hex of a stamp's digest, which names an image this device keeps. @@ -189,3 +281,112 @@ fn document(stamp: &Stamp) -> Option { fn hex(bytes: &[u8]) -> String { bytes.iter().map(|byte| format!("{byte:02x}")).collect() } + +#[cfg(test)] +mod tests { + use super::*; + use onestore::op::{Edit, Op, PageOp}; + + /// A section file in a scratch folder, its image, and the transaction one edit of it + /// publishes. + fn edited(folder: &Path) -> (PathBuf, Vec, Transaction) { + let file = folder.join("Notes.one"); + let image = onestore::create_section("Notes.one", "Text", "Author").unwrap(); + std::fs::write(&file, &image).unwrap(); + let arena = onestore::Arena::default(); + let mut section = onestore::Section::open(&arena, image.clone()).unwrap(); + let space = section.pages().unwrap()[0].0; + let edit = Edit { + at: crate::now(), + ops: vec![Op::Page { + space, + op: PageOp::Color(Some(0x00ab_cdef)), + }], + }; + section.apply("Author", &edit).unwrap(); + (file, image, section.seal().unwrap().unwrap()) + } + + fn published(folder: &Path, file: &Path, transaction: &Transaction) -> Published { + let published = Published::of(&folder.join("replica.sqlite"), "iPhone"); + transaction.commit_file(file).unwrap(); + let mut opened = File::open(file).unwrap(); + published + .keep(Written::InPlace(&mut opened), transaction) + .unwrap(); + published + } + + #[test] + fn nothing_kept_reads_no_stamp() { + let folder = tempfile::tempdir().unwrap(); + let published = Published::of(&folder.path().join("replica.sqlite"), "iPhone"); + let listed = published.versions(|| unreachable!("no stamp is read"), &[]); + assert!(listed.unwrap().is_empty()); + } + + /// The image this device published lists once the file stands otherwise: once reads have + /// had `LAG` to catch up where the file stands as the image it was published on, and not + /// where the provider keeps it as a conflict version itself. + #[test] + fn a_lost_image_lists_once() { + let folder = tempfile::tempdir().unwrap(); + let (file, base, transaction) = edited(folder.path()); + let published = published(folder.path(), &file, &transaction); + let listed = + |conflicts: &[Version]| published.versions(|| file_stamp(&file), conflicts).unwrap(); + assert!(listed(&[]).is_empty()); + let ours = std::fs::read(&file).unwrap(); + std::fs::write(&file, base).unwrap(); + assert!(listed(&[]).is_empty()); + let aged = File::options() + .write(true) + .open(&published.kept()[0].path) + .unwrap(); + aged.set_modified(SystemTime::now() - LAG - Duration::from_secs(1)) + .unwrap(); + assert_eq!(listed(&[]).len(), 1); + // A conflict version not here yet could be this image: it waits for it. + let missing = Version { + id: file + .with_file_name("Not here.one") + .to_string_lossy() + .into_owned(), + device: None, + }; + assert!(listed(&[missing]).is_empty()); + let version = file.with_file_name("Notes (iPhone).one"); + std::fs::write(&version, ours).unwrap(); + let conflict = Version { + id: version.to_string_lossy().into_owned(), + device: Some("iPhone".into()), + }; + assert!(listed(std::slice::from_ref(&conflict)).is_empty()); + published.retired(&version).unwrap(); + assert!(published.kept().is_empty()); + } + + /// A confirmation rewrites the header of the image last published: the kept image follows + /// it, rather than listing as lost and going, which would leave it unprotected. + #[test] + fn a_confirmation_keeps_the_image_kept() { + let folder = tempfile::tempdir().unwrap(); + let (file, _, transaction) = edited(folder.path()); + let published = published(folder.path(), &file, &transaction); + let base = file_stamp(&file).unwrap(); + onestore::confirm_file(&file, &base).unwrap(); + assert_ne!(file_stamp(&file).unwrap(), base); + let mut opened = File::open(&file).unwrap(); + published + .confirmed(Written::InPlace(&mut opened), &base) + .unwrap(); + let kept = published.kept(); + assert_eq!(kept.len(), 1); + assert_eq!( + std::fs::read(&kept[0].path).unwrap(), + std::fs::read(&file).unwrap() + ); + let listed = published.versions(|| file_stamp(&file), &[]).unwrap(); + assert!(listed.is_empty()); + } +} diff --git a/crates/notebook/src/session.rs b/crates/notebook/src/session.rs index 5d48608ab5689b369d0c13ceb6eda102d334c222..1dba2ee954416c3800e455dbd2acd8996bc04c2d 100644 --- a/crates/notebook/src/session.rs +++ b/crates/notebook/src/session.rs @@ -626,7 +626,7 @@ impl Notebook { _ => continue, }; claims - .entry(replica_location(&location, section)) + .entry(replica_location(&location, §ion.path, section.copy)) .or_default() .push((identity, §ion.path)); } @@ -1017,7 +1017,7 @@ impl Notebook { self.catalog .sections() .filter(|section| section.copy) - .map(|section| replica_location(&from, section)), + .map(|section| replica_location(&from, §ion.path, section.copy)), ); for location in locations { let entries = match fs::read_dir(crate::location::folder(&self.cache, &location)) { @@ -1592,11 +1592,13 @@ impl Notebook { }); let _ = self.refresh(); drop(replica); - // Unless something else opened it meanwhile. + // Unless something else opened it meanwhile, or it keeps images of uploads the old + // file may have lost, which name that file and stay with its replica. if let Ok(file) = file && crate::closed(&file).is_ok() + && !crate::published::keeps(&file) { - let _ = remove_replica(&file); + let _ = crate::remove_replica(&file); } Ok(Moved { path: copy, @@ -1817,7 +1819,7 @@ impl Notebook { replica.apply_async(&author, edit).await?; } drop(old); - remove_replica(&path)?; + crate::remove_replica(&path)?; } } Ok(key) @@ -1893,7 +1895,7 @@ impl Notebook { for (replica, held, queued) in self.superseded(&locked) { drop(held); if queued == 0 { - let _ = remove_replica(&replica); + let _ = crate::remove_replica(&replica); } } } @@ -1944,7 +1946,7 @@ impl Notebook { return Err(error); } drop(held); - remove_replica(&replica)?; + crate::remove_replica(&replica)?; let parent = self.folder(folder)?; if let Some(identity) = Entry::of(parent, section).identity { self.edit_toc( @@ -1980,9 +1982,15 @@ impl Notebook { } /// The `location::folder` holding the replica of `section`. + /// A copy's is labelled with its location as it is first named, as one found after the + /// notebook opened is, so that it moves and goes with the notebook (`location::moved`). fn replica_folder(&self, section: &discover::Section) -> PathBuf { - let location = replica_location(&self.storage.location(), section); - crate::location::folder(&self.cache, &location) + let location = replica_location(&self.storage.location(), §ion.path, section.copy); + match section.copy { + true => crate::location::claimed(&self.cache, &location) + .unwrap_or_else(|_| crate::location::folder(&self.cache, &location)), + false => crate::location::folder(&self.cache, &location), + } } /// Every readable section, for `Background::watch`, handing on the files the last @@ -2012,16 +2020,21 @@ impl Notebook { copies: bool, notify: impl Fn() + Send + 'static, ) -> Result { - self.background_with(watched, copies, |file| FileRemote(file.to_owned()), notify) + self.background_with( + watched, + copies, + |file, _| FileRemote(file.to_owned()), + notify, + ) } - /// `background`, reaching each section file through the remote `remote` makes for it - /// (`section_with`). + /// `background`, reaching each section file through the remote `remote` makes for it and + /// its replica, where it has one (`section_with`). pub fn background_with( &self, watched: bool, copies: bool, - remote: impl Fn(&Path) -> R + Clone + Send + 'static, + remote: impl Fn(&Path, Option<&Path>) -> R + Clone + Send + 'static, notify: impl Fn() + Send + 'static, ) -> Result { let Some(root) = self.root.clone() else { @@ -2035,7 +2048,8 @@ impl Notebook { copies, move |_| { let (folder, remote) = (root.clone(), remote.clone()); - let bind = move |path: &str| remote(&folder.join(path)); + let bind = + move |path: &str, replica: Option<&Path>| remote(&folder.join(path), replica); let mut local = discover::Local::open(&root)?; let list = move |folder: String| { use discover::Source; @@ -2056,18 +2070,18 @@ impl Notebook { ) -> Result
{ self.section_with( path, - |file| Ok(FileRemote(file.to_owned())), + |file, _| Ok(FileRemote(file.to_owned())), offline, notify, ) } - /// `section`, reading and publishing through the remote `connect` makes for its file - /// (`Section::open_with`). + /// `section`, reading and publishing through the remote `connect` makes for its file and + /// its replica, which it reads through before there is one (`Section::open_with`). pub fn section_with( &self, path: &str, - connect: impl FnMut(&Path) -> io::Result + Send + 'static, + connect: impl FnMut(&Path, Option<&Path>) -> io::Result + Send + 'static, offline: bool, notify: impl Fn() + Send + 'static, ) -> Result
{ @@ -2085,7 +2099,7 @@ impl Notebook { self.section_unlocked_with( path, key, - |file| Ok(FileRemote(file.to_owned())), + |file, _| Ok(FileRemote(file.to_owned())), offline, notify, ) @@ -2096,7 +2110,7 @@ impl Notebook { &self, path: &str, key: &protected::Key, - connect: impl FnMut(&Path) -> io::Result + Send + 'static, + connect: impl FnMut(&Path, Option<&Path>) -> io::Result + Send + 'static, offline: bool, notify: impl Fn() + Send + 'static, ) -> Result
{ @@ -2107,7 +2121,7 @@ impl Notebook { &self, path: &str, key: Option<&protected::Key>, - connect: impl FnMut(&Path) -> io::Result + Send + 'static, + connect: impl FnMut(&Path, Option<&Path>) -> io::Result + Send + 'static, offline: bool, notify: impl Fn() + Send + 'static, ) -> Result
{ @@ -2146,7 +2160,7 @@ impl Notebook { return Section::start( file, replica, - move || connect(&remote), + move || connect(&remote, Some(&cache)), CHECK, offline, notify, @@ -2412,27 +2426,14 @@ fn catalog_path(folder: &str, name: &str) -> String { /// The location keying the replica of `section` in the notebook at `notebook`: the notebook's, /// so that a section renamed or moved within it keeps its replica, or for a copy of another /// section of the notebook its own file's. -fn replica_location(notebook: &str, section: &discover::Section) -> String { - if section.copy { - format!("{notebook}/{}", section.path) +pub fn replica_location(notebook: &str, path: &str, copy: bool) -> String { + if copy { + format!("{notebook}/{path}") } else { notebook.to_owned() } } -/// Deletes the closed replica at `replica`, with the files SQLite keeps beside it. -fn remove_replica(replica: &Path) -> io::Result<()> { - for suffix in ["", "-wal", "-shm"] { - let mut file = replica.as_os_str().to_owned(); - file.push(suffix); - match fs::remove_file(file) { - Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error), - _ => {} - } - } - Ok(()) -} - /// How often an open section's file is checked where nothing watches it. const CHECK: Duration = Duration::from_secs(2); @@ -2598,7 +2599,7 @@ impl Section { Self::open_with( file, cache, - |file| Ok(FileRemote(file.to_owned())), + |file, _| Ok(FileRemote(file.to_owned())), offline, notify, ) @@ -2610,7 +2611,7 @@ impl Section { pub fn open_with( file: impl AsRef, cache: impl AsRef, - connect: impl FnMut(&Path) -> io::Result + Send + 'static, + connect: impl FnMut(&Path, Option<&Path>) -> io::Result + Send + 'static, offline: bool, notify: impl Fn() + Send + 'static, ) -> Result { @@ -2626,12 +2627,12 @@ impl Section { file: PathBuf, key: Option<&protected::Key>, replica: impl FnOnce(&[u8; 16], &[u8]) -> Result, - mut connect: impl FnMut(&Path) -> io::Result + Send + 'static, + mut connect: impl FnMut(&Path, Option<&Path>) -> io::Result + Send + 'static, interval: Duration, offline: bool, notify: impl Fn() + Send + 'static, ) -> Result { - let source = connect(&file)?.read()?; + let source = connect(&file, None)?.read()?; let store = Store::parse(&source)?; let identity = RevisionIndex::parse(&store)?.root; let cache = replica(&identity.guid, &source)?; @@ -2640,7 +2641,7 @@ impl Section { Self::start( file, replica, - move || connect(&remote), + move || connect(&remote, Some(&cache)), interval, offline, notify, @@ -2717,10 +2718,15 @@ impl Section { offline: bool, notify: impl Fn() + Send + 'static, ) -> Result { + let cache = replica.path(); Self::start( PathBuf::from(&path), replica, - move || Ok(crate::provider::ProviderRemote::new(&documents, &path)), + move || { + let remote = + crate::provider::ProviderRemote::new(&documents, &path, cache.as_deref()); + Ok(remote) + }, Background::UNWATCHED, offline, notify, @@ -2738,7 +2744,11 @@ impl Section { let (cache, location) = (documents.cache.clone(), documents.location()); let cache = cache.as_path(); let file = PathBuf::from(&documents.root); - let remote = move || Ok(crate::provider::ProviderRemote::new(&documents, "")); + let remote = move |replica: Option<&Path>| { + Ok(crate::provider::ProviderRemote::new( + &documents, "", replica, + )) + }; let notify = Arc::new(notify); let (connect, woken) = (remote.clone(), Arc::clone(¬ify)); let replicas = claimed(cache, &location); @@ -2746,7 +2756,7 @@ impl Section { file.clone(), None, replicas, - move |_| connect(), + move |_, replica| connect(replica), Background::UNWATCHED, offline, move || woken(), @@ -2761,8 +2771,8 @@ impl Section { match (replicas.next(), replicas.next()) { (Some(only), None) => Self::start( file, - Replica::open(only)?, - remote, + Replica::open(&only)?, + move || remote(Some(&only)), Background::UNWATCHED, offline, move || notify(), @@ -2872,7 +2882,9 @@ impl Section { /// Follows a section file moved within its notebook, keeping its replica and queued edits. pub fn relocate(&mut self, file: impl AsRef) -> Result<()> { - self.relocate_with(fs::absolute(file)?, |file| Ok(FileRemote(file.to_owned()))) + self.relocate_with(fs::absolute(file)?, |file, _| { + Ok(FileRemote(file.to_owned())) + }) } /// `relocate`, with a host's publication target. Cancellation leaves any in-flight step @@ -2880,17 +2892,21 @@ impl Section { pub fn relocate_with( &mut self, file: impl AsRef, - mut connect: impl FnMut(&Path) -> io::Result + Send + 'static, + mut connect: impl FnMut(&Path, Option<&Path>) -> io::Result + Send + 'static, ) -> Result<()> { let file = file.as_ref().to_owned(); - let target = file.clone(); + let (target, replica) = (file.clone(), self.replica.path()); let previous = self.worker.take(); if let Some(worker) = &previous { worker.cancel(); } self.observed = Arc::new(Mutex::new(None)); - self.worker = - Some(self.start_worker(move || connect(&target), CHECK, false, previous.as_ref())?); + self.worker = Some(self.start_worker( + move || connect(&target, replica.as_deref()), + CHECK, + false, + previous.as_ref(), + )?); self.file = file; Ok(()) } diff --git a/crates/notebook/src/session/tests.rs b/crates/notebook/src/session/tests.rs index 3c29c6406cc0815af75cdeda92e78a5dbe7b59d6..48175733e1771bd91074541e0d8f056bb06c28d6 100644 --- a/crates/notebook/src/session/tests.rs +++ b/crates/notebook/src/session/tests.rs @@ -691,7 +691,7 @@ fn initially_offline_workers_keep_the_original_queue_until_explicitly_woken() { fs::rename(&file, &target).unwrap(); let counter = Arc::clone(&calls); section - .relocate_with(&target, move |file| { + .relocate_with(&target, move |file, _| { counter[0].fetch_add(1, Ordering::SeqCst); Ok(Counted { file: FileRemote(file.to_owned()), diff --git a/crates/notebook/tests/session.rs b/crates/notebook/tests/session.rs index 86d5ceeb5e2823f5a1bf4c9a9a2d31ada397bac5..f4023a842c9959be3e6d3675c4c728f2cf785993 100644 --- a/crates/notebook/tests/session.rs +++ b/crates/notebook/tests/session.rs @@ -1360,7 +1360,7 @@ fn a_moved_section_keeps_its_replica_while_the_old_remote_is_stalled() { let mut section = Section::open_with( &file, directory.path().join("cache"), - move |file| { + move |file, _| { let finishing = (!std::mem::replace(&mut opening, false)).then(|| finished.clone()); if finishing.is_some() { connected.send(()).unwrap(); @@ -1460,7 +1460,7 @@ fn an_open_section_the_background_holds_checks_its_file_when_the_watch_reports_i let section = notebook .section_with( "Second.one", - move |file| Ok(Counted(file.to_owned(), Arc::clone(&counter))), + move |file, _| Ok(Counted(file.to_owned(), Arc::clone(&counter))), false, || {}, ) @@ -1544,7 +1544,7 @@ fn the_background_takes_each_file_discovery_read_rather_than_reading_it_again() let mut notebook = two_sections(directory.path()); let reads = Arc::new(AtomicUsize::new(0)); let counter = Arc::clone(&reads); - let remote = move |file: &Path| Reads(file.to_owned(), Arc::clone(&counter)); + let remote = move |file: &Path, _: Option<&Path>| Reads(file.to_owned(), Arc::clone(&counter)); let synced = |background: ¬ebook::session::Background| { let status = background.status(); status.len() == 2 && status.iter().all(|(_, status)| status.synced.is_some()) diff --git a/crates/snowbound/Cargo.toml b/crates/snowbound/Cargo.toml index bd761c74e67be62f27d2ccc52dac9f32fcd62e7f..1989f28adf6464a4a6c0c55eb5611eb5b34b479f 100644 --- a/crates/snowbound/Cargo.toml +++ b/crates/snowbound/Cargo.toml @@ -65,7 +65,7 @@ web-sys = { version = "0.3", features = ["console", "Document", "Element", "Html libc = "0.2" objc2 = "0.5.2" objc2-app-kit = { version = "0.2.2", default-features = false, features = ["NSAlert", "NSButton", "NSCell", "NSControl", "NSView", "NSResponder", "NSWindow", "NSApplication", "NSEvent", "NSMenu", "NSMenuItem", "NSColor", "NSColorSpace", "NSDatePicker", "NSDatePickerCell", "NSOpenPanel", "NSPanel", "NSSavePanel", "NSWorkspace"] } -objc2-foundation = { version = "0.2.2", default-features = false, features = ["NSString", "NSThread", "NSAttributedString", "NSDate", "NSCalendar", "NSDateFormatter", "NSGeometry", "NSLocale", "NSPathUtilities", "NSTimeZone", "NSURL", "NSArray", "NSRange", "NSError", "NSFileCoordinator", "NSFilePresenter", "NSFileVersion", "NSFileManager", "NSHost", "NSNotification", "NSOperation", "block2"] } +objc2-foundation = { version = "0.2.2", default-features = false, features = ["NSString", "NSThread", "NSAttributedString", "NSDate", "NSCalendar", "NSDateFormatter", "NSGeometry", "NSLocale", "NSPathUtilities", "NSTimeZone", "NSURL", "NSArray", "NSRange", "NSError", "NSFileCoordinator", "NSFilePresenter", "NSFileVersion", "NSFileManager", "NSNotification", "NSOperation", "block2"] } block2 = "0.5.1" [target.'cfg(not(any(target_os = "macos", target_arch = "wasm32")))'.dependencies] diff --git a/crates/snowbound/src/icloud_linux.rs b/crates/snowbound/src/icloud_linux.rs index 4001c4eb99f308b8884b4ce5b57c79712eabcc7d..e302c40b808d7eb666669bdf6bc0cc6744123541 100644 --- a/crates/snowbound/src/icloud_linux.rs +++ b/crates/snowbound/src/icloud_linux.rs @@ -24,7 +24,6 @@ pub fn drive() -> Option { pub fn section( notebook: &Notebook, _: &Path, - _: &Path, path: &str, key: Option<&onestore::protected::Key>, offline: bool, @@ -47,8 +46,6 @@ pub fn lone_section( pub fn background( notebook: &Notebook, - _: &Path, - _: &Path, notify: impl Fn() + Send + 'static, ) -> Result { notebook.background(false, true, notify) diff --git a/crates/snowbound/src/icloud_macos.rs b/crates/snowbound/src/icloud_macos.rs index 066b632c32b797d47147231d73f17af012078bb0..5f96905974075f421fa5cbe625afac195c248fbf 100644 --- a/crates/snowbound/src/icloud_macos.rs +++ b/crates/snowbound/src/icloud_macos.rs @@ -14,8 +14,6 @@ use objc2::{ rc::Retained, runtime::{AnyObject, NSObject, NSObjectProtocol, ProtocolObject}, }; -#[allow(deprecated)] -use objc2_foundation::NSHost; use objc2_foundation::{ NSFileCoordinator, NSFileCoordinatorReadingOptions, NSFileCoordinatorWritingOptions, NSFileManager, NSFilePresenter, NSFileVersion, NSNotification, NSNotificationCenter, @@ -249,6 +247,44 @@ pub struct Coordinated { published: Option, } +/// This Mac's name as Sharing shows it, as iCloud Drive names its conflict versions. +fn computer() -> &'static str { + static NAME: OnceLock = OnceLock::new(); + NAME.get_or_init(|| { + #[link(name = "SystemConfiguration", kind = "framework")] + unsafe extern "C" { + fn SCDynamicStoreCopyComputerName( + store: *const std::ffi::c_void, + encoding: *mut u32, + ) -> *mut NSString; + } + // SAFETY: a null store asks the system's; the name, a CFString the caller owns, is + // toll-free bridged to NSString. + let name = unsafe { + Retained::from_raw(SCDynamicStoreCopyComputerName( + std::ptr::null(), + std::ptr::null_mut(), + )) + }; + name.map_or_else(|| "Mac".to_owned(), |name| name.to_string()) + }) +} + +/// The header and length of `file`, once iCloud Drive has it here. +fn stamp(file: &Path) -> io::Result { + present(file)?; + coordinated(file, false, || { + use std::io::Read; + let mut opened = std::fs::File::open(file)?; + let mut header = [0; 1024]; + opened.read_exact(&mut header)?; + Ok(Stamp { + header, + length: opened.metadata()?.len(), + }) + })? +} + fn uncommitted(error: io::Error) -> CommitError { CommitError { state: CommitState::NotCommitted, @@ -257,19 +293,11 @@ fn uncommitted(error: io::Error) -> CommitError { } impl Coordinated { - /// The file `file` of the notebook folder, or lone section, at `notebook`, whose replicas - /// `cache` holds. - pub fn new(file: &Path, notebook: &Path, cache: &Path) -> Self { - static DEVICE: OnceLock> = OnceLock::new(); - // The computer's name as Sharing shows it, as iCloud names this Mac's versions; the - // Network framework that deprecates NSHost has no such name. - #[allow(deprecated)] - let device = DEVICE.get_or_init(|| { - unsafe { NSHost::currentHost().localizedName() }.map(|name| name.to_string()) - }); + /// The section file `file`, its replica at `replica` where it has one. + pub fn new(file: &Path, replica: Option<&Path>) -> Self { Self { file: file.to_owned(), - published: Published::of(cache, notebook, file, device.clone()), + published: replica.map(|replica| Published::of(replica, computer())), } } @@ -301,17 +329,7 @@ impl Remote for Coordinated { } fn stamp(&mut self) -> io::Result { - present(&self.file)?; - coordinated(&self.file, false, || { - use std::io::Read; - let mut file = std::fs::File::open(&self.file)?; - let mut header = [0; 1024]; - file.read_exact(&mut header)?; - Ok(Stamp { - header, - length: file.metadata()?.len(), - }) - })? + stamp(&self.file) } fn publish(&mut self, transaction: &Transaction) -> Result<(), CommitError> { @@ -330,7 +348,12 @@ impl Remote for Coordinated { fn confirm(&mut self, base: &Stamp) -> Result<(), CommitError> { coordinated(&self.file, true, || { - onestore::confirm_file(&self.file, base) + onestore::confirm_file(&self.file, base)?; + if let Some(published) = &self.published { + let _ = std::fs::File::open(&self.file) + .and_then(|mut file| published.confirmed(Written::InPlace(&mut file), base)); + } + Ok(()) }) .map_err(uncommitted)? } @@ -346,13 +369,9 @@ impl Remote for Coordinated { }) }) .collect(); - if self.published.is_some() { - let stamp = self.stamp()?; - listed.extend( - self.published - .iter() - .flat_map(|published| published.versions(&stamp)), - ); + if let Some(published) = &self.published { + let lost = published.versions(|| stamp(&self.file), &listed)?; + listed.extend(lost); } Ok(listed) } @@ -382,12 +401,14 @@ impl Remote for Coordinated { && published.holds(id) { if keep { - let device = published.device().unwrap_or("this Mac"); - let beside = notebook::published::beside(&self.file, device); + let beside = notebook::published::beside(&self.file, published.device()); coordinated(&beside, true, || std::fs::copy(id, &beside))??; } return std::fs::remove_file(id); } + if let Some(published) = &self.published { + let _ = published.retired(Path::new(id)); + } let version = match self.conflict(id) { Ok(version) => version, // Another device resolved it first. @@ -419,7 +440,6 @@ impl Remote for Coordinated { pub fn section( notebook: &Notebook, root: &Path, - cache: &Path, path: &str, key: Option<&onestore::protected::Key>, offline: bool, @@ -428,8 +448,7 @@ pub fn section( if !notebook.replica_path(path)?.exists() { fetch(&root.join(path))?; } - let (root, cache) = (root.to_owned(), cache.to_owned()); - let connect = move |file: &Path| Ok(Coordinated::new(file, &root, &cache)); + let connect = |file: &Path, replica: Option<&Path>| Ok(Coordinated::new(file, replica)); let section = match key { Some(key) => notebook.section_unlocked_with(path, key, connect, offline, notify)?, None => notebook.section_with(path, connect, offline, notify)?, @@ -458,10 +477,7 @@ pub fn lone_section( let section = Section::open_with( file, cache, - { - let cache = cache.to_owned(); - move |file: &Path| Ok(Coordinated::new(file, file, &cache)) - }, + |file, replica| Ok(Coordinated::new(file, replica)), offline, notify, )?; @@ -475,13 +491,9 @@ pub fn lone_section( /// presenter or the folder's watch reports it. pub fn background( notebook: &Notebook, - root: &Path, - cache: &Path, notify: impl Fn() + Send + 'static, ) -> Result { - let (root, cache) = (root.to_owned(), cache.to_owned()); - let remote = move |file: &Path| Coordinated::new(file, &root, &cache); - notebook.background_with(false, true, remote, notify) + notebook.background_with(false, true, Coordinated::new, notify) } struct Ivars { @@ -655,7 +667,7 @@ mod tests { let notebook = Notebook::create(&root, &cache, Notebook::NEW_COLOR, &page).unwrap(); assert!(ubiquitous(&root)); let path = notebook.catalog().sections[0].path.clone(); - let section = section(¬ebook, &root, &cache, &path, None, false, || {}).unwrap(); + let section = section(¬ebook, &root, &path, None, false, || {}).unwrap(); let (space, ..) = section.pages().unwrap()[0].clone(); let title = section .page(space) @@ -687,7 +699,7 @@ mod tests { .unwrap(); // Wakes skip the pause in typing, as leaving the window does. let file = root.join(&path); - let mut remote = Coordinated::new(&file, &root, &cache); + let mut remote = Coordinated::new(&file, None); let deadline = std::time::Instant::now() + Duration::from_secs(30); loop { section.wake(); @@ -725,7 +737,7 @@ mod tests { let notebook = Notebook::create(&root, &cache, Notebook::NEW_COLOR, &page).unwrap(); let path = notebook.catalog().sections[0].path.clone(); let file = root.join(&path); - let section = section(¬ebook, &root, &cache, &path, None, false, || {}).unwrap(); + let section = section(¬ebook, &root, &path, None, false, || {}).unwrap(); let space = section.pages().unwrap()[0].0; let stored = || { let arena = onestore::Arena::default(); diff --git a/crates/snowbound/src/library.rs b/crates/snowbound/src/library.rs index c600c4d50a8fe7786a3322030772f4e43d907b27..ef1e660a705ddcd990b2238a8e34172603bfdbca 100644 --- a/crates/snowbound/src/library.rs +++ b/crates/snowbound/src/library.rs @@ -413,7 +413,7 @@ impl Library { } let mut notebook = Notebook::open(location, cache); let (background, watch) = match &mut notebook { - Ok(notebook) => local_background(notebook, location, cache), + Ok(notebook) => local_background(notebook, location), Err(_) => (None, None), }; let name = display_name(cache, location).unwrap_or_else(|| file_name(Path::new(location))); @@ -819,7 +819,7 @@ impl Library { /// A notebook `Notebook::create` just made in the folder at `location`. pub fn created(location: &str, mut notebook: Notebook, cache: &Path) -> Self { - let (background, watch) = local_background(&mut notebook, location, cache); + let (background, watch) = local_background(&mut notebook, location); Self { background, watch, @@ -902,7 +902,7 @@ impl Library { #[cfg(feature = "live")] if let Some(joined) = &self.joined { let guest = Arc::clone(&joined.guest); - section.relocate_with(path, move |file| { + section.relocate_with(path, move |file, _| { Ok(notebook::live::share::HostedRemote::new( &guest, &file.to_string_lossy(), @@ -921,7 +921,7 @@ impl Library { root => format!("{root}/{path}"), }; let server = Arc::clone(server); - section.relocate_with(file, move |file| { + section.relocate_with(file, move |file, _| { Ok(notebook::SmbRemote::new( server.connect()?, file.to_string_lossy(), @@ -935,9 +935,8 @@ impl Library { let file = Path::new(&self.location).join(path); #[cfg(target_os = "macos")] if self.in_icloud() { - let (root, cache) = (PathBuf::from(&self.location), self.cache.clone()); - section.relocate_with(file, move |file| { - Ok(crate::icloud::Coordinated::new(file, &root, &cache)) + section.relocate_with(file, |file, replica| { + Ok(crate::icloud::Coordinated::new(file, replica)) })?; } else { section.relocate(file)?; @@ -1025,7 +1024,6 @@ impl Library { crate::icloud::section( notebook, root, - &self.cache, path, key.as_ref(), offline(), @@ -1535,13 +1533,12 @@ fn read_tag_art(notebook: &Notebook) -> TagArt { fn local_background( notebook: &mut Notebook, location: &str, - cache: &Path, ) -> (Option>, Option>) { let reports = Arc::new(OnceLock::new()); let watch = watcher(location, Arc::clone(&reports)); let copies = !crate::watch::on_this_computer(Path::new(location)); let background = match crate::icloud::ubiquitous(Path::new(location)) { - true => crate::icloud::background(notebook, Path::new(location), cache, notify_background), + true => crate::icloud::background(notebook, notify_background), false => notebook.background(watch.is_some(), copies, notify_background), }; let Some(background) = background @@ -1772,7 +1769,7 @@ mod tests { let section = Section::open_with( &file, root.join("cache"), - move |file| { + move |file, _| { Ok(Stalled { file: file.to_owned(), entered: Arc::clone(&entered), diff --git a/crates/snowbound/src/undo.rs b/crates/snowbound/src/undo.rs index fe53e9d323ff5dc6521ec52ca7765a53d1aa5a98..3579d489779e6af443ca7051c39225376bd0683f 100644 --- a/crates/snowbound/src/undo.rs +++ b/crates/snowbound/src/undo.rs @@ -1872,7 +1872,7 @@ mod tests { let arrived = Arc::clone(b.replica()); let (entered, holding) = std::sync::mpsc::channel(); let (release, released) = std::sync::mpsc::channel::<()>(); - b.relocate_with(fixture.folder.join("Notebook/B.one"), move |_| { + b.relocate_with(fixture.folder.join("Notebook/B.one"), move |_, _| { let _ = entered.send(()); let _ = released.recv(); Err::(std::io::ErrorKind::TimedOut.into()) -- 2.54.1