authoravatarclover caruso<git@paperclover.net>2026-10-19 11:44:38-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-19 23:39:58-07:00
log02876d691c955ff0594f528e889ff7e7a057fdd9
treea1dd96b38792a7806ccc90ee643d78bbbca98287
parent82d8d2ab063d1f2aa562cb75c3beedea691e2c15
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

fix(ios): merge back uploads iCloud Drive lost

iCloud Drive syncs whole files with no lock, so another device's upload can land over the phone's with no conflict version kept, and the phone's next sync took the file as it stood: its published edits were gone. Android's document providers already kept each published image until the document held it (c345a48e); that store moves into notebook::published, and the iOS file remote uses it for notebooks a provider keeps elsewhere. Each image the phone publishes is kept in the cache until the file is seen otherwise, then listed beside the host's conflict versions: one the file holds merges as nothing and goes, one it lost merges back with conflict pages named for this device, and one that cannot merge is kept beside the file as "Name (Device).one", as a conflict version is. sb_set_versions takes the device's name for these. The iCloud launch test gains the lost upload, with and without shared history; it fails without the store. - Edits you make on iPhone to an iCloud Drive notebook come back if iCloud lets another device's upload replace them, instead of being lost. Assisted-by: claude-opus-5.5

11 files changed, 375 insertions(+), 203 deletions(-)

apps/ios/Snowbound/ICloud.swift+5-4
...@@ -1,4 +1,4 @@...@@ -1,4 +1,4 @@
1import Foundation1import UIKit
22
3/// Lists the conflict versions iCloud Drive keeps beside a section file, as it keeps another3/// Lists the conflict versions iCloud Drive keeps beside a section file, as it keeps another
4/// device's commit that lost; the library merges each into the file.4/// device's commit that lost; the library merges each into the file.
...@@ -58,10 +58,11 @@ enum ICloud {...@@ -58,10 +58,11 @@ enum ICloud {
58 /// Whether iCloud is signed in with iCloud Drive on, container or not.58 /// Whether iCloud is signed in with iCloud Drive on, container or not.
59 static var signedIn: Bool { FileManager.default.ubiquityIdentityToken != nil }59 static var signedIn: Bool { FileManager.default.ubiquityIdentityToken != nil }
6060
61 /// Has the library merge conflict versions, follows the account, and hears of notebooks61 /// Has the library merge conflict versions and this device's uploads iCloud Drive lost,
62 /// another device adds or removes; call once at launch.62 /// follows the account, and hears of notebooks another device adds or removes; call once
63 /// at launch.
63 static func start() {64 static func start() {
64 sb_set_versions(listVersions, retireVersion)65 sb_set_versions(listVersions, retireVersion, UIDevice.current.name)
65 if automated { return lookUp() }66 if automated { return lookUp() }
66 NotificationCenter.default.addObserver(67 NotificationCenter.default.addObserver(
67 forName: .NSUbiquityIdentityDidChange, object: nil, queue: .main68 forName: .NSUbiquityIdentityDidChange, object: nil, queue: .main
arc/sync.md+8
...@@ -295,6 +295,14 @@ produce byte-identical files. Any device merges any version at once; nothing wai...@@ -295,6 +295,14 @@ produce byte-identical files. Any device merges any version at once; nothing wai
295device that wrote it. A version of another section, or one that cannot be read, is kept beside295device that wrote it. A version of another section, or one that cannot be read, is kept beside
296the file as `Name (Device).one` instead.296the file as `Name (Device).one` instead.
297297
298iCloud Drive can also lose an upload outright: another device's lands over it and no conflict
299version reaches this one. So on the iPhone and on Android each image a section publishes to a
300file a provider keeps stays in the cache (`notebook::published`) until the file is seen
301otherwise, then lists as one more version: one the file holds merges as nothing and goes, one
302it lost merges back as a conflict version does, its conflict pages named for this device. One
303that cannot merge is kept beside the file as `Name (Device).one` on the iPhone, and in the
304cache on Android, where no provider takes a new document beside it.
305
298On a cloud drive every read and write of a section file runs under `NSFileCoordinator`, so the306On a cloud drive every read and write of a section file runs under `NSFileCoordinator`, so the
299iCloud daemon never swaps a file between the stamp check and the append. A file iCloud evicted307iCloud daemon never swaps a file between the stamp check and the append. A file iCloud evicted
300(a `.Name.icloud` placeholder, or on macOS 14 and later a dataless file under its own name) lists308(a `.Name.icloud` placeholder, or on macOS 14 and later a dataless file under its own name) lists
crates/mobile/include/snowbound.h+1-1
...@@ -22,7 +22,7 @@ void sb_set_coordinator(sb_coordinator coordinator);...@@ -22,7 +22,7 @@ void sb_set_coordinator(sb_coordinator coordinator);
22void sb_set_sync_wake(void (*wake)(void));22void sb_set_sync_wake(void (*wake)(void));
23typedef void (*sb_versions)(const char *path, void (*found)(void *, const char *id, const char *device), void *context);23typedef void (*sb_versions)(const char *path, void (*found)(void *, const char *id, const char *device), void *context);
24typedef bool (*sb_retire)(const char *path, const char *id, bool keep);24typedef bool (*sb_retire)(const char *path, const char *id, bool keep);
25void sb_set_versions(sb_versions versions, sb_retire retire);25void sb_set_versions(sb_versions versions, sb_retire retire, const char *device);
26char *sb_tags(void);26char *sb_tags(void);
27char *sb_pens(uint32_t section);27char *sb_pens(uint32_t section);
28bool sb_tag_icon(uint16_t shape, bool checked, uint32_t pixels, uint8_t *rgba);28bool sb_tag_icon(uint16_t shape, bool checked, uint32_t pixels, uint8_t *rgba);
crates/mobile/src/library.rs+117-38
...@@ -9,6 +9,7 @@ use notebook::{...@@ -9,6 +9,7 @@ use notebook::{
9 discover::{Folder, Reason, SectionState},9 discover::{Folder, Reason, SectionState},
10 live::{self, Hello, Reach, share::Guest},10 live::{self, Hello, Reach, share::Guest},
11 provider::Documents,11 provider::Documents,
12 published::{Published, Written},
12 session::{self, Background, Event, Known, Notebook, RECYCLE_BIN, SyncStatus},13 session::{self, Background, Event, Known, Notebook, RECYCLE_BIN, SyncStatus},
13 smb::{Client, Credentials},14 smb::{Client, Credentials},
14};15};
...@@ -103,28 +104,60 @@ pub type Versions = extern "C" fn(...@@ -103,28 +104,60 @@ pub type Versions = extern "C" fn(
103/// beside the file when `keep`; false when it could not.104/// beside the file when `keep`; false when it could not.
104pub type Retire = extern "C" fn(path: *const c_char, id: *const c_char, keep: bool) -> bool;105pub type Retire = extern "C" fn(path: *const c_char, id: *const c_char, keep: bool) -> bool;
105106
106static VERSIONS: OnceLock<(Versions, Retire)> = OnceLock::new();107static VERSIONS: OnceLock<(Versions, Retire, String)> = OnceLock::new();
107108
108/// Merges the conflict versions the host lists into their files from now on, then has the109/// Merges the conflict versions the host lists into their files from now on, then has the
109/// host retire them; set once, before any library opens.110/// host retire them, and merges back this device's publications a file provider lost, which
111/// conflict pages name for `device`; set once, before any library opens.
112///
113/// # Safety
114/// `device` is NUL-terminated UTF-8.
110#[unsafe(no_mangle)]115#[unsafe(no_mangle)]
111pub extern "C" fn sb_set_versions(versions: Versions, retire: Retire) {116pub unsafe extern "C" fn sb_set_versions(
112 let _ = VERSIONS.set((versions, retire));117 versions: Versions,
118 retire: Retire,
119 device: *const c_char,
120) {
121 let _ = VERSIONS.set((versions, retire, string(device)));
122}
123
124/// A local section file read and published under the host's file coordination. One a file
125/// provider keeps elsewhere, which syncs it whole with no lock, keeps each image this device
126/// publishes until the file holds it, and merges one another device's upload replaced back
127/// (`notebook::published`).
128struct Coordinated {
129 file: PathBuf,
130 published: Option<Published>,
113}131}
114132
115/// A local section file read and published under the host's file coordination.133impl Coordinated {
116struct Coordinated(PathBuf);134 fn new(file: &Path, cache: &Path, elsewhere: bool) -> Self {
135 let published = elsewhere
136 .then(|| {
137 notebook::location::claimed(cache, &notebook::location::local(file).ok()?).ok()
138 })
139 .flatten()
140 .map(|folder| {
141 let device = VERSIONS.get().map(|(.., device)| device.clone());
142 Published::new(folder.join("published"), device)
143 });
144 Self {
145 file: file.to_owned(),
146 published,
147 }
148 }
149}
117150
118impl Remote for Coordinated {151impl Remote for Coordinated {
119 fn read(&mut self) -> io::Result<Vec<u8>> {152 fn read(&mut self) -> io::Result<Vec<u8>> {
120 coordinated(&self.0, false, || onestore::read_file(&self.0))?153 coordinated(&self.file, false, || onestore::read_file(&self.file))?
121 }154 }
122155
123 /// The header and length, as the file remote's uncoordinated change detector reads them.156 /// The header and length, as the file remote's uncoordinated change detector reads them.
124 fn stamp(&mut self) -> io::Result<Stamp> {157 fn stamp(&mut self) -> io::Result<Stamp> {
125 coordinated(&self.0, false, || {158 coordinated(&self.file, false, || {
126 use std::io::Read;159 use std::io::Read;
127 let mut file = std::fs::File::open(&self.0)?;160 let mut file = std::fs::File::open(&self.file)?;
128 let mut header = [0; 1024];161 let mut header = [0; 1024];
129 file.read_exact(&mut header)?;162 file.read_exact(&mut header)?;
130 Ok(Stamp {163 Ok(Stamp {
...@@ -135,28 +168,48 @@ impl Remote for Coordinated {...@@ -135,28 +168,48 @@ impl Remote for Coordinated {
135 }168 }
136169
137 fn publish(&mut self, transaction: &Transaction) -> std::result::Result<(), CommitError> {170 fn publish(&mut self, transaction: &Transaction) -> std::result::Result<(), CommitError> {
138 coordinated(&self.0, true, || transaction.commit_file(&self.0)).map_err(uncommitted)?171 coordinated(&self.file, true, || {
172 transaction.commit_file(&self.file)?;
173 if let Some(published) = &self.published {
174 // A copy it could not keep only leaves the publication unprotected.
175 let _ = std::fs::File::open(&self.file)
176 .and_then(|mut file| published.keep(Written::InPlace(&mut file), transaction));
177 }
178 Ok(())
179 })
180 .map_err(uncommitted)?
139 }181 }
140182
141 fn confirm(&mut self, base: &Stamp) -> std::result::Result<(), CommitError> {183 fn confirm(&mut self, base: &Stamp) -> std::result::Result<(), CommitError> {
142 coordinated(&self.0, true, || onestore::confirm_file(&self.0, base)).map_err(uncommitted)?184 coordinated(&self.file, true, || {
185 onestore::confirm_file(&self.file, base)
186 })
187 .map_err(uncommitted)?
143 }188 }
144189
190 /// The host's conflict versions, then this device's publications the file lost.
145 fn versions(&mut self) -> io::Result<Vec<notebook::Version>> {191 fn versions(&mut self) -> io::Result<Vec<notebook::Version>> {
146 let Some((versions, _)) = VERSIONS.get() else {
147 return Ok(Vec::new());
148 };
149 extern "C" fn found(context: *mut c_void, id: *const c_char, device: *const c_char) {
150 // SAFETY: `context` is the list below, alive for the host's call.
151 let listed = unsafe { &mut *context.cast::<Vec<notebook::Version>>() };
152 listed.push(notebook::Version {
153 id: string(id),
154 device: (!device.is_null()).then(|| string(device)),
155 });
156 }
157 let mut listed: Vec<notebook::Version> = Vec::new();192 let mut listed: Vec<notebook::Version> = Vec::new();
158 let path = CString::new(self.0.as_os_str().as_encoded_bytes())?;193 if let Some((versions, ..)) = VERSIONS.get() {
159 versions(path.as_ptr(), found, (&raw mut listed).cast());194 extern "C" fn found(context: *mut c_void, id: *const c_char, device: *const c_char) {
195 // SAFETY: `context` is the list below, alive for the host's call.
196 let listed = unsafe { &mut *context.cast::<Vec<notebook::Version>>() };
197 listed.push(notebook::Version {
198 id: string(id),
199 device: (!device.is_null()).then(|| string(device)),
200 });
201 }
202 let path = CString::new(self.file.as_os_str().as_encoded_bytes())?;
203 versions(path.as_ptr(), found, (&raw mut listed).cast());
204 }
205 if self.published.is_some() {
206 let stamp = self.stamp()?;
207 listed.extend(
208 self.published
209 .iter()
210 .flat_map(|published| published.versions(&stamp)),
211 );
212 }
160 Ok(listed)213 Ok(listed)
161 }214 }
162215
...@@ -165,11 +218,31 @@ impl Remote for Coordinated {...@@ -165,11 +218,31 @@ impl Remote for Coordinated {
165 std::fs::read(id)218 std::fs::read(id)
166 }219 }
167220
221 /// A publication of this device's that cannot merge is kept whole beside the file, as
222 /// the host keeps a conflict version: "Name (Device).one".
168 fn retire(&mut self, id: &str, keep: bool) -> io::Result<()> {223 fn retire(&mut self, id: &str, keep: bool) -> io::Result<()> {
169 let Some((_, retire)) = VERSIONS.get() else {224 if let Some(published) = &self.published
225 && published.holds(id)
226 {
227 if keep {
228 let stem = self.file.file_stem().unwrap_or_default().to_string_lossy();
229 let device = VERSIONS.get().map_or("this device", |(.., device)| device);
230 let beside = (1..)
231 .map(|n| match n {
232 1 => format!("{stem} ({device}).one"),
233 _ => format!("{stem} ({device} {n}).one"),
234 })
235 .map(|name| self.file.with_file_name(name))
236 .find(|beside| std::fs::symlink_metadata(beside).is_err())
237 .ok_or(io::ErrorKind::AlreadyExists)?;
238 coordinated(&beside, true, || std::fs::copy(id, &beside))??;
239 }
240 return std::fs::remove_file(id);
241 }
242 let Some((_, retire, _)) = VERSIONS.get() else {
170 return Ok(());243 return Ok(());
171 };244 };
172 let path = CString::new(self.0.as_os_str().as_encoded_bytes())?;245 let path = CString::new(self.file.as_os_str().as_encoded_bytes())?;
173 let id = CString::new(id)?;246 let id = CString::new(id)?;
174 if retire(path.as_ptr(), id.as_ptr(), keep) {247 if retire(path.as_ptr(), id.as_ptr(), keep) {
175 Ok(())248 Ok(())
...@@ -297,10 +370,10 @@ pub struct Library {...@@ -297,10 +370,10 @@ pub struct Library {
297 offline: AtomicBool,370 offline: AtomicBool,
298 /// Keys of the protected sections unlocked this run, by catalog path; in memory only.371 /// Keys of the protected sections unlocked this run, by catalog path; in memory only.
299 keys: Mutex<HashMap<String, Key>>,372 keys: Mutex<HashMap<String, Key>>,
300 /// How long edits wait for a pause in typing before they publish (`Section::set_pause`):373 /// A file provider keeps the notebook elsewhere too, as iCloud Drive does: edits wait for a
301 /// a file provider such as iCloud Drive uploads each publication, and makes one concurrent374 /// pause in typing before they publish (`Section::set_pause`), as it uploads each
302 /// with another device's a conflict version.375 /// publication and makes one concurrent with another device's a conflict version.
303 pause: Duration,376 elsewhere: bool,
304}377}
305378
306/// The pause edits to a file a provider keeps elsewhere wait for.379/// The pause edits to a file a provider keeps elsewhere wait for.
...@@ -400,10 +473,11 @@ impl Library {...@@ -400,10 +473,11 @@ impl Library {
400 (None, Place::File(path.to_owned()), None)473 (None, Place::File(path.to_owned()), None)
401 } else {474 } else {
402 let mut notebook = Notebook::open(path, cache)?;475 let mut notebook = Notebook::open(path, cache)?;
476 let kept = cache.to_owned();
403 let background = notebook.background_with(477 let background = notebook.background_with(
404 local,478 local,
405 !local,479 !local,
406 |file| Coordinated(file.to_owned()),480 move |file| Coordinated::new(file, &kept, !local),
407 sync_woken,481 sync_woken,
408 )?;482 )?;
409 background.watch(notebook.replicas());483 background.watch(notebook.replicas());
...@@ -422,7 +496,7 @@ impl Library {...@@ -422,7 +496,7 @@ impl Library {
422 open: Mutex::default(),496 open: Mutex::default(),
423 offline: AtomicBool::new(false),497 offline: AtomicBool::new(false),
424 keys: Default::default(),498 keys: Default::default(),
425 pause: if local { Duration::ZERO } else { PAUSE },499 elsewhere: !local,
426 })500 })
427 }501 }
428502
...@@ -445,7 +519,7 @@ impl Library {...@@ -445,7 +519,7 @@ impl Library {
445 open: Mutex::default(),519 open: Mutex::default(),
446 offline: AtomicBool::new(false),520 offline: AtomicBool::new(false),
447 keys: Default::default(),521 keys: Default::default(),
448 pause: Duration::ZERO,522 elsewhere: false,
449 };523 };
450 let reached = library.with_notebook(false, |_| Ok(()));524 let reached = library.with_notebook(false, |_| Ok(()));
451 match library.listed() {525 match library.listed() {
...@@ -504,7 +578,7 @@ impl Library {...@@ -504,7 +578,7 @@ impl Library {
504 open: Mutex::default(),578 open: Mutex::default(),
505 offline: AtomicBool::new(offline),579 offline: AtomicBool::new(offline),
506 keys: Default::default(),580 keys: Default::default(),
507 pause: Duration::ZERO,581 elsewhere: false,
508 })582 })
509 }583 }
510584
...@@ -541,7 +615,7 @@ impl Library {...@@ -541,7 +615,7 @@ impl Library {
541 open: Mutex::default(),615 open: Mutex::default(),
542 offline: AtomicBool::new(false),616 offline: AtomicBool::new(false),
543 keys: Default::default(),617 keys: Default::default(),
544 pause: if local { Duration::ZERO } else { PAUSE },618 elsewhere: !local,
545 })619 })
546 }620 }
547621
...@@ -783,7 +857,11 @@ impl Library {...@@ -783,7 +857,11 @@ impl Library {
783 }857 }
784 };858 };
785 section.set_offline(self.offline.load(Ordering::Relaxed));859 section.set_offline(self.offline.load(Ordering::Relaxed));
786 section.set_pause(self.pause);860 section.set_pause(if self.elsewhere {
861 PAUSE
862 } else {
863 Duration::ZERO
864 });
787 if let Some(background) = &self.background {865 if let Some(background) = &self.background {
788 background.hold(path, &section);866 background.hold(path, &section);
789 }867 }
...@@ -796,18 +874,19 @@ impl Library {...@@ -796,18 +874,19 @@ impl Library {
796 notify: impl Fn() + Send + Sync + 'static,874 notify: impl Fn() + Send + Sync + 'static,
797 ) -> Result<session::Section> {875 ) -> Result<session::Section> {
798 let offline = self.offline.load(Ordering::Relaxed);876 let offline = self.offline.load(Ordering::Relaxed);
877 let (cache, elsewhere) = (self.cache.clone(), self.elsewhere);
878 let connect = move |file: &Path| Ok(Coordinated::new(file, &cache, elsewhere));
799 match &self.place {879 match &self.place {
800 Place::File(file) => Ok(session::Section::open_with(880 Place::File(file) => Ok(session::Section::open_with(
801 file,881 file,
802 &self.cache,882 &self.cache,
803 |file| Ok(Coordinated(file.to_owned())),883 connect,
804 offline,884 offline,
805 notify,885 notify,
806 )?),886 )?),
807 Place::Folder(_) => {887 Place::Folder(_) => {
808 let notebook = self.notebook();888 let notebook = self.notebook();
809 let notebook = notebook.as_ref().ok_or("A lone section has no notebook")?;889 let notebook = notebook.as_ref().ok_or("A lone section has no notebook")?;
810 let connect = |file: &Path| Ok(Coordinated(file.to_owned()));
811 Ok(match self.unlocked().get(path) {890 Ok(match self.unlocked().get(path) {
812 Some(key) => {891 Some(key) => {
813 notebook.section_unlocked_with(path, key, connect, offline, notify)?892 notebook.section_unlocked_with(path, key, connect, offline, notify)?
crates/mobile/src/tests.rs+50-16
...@@ -4655,17 +4655,50 @@ fn every_page(root: &Path, cache: &Path) -> String {...@@ -4655,17 +4655,50 @@ fn every_page(root: &Path, cache: &Path) -> String {
4655 shown4655 shown
4656}4656}
46574657
4658/// The titles and text of every page and conflict page each section file in `root` stores.
4659fn stored(root: &Path) -> String {
4660 let mut text = String::new();
4661 for entry in std::fs::read_dir(root).unwrap() {
4662 let path = entry.unwrap().path();
4663 if path.extension().is_some_and(|extension| extension == "one") {
4664 let arena = onestore::Arena::default();
4665 let mut section =
4666 onestore::Section::open(&arena, std::fs::read(path).unwrap()).unwrap();
4667 let conflicts = section.conflicts().unwrap().into_iter();
4668 let spaces = (section
4669 .pages()
4670 .unwrap()
4671 .into_iter()
4672 .map(|(space, ..)| space))
4673 .chain(
4674 conflicts
4675 .flat_map(|(_, pages)| pages)
4676 .map(|page| page.space),
4677 );
4678 for space in spaces.collect::<Vec<_>>() {
4679 let page = section.page(space).unwrap();
4680 text += &format!("{}\n{}\n", page.title, canvas::search::page_text(&page));
4681 }
4682 }
4683 }
4684 text
4685}
4686
4658/// The phone and the Mac edit one iCloud Drive notebook Snowbound made, each in its own copy:4687/// The phone and the Mac edit one iCloud Drive notebook Snowbound made, each in its own copy:
4659/// the same page, a page each adds and moves to the top, a page the Mac deletes as the phone4688/// the same page, a page each adds and moves to the top, a page the Mac deletes as the phone
4660/// types on it, and a section the phone adds. The Mac's upload wins, so the phone's next4689/// types on it, and a section the phone adds. The Mac's upload wins: iCloud Drive keeps the
4661/// launch finds its own file a conflict version, and opens, lists and merges it through the C4690/// phone's file as a conflict version, or loses it. The phone's next launch opens, lists and
4662/// surface as iOS does, its pages open and typed on in the Reading View all the while.4691/// merges it back through the C surface as iOS does, its pages open and typed on in the
4663/// Nothing either device typed goes missing, merged or set aside whole.4692/// Reading View all the while. Nothing either device typed goes missing, merged or set aside
4693/// whole.
4664#[test]4694#[test]
4665fn an_icloud_notebook_merges_the_other_devices_version_at_launch() {4695fn an_icloud_notebook_merges_the_phones_upload_back_at_launch() {
4666 library::sb_set_versions(cloud_versions, cloud_retire);4696 unsafe {
4667 // Whether the devices' copies already share an edit of the page both type on.4697 library::sb_set_versions(cloud_versions, cloud_retire, c"Clover’s iPhone".as_ptr())
4668 for shared in [false, true] {4698 };
4699 // Whether the devices' copies already share an edit of the page both type on, and
4700 // whether iCloud Drive lost the phone's upload rather than keep it as a version.
4701 for (shared, lost) in [(false, false), (true, false), (false, true), (true, true)] {
4669 let directory = tempfile::tempdir().unwrap();4702 let directory = tempfile::tempdir().unwrap();
4670 let folder = |name: &str| directory.path().join(name);4703 let folder = |name: &str| directory.path().join(name);
4671 let (phone, mac) = (folder("phone"), folder("mac"));4704 let (phone, mac) = (folder("phone"), folder("mac"));
...@@ -4739,12 +4772,11 @@ fn an_icloud_notebook_merges_the_other_devices_version_at_launch() {...@@ -4739,12 +4772,11 @@ fn an_icloud_notebook_merges_the_other_devices_version_at_launch() {
4739 let version = folder("version.one");4772 let version = folder("version.one");
4740 std::fs::rename(&file, &version).unwrap();4773 std::fs::rename(&file, &version).unwrap();
4741 std::fs::copy(mac.join(&path), &file).unwrap();4774 std::fs::copy(mac.join(&path), &file).unwrap();
4742 let device = "Clover’s iPhone".to_owned();4775 if !lost {
4743 CLOUD_VERSIONS.lock().unwrap().push((4776 let device = "Clover’s iPhone".to_owned();
4744 std::fs::canonicalize(&file).unwrap(),4777 let file = std::fs::canonicalize(&file).unwrap();
4745 version,4778 CLOUD_VERSIONS.lock().unwrap().push((file, version, device));
4746 device,4779 }
4747 ));
47484780
4749 // The launch: the notebook opens and lists, the page shown last opens, and the4781 // The launch: the notebook opens and lists, the page shown last opens, and the
4750 // background merges the version meanwhile.4782 // background merges the version meanwhile.
...@@ -4761,8 +4793,10 @@ fn an_icloud_notebook_merges_the_other_devices_version_at_launch() {...@@ -4761,8 +4793,10 @@ fn an_icloud_notebook_merges_the_other_devices_version_at_launch() {
4761 .collect();4793 .collect();
4762 sb_library_sync_now(&library);4794 sb_library_sync_now(&library);
4763 let deadline = Instant::now() + Duration::from_secs(10);4795 let deadline = Instant::now() + Duration::from_secs(10);
4764 while !CLOUD_VERSIONS.lock().unwrap().is_empty() {4796 while !CLOUD_VERSIONS.lock().unwrap().is_empty()
4765 assert!(Instant::now() < deadline, "the version never merged");4797 || !stored(&phone).contains("Phone first words")
4798 {
4799 assert!(Instant::now() < deadline, "the phone's upload never merged");
4766 library::sb_section_poll(&section, &mut 0);4800 library::sb_section_poll(&section, &mut 0);
4767 for view in &mut views {4801 for view in &mut views {
4768 unsafe { sb_insert(view, c"Launch ".as_ptr()) };4802 unsafe { sb_insert(view, c"Launch ".as_ptr()) };
crates/notebook/README.md+5
...@@ -151,6 +151,11 @@ they clash. The revisions the merge writes are named after the version's it merg...@@ -151,6 +151,11 @@ they clash. The revisions the merge writes are named after the version's it merg
151(`Section::seal_as`), so a version merged once, here or on another device, merges as nothing151(`Section::seal_as`), so a version merged once, here or on another device, merges as nothing
152again. It is published on the file's stamp, then retired; one of another section, or one that152again. It is published on the file's stamp, then retired; one of another section, or one that
153cannot be read, is retired with `keep` for the remote to keep beside the file.153cannot be read, is retired with `keep` for the remote to keep beside the file.
154A provider can also lose an upload outright, another device's landing over it with no version
155kept. A remote keeps what it publishes with `published::Published` (`keep` after each
156publication) and lists each image the file no longer stands as among its versions
157(`Published::versions`): one the file holds merges as nothing and is dropped, one it lost
158merges back.
154159
155Publication attempts are recorded before network I/O. An attempt confirms only160Publication attempts are recorded before network I/O. An attempt confirms only
156when the remote holds its revisions, or every page it changed as it changed them,161when the remote holds its revisions, or every page it changed as it changed them,
crates/notebook/src/lib.rs+1
...@@ -33,6 +33,7 @@ mod merge;...@@ -33,6 +33,7 @@ mod merge;
33mod migrate;33mod migrate;
34pub mod package;34pub mod package;
35pub mod provider;35pub mod provider;
36pub mod published;
36mod queue;37mod queue;
37mod recovery;38mod recovery;
38mod resolve;39mod resolve;
crates/notebook/src/location.rs+1-1
...@@ -84,7 +84,7 @@ pub fn forget(cache: &Path, location: &str) -> io::Result<()> {...@@ -84,7 +84,7 @@ pub fn forget(cache: &Path, location: &str) -> io::Result<()> {
84}84}
8585
86/// The folder for `location`, made and labelled with it where it was not.86/// The folder for `location`, made and labelled with it where it was not.
87fn claimed(cache: &Path, location: &str) -> io::Result<PathBuf> {87pub fn claimed(cache: &Path, location: &str) -> io::Result<PathBuf> {
88 let folder = folder(cache, location);88 let folder = folder(cache, location);
89 let label = folder.join(LOCATION);89 let label = folder.join(LOCATION);
90 if fs::metadata(&label).is_err() {90 if fs::metadata(&label).is_err() {
crates/notebook/src/provider.rs+19-140
...@@ -4,7 +4,11 @@...@@ -4,7 +4,11 @@
4//! notebook root (`session::Notebook::open_provider`), a picked document a lone section4//! notebook root (`session::Notebook::open_provider`), a picked document a lone section
5//! (`session::Section::open_provider`); replicas and the catalog's listing stay in the cache.5//! (`session::Section::open_provider`); replicas and the catalog's listing stay in the cache.
66
7use crate::{Error, Result, discover, session::Storage};7use crate::{
8 Error, Result, discover,
9 published::{Published, Written},
10 session::Storage,
11};
8use onestore::{CommitError, CommitIo, CommitState, Stamp, Store, Transaction};12use onestore::{CommitError, CommitIo, CommitState, Stamp, Store, Transaction};
9use std::{13use std::{
10 collections::BTreeMap,14 collections::BTreeMap,
...@@ -12,7 +16,6 @@ use std::{...@@ -12,7 +16,6 @@ use std::{
12 io::{self, Read, Seek, SeekFrom, Write},16 io::{self, Read, Seek, SeekFrom, Write},
13 path::PathBuf,17 path::PathBuf,
14 sync::{Arc, Mutex, MutexGuard, PoisonError},18 sync::{Arc, Mutex, MutexGuard, PoisonError},
15 time::Duration,
16};19};
1720
18/// A document as its provider describes it.21/// A document as its provider describes it.
...@@ -294,12 +297,6 @@ fn whole(file: &mut File, limit: usize) -> io::Result<Vec<u8>> {...@@ -294,12 +297,6 @@ fn whole(file: &mut File, limit: usize) -> io::Result<Vec<u8>> {
294 Ok(bytes)297 Ok(bytes)
295}298}
296299
297/// What a write wrote: the document in place, or its whole image.
298enum Written<'a> {
299 InPlace(&'a mut File),
300 Image(&'a [u8]),
301}
302
303/// A document as a commit writes it: its seekable descriptor, or its image to write whole.300/// A document as a commit writes it: its seekable descriptor, or its image to write whole.
304enum Io {301enum Io {
305 File(File),302 File(File),
...@@ -351,58 +348,12 @@ impl CommitIo for Io {...@@ -351,58 +348,12 @@ impl CommitIo for Io {
351 }348 }
352}349}
353350
354/// The hex of a stamp's digest, which names an image this device keeps.351/// A section a document provider keeps, as its replica publishes to it, each image it
355fn digest(stamp: &Stamp) -> String {352/// publishes kept until the document holds it (`published.rs`).
356 use sha2::Digest;
357 let digest = sha2::Sha256::new()
358 .chain_update(stamp.header)
359 .chain_update(stamp.length.to_le_bytes())
360 .finalize();
361 hex(&digest[..16])
362}
363
364fn hex(bytes: &[u8]) -> String {
365 bytes.iter().map(|byte| format!("{byte:02x}")).collect()
366}
367
368fn file_of(header: &[u8]) -> Option<[u8; 16]> {
369 Some(onestore::Header::parse(header).ok()?.file_id)
370}
371
372/// An image this device keeps, as its name gives it: `<document>-<image>-<base>.one`, the
373/// document's identity, then the digests of the image's stamp and of the stamp of the image
374/// it was published on.
375struct Kept {
376 name: String,
377 document: String,
378 image: String,
379 base: String,
380}
381
382impl Kept {
383 fn named(name: String) -> Option<Self> {
384 let mut parts = name.strip_suffix(".one")?.split('-').map(str::to_owned);
385 let (document, image, base) = (parts.next()?, parts.next()?, parts.next()?);
386 Some(Self {
387 name,
388 document,
389 image,
390 base,
391 })
392 }
393}
394
395/// How long reads may lag this device's own upload before a document still standing as the
396/// image the upload was made on counts as having lost it.
397const LAG: Duration = Duration::from_secs(60);
398
399/// A section a document provider keeps, as its replica publishes to it. Each image it
400/// publishes is kept, and listed as a version once the document is seen otherwise, until the
401/// document holds it: an upload another device's replaced then merges back as a conflict
402/// version does (`resolve.rs`). Only this device's publications are kept so.
403pub(crate) struct ProviderRemote {353pub(crate) struct ProviderRemote {
404 documents: Arc<Documents>,354 documents: Arc<Documents>,
405 path: String,355 path: String,
356 published: Published,
406 /// The image the step read whole for a stamp, which its read takes.357 /// The image the step read whole for a stamp, which its read takes.
407 read: Option<Vec<u8>>,358 read: Option<Vec<u8>>,
408}359}
...@@ -412,74 +363,19 @@ impl ProviderRemote {...@@ -412,74 +363,19 @@ impl ProviderRemote {
412 Self {363 Self {
413 documents: Arc::clone(documents),364 documents: Arc::clone(documents),
414 path: path.to_owned(),365 path: path.to_owned(),
366 published: Published::new(documents.published(), Some(documents.device.clone())),
415 read: None,367 read: None,
416 }368 }
417 }369 }
418370
419 fn kept(&self) -> Vec<(Kept, std::time::SystemTime)> {
420 let entries = crate::fs::read_dir(self.documents.published())
421 .into_iter()
422 .flatten();
423 entries
424 .filter_map(|entry| {
425 let entry = entry.ok()?;
426 let kept = Kept::named(entry.file_name().into_string().ok()?)?;
427 Some((kept, entry.metadata().ok()?.modified().ok()?))
428 })
429 .collect()
430 }
431
432 /// Keeps the image `transaction` published, in place of the one it was published on:
433 /// that kept image with the transaction applied where this device kept it, else read back.
434 fn keep(&self, written: Written<'_>, transaction: &Transaction) -> io::Result<()> {
435 let folder = self.documents.published();
436 let base = digest(transaction.base());
437 let document = file_of(&transaction.base().header).ok_or(io::ErrorKind::InvalidData)?;
438 let on = (self.kept().into_iter())
439 .find(|(kept, _)| kept.document == hex(&document) && kept.image == base);
440 let image = match (written, &on) {
441 (Written::Image(image), _) => image.to_vec(),
442 (Written::InPlace(_), Some((kept, _))) => {
443 let mut image = crate::fs::read(folder.join(&kept.name))?;
444 transaction
445 .apply(&mut image)
446 .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
447 image
448 }
449 (Written::InPlace(file), None) => {
450 file.seek(SeekFrom::Start(0))?;
451 whole(file, crate::MAX_FILE_BYTES)?
452 }
453 };
454 let name = format!(
455 "{}-{}-{base}.one",
456 hex(&document),
457 digest(&transaction.stamp())
458 );
459 crate::fs::create_dir_all(&folder)?;
460 let mut kept = tempfile::NamedTempFile::new_in(&folder)?;
461 kept.write_all(&image)?;
462 kept.as_file().sync_all()?;
463 kept.persist(folder.join(name))
464 .map_err(|error| error.error)?;
465 if let Some((kept, _)) = on {
466 crate::fs::remove_file(folder.join(kept.name))?;
467 }
468 Ok(())
469 }
470
471 /// This device's own upload cut off, as a provider that writes non-atomically leaves it,371 /// This device's own upload cut off, as a provider that writes non-atomically leaves it,
472 /// put back and read again: the newest image kept that the torn bytes begin. Any other372 /// put back and read again: the newest image kept that the torn bytes begin. Any other
473 /// document that does not parse is left as it is.373 /// document that does not parse is left as it is.
474 fn repair(&self, torn: &[u8]) -> io::Result<Vec<u8>> {374 fn repair(&self, torn: &[u8]) -> io::Result<Vec<u8>> {
475 let mut newest = None;375 let image = self
476 for (kept, at) in self.kept() {376 .published
477 let image = crate::fs::read(self.documents.published().join(&kept.name))?;377 .torn(torn)?
478 if image.len() > torn.len() && image.starts_with(torn) {378 .ok_or(io::ErrorKind::WouldBlock)?;
479 newest = newest.max(Some((at, image)));
480 }
481 }
482 let (_, image) = newest.ok_or(io::ErrorKind::WouldBlock)?;
483 let id = self.documents.id(&self.path)?;379 let id = self.documents.id(&self.path)?;
484 self.documents.writable(&id)?;380 self.documents.writable(&id)?;
485 self.documents381 self.documents
...@@ -524,7 +420,7 @@ impl crate::Remote for ProviderRemote {...@@ -524,7 +420,7 @@ impl crate::Remote for ProviderRemote {
524 self.documents.write(420 self.documents.write(
525 &self.path,421 &self.path,
526 |io| transaction.commit(io),422 |io| transaction.commit(io),
527 |written| self.keep(written, transaction),423 |written| self.published.keep(written, transaction),
528 )424 )
529 }425 }
530426
...@@ -534,9 +430,8 @@ impl crate::Remote for ProviderRemote {...@@ -534,9 +430,8 @@ impl crate::Remote for ProviderRemote {
534 .write(&self.path, |io| onestore::confirm(io, base), |_| Ok(()))430 .write(&self.path, |io| onestore::confirm(io, base), |_| Ok(()))
535 }431 }
536432
537 /// The images kept of this document other than the one it stands as; one whose upload433 /// The images kept of this document other than the one it stands as
538 /// the document still stands before is listed only once reads have had `LAG` to catch up.434 /// (`Published::versions`). A document read torn is repaired first, once a step.
539 /// A document read torn is repaired first, once a step.
540 fn versions(&mut self) -> io::Result<Vec<crate::Version>> {435 fn versions(&mut self) -> io::Result<Vec<crate::Version>> {
541 let (mut stamp, mut image) = self.documents.stamp(&self.path)?;436 let (mut stamp, mut image) = self.documents.stamp(&self.path)?;
542 if let Some(torn) = image.as_ref().filter(|image| !parses(image))437 if let Some(torn) = image.as_ref().filter(|image| !parses(image))
...@@ -546,33 +441,17 @@ impl crate::Remote for ProviderRemote {...@@ -546,33 +441,17 @@ impl crate::Remote for ProviderRemote {
546 (stamp, image) = (repaired_stamp, Some(repaired));441 (stamp, image) = (repaired_stamp, Some(repaired));
547 }442 }
548 self.read = image;443 self.read = image;
549 // A header mid-write names no document; the step's read repairs it.444 Ok(self.published.versions(&stamp))
550 let Some(document) = file_of(&stamp.header) else {
551 return Ok(Vec::new());
552 };
553 let (document, current) = (hex(&document), digest(&stamp));
554 let lagging = |kept: &Kept, at: std::time::SystemTime| {
555 kept.base == current && at.elapsed().is_ok_and(|elapsed| elapsed < LAG)
556 };
557 Ok((self.kept().into_iter())
558 .filter(|(kept, at)| {
559 kept.document == document && kept.image != current && !lagging(kept, *at)
560 })
561 .map(|(kept, _)| crate::Version {
562 id: kept.name,
563 device: Some(self.documents.device.clone()),
564 })
565 .collect())
566 }445 }
567446
568 fn version(&mut self, id: &str) -> io::Result<Vec<u8>> {447 fn version(&mut self, id: &str) -> io::Result<Vec<u8>> {
569 crate::fs::read(self.documents.published().join(id))448 crate::fs::read(id)
570 }449 }
571450
572 /// One that cannot merge, of another section under the document's name, is set aside451 /// One that cannot merge, of another section under the document's name, is set aside
573 /// whole, where `versions` lists it no more: no provider here takes a new document beside it.452 /// whole, where `versions` lists it no more: no provider here takes a new document beside it.
574 fn retire(&mut self, id: &str, keep: bool) -> io::Result<()> {453 fn retire(&mut self, id: &str, keep: bool) -> io::Result<()> {
575 let path = self.documents.published().join(id);454 let path = PathBuf::from(id);
576 if keep {455 if keep {
577 crate::fs::rename(&path, path.with_extension("unmerged"))456 crate::fs::rename(&path, path.with_extension("unmerged"))
578 } else {457 } else {
crates/notebook/src/provider/tests.rs+4-3
...@@ -917,9 +917,9 @@ fn kept_images(fake: &Fake) -> Vec<String> {...@@ -917,9 +917,9 @@ fn kept_images(fake: &Fake) -> Vec<String> {
917917
918/// The images a device keeps that Notes.one does not stand as, still to merge.918/// The images a device keeps that Notes.one does not stand as, still to merge.
919fn unmerged(fake: &Fake) -> Vec<String> {919fn unmerged(fake: &Fake) -> Vec<String> {
920 let current = digest(&Stamp::of(&fake.bytes("Notes.one")).unwrap());920 let current = crate::published::digest(&Stamp::of(&fake.bytes("Notes.one")).unwrap());
921 let mut images = kept_images(fake);921 let mut images = kept_images(fake);
922 images.retain(|name| Kept::named(name.clone()).is_some_and(|kept| kept.image != current));922 images.retain(|name| name.split('-').nth(1) != Some(current.as_str()));
923 images923 images
924}924}
925925
...@@ -976,7 +976,8 @@ fn aged(fake: &Fake) {...@@ -976,7 +976,8 @@ fn aged(fake: &Fake) {
976 .write(true)976 .write(true)
977 .open(image.unwrap().path())977 .open(image.unwrap().path())
978 .unwrap();978 .unwrap();
979 let past = std::time::SystemTime::now() - LAG - Duration::from_secs(1);979 let past =
980 std::time::SystemTime::now() - crate::published::LAG - Duration::from_secs(1);
980 file.set_modified(past).unwrap();981 file.set_modified(past).unwrap();
981 }982 }
982 }983 }
crates/notebook/src/published.rs created+164
...@@ -0,0 +1,164 @@
1//! The images this device publishes to section files a provider syncs whole, with no lock or
2//! compare-and-swap, as iCloud Drive and Android's document providers do: another device's
3//! upload can replace this one's. Each image is kept until the file is seen holding it, and
4//! one the file lost is listed as a `Remote::versions` version, so that it merges back as a
5//! conflict version does (`resolve.rs`).
6
7use crate::Version;
8use onestore::{Stamp, Transaction};
9use std::{
10 fs::File,
11 io::{self, Read, Seek, SeekFrom, Write},
12 path::{Path, PathBuf},
13 time::{Duration, SystemTime},
14};
15
16/// How long reads may lag this device's own upload before a file still standing as the image
17/// the upload was made on counts as having lost it.
18pub(crate) const LAG: Duration = Duration::from_secs(60);
19
20/// What a publication wrote: the file in place, or its whole image.
21pub enum Written<'a> {
22 InPlace(&'a mut File),
23 Image(&'a [u8]),
24}
25
26/// The images this device published to the files of one place, kept in `folder` and named
27/// `<document>-<image>-<base>.one`: the file's identity, then the digests of the image's stamp
28/// and of the stamp of the image it was published on. Version ids are their paths.
29pub struct Published {
30 folder: PathBuf,
31 /// This device, as the conflict pages a lost publication makes name it.
32 device: Option<String>,
33}
34
35struct Kept {
36 path: PathBuf,
37 document: String,
38 image: String,
39 base: String,
40 at: SystemTime,
41}
42
43impl Published {
44 pub fn new(folder: impl Into<PathBuf>, device: Option<String>) -> Self {
45 Self {
46 folder: folder.into(),
47 device,
48 }
49 }
50
51 fn kept(&self) -> Vec<Kept> {
52 let entries = crate::fs::read_dir(&self.folder).into_iter().flatten();
53 entries
54 .filter_map(|entry| {
55 let entry = entry.ok()?;
56 let name = entry.file_name().into_string().ok()?;
57 let mut parts = name.strip_suffix(".one")?.split('-').map(str::to_owned);
58 Some(Kept {
59 path: entry.path(),
60 document: parts.next()?,
61 image: parts.next()?,
62 base: parts.next()?,
63 at: entry.metadata().ok()?.modified().ok()?,
64 })
65 })
66 .collect()
67 }
68
69 /// Keeps the image `transaction` published, in place of the one it was published on:
70 /// that kept image with the transaction applied where this device kept it, else read back.
71 pub fn keep(&self, written: Written<'_>, transaction: &Transaction) -> io::Result<()> {
72 let base = digest(transaction.base());
73 let document = document(transaction.base()).ok_or(io::ErrorKind::InvalidData)?;
74 let on =
75 (self.kept().into_iter()).find(|kept| kept.document == document && kept.image == base);
76 let image = match (written, &on) {
77 (Written::Image(image), _) => image.to_vec(),
78 (Written::InPlace(_), Some(kept)) => {
79 let mut image = crate::fs::read(&kept.path)?;
80 transaction
81 .apply(&mut image)
82 .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
83 image
84 }
85 (Written::InPlace(file), None) => {
86 file.seek(SeekFrom::Start(0))?;
87 let mut image = Vec::new();
88 file.take(crate::MAX_FILE_BYTES as u64 + 1)
89 .read_to_end(&mut image)?;
90 if image.len() > crate::MAX_FILE_BYTES {
91 return Err(io::ErrorKind::FileTooLarge.into());
92 }
93 image
94 }
95 };
96 let name = format!("{document}-{}-{base}.one", digest(&transaction.stamp()));
97 crate::fs::create_dir_all(&self.folder)?;
98 let mut kept = tempfile::NamedTempFile::new_in(&self.folder)?;
99 kept.write_all(&image)?;
100 kept.as_file().sync_all()?;
101 kept.persist(self.folder.join(name))
102 .map_err(|error| error.error)?;
103 if let Some(kept) = on {
104 crate::fs::remove_file(kept.path)?;
105 }
106 Ok(())
107 }
108
109 /// The newest image kept that `torn` begins, as a provider writing in place leaves this
110 /// device's upload cut off.
111 pub fn torn(&self, torn: &[u8]) -> io::Result<Option<Vec<u8>>> {
112 let mut newest = None;
113 for kept in self.kept() {
114 let image = crate::fs::read(&kept.path)?;
115 if image.len() > torn.len() && image.starts_with(torn) {
116 newest = newest.max(Some((kept.at, image)));
117 }
118 }
119 Ok(newest.map(|(_, image)| image))
120 }
121
122 /// The images kept of the file `stamp` stamps, other than the one it stands as; one whose
123 /// upload the file still stands before is listed only once reads have had `LAG` to catch up.
124 pub fn versions(&self, stamp: &Stamp) -> Vec<Version> {
125 // A header mid-write names no file.
126 let Some(document) = document(stamp) else {
127 return Vec::new();
128 };
129 let current = digest(stamp);
130 let lagging = |kept: &Kept| {
131 kept.base == current && kept.at.elapsed().is_ok_and(|elapsed| elapsed < LAG)
132 };
133 (self.kept().into_iter())
134 .filter(|kept| kept.document == document && kept.image != current && !lagging(kept))
135 .map(|kept| Version {
136 id: kept.path.to_string_lossy().into_owned(),
137 device: self.device.clone(),
138 })
139 .collect()
140 }
141
142 /// Whether version `id` is an image kept here.
143 pub fn holds(&self, id: &str) -> bool {
144 Path::new(id).parent() == Some(self.folder.as_path())
145 }
146}
147
148/// The hex of a stamp's digest, which names an image this device keeps.
149pub(crate) fn digest(stamp: &Stamp) -> String {
150 use sha2::Digest;
151 let digest = sha2::Sha256::new()
152 .chain_update(stamp.header)
153 .chain_update(stamp.length.to_le_bytes())
154 .finalize();
155 hex(&digest[..16])
156}
157
158fn document(stamp: &Stamp) -> Option<String> {
159 Some(hex(&onestore::Header::parse(&stamp.header).ok()?.file_id))
160}
161
162fn hex(bytes: &[u8]) -> String {
163 bytes.iter().map(|byte| format!("{byte:02x}")).collect()
164}