| author | |
| committer | |
| log | 02876d691c955ff0594f528e889ff7e7a057fdd9 |
| tree | a1dd96b38792a7806ccc90ee643d78bbbca98287 |
| parent | 82d8d2ab063d1f2aa562cb75c3beedea691e2c15 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
iCloud Drive syncs whole files with no lock, so another device's upload can
land over the phone's with no conflict version kept, and the phone's next
sync took the file as it stood: its published edits were gone. Android's
document providers already kept each published image until the document
held it (c345a48e); that store moves into notebook::published, and the iOS
file remote uses it for notebooks a provider keeps elsewhere. Each image
the phone publishes is kept in the cache until the file is seen otherwise,
then listed beside the host's conflict versions: one the file holds merges
as nothing and goes, one it lost merges back with conflict pages named for
this device, and one that cannot merge is kept beside the file as
"Name (Device).one", as a conflict version is. sb_set_versions takes the
device's name for these. The iCloud launch test gains the lost upload, with
and without shared history; it fails without the store.
- Edits you make on iPhone to an iCloud Drive notebook come back if iCloud lets another device's upload replace them, instead of being lost.
Assisted-by: claude-opus-5.511 files changed, 375 insertions(+), 203 deletions(-)
apps/ios/Snowbound/ICloud.swift+5-4| ... | @@ -1,4 +1,4 @@ | ... | @@ -1,4 +1,4 @@ |
| 1 | import Foundation | 1 | import UIKit |
| 2 | 2 | ||
| 3 | /// Lists the conflict versions iCloud Drive keeps beside a section file, as it keeps another | 3 | /// Lists the conflict versions iCloud Drive keeps beside a section file, as it keeps another |
| 4 | /// device's commit that lost; the library merges each into the file. | 4 | /// device's commit that lost; the library merges each into the file. |
| ... | @@ -58,10 +58,11 @@ enum ICloud { | ... | @@ -58,10 +58,11 @@ enum ICloud { |
| 58 | /// Whether iCloud is signed in with iCloud Drive on, container or not. | 58 | /// Whether iCloud is signed in with iCloud Drive on, container or not. |
| 59 | static var signedIn: Bool { FileManager.default.ubiquityIdentityToken != nil } | 59 | static var signedIn: Bool { FileManager.default.ubiquityIdentityToken != nil } |
| 60 | 60 | ||
| 61 | /// Has the library merge conflict versions, follows the account, and hears of notebooks | 61 | /// Has the library merge conflict versions and this device's uploads iCloud Drive lost, |
| 62 | /// another device adds or removes; call once at launch. | 62 | /// follows the account, and hears of notebooks another device adds or removes; call once |
| 63 | /// at launch. | ||
| 63 | static func start() { | 64 | static func start() { |
| 64 | sb_set_versions(listVersions, retireVersion) | 65 | sb_set_versions(listVersions, retireVersion, UIDevice.current.name) |
| 65 | if automated { return lookUp() } | 66 | if automated { return lookUp() } |
| 66 | NotificationCenter.default.addObserver( | 67 | NotificationCenter.default.addObserver( |
| 67 | forName: .NSUbiquityIdentityDidChange, object: nil, queue: .main | 68 | forName: .NSUbiquityIdentityDidChange, object: nil, queue: .main |
arc/sync.md+8| ... | @@ -295,6 +295,14 @@ produce byte-identical files. Any device merges any version at once; nothing wai | ... | @@ -295,6 +295,14 @@ produce byte-identical files. Any device merges any version at once; nothing wai |
| 295 | device that wrote it. A version of another section, or one that cannot be read, is kept beside | 295 | device that wrote it. A version of another section, or one that cannot be read, is kept beside |
| 296 | the file as `Name (Device).one` instead. | 296 | the file as `Name (Device).one` instead. |
| 297 | 297 | ||
| 298 | iCloud Drive can also lose an upload outright: another device's lands over it and no conflict | ||
| 299 | version reaches this one. So on the iPhone and on Android each image a section publishes to a | ||
| 300 | file a provider keeps stays in the cache (`notebook::published`) until the file is seen | ||
| 301 | otherwise, then lists as one more version: one the file holds merges as nothing and goes, one | ||
| 302 | it lost merges back as a conflict version does, its conflict pages named for this device. One | ||
| 303 | that cannot merge is kept beside the file as `Name (Device).one` on the iPhone, and in the | ||
| 304 | cache on Android, where no provider takes a new document beside it. | ||
| 305 | |||
| 298 | On a cloud drive every read and write of a section file runs under `NSFileCoordinator`, so the | 306 | On a cloud drive every read and write of a section file runs under `NSFileCoordinator`, so the |
| 299 | iCloud daemon never swaps a file between the stamp check and the append. A file iCloud evicted | 307 | iCloud daemon never swaps a file between the stamp check and the append. A file iCloud evicted |
| 300 | (a `.Name.icloud` placeholder, or on macOS 14 and later a dataless file under its own name) lists | 308 | (a `.Name.icloud` placeholder, or on macOS 14 and later a dataless file under its own name) lists |
crates/mobile/include/snowbound.h+1-1| ... | @@ -22,7 +22,7 @@ void sb_set_coordinator(sb_coordinator coordinator); | ... | @@ -22,7 +22,7 @@ void sb_set_coordinator(sb_coordinator coordinator); |
| 22 | void sb_set_sync_wake(void (*wake)(void)); | 22 | void sb_set_sync_wake(void (*wake)(void)); |
| 23 | typedef void (*sb_versions)(const char *path, void (*found)(void *, const char *id, const char *device), void *context); | 23 | typedef void (*sb_versions)(const char *path, void (*found)(void *, const char *id, const char *device), void *context); |
| 24 | typedef bool (*sb_retire)(const char *path, const char *id, bool keep); | 24 | typedef bool (*sb_retire)(const char *path, const char *id, bool keep); |
| 25 | void sb_set_versions(sb_versions versions, sb_retire retire); | 25 | void sb_set_versions(sb_versions versions, sb_retire retire, const char *device); |
| 26 | char *sb_tags(void); | 26 | char *sb_tags(void); |
| 27 | char *sb_pens(uint32_t section); | 27 | char *sb_pens(uint32_t section); |
| 28 | bool sb_tag_icon(uint16_t shape, bool checked, uint32_t pixels, uint8_t *rgba); | 28 | bool sb_tag_icon(uint16_t shape, bool checked, uint32_t pixels, uint8_t *rgba); |
crates/mobile/src/library.rs+117-38| ... | @@ -9,6 +9,7 @@ use notebook::{ | ... | @@ -9,6 +9,7 @@ use notebook::{ |
| 9 | discover::{Folder, Reason, SectionState}, | 9 | discover::{Folder, Reason, SectionState}, |
| 10 | live::{self, Hello, Reach, share::Guest}, | 10 | live::{self, Hello, Reach, share::Guest}, |
| 11 | provider::Documents, | 11 | provider::Documents, |
| 12 | published::{Published, Written}, | ||
| 12 | session::{self, Background, Event, Known, Notebook, RECYCLE_BIN, SyncStatus}, | 13 | session::{self, Background, Event, Known, Notebook, RECYCLE_BIN, SyncStatus}, |
| 13 | smb::{Client, Credentials}, | 14 | smb::{Client, Credentials}, |
| 14 | }; | 15 | }; |
| ... | @@ -103,28 +104,60 @@ pub type Versions = extern "C" fn( | ... | @@ -103,28 +104,60 @@ pub type Versions = extern "C" fn( |
| 103 | /// beside the file when `keep`; false when it could not. | 104 | /// beside the file when `keep`; false when it could not. |
| 104 | pub type Retire = extern "C" fn(path: *const c_char, id: *const c_char, keep: bool) -> bool; | 105 | pub type Retire = extern "C" fn(path: *const c_char, id: *const c_char, keep: bool) -> bool; |
| 105 | 106 | ||
| 106 | static VERSIONS: OnceLock<(Versions, Retire)> = OnceLock::new(); | 107 | static VERSIONS: OnceLock<(Versions, Retire, String)> = OnceLock::new(); |
| 107 | 108 | ||
| 108 | /// Merges the conflict versions the host lists into their files from now on, then has the | 109 | /// Merges the conflict versions the host lists into their files from now on, then has the |
| 109 | /// host retire them; set once, before any library opens. | 110 | /// host retire them, and merges back this device's publications a file provider lost, which |
| 111 | /// conflict pages name for `device`; set once, before any library opens. | ||
| 112 | /// | ||
| 113 | /// # Safety | ||
| 114 | /// `device` is NUL-terminated UTF-8. | ||
| 110 | #[unsafe(no_mangle)] | 115 | #[unsafe(no_mangle)] |
| 111 | pub extern "C" fn sb_set_versions(versions: Versions, retire: Retire) { | 116 | pub unsafe extern "C" fn sb_set_versions( |
| 112 | let _ = VERSIONS.set((versions, retire)); | 117 | versions: Versions, |
| 118 | retire: Retire, | ||
| 119 | device: *const c_char, | ||
| 120 | ) { | ||
| 121 | let _ = VERSIONS.set((versions, retire, string(device))); | ||
| 122 | } | ||
| 123 | |||
| 124 | /// A local section file read and published under the host's file coordination. One a file | ||
| 125 | /// provider keeps elsewhere, which syncs it whole with no lock, keeps each image this device | ||
| 126 | /// publishes until the file holds it, and merges one another device's upload replaced back | ||
| 127 | /// (`notebook::published`). | ||
| 128 | struct Coordinated { | ||
| 129 | file: PathBuf, | ||
| 130 | published: Option<Published>, | ||
| 113 | } | 131 | } |
| 114 | 132 | ||
| 115 | /// A local section file read and published under the host's file coordination. | 133 | impl Coordinated { |
| 116 | struct Coordinated(PathBuf); | 134 | fn new(file: &Path, cache: &Path, elsewhere: bool) -> Self { |
| 135 | let published = elsewhere | ||
| 136 | .then(|| { | ||
| 137 | notebook::location::claimed(cache, &notebook::location::local(file).ok()?).ok() | ||
| 138 | }) | ||
| 139 | .flatten() | ||
| 140 | .map(|folder| { | ||
| 141 | let device = VERSIONS.get().map(|(.., device)| device.clone()); | ||
| 142 | Published::new(folder.join("published"), device) | ||
| 143 | }); | ||
| 144 | Self { | ||
| 145 | file: file.to_owned(), | ||
| 146 | published, | ||
| 147 | } | ||
| 148 | } | ||
| 149 | } | ||
| 117 | 150 | ||
| 118 | impl Remote for Coordinated { | 151 | impl Remote for Coordinated { |
| 119 | fn read(&mut self) -> io::Result<Vec<u8>> { | 152 | fn read(&mut self) -> io::Result<Vec<u8>> { |
| 120 | coordinated(&self.0, false, || onestore::read_file(&self.0))? | 153 | coordinated(&self.file, false, || onestore::read_file(&self.file))? |
| 121 | } | 154 | } |
| 122 | 155 | ||
| 123 | /// The header and length, as the file remote's uncoordinated change detector reads them. | 156 | /// The header and length, as the file remote's uncoordinated change detector reads them. |
| 124 | fn stamp(&mut self) -> io::Result<Stamp> { | 157 | fn stamp(&mut self) -> io::Result<Stamp> { |
| 125 | coordinated(&self.0, false, || { | 158 | coordinated(&self.file, false, || { |
| 126 | use std::io::Read; | 159 | use std::io::Read; |
| 127 | let mut file = std::fs::File::open(&self.0)?; | 160 | let mut file = std::fs::File::open(&self.file)?; |
| 128 | let mut header = [0; 1024]; | 161 | let mut header = [0; 1024]; |
| 129 | file.read_exact(&mut header)?; | 162 | file.read_exact(&mut header)?; |
| 130 | Ok(Stamp { | 163 | Ok(Stamp { |
| ... | @@ -135,28 +168,48 @@ impl Remote for Coordinated { | ... | @@ -135,28 +168,48 @@ impl Remote for Coordinated { |
| 135 | } | 168 | } |
| 136 | 169 | ||
| 137 | fn publish(&mut self, transaction: &Transaction) -> std::result::Result<(), CommitError> { | 170 | fn publish(&mut self, transaction: &Transaction) -> std::result::Result<(), CommitError> { |
| 138 | coordinated(&self.0, true, || transaction.commit_file(&self.0)).map_err(uncommitted)? | 171 | coordinated(&self.file, true, || { |
| 172 | transaction.commit_file(&self.file)?; | ||
| 173 | if let Some(published) = &self.published { | ||
| 174 | // A copy it could not keep only leaves the publication unprotected. | ||
| 175 | let _ = std::fs::File::open(&self.file) | ||
| 176 | .and_then(|mut file| published.keep(Written::InPlace(&mut file), transaction)); | ||
| 177 | } | ||
| 178 | Ok(()) | ||
| 179 | }) | ||
| 180 | .map_err(uncommitted)? | ||
| 139 | } | 181 | } |
| 140 | 182 | ||
| 141 | fn confirm(&mut self, base: &Stamp) -> std::result::Result<(), CommitError> { | 183 | fn confirm(&mut self, base: &Stamp) -> std::result::Result<(), CommitError> { |
| 142 | coordinated(&self.0, true, || onestore::confirm_file(&self.0, base)).map_err(uncommitted)? | 184 | coordinated(&self.file, true, || { |
| 185 | onestore::confirm_file(&self.file, base) | ||
| 186 | }) | ||
| 187 | .map_err(uncommitted)? | ||
| 143 | } | 188 | } |
| 144 | 189 | ||
| 190 | /// The host's conflict versions, then this device's publications the file lost. | ||
| 145 | fn versions(&mut self) -> io::Result<Vec<notebook::Version>> { | 191 | fn versions(&mut self) -> io::Result<Vec<notebook::Version>> { |
| 146 | let Some((versions, _)) = VERSIONS.get() else { | ||
| 147 | return Ok(Vec::new()); | ||
| 148 | }; | ||
| 149 | extern "C" fn found(context: *mut c_void, id: *const c_char, device: *const c_char) { | ||
| 150 | // SAFETY: `context` is the list below, alive for the host's call. | ||
| 151 | let listed = unsafe { &mut *context.cast::<Vec<notebook::Version>>() }; | ||
| 152 | listed.push(notebook::Version { | ||
| 153 | id: string(id), | ||
| 154 | device: (!device.is_null()).then(|| string(device)), | ||
| 155 | }); | ||
| 156 | } | ||
| 157 | let mut listed: Vec<notebook::Version> = Vec::new(); | 192 | let mut listed: Vec<notebook::Version> = Vec::new(); |
| 158 | let path = CString::new(self.0.as_os_str().as_encoded_bytes())?; | 193 | if let Some((versions, ..)) = VERSIONS.get() { |
| 159 | versions(path.as_ptr(), found, (&raw mut listed).cast()); | 194 | extern "C" fn found(context: *mut c_void, id: *const c_char, device: *const c_char) { |
| 195 | // SAFETY: `context` is the list below, alive for the host's call. | ||
| 196 | let listed = unsafe { &mut *context.cast::<Vec<notebook::Version>>() }; | ||
| 197 | listed.push(notebook::Version { | ||
| 198 | id: string(id), | ||
| 199 | device: (!device.is_null()).then(|| string(device)), | ||
| 200 | }); | ||
| 201 | } | ||
| 202 | let path = CString::new(self.file.as_os_str().as_encoded_bytes())?; | ||
| 203 | versions(path.as_ptr(), found, (&raw mut listed).cast()); | ||
| 204 | } | ||
| 205 | if self.published.is_some() { | ||
| 206 | let stamp = self.stamp()?; | ||
| 207 | listed.extend( | ||
| 208 | self.published | ||
| 209 | .iter() | ||
| 210 | .flat_map(|published| published.versions(&stamp)), | ||
| 211 | ); | ||
| 212 | } | ||
| 160 | Ok(listed) | 213 | Ok(listed) |
| 161 | } | 214 | } |
| 162 | 215 | ||
| ... | @@ -165,11 +218,31 @@ impl Remote for Coordinated { | ... | @@ -165,11 +218,31 @@ impl Remote for Coordinated { |
| 165 | std::fs::read(id) | 218 | std::fs::read(id) |
| 166 | } | 219 | } |
| 167 | 220 | ||
| 221 | /// A publication of this device's that cannot merge is kept whole beside the file, as | ||
| 222 | /// the host keeps a conflict version: "Name (Device).one". | ||
| 168 | fn retire(&mut self, id: &str, keep: bool) -> io::Result<()> { | 223 | fn retire(&mut self, id: &str, keep: bool) -> io::Result<()> { |
| 169 | let Some((_, retire)) = VERSIONS.get() else { | 224 | if let Some(published) = &self.published |
| 225 | && published.holds(id) | ||
| 226 | { | ||
| 227 | if keep { | ||
| 228 | let stem = self.file.file_stem().unwrap_or_default().to_string_lossy(); | ||
| 229 | let device = VERSIONS.get().map_or("this device", |(.., device)| device); | ||
| 230 | let beside = (1..) | ||
| 231 | .map(|n| match n { | ||
| 232 | 1 => format!("{stem} ({device}).one"), | ||
| 233 | _ => format!("{stem} ({device} {n}).one"), | ||
| 234 | }) | ||
| 235 | .map(|name| self.file.with_file_name(name)) | ||
| 236 | .find(|beside| std::fs::symlink_metadata(beside).is_err()) | ||
| 237 | .ok_or(io::ErrorKind::AlreadyExists)?; | ||
| 238 | coordinated(&beside, true, || std::fs::copy(id, &beside))??; | ||
| 239 | } | ||
| 240 | return std::fs::remove_file(id); | ||
| 241 | } | ||
| 242 | let Some((_, retire, _)) = VERSIONS.get() else { | ||
| 170 | return Ok(()); | 243 | return Ok(()); |
| 171 | }; | 244 | }; |
| 172 | let path = CString::new(self.0.as_os_str().as_encoded_bytes())?; | 245 | let path = CString::new(self.file.as_os_str().as_encoded_bytes())?; |
| 173 | let id = CString::new(id)?; | 246 | let id = CString::new(id)?; |
| 174 | if retire(path.as_ptr(), id.as_ptr(), keep) { | 247 | if retire(path.as_ptr(), id.as_ptr(), keep) { |
| 175 | Ok(()) | 248 | Ok(()) |
| ... | @@ -297,10 +370,10 @@ pub struct Library { | ... | @@ -297,10 +370,10 @@ pub struct Library { |
| 297 | offline: AtomicBool, | 370 | offline: AtomicBool, |
| 298 | /// Keys of the protected sections unlocked this run, by catalog path; in memory only. | 371 | /// Keys of the protected sections unlocked this run, by catalog path; in memory only. |
| 299 | keys: Mutex<HashMap<String, Key>>, | 372 | keys: Mutex<HashMap<String, Key>>, |
| 300 | /// How long edits wait for a pause in typing before they publish (`Section::set_pause`): | 373 | /// A file provider keeps the notebook elsewhere too, as iCloud Drive does: edits wait for a |
| 301 | /// a file provider such as iCloud Drive uploads each publication, and makes one concurrent | 374 | /// pause in typing before they publish (`Section::set_pause`), as it uploads each |
| 302 | /// with another device's a conflict version. | 375 | /// publication and makes one concurrent with another device's a conflict version. |
| 303 | pause: Duration, | 376 | elsewhere: bool, |
| 304 | } | 377 | } |
| 305 | 378 | ||
| 306 | /// The pause edits to a file a provider keeps elsewhere wait for. | 379 | /// The pause edits to a file a provider keeps elsewhere wait for. |
| ... | @@ -400,10 +473,11 @@ impl Library { | ... | @@ -400,10 +473,11 @@ impl Library { |
| 400 | (None, Place::File(path.to_owned()), None) | 473 | (None, Place::File(path.to_owned()), None) |
| 401 | } else { | 474 | } else { |
| 402 | let mut notebook = Notebook::open(path, cache)?; | 475 | let mut notebook = Notebook::open(path, cache)?; |
| 476 | let kept = cache.to_owned(); | ||
| 403 | let background = notebook.background_with( | 477 | let background = notebook.background_with( |
| 404 | local, | 478 | local, |
| 405 | !local, | 479 | !local, |
| 406 | |file| Coordinated(file.to_owned()), | 480 | move |file| Coordinated::new(file, &kept, !local), |
| 407 | sync_woken, | 481 | sync_woken, |
| 408 | )?; | 482 | )?; |
| 409 | background.watch(notebook.replicas()); | 483 | background.watch(notebook.replicas()); |
| ... | @@ -422,7 +496,7 @@ impl Library { | ... | @@ -422,7 +496,7 @@ impl Library { |
| 422 | open: Mutex::default(), | 496 | open: Mutex::default(), |
| 423 | offline: AtomicBool::new(false), | 497 | offline: AtomicBool::new(false), |
| 424 | keys: Default::default(), | 498 | keys: Default::default(), |
| 425 | pause: if local { Duration::ZERO } else { PAUSE }, | 499 | elsewhere: !local, |
| 426 | }) | 500 | }) |
| 427 | } | 501 | } |
| 428 | 502 | ||
| ... | @@ -445,7 +519,7 @@ impl Library { | ... | @@ -445,7 +519,7 @@ impl Library { |
| 445 | open: Mutex::default(), | 519 | open: Mutex::default(), |
| 446 | offline: AtomicBool::new(false), | 520 | offline: AtomicBool::new(false), |
| 447 | keys: Default::default(), | 521 | keys: Default::default(), |
| 448 | pause: Duration::ZERO, | 522 | elsewhere: false, |
| 449 | }; | 523 | }; |
| 450 | let reached = library.with_notebook(false, |_| Ok(())); | 524 | let reached = library.with_notebook(false, |_| Ok(())); |
| 451 | match library.listed() { | 525 | match library.listed() { |
| ... | @@ -504,7 +578,7 @@ impl Library { | ... | @@ -504,7 +578,7 @@ impl Library { |
| 504 | open: Mutex::default(), | 578 | open: Mutex::default(), |
| 505 | offline: AtomicBool::new(offline), | 579 | offline: AtomicBool::new(offline), |
| 506 | keys: Default::default(), | 580 | keys: Default::default(), |
| 507 | pause: Duration::ZERO, | 581 | elsewhere: false, |
| 508 | }) | 582 | }) |
| 509 | } | 583 | } |
| 510 | 584 | ||
| ... | @@ -541,7 +615,7 @@ impl Library { | ... | @@ -541,7 +615,7 @@ impl Library { |
| 541 | open: Mutex::default(), | 615 | open: Mutex::default(), |
| 542 | offline: AtomicBool::new(false), | 616 | offline: AtomicBool::new(false), |
| 543 | keys: Default::default(), | 617 | keys: Default::default(), |
| 544 | pause: if local { Duration::ZERO } else { PAUSE }, | 618 | elsewhere: !local, |
| 545 | }) | 619 | }) |
| 546 | } | 620 | } |
| 547 | 621 | ||
| ... | @@ -783,7 +857,11 @@ impl Library { | ... | @@ -783,7 +857,11 @@ impl Library { |
| 783 | } | 857 | } |
| 784 | }; | 858 | }; |
| 785 | section.set_offline(self.offline.load(Ordering::Relaxed)); | 859 | section.set_offline(self.offline.load(Ordering::Relaxed)); |
| 786 | section.set_pause(self.pause); | 860 | section.set_pause(if self.elsewhere { |
| 861 | PAUSE | ||
| 862 | } else { | ||
| 863 | Duration::ZERO | ||
| 864 | }); | ||
| 787 | if let Some(background) = &self.background { | 865 | if let Some(background) = &self.background { |
| 788 | background.hold(path, &section); | 866 | background.hold(path, &section); |
| 789 | } | 867 | } |
| ... | @@ -796,18 +874,19 @@ impl Library { | ... | @@ -796,18 +874,19 @@ impl Library { |
| 796 | notify: impl Fn() + Send + Sync + 'static, | 874 | notify: impl Fn() + Send + Sync + 'static, |
| 797 | ) -> Result<session::Section> { | 875 | ) -> Result<session::Section> { |
| 798 | let offline = self.offline.load(Ordering::Relaxed); | 876 | let offline = self.offline.load(Ordering::Relaxed); |
| 877 | let (cache, elsewhere) = (self.cache.clone(), self.elsewhere); | ||
| 878 | let connect = move |file: &Path| Ok(Coordinated::new(file, &cache, elsewhere)); | ||
| 799 | match &self.place { | 879 | match &self.place { |
| 800 | Place::File(file) => Ok(session::Section::open_with( | 880 | Place::File(file) => Ok(session::Section::open_with( |
| 801 | file, | 881 | file, |
| 802 | &self.cache, | 882 | &self.cache, |
| 803 | |file| Ok(Coordinated(file.to_owned())), | 883 | connect, |
| 804 | offline, | 884 | offline, |
| 805 | notify, | 885 | notify, |
| 806 | )?), | 886 | )?), |
| 807 | Place::Folder(_) => { | 887 | Place::Folder(_) => { |
| 808 | let notebook = self.notebook(); | 888 | let notebook = self.notebook(); |
| 809 | let notebook = notebook.as_ref().ok_or("A lone section has no notebook")?; | 889 | let notebook = notebook.as_ref().ok_or("A lone section has no notebook")?; |
| 810 | let connect = |file: &Path| Ok(Coordinated(file.to_owned())); | ||
| 811 | Ok(match self.unlocked().get(path) { | 890 | Ok(match self.unlocked().get(path) { |
| 812 | Some(key) => { | 891 | Some(key) => { |
| 813 | notebook.section_unlocked_with(path, key, connect, offline, notify)? | 892 | notebook.section_unlocked_with(path, key, connect, offline, notify)? |
crates/mobile/src/tests.rs+50-16| ... | @@ -4655,17 +4655,50 @@ fn every_page(root: &Path, cache: &Path) -> String { | ... | @@ -4655,17 +4655,50 @@ fn every_page(root: &Path, cache: &Path) -> String { |
| 4655 | shown | 4655 | shown |
| 4656 | } | 4656 | } |
| 4657 | 4657 | ||
| 4658 | /// The titles and text of every page and conflict page each section file in `root` stores. | ||
| 4659 | fn stored(root: &Path) -> String { | ||
| 4660 | let mut text = String::new(); | ||
| 4661 | for entry in std::fs::read_dir(root).unwrap() { | ||
| 4662 | let path = entry.unwrap().path(); | ||
| 4663 | if path.extension().is_some_and(|extension| extension == "one") { | ||
| 4664 | let arena = onestore::Arena::default(); | ||
| 4665 | let mut section = | ||
| 4666 | onestore::Section::open(&arena, std::fs::read(path).unwrap()).unwrap(); | ||
| 4667 | let conflicts = section.conflicts().unwrap().into_iter(); | ||
| 4668 | let spaces = (section | ||
| 4669 | .pages() | ||
| 4670 | .unwrap() | ||
| 4671 | .into_iter() | ||
| 4672 | .map(|(space, ..)| space)) | ||
| 4673 | .chain( | ||
| 4674 | conflicts | ||
| 4675 | .flat_map(|(_, pages)| pages) | ||
| 4676 | .map(|page| page.space), | ||
| 4677 | ); | ||
| 4678 | for space in spaces.collect::<Vec<_>>() { | ||
| 4679 | let page = section.page(space).unwrap(); | ||
| 4680 | text += &format!("{}\n{}\n", page.title, canvas::search::page_text(&page)); | ||
| 4681 | } | ||
| 4682 | } | ||
| 4683 | } | ||
| 4684 | text | ||
| 4685 | } | ||
| 4686 | |||
| 4658 | /// The phone and the Mac edit one iCloud Drive notebook Snowbound made, each in its own copy: | 4687 | /// The phone and the Mac edit one iCloud Drive notebook Snowbound made, each in its own copy: |
| 4659 | /// the same page, a page each adds and moves to the top, a page the Mac deletes as the phone | 4688 | /// the same page, a page each adds and moves to the top, a page the Mac deletes as the phone |
| 4660 | /// types on it, and a section the phone adds. The Mac's upload wins, so the phone's next | 4689 | /// types on it, and a section the phone adds. The Mac's upload wins: iCloud Drive keeps the |
| 4661 | /// launch finds its own file a conflict version, and opens, lists and merges it through the C | 4690 | /// phone's file as a conflict version, or loses it. The phone's next launch opens, lists and |
| 4662 | /// surface as iOS does, its pages open and typed on in the Reading View all the while. | 4691 | /// merges it back through the C surface as iOS does, its pages open and typed on in the |
| 4663 | /// Nothing either device typed goes missing, merged or set aside whole. | 4692 | /// Reading View all the while. Nothing either device typed goes missing, merged or set aside |
| 4693 | /// whole. | ||
| 4664 | #[test] | 4694 | #[test] |
| 4665 | fn an_icloud_notebook_merges_the_other_devices_version_at_launch() { | 4695 | fn an_icloud_notebook_merges_the_phones_upload_back_at_launch() { |
| 4666 | library::sb_set_versions(cloud_versions, cloud_retire); | 4696 | unsafe { |
| 4667 | // Whether the devices' copies already share an edit of the page both type on. | 4697 | library::sb_set_versions(cloud_versions, cloud_retire, c"Clover’s iPhone".as_ptr()) |
| 4668 | for shared in [false, true] { | 4698 | }; |
| 4699 | // Whether the devices' copies already share an edit of the page both type on, and | ||
| 4700 | // whether iCloud Drive lost the phone's upload rather than keep it as a version. | ||
| 4701 | for (shared, lost) in [(false, false), (true, false), (false, true), (true, true)] { | ||
| 4669 | let directory = tempfile::tempdir().unwrap(); | 4702 | let directory = tempfile::tempdir().unwrap(); |
| 4670 | let folder = |name: &str| directory.path().join(name); | 4703 | let folder = |name: &str| directory.path().join(name); |
| 4671 | let (phone, mac) = (folder("phone"), folder("mac")); | 4704 | let (phone, mac) = (folder("phone"), folder("mac")); |
| ... | @@ -4739,12 +4772,11 @@ fn an_icloud_notebook_merges_the_other_devices_version_at_launch() { | ... | @@ -4739,12 +4772,11 @@ fn an_icloud_notebook_merges_the_other_devices_version_at_launch() { |
| 4739 | let version = folder("version.one"); | 4772 | let version = folder("version.one"); |
| 4740 | std::fs::rename(&file, &version).unwrap(); | 4773 | std::fs::rename(&file, &version).unwrap(); |
| 4741 | std::fs::copy(mac.join(&path), &file).unwrap(); | 4774 | std::fs::copy(mac.join(&path), &file).unwrap(); |
| 4742 | let device = "Clover’s iPhone".to_owned(); | 4775 | if !lost { |
| 4743 | CLOUD_VERSIONS.lock().unwrap().push(( | 4776 | let device = "Clover’s iPhone".to_owned(); |
| 4744 | std::fs::canonicalize(&file).unwrap(), | 4777 | let file = std::fs::canonicalize(&file).unwrap(); |
| 4745 | version, | 4778 | CLOUD_VERSIONS.lock().unwrap().push((file, version, device)); |
| 4746 | device, | 4779 | } |
| 4747 | )); | ||
| 4748 | 4780 | ||
| 4749 | // The launch: the notebook opens and lists, the page shown last opens, and the | 4781 | // The launch: the notebook opens and lists, the page shown last opens, and the |
| 4750 | // background merges the version meanwhile. | 4782 | // background merges the version meanwhile. |
| ... | @@ -4761,8 +4793,10 @@ fn an_icloud_notebook_merges_the_other_devices_version_at_launch() { | ... | @@ -4761,8 +4793,10 @@ fn an_icloud_notebook_merges_the_other_devices_version_at_launch() { |
| 4761 | .collect(); | 4793 | .collect(); |
| 4762 | sb_library_sync_now(&library); | 4794 | sb_library_sync_now(&library); |
| 4763 | let deadline = Instant::now() + Duration::from_secs(10); | 4795 | let deadline = Instant::now() + Duration::from_secs(10); |
| 4764 | while !CLOUD_VERSIONS.lock().unwrap().is_empty() { | 4796 | while !CLOUD_VERSIONS.lock().unwrap().is_empty() |
| 4765 | assert!(Instant::now() < deadline, "the version never merged"); | 4797 | || !stored(&phone).contains("Phone first words") |
| 4798 | { | ||
| 4799 | assert!(Instant::now() < deadline, "the phone's upload never merged"); | ||
| 4766 | library::sb_section_poll(&section, &mut 0); | 4800 | library::sb_section_poll(&section, &mut 0); |
| 4767 | for view in &mut views { | 4801 | for view in &mut views { |
| 4768 | unsafe { sb_insert(view, c"Launch ".as_ptr()) }; | 4802 | unsafe { sb_insert(view, c"Launch ".as_ptr()) }; |
crates/notebook/README.md+5| ... | @@ -151,6 +151,11 @@ they clash. The revisions the merge writes are named after the version's it merg | ... | @@ -151,6 +151,11 @@ they clash. The revisions the merge writes are named after the version's it merg |
| 151 | (`Section::seal_as`), so a version merged once, here or on another device, merges as nothing | 151 | (`Section::seal_as`), so a version merged once, here or on another device, merges as nothing |
| 152 | again. It is published on the file's stamp, then retired; one of another section, or one that | 152 | again. It is published on the file's stamp, then retired; one of another section, or one that |
| 153 | cannot be read, is retired with `keep` for the remote to keep beside the file. | 153 | cannot be read, is retired with `keep` for the remote to keep beside the file. |
| 154 | A provider can also lose an upload outright, another device's landing over it with no version | ||
| 155 | kept. A remote keeps what it publishes with `published::Published` (`keep` after each | ||
| 156 | publication) and lists each image the file no longer stands as among its versions | ||
| 157 | (`Published::versions`): one the file holds merges as nothing and is dropped, one it lost | ||
| 158 | merges back. | ||
| 154 | 159 | ||
| 155 | Publication attempts are recorded before network I/O. An attempt confirms only | 160 | Publication attempts are recorded before network I/O. An attempt confirms only |
| 156 | when the remote holds its revisions, or every page it changed as it changed them, | 161 | when the remote holds its revisions, or every page it changed as it changed them, |
crates/notebook/src/lib.rs+1| ... | @@ -33,6 +33,7 @@ mod merge; | ... | @@ -33,6 +33,7 @@ mod merge; |
| 33 | mod migrate; | 33 | mod migrate; |
| 34 | pub mod package; | 34 | pub mod package; |
| 35 | pub mod provider; | 35 | pub mod provider; |
| 36 | pub mod published; | ||
| 36 | mod queue; | 37 | mod queue; |
| 37 | mod recovery; | 38 | mod recovery; |
| 38 | mod resolve; | 39 | mod resolve; |
crates/notebook/src/location.rs+1-1| ... | @@ -84,7 +84,7 @@ pub fn forget(cache: &Path, location: &str) -> io::Result<()> { | ... | @@ -84,7 +84,7 @@ pub fn forget(cache: &Path, location: &str) -> io::Result<()> { |
| 84 | } | 84 | } |
| 85 | 85 | ||
| 86 | /// The folder for `location`, made and labelled with it where it was not. | 86 | /// The folder for `location`, made and labelled with it where it was not. |
| 87 | fn claimed(cache: &Path, location: &str) -> io::Result<PathBuf> { | 87 | pub fn claimed(cache: &Path, location: &str) -> io::Result<PathBuf> { |
| 88 | let folder = folder(cache, location); | 88 | let folder = folder(cache, location); |
| 89 | let label = folder.join(LOCATION); | 89 | let label = folder.join(LOCATION); |
| 90 | if fs::metadata(&label).is_err() { | 90 | if fs::metadata(&label).is_err() { |
crates/notebook/src/provider.rs+19-140| ... | @@ -4,7 +4,11 @@ | ... | @@ -4,7 +4,11 @@ |
| 4 | //! notebook root (`session::Notebook::open_provider`), a picked document a lone section | 4 | //! notebook root (`session::Notebook::open_provider`), a picked document a lone section |
| 5 | //! (`session::Section::open_provider`); replicas and the catalog's listing stay in the cache. | 5 | //! (`session::Section::open_provider`); replicas and the catalog's listing stay in the cache. |
| 6 | 6 | ||
| 7 | use crate::{Error, Result, discover, session::Storage}; | 7 | use crate::{ |
| 8 | Error, Result, discover, | ||
| 9 | published::{Published, Written}, | ||
| 10 | session::Storage, | ||
| 11 | }; | ||
| 8 | use onestore::{CommitError, CommitIo, CommitState, Stamp, Store, Transaction}; | 12 | use onestore::{CommitError, CommitIo, CommitState, Stamp, Store, Transaction}; |
| 9 | use std::{ | 13 | use std::{ |
| 10 | collections::BTreeMap, | 14 | collections::BTreeMap, |
| ... | @@ -12,7 +16,6 @@ use std::{ | ... | @@ -12,7 +16,6 @@ use std::{ |
| 12 | io::{self, Read, Seek, SeekFrom, Write}, | 16 | io::{self, Read, Seek, SeekFrom, Write}, |
| 13 | path::PathBuf, | 17 | path::PathBuf, |
| 14 | sync::{Arc, Mutex, MutexGuard, PoisonError}, | 18 | sync::{Arc, Mutex, MutexGuard, PoisonError}, |
| 15 | time::Duration, | ||
| 16 | }; | 19 | }; |
| 17 | 20 | ||
| 18 | /// A document as its provider describes it. | 21 | /// A document as its provider describes it. |
| ... | @@ -294,12 +297,6 @@ fn whole(file: &mut File, limit: usize) -> io::Result<Vec<u8>> { | ... | @@ -294,12 +297,6 @@ fn whole(file: &mut File, limit: usize) -> io::Result<Vec<u8>> { |
| 294 | Ok(bytes) | 297 | Ok(bytes) |
| 295 | } | 298 | } |
| 296 | 299 | ||
| 297 | /// What a write wrote: the document in place, or its whole image. | ||
| 298 | enum Written<'a> { | ||
| 299 | InPlace(&'a mut File), | ||
| 300 | Image(&'a [u8]), | ||
| 301 | } | ||
| 302 | |||
| 303 | /// A document as a commit writes it: its seekable descriptor, or its image to write whole. | 300 | /// A document as a commit writes it: its seekable descriptor, or its image to write whole. |
| 304 | enum Io { | 301 | enum Io { |
| 305 | File(File), | 302 | File(File), |
| ... | @@ -351,58 +348,12 @@ impl CommitIo for Io { | ... | @@ -351,58 +348,12 @@ impl CommitIo for Io { |
| 351 | } | 348 | } |
| 352 | } | 349 | } |
| 353 | 350 | ||
| 354 | /// The hex of a stamp's digest, which names an image this device keeps. | 351 | /// A section a document provider keeps, as its replica publishes to it, each image it |
| 355 | fn digest(stamp: &Stamp) -> String { | 352 | /// publishes kept until the document holds it (`published.rs`). |
| 356 | use sha2::Digest; | ||
| 357 | let digest = sha2::Sha256::new() | ||
| 358 | .chain_update(stamp.header) | ||
| 359 | .chain_update(stamp.length.to_le_bytes()) | ||
| 360 | .finalize(); | ||
| 361 | hex(&digest[..16]) | ||
| 362 | } | ||
| 363 | |||
| 364 | fn hex(bytes: &[u8]) -> String { | ||
| 365 | bytes.iter().map(|byte| format!("{byte:02x}")).collect() | ||
| 366 | } | ||
| 367 | |||
| 368 | fn file_of(header: &[u8]) -> Option<[u8; 16]> { | ||
| 369 | Some(onestore::Header::parse(header).ok()?.file_id) | ||
| 370 | } | ||
| 371 | |||
| 372 | /// An image this device keeps, as its name gives it: `<document>-<image>-<base>.one`, the | ||
| 373 | /// document's identity, then the digests of the image's stamp and of the stamp of the image | ||
| 374 | /// it was published on. | ||
| 375 | struct Kept { | ||
| 376 | name: String, | ||
| 377 | document: String, | ||
| 378 | image: String, | ||
| 379 | base: String, | ||
| 380 | } | ||
| 381 | |||
| 382 | impl Kept { | ||
| 383 | fn named(name: String) -> Option<Self> { | ||
| 384 | let mut parts = name.strip_suffix(".one")?.split('-').map(str::to_owned); | ||
| 385 | let (document, image, base) = (parts.next()?, parts.next()?, parts.next()?); | ||
| 386 | Some(Self { | ||
| 387 | name, | ||
| 388 | document, | ||
| 389 | image, | ||
| 390 | base, | ||
| 391 | }) | ||
| 392 | } | ||
| 393 | } | ||
| 394 | |||
| 395 | /// How long reads may lag this device's own upload before a document still standing as the | ||
| 396 | /// image the upload was made on counts as having lost it. | ||
| 397 | const LAG: Duration = Duration::from_secs(60); | ||
| 398 | |||
| 399 | /// A section a document provider keeps, as its replica publishes to it. Each image it | ||
| 400 | /// publishes is kept, and listed as a version once the document is seen otherwise, until the | ||
| 401 | /// document holds it: an upload another device's replaced then merges back as a conflict | ||
| 402 | /// version does (`resolve.rs`). Only this device's publications are kept so. | ||
| 403 | pub(crate) struct ProviderRemote { | 353 | pub(crate) struct ProviderRemote { |
| 404 | documents: Arc<Documents>, | 354 | documents: Arc<Documents>, |
| 405 | path: String, | 355 | path: String, |
| 356 | published: Published, | ||
| 406 | /// The image the step read whole for a stamp, which its read takes. | 357 | /// The image the step read whole for a stamp, which its read takes. |
| 407 | read: Option<Vec<u8>>, | 358 | read: Option<Vec<u8>>, |
| 408 | } | 359 | } |
| ... | @@ -412,74 +363,19 @@ impl ProviderRemote { | ... | @@ -412,74 +363,19 @@ impl ProviderRemote { |
| 412 | Self { | 363 | Self { |
| 413 | documents: Arc::clone(documents), | 364 | documents: Arc::clone(documents), |
| 414 | path: path.to_owned(), | 365 | path: path.to_owned(), |
| 366 | published: Published::new(documents.published(), Some(documents.device.clone())), | ||
| 415 | read: None, | 367 | read: None, |
| 416 | } | 368 | } |
| 417 | } | 369 | } |
| 418 | 370 | ||
| 419 | fn kept(&self) -> Vec<(Kept, std::time::SystemTime)> { | ||
| 420 | let entries = crate::fs::read_dir(self.documents.published()) | ||
| 421 | .into_iter() | ||
| 422 | .flatten(); | ||
| 423 | entries | ||
| 424 | .filter_map(|entry| { | ||
| 425 | let entry = entry.ok()?; | ||
| 426 | let kept = Kept::named(entry.file_name().into_string().ok()?)?; | ||
| 427 | Some((kept, entry.metadata().ok()?.modified().ok()?)) | ||
| 428 | }) | ||
| 429 | .collect() | ||
| 430 | } | ||
| 431 | |||
| 432 | /// Keeps the image `transaction` published, in place of the one it was published on: | ||
| 433 | /// that kept image with the transaction applied where this device kept it, else read back. | ||
| 434 | fn keep(&self, written: Written<'_>, transaction: &Transaction) -> io::Result<()> { | ||
| 435 | let folder = self.documents.published(); | ||
| 436 | let base = digest(transaction.base()); | ||
| 437 | let document = file_of(&transaction.base().header).ok_or(io::ErrorKind::InvalidData)?; | ||
| 438 | let on = (self.kept().into_iter()) | ||
| 439 | .find(|(kept, _)| kept.document == hex(&document) && kept.image == base); | ||
| 440 | let image = match (written, &on) { | ||
| 441 | (Written::Image(image), _) => image.to_vec(), | ||
| 442 | (Written::InPlace(_), Some((kept, _))) => { | ||
| 443 | let mut image = crate::fs::read(folder.join(&kept.name))?; | ||
| 444 | transaction | ||
| 445 | .apply(&mut image) | ||
| 446 | .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; | ||
| 447 | image | ||
| 448 | } | ||
| 449 | (Written::InPlace(file), None) => { | ||
| 450 | file.seek(SeekFrom::Start(0))?; | ||
| 451 | whole(file, crate::MAX_FILE_BYTES)? | ||
| 452 | } | ||
| 453 | }; | ||
| 454 | let name = format!( | ||
| 455 | "{}-{}-{base}.one", | ||
| 456 | hex(&document), | ||
| 457 | digest(&transaction.stamp()) | ||
| 458 | ); | ||
| 459 | crate::fs::create_dir_all(&folder)?; | ||
| 460 | let mut kept = tempfile::NamedTempFile::new_in(&folder)?; | ||
| 461 | kept.write_all(&image)?; | ||
| 462 | kept.as_file().sync_all()?; | ||
| 463 | kept.persist(folder.join(name)) | ||
| 464 | .map_err(|error| error.error)?; | ||
| 465 | if let Some((kept, _)) = on { | ||
| 466 | crate::fs::remove_file(folder.join(kept.name))?; | ||
| 467 | } | ||
| 468 | Ok(()) | ||
| 469 | } | ||
| 470 | |||
| 471 | /// This device's own upload cut off, as a provider that writes non-atomically leaves it, | 371 | /// This device's own upload cut off, as a provider that writes non-atomically leaves it, |
| 472 | /// put back and read again: the newest image kept that the torn bytes begin. Any other | 372 | /// put back and read again: the newest image kept that the torn bytes begin. Any other |
| 473 | /// document that does not parse is left as it is. | 373 | /// document that does not parse is left as it is. |
| 474 | fn repair(&self, torn: &[u8]) -> io::Result<Vec<u8>> { | 374 | fn repair(&self, torn: &[u8]) -> io::Result<Vec<u8>> { |
| 475 | let mut newest = None; | 375 | let image = self |
| 476 | for (kept, at) in self.kept() { | 376 | .published |
| 477 | let image = crate::fs::read(self.documents.published().join(&kept.name))?; | 377 | .torn(torn)? |
| 478 | if image.len() > torn.len() && image.starts_with(torn) { | 378 | .ok_or(io::ErrorKind::WouldBlock)?; |
| 479 | newest = newest.max(Some((at, image))); | ||
| 480 | } | ||
| 481 | } | ||
| 482 | let (_, image) = newest.ok_or(io::ErrorKind::WouldBlock)?; | ||
| 483 | let id = self.documents.id(&self.path)?; | 379 | let id = self.documents.id(&self.path)?; |
| 484 | self.documents.writable(&id)?; | 380 | self.documents.writable(&id)?; |
| 485 | self.documents | 381 | self.documents |
| ... | @@ -524,7 +420,7 @@ impl crate::Remote for ProviderRemote { | ... | @@ -524,7 +420,7 @@ impl crate::Remote for ProviderRemote { |
| 524 | self.documents.write( | 420 | self.documents.write( |
| 525 | &self.path, | 421 | &self.path, |
| 526 | |io| transaction.commit(io), | 422 | |io| transaction.commit(io), |
| 527 | |written| self.keep(written, transaction), | 423 | |written| self.published.keep(written, transaction), |
| 528 | ) | 424 | ) |
| 529 | } | 425 | } |
| 530 | 426 | ||
| ... | @@ -534,9 +430,8 @@ impl crate::Remote for ProviderRemote { | ... | @@ -534,9 +430,8 @@ impl crate::Remote for ProviderRemote { |
| 534 | .write(&self.path, |io| onestore::confirm(io, base), |_| Ok(())) | 430 | .write(&self.path, |io| onestore::confirm(io, base), |_| Ok(())) |
| 535 | } | 431 | } |
| 536 | 432 | ||
| 537 | /// The images kept of this document other than the one it stands as; one whose upload | 433 | /// The images kept of this document other than the one it stands as |
| 538 | /// the document still stands before is listed only once reads have had `LAG` to catch up. | 434 | /// (`Published::versions`). A document read torn is repaired first, once a step. |
| 539 | /// A document read torn is repaired first, once a step. | ||
| 540 | fn versions(&mut self) -> io::Result<Vec<crate::Version>> { | 435 | fn versions(&mut self) -> io::Result<Vec<crate::Version>> { |
| 541 | let (mut stamp, mut image) = self.documents.stamp(&self.path)?; | 436 | let (mut stamp, mut image) = self.documents.stamp(&self.path)?; |
| 542 | if let Some(torn) = image.as_ref().filter(|image| !parses(image)) | 437 | if let Some(torn) = image.as_ref().filter(|image| !parses(image)) |
| ... | @@ -546,33 +441,17 @@ impl crate::Remote for ProviderRemote { | ... | @@ -546,33 +441,17 @@ impl crate::Remote for ProviderRemote { |
| 546 | (stamp, image) = (repaired_stamp, Some(repaired)); | 441 | (stamp, image) = (repaired_stamp, Some(repaired)); |
| 547 | } | 442 | } |
| 548 | self.read = image; | 443 | self.read = image; |
| 549 | // A header mid-write names no document; the step's read repairs it. | 444 | Ok(self.published.versions(&stamp)) |
| 550 | let Some(document) = file_of(&stamp.header) else { | ||
| 551 | return Ok(Vec::new()); | ||
| 552 | }; | ||
| 553 | let (document, current) = (hex(&document), digest(&stamp)); | ||
| 554 | let lagging = |kept: &Kept, at: std::time::SystemTime| { | ||
| 555 | kept.base == current && at.elapsed().is_ok_and(|elapsed| elapsed < LAG) | ||
| 556 | }; | ||
| 557 | Ok((self.kept().into_iter()) | ||
| 558 | .filter(|(kept, at)| { | ||
| 559 | kept.document == document && kept.image != current && !lagging(kept, *at) | ||
| 560 | }) | ||
| 561 | .map(|(kept, _)| crate::Version { | ||
| 562 | id: kept.name, | ||
| 563 | device: Some(self.documents.device.clone()), | ||
| 564 | }) | ||
| 565 | .collect()) | ||
| 566 | } | 445 | } |
| 567 | 446 | ||
| 568 | fn version(&mut self, id: &str) -> io::Result<Vec<u8>> { | 447 | fn version(&mut self, id: &str) -> io::Result<Vec<u8>> { |
| 569 | crate::fs::read(self.documents.published().join(id)) | 448 | crate::fs::read(id) |
| 570 | } | 449 | } |
| 571 | 450 | ||
| 572 | /// One that cannot merge, of another section under the document's name, is set aside | 451 | /// One that cannot merge, of another section under the document's name, is set aside |
| 573 | /// whole, where `versions` lists it no more: no provider here takes a new document beside it. | 452 | /// whole, where `versions` lists it no more: no provider here takes a new document beside it. |
| 574 | fn retire(&mut self, id: &str, keep: bool) -> io::Result<()> { | 453 | fn retire(&mut self, id: &str, keep: bool) -> io::Result<()> { |
| 575 | let path = self.documents.published().join(id); | 454 | let path = PathBuf::from(id); |
| 576 | if keep { | 455 | if keep { |
| 577 | crate::fs::rename(&path, path.with_extension("unmerged")) | 456 | crate::fs::rename(&path, path.with_extension("unmerged")) |
| 578 | } else { | 457 | } else { |
crates/notebook/src/provider/tests.rs+4-3| ... | @@ -917,9 +917,9 @@ fn kept_images(fake: &Fake) -> Vec<String> { | ... | @@ -917,9 +917,9 @@ fn kept_images(fake: &Fake) -> Vec<String> { |
| 917 | 917 | ||
| 918 | /// The images a device keeps that Notes.one does not stand as, still to merge. | 918 | /// The images a device keeps that Notes.one does not stand as, still to merge. |
| 919 | fn unmerged(fake: &Fake) -> Vec<String> { | 919 | fn unmerged(fake: &Fake) -> Vec<String> { |
| 920 | let current = digest(&Stamp::of(&fake.bytes("Notes.one")).unwrap()); | 920 | let current = crate::published::digest(&Stamp::of(&fake.bytes("Notes.one")).unwrap()); |
| 921 | let mut images = kept_images(fake); | 921 | let mut images = kept_images(fake); |
| 922 | images.retain(|name| Kept::named(name.clone()).is_some_and(|kept| kept.image != current)); | 922 | images.retain(|name| name.split('-').nth(1) != Some(current.as_str())); |
| 923 | images | 923 | images |
| 924 | } | 924 | } |
| 925 | 925 | ||
| ... | @@ -976,7 +976,8 @@ fn aged(fake: &Fake) { | ... | @@ -976,7 +976,8 @@ fn aged(fake: &Fake) { |
| 976 | .write(true) | 976 | .write(true) |
| 977 | .open(image.unwrap().path()) | 977 | .open(image.unwrap().path()) |
| 978 | .unwrap(); | 978 | .unwrap(); |
| 979 | let past = std::time::SystemTime::now() - LAG - Duration::from_secs(1); | 979 | let past = |
| 980 | std::time::SystemTime::now() - crate::published::LAG - Duration::from_secs(1); | ||
| 980 | file.set_modified(past).unwrap(); | 981 | file.set_modified(past).unwrap(); |
| 981 | } | 982 | } |
| 982 | } | 983 | } |
crates/notebook/src/published.rs created+164| ... | @@ -0,0 +1,164 @@ | ||
| 1 | //! The images this device publishes to section files a provider syncs whole, with no lock or | ||
| 2 | //! compare-and-swap, as iCloud Drive and Android's document providers do: another device's | ||
| 3 | //! upload can replace this one's. Each image is kept until the file is seen holding it, and | ||
| 4 | //! one the file lost is listed as a `Remote::versions` version, so that it merges back as a | ||
| 5 | //! conflict version does (`resolve.rs`). | ||
| 6 | |||
| 7 | use crate::Version; | ||
| 8 | use onestore::{Stamp, Transaction}; | ||
| 9 | use std::{ | ||
| 10 | fs::File, | ||
| 11 | io::{self, Read, Seek, SeekFrom, Write}, | ||
| 12 | path::{Path, PathBuf}, | ||
| 13 | time::{Duration, SystemTime}, | ||
| 14 | }; | ||
| 15 | |||
| 16 | /// How long reads may lag this device's own upload before a file still standing as the image | ||
| 17 | /// the upload was made on counts as having lost it. | ||
| 18 | pub(crate) const LAG: Duration = Duration::from_secs(60); | ||
| 19 | |||
| 20 | /// What a publication wrote: the file in place, or its whole image. | ||
| 21 | pub enum Written<'a> { | ||
| 22 | InPlace(&'a mut File), | ||
| 23 | Image(&'a [u8]), | ||
| 24 | } | ||
| 25 | |||
| 26 | /// The images this device published to the files of one place, kept in `folder` and named | ||
| 27 | /// `<document>-<image>-<base>.one`: the file's identity, then the digests of the image's stamp | ||
| 28 | /// and of the stamp of the image it was published on. Version ids are their paths. | ||
| 29 | pub struct Published { | ||
| 30 | folder: PathBuf, | ||
| 31 | /// This device, as the conflict pages a lost publication makes name it. | ||
| 32 | device: Option<String>, | ||
| 33 | } | ||
| 34 | |||
| 35 | struct Kept { | ||
| 36 | path: PathBuf, | ||
| 37 | document: String, | ||
| 38 | image: String, | ||
| 39 | base: String, | ||
| 40 | at: SystemTime, | ||
| 41 | } | ||
| 42 | |||
| 43 | impl Published { | ||
| 44 | pub fn new(folder: impl Into<PathBuf>, device: Option<String>) -> Self { | ||
| 45 | Self { | ||
| 46 | folder: folder.into(), | ||
| 47 | device, | ||
| 48 | } | ||
| 49 | } | ||
| 50 | |||
| 51 | fn kept(&self) -> Vec<Kept> { | ||
| 52 | let entries = crate::fs::read_dir(&self.folder).into_iter().flatten(); | ||
| 53 | entries | ||
| 54 | .filter_map(|entry| { | ||
| 55 | let entry = entry.ok()?; | ||
| 56 | let name = entry.file_name().into_string().ok()?; | ||
| 57 | let mut parts = name.strip_suffix(".one")?.split('-').map(str::to_owned); | ||
| 58 | Some(Kept { | ||
| 59 | path: entry.path(), | ||
| 60 | document: parts.next()?, | ||
| 61 | image: parts.next()?, | ||
| 62 | base: parts.next()?, | ||
| 63 | at: entry.metadata().ok()?.modified().ok()?, | ||
| 64 | }) | ||
| 65 | }) | ||
| 66 | .collect() | ||
| 67 | } | ||
| 68 | |||
| 69 | /// Keeps the image `transaction` published, in place of the one it was published on: | ||
| 70 | /// that kept image with the transaction applied where this device kept it, else read back. | ||
| 71 | pub fn keep(&self, written: Written<'_>, transaction: &Transaction) -> io::Result<()> { | ||
| 72 | let base = digest(transaction.base()); | ||
| 73 | let document = document(transaction.base()).ok_or(io::ErrorKind::InvalidData)?; | ||
| 74 | let on = | ||
| 75 | (self.kept().into_iter()).find(|kept| kept.document == document && kept.image == base); | ||
| 76 | let image = match (written, &on) { | ||
| 77 | (Written::Image(image), _) => image.to_vec(), | ||
| 78 | (Written::InPlace(_), Some(kept)) => { | ||
| 79 | let mut image = crate::fs::read(&kept.path)?; | ||
| 80 | transaction | ||
| 81 | .apply(&mut image) | ||
| 82 | .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; | ||
| 83 | image | ||
| 84 | } | ||
| 85 | (Written::InPlace(file), None) => { | ||
| 86 | file.seek(SeekFrom::Start(0))?; | ||
| 87 | let mut image = Vec::new(); | ||
| 88 | file.take(crate::MAX_FILE_BYTES as u64 + 1) | ||
| 89 | .read_to_end(&mut image)?; | ||
| 90 | if image.len() > crate::MAX_FILE_BYTES { | ||
| 91 | return Err(io::ErrorKind::FileTooLarge.into()); | ||
| 92 | } | ||
| 93 | image | ||
| 94 | } | ||
| 95 | }; | ||
| 96 | let name = format!("{document}-{}-{base}.one", digest(&transaction.stamp())); | ||
| 97 | crate::fs::create_dir_all(&self.folder)?; | ||
| 98 | let mut kept = tempfile::NamedTempFile::new_in(&self.folder)?; | ||
| 99 | kept.write_all(&image)?; | ||
| 100 | kept.as_file().sync_all()?; | ||
| 101 | kept.persist(self.folder.join(name)) | ||
| 102 | .map_err(|error| error.error)?; | ||
| 103 | if let Some(kept) = on { | ||
| 104 | crate::fs::remove_file(kept.path)?; | ||
| 105 | } | ||
| 106 | Ok(()) | ||
| 107 | } | ||
| 108 | |||
| 109 | /// The newest image kept that `torn` begins, as a provider writing in place leaves this | ||
| 110 | /// device's upload cut off. | ||
| 111 | pub fn torn(&self, torn: &[u8]) -> io::Result<Option<Vec<u8>>> { | ||
| 112 | let mut newest = None; | ||
| 113 | for kept in self.kept() { | ||
| 114 | let image = crate::fs::read(&kept.path)?; | ||
| 115 | if image.len() > torn.len() && image.starts_with(torn) { | ||
| 116 | newest = newest.max(Some((kept.at, image))); | ||
| 117 | } | ||
| 118 | } | ||
| 119 | Ok(newest.map(|(_, image)| image)) | ||
| 120 | } | ||
| 121 | |||
| 122 | /// The images kept of the file `stamp` stamps, other than the one it stands as; one whose | ||
| 123 | /// upload the file still stands before is listed only once reads have had `LAG` to catch up. | ||
| 124 | pub fn versions(&self, stamp: &Stamp) -> Vec<Version> { | ||
| 125 | // A header mid-write names no file. | ||
| 126 | let Some(document) = document(stamp) else { | ||
| 127 | return Vec::new(); | ||
| 128 | }; | ||
| 129 | let current = digest(stamp); | ||
| 130 | let lagging = |kept: &Kept| { | ||
| 131 | kept.base == current && kept.at.elapsed().is_ok_and(|elapsed| elapsed < LAG) | ||
| 132 | }; | ||
| 133 | (self.kept().into_iter()) | ||
| 134 | .filter(|kept| kept.document == document && kept.image != current && !lagging(kept)) | ||
| 135 | .map(|kept| Version { | ||
| 136 | id: kept.path.to_string_lossy().into_owned(), | ||
| 137 | device: self.device.clone(), | ||
| 138 | }) | ||
| 139 | .collect() | ||
| 140 | } | ||
| 141 | |||
| 142 | /// Whether version `id` is an image kept here. | ||
| 143 | pub fn holds(&self, id: &str) -> bool { | ||
| 144 | Path::new(id).parent() == Some(self.folder.as_path()) | ||
| 145 | } | ||
| 146 | } | ||
| 147 | |||
| 148 | /// The hex of a stamp's digest, which names an image this device keeps. | ||
| 149 | pub(crate) fn digest(stamp: &Stamp) -> String { | ||
| 150 | use sha2::Digest; | ||
| 151 | let digest = sha2::Sha256::new() | ||
| 152 | .chain_update(stamp.header) | ||
| 153 | .chain_update(stamp.length.to_le_bytes()) | ||
| 154 | .finalize(); | ||
| 155 | hex(&digest[..16]) | ||
| 156 | } | ||
| 157 | |||
| 158 | fn document(stamp: &Stamp) -> Option<String> { | ||
| 159 | Some(hex(&onestore::Header::parse(&stamp.header).ok()?.file_id)) | ||
| 160 | } | ||
| 161 | |||
| 162 | fn hex(bytes: &[u8]) -> String { | ||
| 163 | bytes.iter().map(|byte| format!("{byte:02x}")).collect() | ||
| 164 | } | ||