| 1 | //! The images this device publishes to section files a provider syncs whole, with no lock or |
| 2 | //! compare-and-swap, as iCloud Drive and Android's document providers do: another device's |
| 3 | //! upload can replace this one's. Each image is kept beside the section's replica until the |
| 4 | //! file is seen holding it, and one the file lost is listed as a `Remote::versions` version, |
| 5 | //! so that it merges back as a conflict version does (`resolve.rs`). They move and go with |
| 6 | //! the replica (`location::moved`, `remove_replica`). |
| 7 | |
| 8 | use crate::Version; |
| 9 | use onestore::{Stamp, Transaction}; |
| 10 | use std::{ |
| 11 | fs::File, |
| 12 | io::{self, Read, Seek, SeekFrom, Write}, |
| 13 | path::{Path, PathBuf}, |
| 14 | time::{Duration, SystemTime}, |
| 15 | }; |
| 16 | |
| 17 | /// How long reads may lag this device's own upload before a file still standing as the image |
| 18 | /// the upload was made on counts as having lost it. |
| 19 | pub(crate) const LAG: Duration = Duration::from_secs(60); |
| 20 | |
| 21 | /// What a publication wrote: the file in place, or its whole image. |
| 22 | pub enum Written<'a> { |
| 23 | InPlace(&'a mut File), |
| 24 | Image(&'a [u8]), |
| 25 | } |
| 26 | |
| 27 | /// The images this device published to one section file, named `<document>-<image>-<base>.one`: |
| 28 | /// the file's identity, then the digests of the image's stamp and of the stamp of the image it |
| 29 | /// was published on. Version ids are their paths. |
| 30 | pub struct Published { |
| 31 | folder: PathBuf, |
| 32 | /// This device, as the conflict pages a lost publication makes name it. |
| 33 | device: String, |
| 34 | } |
| 35 | |
| 36 | struct Kept { |
| 37 | path: PathBuf, |
| 38 | document: String, |
| 39 | image: String, |
| 40 | base: String, |
| 41 | at: SystemTime, |
| 42 | } |
| 43 | |
| 44 | /// The folder beside the replica at `replica` holding its section's images. |
| 45 | pub(crate) fn folder(replica: &Path) -> PathBuf { |
| 46 | replica.with_extension("published") |
| 47 | } |
| 48 | |
| 49 | impl Published { |
| 50 | /// The images kept of the section whose replica is at `replica`. |
| 51 | pub fn of(replica: &Path, device: impl Into<String>) -> Self { |
| 52 | Self { |
| 53 | folder: folder(replica), |
| 54 | device: device.into(), |
| 55 | } |
| 56 | } |
| 57 | |
| 58 | pub fn device(&self) -> &str { |
| 59 | &self.device |
| 60 | } |
| 61 | |
| 62 | fn kept(&self) -> Vec<Kept> { |
| 63 | let entries = crate::fs::read_dir(&self.folder).into_iter().flatten(); |
| 64 | entries |
| 65 | .filter_map(|entry| { |
| 66 | let entry = entry.ok()?; |
| 67 | let name = entry.file_name().into_string().ok()?; |
| 68 | let mut parts = name.strip_suffix(".one")?.split('-').map(str::to_owned); |
| 69 | Some(Kept { |
| 70 | path: entry.path(), |
| 71 | document: parts.next()?, |
| 72 | image: parts.next()?, |
| 73 | base: parts.next()?, |
| 74 | at: entry.metadata().ok()?.modified().ok()?, |
| 75 | }) |
| 76 | }) |
| 77 | .collect() |
| 78 | } |
| 79 | |
| 80 | fn put(&self, image: &[u8], name: String, replacing: Option<Kept>) -> io::Result<()> { |
| 81 | crate::fs::create_dir_all(&self.folder)?; |
| 82 | let mut kept = tempfile::NamedTempFile::new_in(&self.folder)?; |
| 83 | kept.write_all(image)?; |
| 84 | kept.as_file().sync_all()?; |
| 85 | kept.persist(self.folder.join(name)) |
| 86 | .map_err(|error| error.error)?; |
| 87 | if let Some(kept) = replacing { |
| 88 | crate::fs::remove_file(kept.path)?; |
| 89 | } |
| 90 | Ok(()) |
| 91 | } |
| 92 | |
| 93 | /// Keeps the image `transaction` published, in place of the one it was published on: |
| 94 | /// that kept image with the transaction applied where this device kept it, else read back. |
| 95 | pub fn keep(&self, written: Written<'_>, transaction: &Transaction) -> io::Result<()> { |
| 96 | let base = digest(transaction.base()); |
| 97 | let document = document(transaction.base()).ok_or(io::ErrorKind::InvalidData)?; |
| 98 | let on = |
| 99 | (self.kept().into_iter()).find(|kept| kept.document == document && kept.image == base); |
| 100 | let image = match (written, &on) { |
| 101 | (Written::Image(image), _) => image.to_vec(), |
| 102 | (Written::InPlace(_), Some(kept)) => { |
| 103 | let mut image = crate::fs::read(&kept.path)?; |
| 104 | transaction |
| 105 | .apply(&mut image) |
| 106 | .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; |
| 107 | image |
| 108 | } |
| 109 | (Written::InPlace(file), None) => { |
| 110 | file.seek(SeekFrom::Start(0))?; |
| 111 | whole(file, crate::MAX_FILE_BYTES)? |
| 112 | } |
| 113 | }; |
| 114 | let name = format!("{document}-{}-{base}.one", digest(&transaction.stamp())); |
| 115 | self.put(&image, name, on) |
| 116 | } |
| 117 | |
| 118 | /// Follows the image kept as `base` through a confirmation (`onestore::confirm`), which |
| 119 | /// rewrites only the header: the image `written` now begins with. |
| 120 | pub fn confirmed(&self, written: Written<'_>, base: &Stamp) -> io::Result<()> { |
| 121 | let image = digest(base); |
| 122 | let Some(kept) = (self.kept().into_iter()).find(|kept| kept.image == image) else { |
| 123 | return Ok(()); |
| 124 | }; |
| 125 | let mut header = [0; 1024]; |
| 126 | match written { |
| 127 | Written::InPlace(file) => { |
| 128 | file.seek(SeekFrom::Start(0))?; |
| 129 | file.read_exact(&mut header)?; |
| 130 | } |
| 131 | Written::Image(image) => { |
| 132 | header.copy_from_slice(image.get(..1024).ok_or(io::ErrorKind::UnexpectedEof)?) |
| 133 | } |
| 134 | } |
| 135 | let mut confirmed = crate::fs::read(&kept.path)?; |
| 136 | confirmed |
| 137 | .get_mut(..1024) |
| 138 | .ok_or(io::ErrorKind::InvalidData)? |
| 139 | .copy_from_slice(&header); |
| 140 | let stamp = Stamp { |
| 141 | header, |
| 142 | length: confirmed.len() as u64, |
| 143 | }; |
| 144 | let name = format!("{}-{}-{}.one", kept.document, digest(&stamp), kept.base); |
| 145 | self.put(&confirmed, name, Some(kept)) |
| 146 | } |
| 147 | |
| 148 | /// The newest image kept that `torn` begins, as a provider writing in place leaves this |
| 149 | /// device's upload cut off. |
| 150 | pub(crate) fn torn(&self, torn: &[u8]) -> io::Result<Option<Vec<u8>>> { |
| 151 | let mut newest = None; |
| 152 | for kept in self.kept() { |
| 153 | let image = crate::fs::read(&kept.path)?; |
| 154 | if image.len() > torn.len() && image.starts_with(torn) { |
| 155 | newest = newest.max(Some((kept.at, image))); |
| 156 | } |
| 157 | } |
| 158 | Ok(newest.map(|(_, image)| image)) |
| 159 | } |
| 160 | |
| 161 | /// The images kept other than the one the file stands as, by `stamp`, which is read only |
| 162 | /// where any is kept; one whose upload the file still stands before is listed only once |
| 163 | /// reads have had `LAG` to catch up. One the provider keeps as a conflict version too, in |
| 164 | /// `conflicts`, is left to that version, so that it is not merged, or kept beside, twice; |
| 165 | /// none is listed while a conflict version cannot be read. |
| 166 | pub fn versions( |
| 167 | &self, |
| 168 | stamp: impl FnOnce() -> io::Result<Stamp>, |
| 169 | conflicts: &[Version], |
| 170 | ) -> io::Result<Vec<Version>> { |
| 171 | let kept = self.kept(); |
| 172 | if kept.is_empty() { |
| 173 | return Ok(Vec::new()); |
| 174 | } |
| 175 | let stamp = stamp()?; |
| 176 | // A header mid-write names no file. |
| 177 | let Some(document) = document(&stamp) else { |
| 178 | return Ok(Vec::new()); |
| 179 | }; |
| 180 | let current = digest(&stamp); |
| 181 | let lagging = |kept: &Kept| { |
| 182 | kept.base == current && kept.at.elapsed().is_ok_and(|elapsed| elapsed < LAG) |
| 183 | }; |
| 184 | // A conflict version not read yet, as one iCloud Drive has not brought down, may be |
| 185 | // one of these: they wait for it, rather than be merged or kept beside the file twice. |
| 186 | let Ok(conflicts) = (conflicts.iter()) |
| 187 | .map(|version| Ok(digest(&file_stamp(Path::new(&version.id))?))) |
| 188 | .collect::<io::Result<Vec<String>>>() |
| 189 | else { |
| 190 | return Ok(Vec::new()); |
| 191 | }; |
| 192 | Ok(kept |
| 193 | .into_iter() |
| 194 | .filter(|kept| { |
| 195 | kept.document == document |
| 196 | && kept.image != current |
| 197 | && !lagging(kept) |
| 198 | && !conflicts.contains(&kept.image) |
| 199 | }) |
| 200 | .map(|kept| Version { |
| 201 | id: kept.path.to_string_lossy().into_owned(), |
| 202 | device: Some(self.device.clone()), |
| 203 | }) |
| 204 | .collect()) |
| 205 | } |
| 206 | |
| 207 | /// The provider's conflict version at `version` is retired, merged or kept beside the |
| 208 | /// file: an image kept here that it is goes with it, so that it is not merged again. |
| 209 | pub fn retired(&self, version: &Path) -> io::Result<()> { |
| 210 | let image = digest(&file_stamp(version)?); |
| 211 | for kept in self.kept().into_iter().filter(|kept| kept.image == image) { |
| 212 | crate::fs::remove_file(kept.path)?; |
| 213 | } |
| 214 | Ok(()) |
| 215 | } |
| 216 | |
| 217 | /// Whether version `id` is an image kept here. |
| 218 | pub fn holds(&self, id: &str) -> bool { |
| 219 | Path::new(id).parent() == Some(self.folder.as_path()) |
| 220 | } |
| 221 | } |
| 222 | |
| 223 | /// The stamp of the file at `path`: its header and length. |
| 224 | fn file_stamp(path: &Path) -> io::Result<Stamp> { |
| 225 | let mut file = File::open(path)?; |
| 226 | let mut header = [0; 1024]; |
| 227 | file.read_exact(&mut header)?; |
| 228 | Ok(Stamp { |
| 229 | header, |
| 230 | length: file.metadata()?.len(), |
| 231 | }) |
| 232 | } |
| 233 | |
| 234 | /// Whether the section whose replica is at `replica` keeps images of its publications. |
| 235 | pub(crate) fn keeps(replica: &Path) -> bool { |
| 236 | crate::fs::read_dir(folder(replica)).is_ok_and(|mut images| images.next().is_some()) |
| 237 | } |
| 238 | |
| 239 | /// Moves the images a provider notebook kept in one folder for all its sections (`legacy`, |
| 240 | /// as before they were kept beside each replica) to the replica `replica` names for each |
| 241 | /// image's file identity, once: an image may be a publication the document lost, not yet |
| 242 | /// merged. One whose file the notebook no longer lists stays where it was. |
| 243 | pub(crate) fn adopt( |
| 244 | legacy: &Path, |
| 245 | replica: impl Fn(&[u8; 16]) -> Option<PathBuf>, |
| 246 | ) -> io::Result<()> { |
| 247 | let Ok(entries) = crate::fs::read_dir(legacy) else { |
| 248 | return Ok(()); |
| 249 | }; |
| 250 | for entry in entries { |
| 251 | let image = entry?.path(); |
| 252 | let file = (image.file_name().and_then(|name| name.to_str())) |
| 253 | .and_then(|name| name.strip_suffix(".one")?.split('-').next()) |
| 254 | .and_then(identity); |
| 255 | let Some(replica) = file.and_then(|file| replica(&file)) else { |
| 256 | continue; |
| 257 | }; |
| 258 | let store = folder(&replica); |
| 259 | crate::fs::create_dir_all(&store)?; |
| 260 | let name = image.file_name().unwrap_or_default(); |
| 261 | if crate::fs::metadata(store.join(name)).is_err() { |
| 262 | crate::fs::rename(&image, store.join(name))?; |
| 263 | } |
| 264 | } |
| 265 | let _ = crate::fs::remove_dir(legacy); |
| 266 | Ok(()) |
| 267 | } |
| 268 | |
| 269 | /// The file identity a kept image's name begins with. |
| 270 | fn identity(hex: &str) -> Option<[u8; 16]> { |
| 271 | let mut identity = [0; 16]; |
| 272 | for (at, byte) in identity.iter_mut().enumerate() { |
| 273 | *byte = u8::from_str_radix(hex.get(at * 2..at * 2 + 2)?, 16).ok()?; |
| 274 | } |
| 275 | (hex.len() == 32).then_some(identity) |
| 276 | } |
| 277 | |
| 278 | /// Where a version of `file` that cannot merge is kept beside it, as iCloud Drive keeps a |
| 279 | /// conflict version: "Name (Device).one", numbered past any name taken. |
| 280 | pub fn beside(file: &Path, device: &str) -> PathBuf { |
| 281 | let stem = file.file_stem().unwrap_or_default().to_string_lossy(); |
| 282 | let mut n = 1; |
| 283 | loop { |
| 284 | let name = match n { |
| 285 | 1 => format!("{stem} ({device}).one"), |
| 286 | n => format!("{stem} ({device} {n}).one"), |
| 287 | }; |
| 288 | let beside = file.with_file_name(name); |
| 289 | if std::fs::symlink_metadata(&beside).is_err() { |
| 290 | return beside; |
| 291 | } |
| 292 | n += 1; |
| 293 | } |
| 294 | } |
| 295 | |
| 296 | /// All of `file`, at most `limit` bytes. |
| 297 | pub(crate) fn whole(file: &mut File, limit: usize) -> io::Result<Vec<u8>> { |
| 298 | let mut bytes = Vec::new(); |
| 299 | file.take(limit as u64 + 1).read_to_end(&mut bytes)?; |
| 300 | if bytes.len() > limit { |
| 301 | return Err(io::ErrorKind::FileTooLarge.into()); |
| 302 | } |
| 303 | Ok(bytes) |
| 304 | } |
| 305 | |
| 306 | /// The hex of a stamp's digest, which names an image this device keeps. |
| 307 | pub(crate) fn digest(stamp: &Stamp) -> String { |
| 308 | use sha2::Digest; |
| 309 | let digest = sha2::Sha256::new() |
| 310 | .chain_update(stamp.header) |
| 311 | .chain_update(stamp.length.to_le_bytes()) |
| 312 | .finalize(); |
| 313 | hex(&digest[..16]) |
| 314 | } |
| 315 | |
| 316 | fn document(stamp: &Stamp) -> Option<String> { |
| 317 | Some(hex(&onestore::Header::parse(&stamp.header).ok()?.file_id)) |
| 318 | } |
| 319 | |
| 320 | fn hex(bytes: &[u8]) -> String { |
| 321 | bytes.iter().map(|byte| format!("{byte:02x}")).collect() |
| 322 | } |
| 323 | |
| 324 | #[cfg(test)] |
| 325 | mod tests { |
| 326 | use super::*; |
| 327 | use onestore::op::{Edit, Op, PageOp}; |
| 328 | |
| 329 | /// A section file in a scratch folder, its image, and the transaction one edit of it |
| 330 | /// publishes. |
| 331 | fn edited(folder: &Path) -> (PathBuf, Vec<u8>, Transaction) { |
| 332 | let file = folder.join("Notes.one"); |
| 333 | let image = onestore::create_section("Notes.one", "Text", "Author").unwrap(); |
| 334 | std::fs::write(&file, &image).unwrap(); |
| 335 | let arena = onestore::Arena::default(); |
| 336 | let mut section = onestore::Section::open(&arena, image.clone()).unwrap(); |
| 337 | let space = section.pages().unwrap()[0].0; |
| 338 | let edit = Edit { |
| 339 | at: crate::now(), |
| 340 | ops: vec![Op::Page { |
| 341 | space, |
| 342 | op: PageOp::Color(Some(0x00ab_cdef)), |
| 343 | }], |
| 344 | }; |
| 345 | section.apply("Author", &edit).unwrap(); |
| 346 | (file, image, section.seal().unwrap().unwrap()) |
| 347 | } |
| 348 | |
| 349 | fn published(folder: &Path, file: &Path, transaction: &Transaction) -> Published { |
| 350 | let published = Published::of(&folder.join("replica.sqlite"), "iPhone"); |
| 351 | transaction.commit_file(file).unwrap(); |
| 352 | let mut opened = File::open(file).unwrap(); |
| 353 | published |
| 354 | .keep(Written::InPlace(&mut opened), transaction) |
| 355 | .unwrap(); |
| 356 | published |
| 357 | } |
| 358 | |
| 359 | #[test] |
| 360 | fn nothing_kept_reads_no_stamp() { |
| 361 | let folder = tempfile::tempdir().unwrap(); |
| 362 | let published = Published::of(&folder.path().join("replica.sqlite"), "iPhone"); |
| 363 | let listed = published.versions(|| unreachable!("no stamp is read"), &[]); |
| 364 | assert!(listed.unwrap().is_empty()); |
| 365 | } |
| 366 | |
| 367 | /// The image this device published lists once the file stands otherwise: once reads have |
| 368 | /// had `LAG` to catch up where the file stands as the image it was published on, and not |
| 369 | /// where the provider keeps it as a conflict version itself. |
| 370 | #[test] |
| 371 | fn a_lost_image_lists_once() { |
| 372 | let folder = tempfile::tempdir().unwrap(); |
| 373 | let (file, base, transaction) = edited(folder.path()); |
| 374 | let published = published(folder.path(), &file, &transaction); |
| 375 | let listed = |
| 376 | |conflicts: &[Version]| published.versions(|| file_stamp(&file), conflicts).unwrap(); |
| 377 | assert!(listed(&[]).is_empty()); |
| 378 | let ours = std::fs::read(&file).unwrap(); |
| 379 | std::fs::write(&file, base).unwrap(); |
| 380 | assert!(listed(&[]).is_empty()); |
| 381 | let aged = File::options() |
| 382 | .write(true) |
| 383 | .open(&published.kept()[0].path) |
| 384 | .unwrap(); |
| 385 | aged.set_modified(SystemTime::now() - LAG - Duration::from_secs(1)) |
| 386 | .unwrap(); |
| 387 | assert_eq!(listed(&[]).len(), 1); |
| 388 | // A conflict version not here yet could be this image: it waits for it. |
| 389 | let missing = Version { |
| 390 | id: file |
| 391 | .with_file_name("Not here.one") |
| 392 | .to_string_lossy() |
| 393 | .into_owned(), |
| 394 | device: None, |
| 395 | }; |
| 396 | assert!(listed(&[missing]).is_empty()); |
| 397 | let version = file.with_file_name("Notes (iPhone).one"); |
| 398 | std::fs::write(&version, ours).unwrap(); |
| 399 | let conflict = Version { |
| 400 | id: version.to_string_lossy().into_owned(), |
| 401 | device: Some("iPhone".into()), |
| 402 | }; |
| 403 | assert!(listed(std::slice::from_ref(&conflict)).is_empty()); |
| 404 | published.retired(&version).unwrap(); |
| 405 | assert!(published.kept().is_empty()); |
| 406 | } |
| 407 | |
| 408 | /// A confirmation rewrites the header of the image last published: the kept image follows |
| 409 | /// it, rather than listing as lost and going, which would leave it unprotected. |
| 410 | #[test] |
| 411 | fn a_confirmation_keeps_the_image_kept() { |
| 412 | let folder = tempfile::tempdir().unwrap(); |
| 413 | let (file, _, transaction) = edited(folder.path()); |
| 414 | let published = published(folder.path(), &file, &transaction); |
| 415 | let base = file_stamp(&file).unwrap(); |
| 416 | onestore::confirm_file(&file, &base).unwrap(); |
| 417 | assert_ne!(file_stamp(&file).unwrap(), base); |
| 418 | let mut opened = File::open(&file).unwrap(); |
| 419 | published |
| 420 | .confirmed(Written::InPlace(&mut opened), &base) |
| 421 | .unwrap(); |
| 422 | let kept = published.kept(); |
| 423 | assert_eq!(kept.len(), 1); |
| 424 | assert_eq!( |
| 425 | std::fs::read(&kept[0].path).unwrap(), |
| 426 | std::fs::read(&file).unwrap() |
| 427 | ); |
| 428 | let listed = published.versions(|| file_stamp(&file), &[]).unwrap(); |
| 429 | assert!(listed.is_empty()); |
| 430 | } |
| 431 | } |