1//! The images this device publishes to section files a provider syncs whole, with no lock or
2//! compare-and-swap, as iCloud Drive and Android's document providers do: another device's
3//! upload can replace this one's. Each image is kept beside the section's replica until the
4//! file is seen holding it, and one the file lost is listed as a `Remote::versions` version,
5//! so that it merges back as a conflict version does (`resolve.rs`). They move and go with
6//! the replica (`location::moved`, `remove_replica`).
7
8use crate::Version;
9use onestore::{Stamp, Transaction};
10use std::{
11 fs::File,
12 io::{self, Read, Seek, SeekFrom, Write},
13 path::{Path, PathBuf},
14 time::{Duration, SystemTime},
15};
16
17/// How long reads may lag this device's own upload before a file still standing as the image
18/// the upload was made on counts as having lost it.
19pub(crate) const LAG: Duration = Duration::from_secs(60);
20
21/// What a publication wrote: the file in place, or its whole image.
22pub enum Written<'a> {
23 InPlace(&'a mut File),
24 Image(&'a [u8]),
25}
26
27/// The images this device published to one section file, named `<document>-<image>-<base>.one`:
28/// the file's identity, then the digests of the image's stamp and of the stamp of the image it
29/// was published on. Version ids are their paths.
30pub struct Published {
31 folder: PathBuf,
32 /// This device, as the conflict pages a lost publication makes name it.
33 device: String,
34}
35
36struct Kept {
37 path: PathBuf,
38 document: String,
39 image: String,
40 base: String,
41 at: SystemTime,
42}
43
44/// The folder beside the replica at `replica` holding its section's images.
45pub(crate) fn folder(replica: &Path) -> PathBuf {
46 replica.with_extension("published")
47}
48
49impl Published {
50 /// The images kept of the section whose replica is at `replica`.
51 pub fn of(replica: &Path, device: impl Into<String>) -> Self {
52 Self {
53 folder: folder(replica),
54 device: device.into(),
55 }
56 }
57
58 pub fn device(&self) -> &str {
59 &self.device
60 }
61
62 fn kept(&self) -> Vec<Kept> {
63 let entries = crate::fs::read_dir(&self.folder).into_iter().flatten();
64 entries
65 .filter_map(|entry| {
66 let entry = entry.ok()?;
67 let name = entry.file_name().into_string().ok()?;
68 let mut parts = name.strip_suffix(".one")?.split('-').map(str::to_owned);
69 Some(Kept {
70 path: entry.path(),
71 document: parts.next()?,
72 image: parts.next()?,
73 base: parts.next()?,
74 at: entry.metadata().ok()?.modified().ok()?,
75 })
76 })
77 .collect()
78 }
79
80 fn put(&self, image: &[u8], name: String, replacing: Option<Kept>) -> io::Result<()> {
81 crate::fs::create_dir_all(&self.folder)?;
82 let mut kept = tempfile::NamedTempFile::new_in(&self.folder)?;
83 kept.write_all(image)?;
84 kept.as_file().sync_all()?;
85 kept.persist(self.folder.join(name))
86 .map_err(|error| error.error)?;
87 if let Some(kept) = replacing {
88 crate::fs::remove_file(kept.path)?;
89 }
90 Ok(())
91 }
92
93 /// Keeps the image `transaction` published, in place of the one it was published on:
94 /// that kept image with the transaction applied where this device kept it, else read back.
95 pub fn keep(&self, written: Written<'_>, transaction: &Transaction) -> io::Result<()> {
96 let base = digest(transaction.base());
97 let document = document(transaction.base()).ok_or(io::ErrorKind::InvalidData)?;
98 let on =
99 (self.kept().into_iter()).find(|kept| kept.document == document && kept.image == base);
100 let image = match (written, &on) {
101 (Written::Image(image), _) => image.to_vec(),
102 (Written::InPlace(_), Some(kept)) => {
103 let mut image = crate::fs::read(&kept.path)?;
104 transaction
105 .apply(&mut image)
106 .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?;
107 image
108 }
109 (Written::InPlace(file), None) => {
110 file.seek(SeekFrom::Start(0))?;
111 whole(file, crate::MAX_FILE_BYTES)?
112 }
113 };
114 let name = format!("{document}-{}-{base}.one", digest(&transaction.stamp()));
115 self.put(&image, name, on)
116 }
117
118 /// Follows the image kept as `base` through a confirmation (`onestore::confirm`), which
119 /// rewrites only the header: the image `written` now begins with.
120 pub fn confirmed(&self, written: Written<'_>, base: &Stamp) -> io::Result<()> {
121 let image = digest(base);
122 let Some(kept) = (self.kept().into_iter()).find(|kept| kept.image == image) else {
123 return Ok(());
124 };
125 let mut header = [0; 1024];
126 match written {
127 Written::InPlace(file) => {
128 file.seek(SeekFrom::Start(0))?;
129 file.read_exact(&mut header)?;
130 }
131 Written::Image(image) => {
132 header.copy_from_slice(image.get(..1024).ok_or(io::ErrorKind::UnexpectedEof)?)
133 }
134 }
135 let mut confirmed = crate::fs::read(&kept.path)?;
136 confirmed
137 .get_mut(..1024)
138 .ok_or(io::ErrorKind::InvalidData)?
139 .copy_from_slice(&header);
140 let stamp = Stamp {
141 header,
142 length: confirmed.len() as u64,
143 };
144 let name = format!("{}-{}-{}.one", kept.document, digest(&stamp), kept.base);
145 self.put(&confirmed, name, Some(kept))
146 }
147
148 /// The newest image kept that `torn` begins, as a provider writing in place leaves this
149 /// device's upload cut off.
150 pub(crate) fn torn(&self, torn: &[u8]) -> io::Result<Option<Vec<u8>>> {
151 let mut newest = None;
152 for kept in self.kept() {
153 let image = crate::fs::read(&kept.path)?;
154 if image.len() > torn.len() && image.starts_with(torn) {
155 newest = newest.max(Some((kept.at, image)));
156 }
157 }
158 Ok(newest.map(|(_, image)| image))
159 }
160
161 /// The images kept other than the one the file stands as, by `stamp`, which is read only
162 /// where any is kept; one whose upload the file still stands before is listed only once
163 /// reads have had `LAG` to catch up. One the provider keeps as a conflict version too, in
164 /// `conflicts`, is left to that version, so that it is not merged, or kept beside, twice;
165 /// none is listed while a conflict version cannot be read.
166 pub fn versions(
167 &self,
168 stamp: impl FnOnce() -> io::Result<Stamp>,
169 conflicts: &[Version],
170 ) -> io::Result<Vec<Version>> {
171 let kept = self.kept();
172 if kept.is_empty() {
173 return Ok(Vec::new());
174 }
175 let stamp = stamp()?;
176 // A header mid-write names no file.
177 let Some(document) = document(&stamp) else {
178 return Ok(Vec::new());
179 };
180 let current = digest(&stamp);
181 let lagging = |kept: &Kept| {
182 kept.base == current && kept.at.elapsed().is_ok_and(|elapsed| elapsed < LAG)
183 };
184 // A conflict version not read yet, as one iCloud Drive has not brought down, may be
185 // one of these: they wait for it, rather than be merged or kept beside the file twice.
186 let Ok(conflicts) = (conflicts.iter())
187 .map(|version| Ok(digest(&file_stamp(Path::new(&version.id))?)))
188 .collect::<io::Result<Vec<String>>>()
189 else {
190 return Ok(Vec::new());
191 };
192 Ok(kept
193 .into_iter()
194 .filter(|kept| {
195 kept.document == document
196 && kept.image != current
197 && !lagging(kept)
198 && !conflicts.contains(&kept.image)
199 })
200 .map(|kept| Version {
201 id: kept.path.to_string_lossy().into_owned(),
202 device: Some(self.device.clone()),
203 })
204 .collect())
205 }
206
207 /// The provider's conflict version at `version` is retired, merged or kept beside the
208 /// file: an image kept here that it is goes with it, so that it is not merged again.
209 pub fn retired(&self, version: &Path) -> io::Result<()> {
210 let image = digest(&file_stamp(version)?);
211 for kept in self.kept().into_iter().filter(|kept| kept.image == image) {
212 crate::fs::remove_file(kept.path)?;
213 }
214 Ok(())
215 }
216
217 /// Whether version `id` is an image kept here.
218 pub fn holds(&self, id: &str) -> bool {
219 Path::new(id).parent() == Some(self.folder.as_path())
220 }
221}
222
223/// The stamp of the file at `path`: its header and length.
224fn file_stamp(path: &Path) -> io::Result<Stamp> {
225 let mut file = File::open(path)?;
226 let mut header = [0; 1024];
227 file.read_exact(&mut header)?;
228 Ok(Stamp {
229 header,
230 length: file.metadata()?.len(),
231 })
232}
233
234/// Whether the section whose replica is at `replica` keeps images of its publications.
235pub(crate) fn keeps(replica: &Path) -> bool {
236 crate::fs::read_dir(folder(replica)).is_ok_and(|mut images| images.next().is_some())
237}
238
239/// Moves the images a provider notebook kept in one folder for all its sections (`legacy`,
240/// as before they were kept beside each replica) to the replica `replica` names for each
241/// image's file identity, once: an image may be a publication the document lost, not yet
242/// merged. One whose file the notebook no longer lists stays where it was.
243pub(crate) fn adopt(
244 legacy: &Path,
245 replica: impl Fn(&[u8; 16]) -> Option<PathBuf>,
246) -> io::Result<()> {
247 let Ok(entries) = crate::fs::read_dir(legacy) else {
248 return Ok(());
249 };
250 for entry in entries {
251 let image = entry?.path();
252 let file = (image.file_name().and_then(|name| name.to_str()))
253 .and_then(|name| name.strip_suffix(".one")?.split('-').next())
254 .and_then(identity);
255 let Some(replica) = file.and_then(|file| replica(&file)) else {
256 continue;
257 };
258 let store = folder(&replica);
259 crate::fs::create_dir_all(&store)?;
260 let name = image.file_name().unwrap_or_default();
261 if crate::fs::metadata(store.join(name)).is_err() {
262 crate::fs::rename(&image, store.join(name))?;
263 }
264 }
265 let _ = crate::fs::remove_dir(legacy);
266 Ok(())
267}
268
269/// The file identity a kept image's name begins with.
270fn identity(hex: &str) -> Option<[u8; 16]> {
271 let mut identity = [0; 16];
272 for (at, byte) in identity.iter_mut().enumerate() {
273 *byte = u8::from_str_radix(hex.get(at * 2..at * 2 + 2)?, 16).ok()?;
274 }
275 (hex.len() == 32).then_some(identity)
276}
277
278/// Where a version of `file` that cannot merge is kept beside it, as iCloud Drive keeps a
279/// conflict version: "Name (Device).one", numbered past any name taken.
280pub fn beside(file: &Path, device: &str) -> PathBuf {
281 let stem = file.file_stem().unwrap_or_default().to_string_lossy();
282 let mut n = 1;
283 loop {
284 let name = match n {
285 1 => format!("{stem} ({device}).one"),
286 n => format!("{stem} ({device} {n}).one"),
287 };
288 let beside = file.with_file_name(name);
289 if std::fs::symlink_metadata(&beside).is_err() {
290 return beside;
291 }
292 n += 1;
293 }
294}
295
296/// All of `file`, at most `limit` bytes.
297pub(crate) fn whole(file: &mut File, limit: usize) -> io::Result<Vec<u8>> {
298 let mut bytes = Vec::new();
299 file.take(limit as u64 + 1).read_to_end(&mut bytes)?;
300 if bytes.len() > limit {
301 return Err(io::ErrorKind::FileTooLarge.into());
302 }
303 Ok(bytes)
304}
305
306/// The hex of a stamp's digest, which names an image this device keeps.
307pub(crate) fn digest(stamp: &Stamp) -> String {
308 use sha2::Digest;
309 let digest = sha2::Sha256::new()
310 .chain_update(stamp.header)
311 .chain_update(stamp.length.to_le_bytes())
312 .finalize();
313 hex(&digest[..16])
314}
315
316fn document(stamp: &Stamp) -> Option<String> {
317 Some(hex(&onestore::Header::parse(&stamp.header).ok()?.file_id))
318}
319
320fn hex(bytes: &[u8]) -> String {
321 bytes.iter().map(|byte| format!("{byte:02x}")).collect()
322}
323
324#[cfg(test)]
325mod tests {
326 use super::*;
327 use onestore::op::{Edit, Op, PageOp};
328
329 /// A section file in a scratch folder, its image, and the transaction one edit of it
330 /// publishes.
331 fn edited(folder: &Path) -> (PathBuf, Vec<u8>, Transaction) {
332 let file = folder.join("Notes.one");
333 let image = onestore::create_section("Notes.one", "Text", "Author").unwrap();
334 std::fs::write(&file, &image).unwrap();
335 let arena = onestore::Arena::default();
336 let mut section = onestore::Section::open(&arena, image.clone()).unwrap();
337 let space = section.pages().unwrap()[0].0;
338 let edit = Edit {
339 at: crate::now(),
340 ops: vec![Op::Page {
341 space,
342 op: PageOp::Color(Some(0x00ab_cdef)),
343 }],
344 };
345 section.apply("Author", &edit).unwrap();
346 (file, image, section.seal().unwrap().unwrap())
347 }
348
349 fn published(folder: &Path, file: &Path, transaction: &Transaction) -> Published {
350 let published = Published::of(&folder.join("replica.sqlite"), "iPhone");
351 transaction.commit_file(file).unwrap();
352 let mut opened = File::open(file).unwrap();
353 published
354 .keep(Written::InPlace(&mut opened), transaction)
355 .unwrap();
356 published
357 }
358
359 #[test]
360 fn nothing_kept_reads_no_stamp() {
361 let folder = tempfile::tempdir().unwrap();
362 let published = Published::of(&folder.path().join("replica.sqlite"), "iPhone");
363 let listed = published.versions(|| unreachable!("no stamp is read"), &[]);
364 assert!(listed.unwrap().is_empty());
365 }
366
367 /// The image this device published lists once the file stands otherwise: once reads have
368 /// had `LAG` to catch up where the file stands as the image it was published on, and not
369 /// where the provider keeps it as a conflict version itself.
370 #[test]
371 fn a_lost_image_lists_once() {
372 let folder = tempfile::tempdir().unwrap();
373 let (file, base, transaction) = edited(folder.path());
374 let published = published(folder.path(), &file, &transaction);
375 let listed =
376 |conflicts: &[Version]| published.versions(|| file_stamp(&file), conflicts).unwrap();
377 assert!(listed(&[]).is_empty());
378 let ours = std::fs::read(&file).unwrap();
379 std::fs::write(&file, base).unwrap();
380 assert!(listed(&[]).is_empty());
381 let aged = File::options()
382 .write(true)
383 .open(&published.kept()[0].path)
384 .unwrap();
385 aged.set_modified(SystemTime::now() - LAG - Duration::from_secs(1))
386 .unwrap();
387 assert_eq!(listed(&[]).len(), 1);
388 // A conflict version not here yet could be this image: it waits for it.
389 let missing = Version {
390 id: file
391 .with_file_name("Not here.one")
392 .to_string_lossy()
393 .into_owned(),
394 device: None,
395 };
396 assert!(listed(&[missing]).is_empty());
397 let version = file.with_file_name("Notes (iPhone).one");
398 std::fs::write(&version, ours).unwrap();
399 let conflict = Version {
400 id: version.to_string_lossy().into_owned(),
401 device: Some("iPhone".into()),
402 };
403 assert!(listed(std::slice::from_ref(&conflict)).is_empty());
404 published.retired(&version).unwrap();
405 assert!(published.kept().is_empty());
406 }
407
408 /// A confirmation rewrites the header of the image last published: the kept image follows
409 /// it, rather than listing as lost and going, which would leave it unprotected.
410 #[test]
411 fn a_confirmation_keeps_the_image_kept() {
412 let folder = tempfile::tempdir().unwrap();
413 let (file, _, transaction) = edited(folder.path());
414 let published = published(folder.path(), &file, &transaction);
415 let base = file_stamp(&file).unwrap();
416 onestore::confirm_file(&file, &base).unwrap();
417 assert_ne!(file_stamp(&file).unwrap(), base);
418 let mut opened = File::open(&file).unwrap();
419 published
420 .confirmed(Written::InPlace(&mut opened), &base)
421 .unwrap();
422 let kept = published.kept();
423 assert_eq!(kept.len(), 1);
424 assert_eq!(
425 std::fs::read(&kept[0].path).unwrap(),
426 std::fs::read(&file).unwrap()
427 );
428 let listed = published.versions(|| file_stamp(&file), &[]).unwrap();
429 assert!(listed.is_empty());
430 }
431}