| author | |
| committer | |
| log | 53d905dfd53180623af4d00f4461b218223117e5 |
| tree | 85e8f8470ecbd90f5eab7db5145ac0ce684b84a1 |
| parent | 650ea9ccbc132590b96c6c067d62f2dc753117b8 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
A Live Share host takes a batch sent again for one it already landed by finding the batch's revisions in the section, or, since OneNote's Optimize drops a revision a later one supersedes, in a record it held in memory. A host that restarted lost that record, so a guest sending again a batch whose answer it never heard, after the host had undone the batch and Optimize had dropped its revision, was refused for want of a base and replayed its queue on the section, bringing back what the host had undone. The host now keeps, with the share's devices in what it already saves to take the share up again, each device's last commit to each section: a batch's revisions, or a protected section's transaction by its stamp. A device's queue sends nothing more to a section until its last commit there is settled, so that one commit is all it may ever send again, and the record holds one per device and section, going with the device when it is removed. The host notes a commit once its guests have their replies and saves the share at most every 2 seconds, off the section's writer, and once more when the host is dropped; a share that stopped is not saved again, as its app forgets it.
The record misses a commit if the host dies within those 2 seconds, if its own commit ends without a clear answer, or if the section is renamed before the batch comes again; each needs Optimize to have dropped the batch's revision as well before a resend could bring undone text back.
A test lands a batch whose answer is lost, undoes it on the host as Optimize leaves it, restarts the host, and checks that the batch sent again stays undone, where before it came back; another checks that a share stopped just after a batch landed is not saved again.
Assisted-by: claude-opus-5.56 files changed, 361 insertions(+), 150 deletions(-)
arc/sync.md+6-4| ... | @@ -130,10 +130,12 @@ the first network byte. If the reply is lost, the attempt confirms only when | ... | @@ -130,10 +130,12 @@ the first network byte. If the reply is lost, the attempt confirms only when |
| 130 | the remote holds its revisions (or every page it changed, as it changed them). | 130 | the remote holds its revisions (or every page it changed, as it changed them). |
| 131 | Otherwise it waits as `AwaitingConfirmation`. A Live Share host lands a batch at | 131 | Otherwise it waits as `AwaitingConfirmation`. A Live Share host lands a batch at |
| 132 | most once, as it looks for the batch's revisions before applying it, and commits a | 132 | most once, as it looks for the batch's revisions before applying it, and commits a |
| 133 | protected section's sealed transaction only on the image it was made on, so the next | 133 | protected section's sealed transaction only on the image it was made on. OneNote's |
| 134 | step sends it the same batch again. Refused because the host no longer holds that | 134 | Optimize drops a revision a later one supersedes, so the host also keeps, with the |
| 135 | image, the batch has landed if the section holds its revisions, and else never will: | 135 | share, each device's last commit to each section, the only one it may send again. |
| 136 | the queue replays on the section as it is. Any other remote, where a late write could land | 136 | The next step therefore sends it the same batch again. Refused because the host no |
| 137 | longer holds that image, the batch has landed if the section holds its revisions, | ||
| 138 | and else never will: the queue replays on the section as it is. Any other remote, where a late write could land | ||
| 137 | a second copy, waits for `release`, which exports a recovery archive first, then | 139 | a second copy, waits for `release`, which exports a recovery archive first, then |
| 138 | republishes or abandons. | 140 | republishes or abandons. |
| 139 | 141 |
crates/notebook/src/live/share.rs+46-6| ... | @@ -14,7 +14,7 @@ use crate::{Error, Result, background::Reports, discover, session::Storage}; | ... | @@ -14,7 +14,7 @@ use crate::{Error, Result, background::Reports, discover, session::Storage}; |
| 14 | use onestore::{CommitError, CommitState, RevisionIndex, Stamp, Store, Transaction}; | 14 | use onestore::{CommitError, CommitState, RevisionIndex, Stamp, Store, Transaction}; |
| 15 | use sha2::{Digest, Sha256}; | 15 | use sha2::{Digest, Sha256}; |
| 16 | use std::{ | 16 | use std::{ |
| 17 | collections::{BTreeMap, HashMap, HashSet}, | 17 | collections::{BTreeMap, HashMap}, |
| 18 | io, | 18 | io, |
| 19 | path::PathBuf, | 19 | path::PathBuf, |
| 20 | sync::{ | 20 | sync::{ |
| ... | @@ -98,6 +98,18 @@ pub struct Device { | ... | @@ -98,6 +98,18 @@ pub struct Device { |
| 98 | pub secret: [u8; 16], | 98 | pub secret: [u8; 16], |
| 99 | pub name: String, | 99 | pub name: String, |
| 100 | pub device: Option<String>, | 100 | pub device: Option<String>, |
| 101 | /// The device's last commit to each section, by path, the only one it may send again. | ||
| 102 | #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] | ||
| 103 | pub landed: BTreeMap<String, Landed>, | ||
| 104 | } | ||
| 105 | |||
| 106 | /// A commit a device landed, as it would send it again: a batch's revisions, which OneNote's | ||
| 107 | /// Optimize may drop from the section once later ones supersede them, or a whole transaction's | ||
| 108 | /// stamp after it, as `digest` hashes it. | ||
| 109 | #[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] | ||
| 110 | pub enum Landed { | ||
| 111 | Batch(BTreeMap<onestore::ExGuid, onestore::ExGuid>), | ||
| 112 | Commit(u64), | ||
| 101 | } | 113 | } |
| 102 | 114 | ||
| 103 | impl Sharing { | 115 | impl Sharing { |
| ... | @@ -294,9 +306,8 @@ struct Served { | ... | @@ -294,9 +306,8 @@ struct Served { |
| 294 | puts: Mutex<ByGuest<Vec<u8>>>, | 306 | puts: Mutex<ByGuest<Vec<u8>>>, |
| 295 | guests: Mutex<BTreeMap<[u8; 16], Admitted>>, | 307 | guests: Mutex<BTreeMap<[u8; 16], Admitted>>, |
| 296 | writers: Mutex<HashMap<String, mpsc::SyncSender<batch::Waiting>>>, | 308 | writers: Mutex<HashMap<String, mpsc::SyncSender<batch::Waiting>>>, |
| 297 | /// The revisions guests' batches landed as, by page: OneNote's Optimize drops one a | 309 | /// The share, which keeps what each device last landed. |
| 298 | /// later revision supersedes, and a batch sent again must still find it. | 310 | members: Weak<membership::Members>, |
| 299 | landed: Mutex<HashSet<(onestore::ExGuid, onestore::ExGuid)>>, | ||
| 300 | /// Hears the paths guests changed, as the host's own notebook should. | 311 | /// Hears the paths guests changed, as the host's own notebook should. |
| 301 | host: Mutex<Option<crate::session::Listener>>, | 312 | host: Mutex<Option<crate::session::Listener>>, |
| 302 | /// The share's room, to tell guests what changed. | 313 | /// The share's room, to tell guests what changed. |
| ... | @@ -308,6 +319,8 @@ struct Served { | ... | @@ -308,6 +319,8 @@ struct Served { |
| 308 | /// their images. | 319 | /// their images. |
| 309 | struct Admitted { | 320 | struct Admitted { |
| 310 | hello: Arc<Hello>, | 321 | hello: Arc<Hello>, |
| 322 | /// The device's access secret. | ||
| 323 | device: [u8; 16], | ||
| 311 | line: Line, | 324 | line: Line, |
| 312 | queue: mpsc::SyncSender<(u16, Vec<u8>)>, | 325 | queue: mpsc::SyncSender<(u16, Vec<u8>)>, |
| 313 | starts: f64, | 326 | starts: f64, |
| ... | @@ -339,11 +352,13 @@ fn refused(kind: io::ErrorKind, message: &str) -> Error { | ... | @@ -339,11 +352,13 @@ fn refused(kind: io::ErrorKind, message: &str) -> Error { |
| 339 | } | 352 | } |
| 340 | 353 | ||
| 341 | impl Served { | 354 | impl Served { |
| 342 | /// Serves `peer` on `line`, through workers of its own that end as it leaves. | 355 | /// Serves `peer`, of device `device`, on `line`, through workers of its own that end as |
| 356 | /// it leaves. | ||
| 343 | fn admit( | 357 | fn admit( |
| 344 | self: &Arc<Self>, | 358 | self: &Arc<Self>, |
| 345 | source: &Arc<super::Shared>, | 359 | source: &Arc<super::Shared>, |
| 346 | hello: &Arc<Hello>, | 360 | hello: &Arc<Hello>, |
| 361 | device: [u8; 16], | ||
| 347 | line: &Line, | 362 | line: &Line, |
| 348 | ) -> bool { | 363 | ) -> bool { |
| 349 | let peer = hello.peer; | 364 | let peer = hello.peer; |
| ... | @@ -351,6 +366,7 @@ impl Served { | ... | @@ -351,6 +366,7 @@ impl Served { |
| 351 | let waiting = Arc::new(Mutex::new(waiting)); | 366 | let waiting = Arc::new(Mutex::new(waiting)); |
| 352 | let guest = Admitted { | 367 | let guest = Admitted { |
| 353 | hello: Arc::clone(hello), | 368 | hello: Arc::clone(hello), |
| 369 | device, | ||
| 354 | line: line.clone(), | 370 | line: line.clone(), |
| 355 | queue, | 371 | queue, |
| 356 | starts: BURST, | 372 | starts: BURST, |
| ... | @@ -433,6 +449,25 @@ impl Served { | ... | @@ -433,6 +449,25 @@ impl Served { |
| 433 | drop(retired); | 449 | drop(retired); |
| 434 | } | 450 | } |
| 435 | 451 | ||
| 452 | /// What the device of guest `peer` last landed on the section at `path`. | ||
| 453 | fn landed(&self, peer: &[u8; 16], path: &str) -> Option<Landed> { | ||
| 454 | let device = self.guests.lock().unwrap().get(peer)?.device; | ||
| 455 | self.members.upgrade()?.landed(&device, path) | ||
| 456 | } | ||
| 457 | |||
| 458 | /// Notes `landed` as what the device of guest `peer` last landed on the section at `path`. | ||
| 459 | fn remember(&self, peer: &[u8; 16], path: &str, landed: Landed) { | ||
| 460 | let device = self | ||
| 461 | .guests | ||
| 462 | .lock() | ||
| 463 | .unwrap() | ||
| 464 | .get(peer) | ||
| 465 | .map(|guest| guest.device); | ||
| 466 | if let (Some(device), Some(members)) = (device, self.members.upgrade()) { | ||
| 467 | members.remember(&device, path, landed); | ||
| 468 | } | ||
| 469 | } | ||
| 470 | |||
| 436 | /// Tells every guest, and the host, that a guest changed the files at `paths`. | 471 | /// Tells every guest, and the host, that a guest changed the files at `paths`. |
| 437 | fn changed(&self, paths: &[String]) { | 472 | fn changed(&self, paths: &[String]) { |
| 438 | self.tell(paths); | 473 | self.tell(paths); |
| ... | @@ -775,7 +810,11 @@ impl Served { | ... | @@ -775,7 +810,11 @@ impl Served { |
| 775 | Error::Io(error) => not_committed(error), | 810 | Error::Io(error) => not_committed(error), |
| 776 | error => error, | 811 | error => error, |
| 777 | })?; | 812 | })?; |
| 813 | let landed = Landed::Commit(digest(&transaction.stamp())); | ||
| 778 | if Stamp::of(&image).ok().as_ref() != Some(transaction.base()) { | 814 | if Stamp::of(&image).ok().as_ref() != Some(transaction.base()) { |
| 815 | if self.landed(guest, path).as_ref() == Some(&landed) { | ||
| 816 | return Ok(()); | ||
| 817 | } | ||
| 779 | return Err(not_committed(io::Error::new( | 818 | return Err(not_committed(io::Error::new( |
| 780 | io::ErrorKind::StaleNetworkFileHandle, | 819 | io::ErrorKind::StaleNetworkFileHandle, |
| 781 | "The section changed since", | 820 | "The section changed since", |
| ... | @@ -793,6 +832,7 @@ impl Served { | ... | @@ -793,6 +832,7 @@ impl Served { |
| 793 | ))); | 832 | ))); |
| 794 | } | 833 | } |
| 795 | self.storage.commit(path, transaction)?; | 834 | self.storage.commit(path, transaction)?; |
| 835 | self.remember(guest, path, landed); | ||
| 796 | self.tell_delta(path, &image, &next, Some(guest)); | 836 | self.tell_delta(path, &image, &next, Some(guest)); |
| 797 | self.keep(path, Arc::new(next)); | 837 | self.keep(path, Arc::new(next)); |
| 798 | Ok(()) | 838 | Ok(()) |
| ... | @@ -2165,7 +2205,7 @@ mod tests { | ... | @@ -2165,7 +2205,7 @@ mod tests { |
| 2165 | puts: Mutex::default(), | 2205 | puts: Mutex::default(), |
| 2166 | guests: Mutex::default(), | 2206 | guests: Mutex::default(), |
| 2167 | writers: Mutex::default(), | 2207 | writers: Mutex::default(), |
| 2168 | landed: Mutex::default(), | 2208 | members: Weak::new(), |
| 2169 | host: Mutex::default(), | 2209 | host: Mutex::default(), |
| 2170 | room: Mutex::default(), | 2210 | room: Mutex::default(), |
| 2171 | }); | 2211 | }); |
crates/notebook/src/live/share/batch.rs+13-132| ... | @@ -132,7 +132,7 @@ impl Served { | ... | @@ -132,7 +132,7 @@ impl Served { |
| 132 | match extended { | 132 | match extended { |
| 133 | Ok(()) => { | 133 | Ok(()) => { |
| 134 | image = Arc::new(next); | 134 | image = Arc::new(next); |
| 135 | landing.extend(revisions); | 135 | landing.push((batch.peer, revisions)); |
| 136 | accepted.push(batch); | 136 | accepted.push(batch); |
| 137 | } | 137 | } |
| 138 | Err(error) => { | 138 | Err(error) => { |
| ... | @@ -168,7 +168,6 @@ impl Served { | ... | @@ -168,7 +168,6 @@ impl Served { |
| 168 | }), | 168 | }), |
| 169 | }; | 169 | }; |
| 170 | if result.is_ok() { | 170 | if result.is_ok() { |
| 171 | self.landed.lock().unwrap().extend(landing); | ||
| 172 | self.tell_delta(path, &before, &image, None); | 171 | self.tell_delta(path, &before, &image, None); |
| 173 | self.keep(path, Arc::clone(&image)); | 172 | self.keep(path, Arc::clone(&image)); |
| 174 | self.changed(&[path.to_owned()]); | 173 | self.changed(&[path.to_owned()]); |
| ... | @@ -190,6 +189,11 @@ impl Served { | ... | @@ -190,6 +189,11 @@ impl Served { |
| 190 | }; | 189 | }; |
| 191 | let _ = batch.reply.send(reply); | 190 | let _ = batch.reply.send(reply); |
| 192 | } | 191 | } |
| 192 | if result.is_ok() { | ||
| 193 | for (peer, revisions) in landing { | ||
| 194 | self.remember(&peer, path, Landed::Batch(revisions)); | ||
| 195 | } | ||
| 196 | } | ||
| 193 | } | 197 | } |
| 194 | 198 | ||
| 195 | fn prepare_batch( | 199 | fn prepare_batch( |
| ... | @@ -222,15 +226,17 @@ impl Served { | ... | @@ -222,15 +226,17 @@ impl Served { |
| 222 | } | 226 | } |
| 223 | let store = Store::parse(image)?; | 227 | let store = Store::parse(image)?; |
| 224 | let index = RevisionIndex::parse(&store)?; | 228 | let index = RevisionIndex::parse(&store)?; |
| 225 | let landed = self.landed.lock().unwrap(); | 229 | let landed = match self.landed(&batch.peer, path) { |
| 230 | Some(Landed::Batch(landed)) => landed, | ||
| 231 | _ => BTreeMap::new(), | ||
| 232 | }; | ||
| 226 | if edits.revisions.iter().all(|(space, revision)| { | 233 | if edits.revisions.iter().all(|(space, revision)| { |
| 227 | landed.contains(&(*space, *revision)) | 234 | landed.get(space) == Some(revision) |
| 228 | || (index.spaces.get(space)) | 235 | || (index.spaces.get(space)) |
| 229 | .is_some_and(|space| space.revisions.contains_key(revision)) | 236 | .is_some_and(|space| space.revisions.contains_key(revision)) |
| 230 | }) { | 237 | }) { |
| 231 | return Ok((None, edits.revisions)); | 238 | return Ok((None, edits.revisions)); |
| 232 | } | 239 | } |
| 233 | drop(landed); | ||
| 234 | let names = edits | 240 | let names = edits |
| 235 | .revisions | 241 | .revisions |
| 236 | .iter() | 242 | .iter() |
| ... | @@ -345,7 +351,7 @@ mod tests { | ... | @@ -345,7 +351,7 @@ mod tests { |
| 345 | puts: Mutex::default(), | 351 | puts: Mutex::default(), |
| 346 | guests: Mutex::default(), | 352 | guests: Mutex::default(), |
| 347 | writers: Mutex::default(), | 353 | writers: Mutex::default(), |
| 348 | landed: Mutex::default(), | 354 | members: Weak::new(), |
| 349 | host: Mutex::default(), | 355 | host: Mutex::default(), |
| 350 | room: Mutex::default(), | 356 | room: Mutex::default(), |
| 351 | }); | 357 | }); |
| ... | @@ -359,6 +365,7 @@ mod tests { | ... | @@ -359,6 +365,7 @@ mod tests { |
| 359 | peer, | 365 | peer, |
| 360 | ..Hello::new("Guest".into(), None).unwrap() | 366 | ..Hello::new("Guest".into(), None).unwrap() |
| 361 | }), | 367 | }), |
| 368 | device: [0; 16], | ||
| 362 | line: crate::live::Line(line), | 369 | line: crate::live::Line(line), |
| 363 | queue, | 370 | queue, |
| 364 | starts: BURST, | 371 | starts: BURST, |
| ... | @@ -474,132 +481,6 @@ mod tests { | ... | @@ -474,132 +481,6 @@ mod tests { |
| 474 | assert_eq!(served.writers.lock().unwrap().len(), 1); | 481 | assert_eq!(served.writers.lock().unwrap().len(), 1); |
| 475 | } | 482 | } |
| 476 | 483 | ||
| 477 | /// OneNote's Optimize drops a revision a later one superseded, as the page-versions | ||
| 478 | /// corpus shows; a batch sent again after its revision went that way is not applied | ||
| 479 | /// again. | ||
| 480 | #[cfg(not(target_arch = "wasm32"))] | ||
| 481 | #[test] | ||
| 482 | fn a_batch_whose_revision_optimize_dropped_lands_once() { | ||
| 483 | let directory = tempfile::tempdir().unwrap(); | ||
| 484 | let folder = directory.path().join("notebook"); | ||
| 485 | std::fs::create_dir(&folder).unwrap(); | ||
| 486 | let path = "Garden.one"; | ||
| 487 | let base = onestore::create_section(path, "Original text", "Fixture").unwrap(); | ||
| 488 | std::fs::write(folder.join(path), &base).unwrap(); | ||
| 489 | let served = Arc::new(Served { | ||
| 490 | storage: crate::session::Notebook::open(&folder, directory.path().join("cache")) | ||
| 491 | .unwrap() | ||
| 492 | .into_storage(), | ||
| 493 | images: Mutex::default(), | ||
| 494 | snapshots: Mutex::default(), | ||
| 495 | puts: Mutex::default(), | ||
| 496 | guests: Mutex::default(), | ||
| 497 | writers: Mutex::default(), | ||
| 498 | landed: Mutex::default(), | ||
| 499 | host: Mutex::default(), | ||
| 500 | room: Mutex::default(), | ||
| 501 | }); | ||
| 502 | let peer = [1; 16]; | ||
| 503 | let (line, _outbound) = mpsc::channel(); | ||
| 504 | let (queue, _requests) = mpsc::sync_channel(1); | ||
| 505 | served.guests.lock().unwrap().insert( | ||
| 506 | peer, | ||
| 507 | Admitted { | ||
| 508 | hello: Arc::new(Hello { | ||
| 509 | peer, | ||
| 510 | ..Hello::new("Guest".into(), None).unwrap() | ||
| 511 | }), | ||
| 512 | line: crate::live::Line(line), | ||
| 513 | queue, | ||
| 514 | starts: BURST, | ||
| 515 | counted: Instant::now(), | ||
| 516 | held: Vec::new(), | ||
| 517 | }, | ||
| 518 | ); | ||
| 519 | let store = Store::parse(&base).unwrap(); | ||
| 520 | let index = RevisionIndex::parse(&store).unwrap(); | ||
| 521 | let document = onestore::document::Document::parse(&index).unwrap(); | ||
| 522 | let (space, text) = document | ||
| 523 | .spaces | ||
| 524 | .iter() | ||
| 525 | .find_map(|(space, object)| { | ||
| 526 | object.revisions[&object.contexts[&ExGuid::default()]] | ||
| 527 | .nodes | ||
| 528 | .iter() | ||
| 529 | .find_map(|(id, node)| { | ||
| 530 | matches!(node.kind, onestore::document::Kind::RichText { .. }) | ||
| 531 | .then_some((*space, *id)) | ||
| 532 | }) | ||
| 533 | }) | ||
| 534 | .unwrap(); | ||
| 535 | let typed = |range: std::ops::Range<u32>, with: &str| Edit { | ||
| 536 | at: 134_000_000_000_000_000, | ||
| 537 | ops: vec![Op::Page { | ||
| 538 | space, | ||
| 539 | op: PageOp::Text { | ||
| 540 | text, | ||
| 541 | range, | ||
| 542 | with: with.into(), | ||
| 543 | }, | ||
| 544 | }], | ||
| 545 | }; | ||
| 546 | let arena = Arena::default(); | ||
| 547 | let mut section = Section::open(&arena, base.clone()).unwrap(); | ||
| 548 | section.apply("Guest", &typed(13..13, "X")).unwrap(); | ||
| 549 | let transaction = section.seal().unwrap().unwrap(); | ||
| 550 | let request = Request { | ||
| 551 | path: path.into(), | ||
| 552 | stamp: Some(transaction.base().into()), | ||
| 553 | bytes: Some( | ||
| 554 | encode( | ||
| 555 | &Edits { | ||
| 556 | edits: vec![("Guest".into(), typed(13..13, "X"))], | ||
| 557 | revisions: section.newest().filter(|(at, _)| *at == space).collect(), | ||
| 558 | }, | ||
| 559 | LIMIT, | ||
| 560 | ) | ||
| 561 | .unwrap(), | ||
| 562 | ), | ||
| 563 | ..Request::default() | ||
| 564 | }; | ||
| 565 | assert!( | ||
| 566 | served | ||
| 567 | .batch(&peer, request.clone()) | ||
| 568 | .unwrap() | ||
| 569 | .failure | ||
| 570 | .is_none() | ||
| 571 | ); | ||
| 572 | |||
| 573 | // The host deletes what the batch typed, and Optimize keeps the page's newest | ||
| 574 | // revision alone: the file holds neither the text nor the batch's revision. | ||
| 575 | let arena = Arena::default(); | ||
| 576 | let mut optimized = Section::open(&arena, base.clone()).unwrap(); | ||
| 577 | optimized.apply("Guest", &typed(13..13, "X")).unwrap(); | ||
| 578 | optimized.apply("Host", &typed(13..14, "")).unwrap(); | ||
| 579 | let mut image = base; | ||
| 580 | optimized | ||
| 581 | .seal() | ||
| 582 | .unwrap() | ||
| 583 | .unwrap() | ||
| 584 | .apply(&mut image) | ||
| 585 | .unwrap(); | ||
| 586 | std::fs::write(folder.join(path), &image).unwrap(); | ||
| 587 | let replayed = served.batch(&peer, request).unwrap(); | ||
| 588 | assert!(replayed.failure.is_none(), "{:?}", replayed.failure); | ||
| 589 | let stored = std::fs::read(folder.join(path)).unwrap(); | ||
| 590 | let arena = Arena::default(); | ||
| 591 | let section = Section::open(&arena, stored).unwrap(); | ||
| 592 | let texts: Vec<String> = (section.page(space).unwrap().objects.iter()) | ||
| 593 | .filter_map(|object| match object { | ||
| 594 | PageObject::Outline(outline) => Some(outline.paragraphs.iter()), | ||
| 595 | _ => None, | ||
| 596 | }) | ||
| 597 | .flatten() | ||
| 598 | .filter_map(|paragraph| Some(paragraph.text()?.text.text().to_owned())) | ||
| 599 | .collect(); | ||
| 600 | assert_eq!(texts, ["Original text"]); | ||
| 601 | } | ||
| 602 | |||
| 603 | #[test] | 484 | #[test] |
| 604 | fn attachment_encoding_stops_at_the_upload_budget() { | 485 | fn attachment_encoding_stops_at_the_upload_budget() { |
| 605 | let id = ExGuid { | 486 | let id = ExGuid { |
crates/notebook/src/live/share/handover_tests.rs+4-3| ... | @@ -437,7 +437,7 @@ fn served() -> (tempfile::TempDir, Arc<Served>) { | ... | @@ -437,7 +437,7 @@ fn served() -> (tempfile::TempDir, Arc<Served>) { |
| 437 | puts: Mutex::default(), | 437 | puts: Mutex::default(), |
| 438 | guests: Mutex::default(), | 438 | guests: Mutex::default(), |
| 439 | writers: Mutex::default(), | 439 | writers: Mutex::default(), |
| 440 | landed: Mutex::default(), | 440 | members: Weak::new(), |
| 441 | host: Mutex::default(), | 441 | host: Mutex::default(), |
| 442 | room: Mutex::default(), | 442 | room: Mutex::default(), |
| 443 | }); | 443 | }); |
| ... | @@ -450,7 +450,7 @@ fn handover_copied_host_request_cannot_enter_promoted_queue() { | ... | @@ -450,7 +450,7 @@ fn handover_copied_host_request_cannot_enter_promoted_queue() { |
| 450 | let mut fixture = Handover::new(None); | 450 | let mut fixture = Handover::new(None); |
| 451 | let serving = Arc::clone(&served); | 451 | let serving = Arc::clone(&served); |
| 452 | let events = fixture.intercept(move |source, event| match event { | 452 | let events = fixture.intercept(move |source, event| match event { |
| 453 | Event::Met(hello, line) => assert!(serving.admit(source, hello, line)), | 453 | Event::Met(hello, line) => assert!(serving.admit(source, hello, [0; 16], line)), |
| 454 | Event::Frame { from, kind, body } => serving.queue(source, from, *kind, body), | 454 | Event::Frame { from, kind, body } => serving.queue(source, from, *kind, body), |
| 455 | Event::Left(hello) => serving.forget(&hello.peer, Some((source, hello))), | 455 | Event::Left(hello) => serving.forget(&hello.peer, Some((source, hello))), |
| 456 | _ => {} | 456 | _ => {} |
| ... | @@ -523,7 +523,7 @@ fn handover_old_left_preserves_promoted_host_uploads_and_snapshots() { | ... | @@ -523,7 +523,7 @@ fn handover_old_left_preserves_promoted_host_uploads_and_snapshots() { |
| 523 | let gate = Mutex::new(Some(gate)); | 523 | let gate = Mutex::new(Some(gate)); |
| 524 | let serving = Arc::clone(&served); | 524 | let serving = Arc::clone(&served); |
| 525 | let events = fixture.intercept(move |source, event| match event { | 525 | let events = fixture.intercept(move |source, event| match event { |
| 526 | Event::Met(hello, line) => assert!(serving.admit(source, hello, line)), | 526 | Event::Met(hello, line) => assert!(serving.admit(source, hello, [0; 16], line)), |
| 527 | Event::Left(hello) => { | 527 | Event::Left(hello) => { |
| 528 | if let Some(gate) = gate.lock().unwrap().take() { | 528 | if let Some(gate) = gate.lock().unwrap().take() { |
| 529 | gate.wait().unwrap(); | 529 | gate.wait().unwrap(); |
| ... | @@ -802,6 +802,7 @@ fn handover_retired_membership_rejects_deferred_access_met() { | ... | @@ -802,6 +802,7 @@ fn handover_retired_membership_rejects_deferred_access_met() { |
| 802 | secret, | 802 | secret, |
| 803 | name: "Retired device".into(), | 803 | name: "Retired device".into(), |
| 804 | device: None, | 804 | device: None, |
| 805 | landed: BTreeMap::new(), | ||
| 805 | }], | 806 | }], |
| 806 | ..Sharing::new("").unwrap() | 807 | ..Sharing::new("").unwrap() |
| 807 | }; | 808 | }; |
crates/notebook/src/live/share/membership.rs+62-4| ... | @@ -6,7 +6,7 @@ pub struct Host { | ... | @@ -6,7 +6,7 @@ pub struct Host { |
| 6 | } | 6 | } |
| 7 | 7 | ||
| 8 | #[allow(clippy::type_complexity)] | 8 | #[allow(clippy::type_complexity)] |
| 9 | struct Members { | 9 | pub(super) struct Members { |
| 10 | me: Hello, | 10 | me: Hello, |
| 11 | notebook: String, | 11 | notebook: String, |
| 12 | reach: Option<Reach>, | 12 | reach: Option<Reach>, |
| ... | @@ -19,6 +19,17 @@ struct Members { | ... | @@ -19,6 +19,17 @@ struct Members { |
| 19 | served: Arc<Served>, | 19 | served: Arc<Served>, |
| 20 | events: Arc<dyn Fn() + Send + Sync>, | 20 | events: Arc<dyn Fn() + Send + Sync>, |
| 21 | keep: Box<dyn Fn(&Sharing) -> io::Result<()> + Send + Sync>, | 21 | keep: Box<dyn Fn(&Sharing) -> io::Result<()> + Send + Sync>, |
| 22 | /// Commits noted in `sharing` that it does not keep yet. | ||
| 23 | unkept: AtomicBool, | ||
| 24 | } | ||
| 25 | |||
| 26 | /// How long a commit noted waits to be kept, so that a burst of them is kept once. | ||
| 27 | const KEEP_LANDED: Duration = Duration::from_secs(2); | ||
| 28 | |||
| 29 | impl Drop for Members { | ||
| 30 | fn drop(&mut self) { | ||
| 31 | self.keep_landed(); | ||
| 32 | } | ||
| 22 | } | 33 | } |
| 23 | 34 | ||
| 24 | impl Host { | 35 | impl Host { |
| ... | @@ -35,7 +46,7 @@ impl Host { | ... | @@ -35,7 +46,7 @@ impl Host { |
| 35 | keep: impl Fn(&Sharing) -> io::Result<()> + Send + Sync + 'static, | 46 | keep: impl Fn(&Sharing) -> io::Result<()> + Send + Sync + 'static, |
| 36 | ) -> io::Result<Self> { | 47 | ) -> io::Result<Self> { |
| 37 | keep(&sharing)?; | 48 | keep(&sharing)?; |
| 38 | let members = Arc::new(Members { | 49 | let members = Arc::new_cyclic(|members| Members { |
| 39 | me, | 50 | me, |
| 40 | notebook: notebook.to_owned(), | 51 | notebook: notebook.to_owned(), |
| 41 | reach, | 52 | reach, |
| ... | @@ -52,12 +63,13 @@ impl Host { | ... | @@ -52,12 +63,13 @@ impl Host { |
| 52 | puts: Mutex::default(), | 63 | puts: Mutex::default(), |
| 53 | guests: Mutex::default(), | 64 | guests: Mutex::default(), |
| 54 | writers: Mutex::default(), | 65 | writers: Mutex::default(), |
| 55 | landed: Mutex::default(), | 66 | members: members.clone(), |
| 56 | host: Mutex::default(), | 67 | host: Mutex::default(), |
| 57 | room: Mutex::default(), | 68 | room: Mutex::default(), |
| 58 | }), | 69 | }), |
| 59 | events: Arc::new(events), | 70 | events: Arc::new(events), |
| 60 | keep: Box::new(keep), | 71 | keep: Box::new(keep), |
| 72 | unkept: AtomicBool::new(false), | ||
| 61 | }); | 73 | }); |
| 62 | let sharing = members.sharing.lock().unwrap().clone(); | 74 | let sharing = members.sharing.lock().unwrap().clone(); |
| 63 | members.presence(sharing.secret)?; | 75 | members.presence(sharing.secret)?; |
| ... | @@ -271,6 +283,51 @@ impl Host { | ... | @@ -271,6 +283,51 @@ impl Host { |
| 271 | } | 283 | } |
| 272 | 284 | ||
| 273 | impl Members { | 285 | impl Members { |
| 286 | /// What device `secret` last landed on the section at `path`. | ||
| 287 | pub(super) fn landed(&self, secret: &[u8; 16], path: &str) -> Option<Landed> { | ||
| 288 | let sharing = self.sharing.lock().unwrap(); | ||
| 289 | let device = sharing | ||
| 290 | .members | ||
| 291 | .iter() | ||
| 292 | .find(|device| device.secret == *secret)?; | ||
| 293 | device.landed.get(path).cloned() | ||
| 294 | } | ||
| 295 | |||
| 296 | /// Notes `landed` as what device `secret` last landed on the section at `path`, in place | ||
| 297 | /// of the commit before, which the device has since heard of, and keeps the share soon. | ||
| 298 | pub(super) fn remember(self: &Arc<Self>, secret: &[u8; 16], path: &str, landed: Landed) { | ||
| 299 | let mut sharing = self.sharing.lock().unwrap(); | ||
| 300 | let Some(device) = (sharing.members.iter_mut()).find(|device| device.secret == *secret) | ||
| 301 | else { | ||
| 302 | return; | ||
| 303 | }; | ||
| 304 | device.landed.insert(path.to_owned(), landed); | ||
| 305 | drop(sharing); | ||
| 306 | if self.unkept.swap(true, Ordering::AcqRel) { | ||
| 307 | return; | ||
| 308 | } | ||
| 309 | let members = Arc::downgrade(self); | ||
| 310 | thread::spawn(move || { | ||
| 311 | thread::sleep(KEEP_LANDED); | ||
| 312 | if let Some(members) = members.upgrade() { | ||
| 313 | members.keep_landed(); | ||
| 314 | } | ||
| 315 | }); | ||
| 316 | } | ||
| 317 | |||
| 318 | /// Keeps the share where commits it noted since are not kept yet, unless it stopped, | ||
| 319 | /// whose host keeps it no more. | ||
| 320 | fn keep_landed(&self) { | ||
| 321 | let _change = self.changes.lock().unwrap(); | ||
| 322 | let sharing = self.sharing.lock().unwrap(); | ||
| 323 | if self.unkept.swap(false, Ordering::AcqRel) | ||
| 324 | && self.room.lock().unwrap().is_some() | ||
| 325 | && let Err(error) = (self.keep)(&sharing) | ||
| 326 | { | ||
| 327 | eprintln!("Live Share: could not keep guests' commits: {error}"); | ||
| 328 | } | ||
| 329 | } | ||
| 330 | |||
| 274 | fn welcome(&self, sharing: &Sharing, secret: [u8; 16]) -> Welcome { | 331 | fn welcome(&self, sharing: &Sharing, secret: [u8; 16]) -> Welcome { |
| 275 | Welcome { | 332 | Welcome { |
| 276 | share: sharing.share, | 333 | share: sharing.share, |
| ... | @@ -316,7 +373,7 @@ impl Members { | ... | @@ -316,7 +373,7 @@ impl Members { |
| 316 | if sharing.members.iter().any(|device| device.secret == secret) { | 373 | if sharing.members.iter().any(|device| device.secret == secret) { |
| 317 | Ok(members | 374 | Ok(members |
| 318 | .served | 375 | .served |
| 319 | .admit(source, hello, line) | 376 | .admit(source, hello, secret, line) |
| 320 | .then(|| members.welcome(&sharing, secret))) | 377 | .then(|| members.welcome(&sharing, secret))) |
| 321 | } else { | 378 | } else { |
| 322 | Err(()) | 379 | Err(()) |
| ... | @@ -366,6 +423,7 @@ impl Members { | ... | @@ -366,6 +423,7 @@ impl Members { |
| 366 | secret, | 423 | secret, |
| 367 | name: hello.name.clone(), | 424 | name: hello.name.clone(), |
| 368 | device: hello.device.clone(), | 425 | device: hello.device.clone(), |
| 426 | landed: BTreeMap::new(), | ||
| 369 | }); | 427 | }); |
| 370 | (self.keep)(&next)?; | 428 | (self.keep)(&next)?; |
| 371 | *sharing = next; | 429 | *sharing = next; |
crates/notebook/tests/live_batch.rs+230-1| ... | @@ -383,7 +383,12 @@ fn edit_receipts_survive_guest_and_host_process_death() { | ... | @@ -383,7 +383,12 @@ fn edit_receipts_survive_guest_and_host_process_death() { |
| 383 | serde_json::from_slice(&std::fs::read(root.join("sharing.json")).unwrap()).unwrap(); | 383 | serde_json::from_slice(&std::fs::read(root.join("sharing.json")).unwrap()).unwrap(); |
| 384 | assert_eq!(restarted.share, sharing.share); | 384 | assert_eq!(restarted.share, sharing.share); |
| 385 | assert_eq!(restarted.secret, sharing.secret); | 385 | assert_eq!(restarted.secret, sharing.secret); |
| 386 | assert_eq!(restarted.members, sharing.members); | 386 | let devices = |sharing: &Sharing| { |
| 387 | (sharing.members.iter()) | ||
| 388 | .map(|device| (device.secret, device.name.clone())) | ||
| 389 | .collect::<Vec<_>>() | ||
| 390 | }; | ||
| 391 | assert_eq!(devices(&restarted), devices(&sharing)); | ||
| 387 | assert_eq!(std::fs::read(root.join("guest.json")).unwrap(), credentials); | 392 | assert_eq!(std::fs::read(root.join("guest.json")).unwrap(), credentials); |
| 388 | let replica = Replica::open(&database).unwrap(); | 393 | let replica = Replica::open(&database).unwrap(); |
| 389 | assert!(matches!( | 394 | assert!(matches!( |
| ... | @@ -921,3 +926,227 @@ fn a_protected_section_s_uncertain_commit_lands_once() { | ... | @@ -921,3 +926,227 @@ fn a_protected_section_s_uncertain_commit_lands_once() { |
| 921 | }); | 926 | }); |
| 922 | assert_eq!(stored(), ["BASealed text"]); | 927 | assert_eq!(stored(), ["BASealed text"]); |
| 923 | } | 928 | } |
| 929 | |||
| 930 | /// A batch whose answer was lost after it landed, which the host then undid and OneNote's | ||
| 931 | /// Optimize dropped the revision of, is still known as landed by a host that restarted | ||
| 932 | /// before the guest sent it again: what the host undid stays undone. | ||
| 933 | #[test] | ||
| 934 | fn a_restarted_host_knows_a_landed_batch_optimize_dropped() { | ||
| 935 | use notebook::{ | ||
| 936 | EditStatus, PendingEdit, Remote, | ||
| 937 | live::share::{Guest, Host}, | ||
| 938 | session::Notebook, | ||
| 939 | }; | ||
| 940 | use onestore::{CommitError, CommitState, ExGuid, Stamp, Transaction}; | ||
| 941 | use std::{collections::BTreeMap, sync::Mutex}; | ||
| 942 | |||
| 943 | /// A remote whose batches land but whose answers are lost. | ||
| 944 | struct Unanswered(HostedRemote); | ||
| 945 | impl Remote for Unanswered { | ||
| 946 | fn read(&mut self) -> std::io::Result<Vec<u8>> { | ||
| 947 | self.0.read() | ||
| 948 | } | ||
| 949 | fn stamp(&mut self) -> std::io::Result<Stamp> { | ||
| 950 | self.0.stamp() | ||
| 951 | } | ||
| 952 | fn publish(&mut self, transaction: &Transaction) -> Result<(), CommitError> { | ||
| 953 | self.0.publish(transaction) | ||
| 954 | } | ||
| 955 | fn confirm(&mut self, stamp: &Stamp) -> Result<(), CommitError> { | ||
| 956 | self.0.confirm(stamp) | ||
| 957 | } | ||
| 958 | fn accepts_edits(&self) -> bool { | ||
| 959 | self.0.accepts_edits() | ||
| 960 | } | ||
| 961 | fn publish_edits( | ||
| 962 | &mut self, | ||
| 963 | transaction: &Transaction, | ||
| 964 | edits: &[PendingEdit], | ||
| 965 | revisions: &BTreeMap<ExGuid, ExGuid>, | ||
| 966 | ) -> Result<(), CommitError> { | ||
| 967 | self.0.publish_edits(transaction, edits, revisions)?; | ||
| 968 | Err(CommitError { | ||
| 969 | state: CommitState::Unknown, | ||
| 970 | error: std::io::ErrorKind::BrokenPipe.into(), | ||
| 971 | }) | ||
| 972 | } | ||
| 973 | } | ||
| 974 | |||
| 975 | let directory = tempfile::tempdir().unwrap(); | ||
| 976 | let folder = notebook(directory.path()); | ||
| 977 | let url = relay(Default::default()); | ||
| 978 | let kept = Arc::new(Mutex::new(Sharing::new("").unwrap())); | ||
| 979 | let start = |sharing: Sharing| { | ||
| 980 | let kept = Arc::clone(&kept); | ||
| 981 | Host::start( | ||
| 982 | Notebook::open(&folder, directory.path().join("host")) | ||
| 983 | .unwrap() | ||
| 984 | .into_storage(), | ||
| 985 | hello("Ada"), | ||
| 986 | sharing, | ||
| 987 | "Garden", | ||
| 988 | None, | ||
| 989 | Some(&url), | ||
| 990 | || {}, | ||
| 991 | move |sharing| { | ||
| 992 | *kept.lock().unwrap() = sharing.clone(); | ||
| 993 | Ok(()) | ||
| 994 | }, | ||
| 995 | ) | ||
| 996 | .unwrap() | ||
| 997 | }; | ||
| 998 | let sharing = kept.lock().unwrap().clone(); | ||
| 999 | let host = start(sharing); | ||
| 1000 | let welcome = | ||
| 1001 | notebook::live::share::join(hello("Grace"), &code(&host), "", None, Some(&url)).unwrap(); | ||
| 1002 | let guest = Guest::start( | ||
| 1003 | hello("Grace"), | ||
| 1004 | welcome.share, | ||
| 1005 | welcome.secret, | ||
| 1006 | None, | ||
| 1007 | Some(&url), | ||
| 1008 | || {}, | ||
| 1009 | ) | ||
| 1010 | .unwrap(); | ||
| 1011 | until("the host was never met", || guest.host().is_some()); | ||
| 1012 | let file = folder.join("Garden.one"); | ||
| 1013 | let original = std::fs::read(&file).unwrap(); | ||
| 1014 | let (space, text, _) = server::text(&original); | ||
| 1015 | let typed = |range: std::ops::Range<u32>, with: &str| Edit { | ||
| 1016 | at: 134_000_000_000_000_000, | ||
| 1017 | ops: vec![Op::Page { | ||
| 1018 | space, | ||
| 1019 | op: PageOp::Text { | ||
| 1020 | text, | ||
| 1021 | range, | ||
| 1022 | with: with.into(), | ||
| 1023 | }, | ||
| 1024 | }], | ||
| 1025 | }; | ||
| 1026 | let replica = Replica::open_or_create(directory.path().join("grace.sqlite"), None, || { | ||
| 1027 | Ok(original.clone()) | ||
| 1028 | }) | ||
| 1029 | .unwrap(); | ||
| 1030 | let published = |id| { | ||
| 1031 | matches!( | ||
| 1032 | replica.status(id).unwrap(), | ||
| 1033 | Some(EditStatus::Published { .. }) | ||
| 1034 | ) | ||
| 1035 | }; | ||
| 1036 | let first = replica.apply("Grace", typed(0..0, "A")).unwrap(); | ||
| 1037 | until("the first edit was never published", || { | ||
| 1038 | let _ = replica.sync_once(&mut HostedRemote::new(&guest, "Garden.one")); | ||
| 1039 | published(first) | ||
| 1040 | }); | ||
| 1041 | let second = replica.apply("Grace", typed(0..0, "B")).unwrap(); | ||
| 1042 | until("the batch never landed", || { | ||
| 1043 | replica | ||
| 1044 | .sync_once(&mut Unanswered(HostedRemote::new(&guest, "Garden.one"))) | ||
| 1045 | .is_err() | ||
| 1046 | }); | ||
| 1047 | assert_eq!( | ||
| 1048 | server::text(&std::fs::read(&file).unwrap()).2, | ||
| 1049 | "BAOriginal text" | ||
| 1050 | ); | ||
| 1051 | |||
| 1052 | // The host deletes what the batch typed and quits; Optimize keeps the page's newest | ||
| 1053 | // revision alone, so the section holds neither the text nor the batch's revision. | ||
| 1054 | drop(host); | ||
| 1055 | let arena = onestore::Arena::default(); | ||
| 1056 | let mut optimized = onestore::Section::open(&arena, original.clone()).unwrap(); | ||
| 1057 | optimized.apply("Ada", &typed(0..0, "A")).unwrap(); | ||
| 1058 | let mut image = original; | ||
| 1059 | optimized | ||
| 1060 | .seal() | ||
| 1061 | .unwrap() | ||
| 1062 | .unwrap() | ||
| 1063 | .apply(&mut image) | ||
| 1064 | .unwrap(); | ||
| 1065 | std::fs::write(&file, image).unwrap(); | ||
| 1066 | let sharing = kept.lock().unwrap().clone(); | ||
| 1067 | let _host = start(sharing); | ||
| 1068 | until("the batch sent again was never confirmed", || { | ||
| 1069 | let _ = replica.sync_once(&mut HostedRemote::new(&guest, "Garden.one")); | ||
| 1070 | published(second) | ||
| 1071 | }); | ||
| 1072 | assert_eq!( | ||
| 1073 | server::text(&std::fs::read(&file).unwrap()).2, | ||
| 1074 | "AOriginal text" | ||
| 1075 | ); | ||
| 1076 | assert_eq!( | ||
| 1077 | server::text(&replica.snapshot().unwrap()).2, | ||
| 1078 | "AOriginal text" | ||
| 1079 | ); | ||
| 1080 | } | ||
| 1081 | |||
| 1082 | /// A host that stops sharing keeps nothing of it after, as its app forgets the share: what | ||
| 1083 | /// a guest's last batch noted is not kept once the share has stopped. | ||
| 1084 | #[test] | ||
| 1085 | fn a_stopped_share_is_not_kept_again() { | ||
| 1086 | use notebook::{EditStatus, live::share::Guest}; | ||
| 1087 | use std::sync::atomic::{AtomicUsize, Ordering}; | ||
| 1088 | |||
| 1089 | let directory = tempfile::tempdir().unwrap(); | ||
| 1090 | let folder = notebook(directory.path()); | ||
| 1091 | let url = relay(Default::default()); | ||
| 1092 | let keeps = Arc::new(AtomicUsize::new(0)); | ||
| 1093 | let counted = Arc::clone(&keeps); | ||
| 1094 | let host = notebook::live::share::Host::start( | ||
| 1095 | notebook::session::Notebook::open(&folder, directory.path().join("host")) | ||
| 1096 | .unwrap() | ||
| 1097 | .into_storage(), | ||
| 1098 | hello("Ada"), | ||
| 1099 | Sharing::new("").unwrap(), | ||
| 1100 | "Garden", | ||
| 1101 | None, | ||
| 1102 | Some(&url), | ||
| 1103 | || {}, | ||
| 1104 | move |_| { | ||
| 1105 | counted.fetch_add(1, Ordering::SeqCst); | ||
| 1106 | Ok(()) | ||
| 1107 | }, | ||
| 1108 | ) | ||
| 1109 | .unwrap(); | ||
| 1110 | let welcome = | ||
| 1111 | notebook::live::share::join(hello("Grace"), &code(&host), "", None, Some(&url)).unwrap(); | ||
| 1112 | let guest = Guest::start( | ||
| 1113 | hello("Grace"), | ||
| 1114 | welcome.share, | ||
| 1115 | welcome.secret, | ||
| 1116 | None, | ||
| 1117 | Some(&url), | ||
| 1118 | || {}, | ||
| 1119 | ) | ||
| 1120 | .unwrap(); | ||
| 1121 | until("the host was never met", || guest.host().is_some()); | ||
| 1122 | let original = std::fs::read(folder.join("Garden.one")).unwrap(); | ||
| 1123 | let (space, text, _) = server::text(&original); | ||
| 1124 | let replica = Replica::open_or_create(directory.path().join("grace.sqlite"), None, || { | ||
| 1125 | Ok(original.clone()) | ||
| 1126 | }) | ||
| 1127 | .unwrap(); | ||
| 1128 | let edit = Edit { | ||
| 1129 | at: 134_000_000_000_000_000, | ||
| 1130 | ops: vec![Op::Page { | ||
| 1131 | space, | ||
| 1132 | op: PageOp::Text { | ||
| 1133 | text, | ||
| 1134 | range: 0..0, | ||
| 1135 | with: "A".into(), | ||
| 1136 | }, | ||
| 1137 | }], | ||
| 1138 | }; | ||
| 1139 | let id = replica.apply("Grace", edit).unwrap(); | ||
| 1140 | until("the edit was never published", || { | ||
| 1141 | let _ = replica.sync_once(&mut HostedRemote::new(&guest, "Garden.one")); | ||
| 1142 | matches!( | ||
| 1143 | replica.status(id).unwrap(), | ||
| 1144 | Some(EditStatus::Published { .. }) | ||
| 1145 | ) | ||
| 1146 | }); | ||
| 1147 | host.stop(); | ||
| 1148 | let stopped = keeps.load(Ordering::SeqCst); | ||
| 1149 | std::thread::sleep(std::time::Duration::from_secs(3)); | ||
| 1150 | drop(host); | ||
| 1151 | assert_eq!(keeps.load(Ordering::SeqCst), stopped); | ||
| 1152 | } |