1//! Commits whose WAL write or sync fails, and caches that will not reopen after one, through a
2//! shim over the platform's VFS. Its own binary, as the shim becomes the process's default VFS.
3#![cfg(unix)]
4
5use notebook::{
6 EditStatus, Error, Replica,
7 session::{Event, Section, SyncState},
8};
9use onestore::{
10 PageCreation,
11 op::{Edit, Op, PageOp, SectionOp},
12};
13use std::{
14 io,
15 path::{Path, PathBuf},
16 sync::{MutexGuard, PoisonError, atomic::Ordering, mpsc},
17 time::{Duration, Instant},
18};
19
20#[path = "support/vfs.rs"]
21mod vfs;
22use vfs::{
23 ARMED, EXCLUSIVE, FAILING_OPENS, KEEP_WAL, SYNC_AFTER, SYNC_BEFORE, WAL_WRITES_AFTER_FAULT,
24 WRITE_COMMIT, install,
25};
26
27/// Holds the shim for one test, disarmed of what a failed test left.
28fn exclusive() -> MutexGuard<'static, ()> {
29 install();
30 let exclusive = EXCLUSIVE.lock().unwrap_or_else(PoisonError::into_inner);
31 ARMED.store(0, Ordering::SeqCst);
32 FAILING_OPENS.store(0, Ordering::SeqCst);
33 exclusive
34}
35
36/// A section file in `root` and its cache, which the returned path names.
37fn seed(root: &Path) -> PathBuf {
38 let source = onestore::create_section("Fault.one", "Original", "Fixture").unwrap();
39 std::fs::write(root.join("Fault.one"), &source).unwrap();
40 let cache = root.join("replica.sqlite");
41 drop(Replica::create(&cache, &source).unwrap());
42 cache
43}
44
45fn colour(space: onestore::ExGuid, colour: u32) -> Edit {
46 Edit {
47 at: 133_000_000_000_000_000 + u64::from(colour),
48 ops: vec![Op::Page {
49 space,
50 op: PageOp::Color(Some(colour)),
51 }],
52 }
53}
54
55/// The queued edits a cold open of `cache` reads, with the colour each gives the page.
56fn colours(cache: &Path) -> Vec<(u64, u32)> {
57 Replica::open(cache)
58 .unwrap()
59 .pending()
60 .unwrap()
61 .iter()
62 .map(|edit| match &edit.edit.ops[..] {
63 [
64 Op::Page {
65 op: PageOp::Color(Some(colour)),
66 ..
67 },
68 ] => (edit.id, *colour),
69 ops => panic!("{ops:?}"),
70 })
71 .collect()
72}
73
74fn until(section: &Section, seen: impl Fn(&Event) -> bool) {
75 let deadline = Instant::now() + Duration::from_secs(10);
76 while !section.events().iter().any(&seen) {
77 assert!(Instant::now() < deadline);
78 std::thread::sleep(Duration::from_millis(10));
79 }
80}
81
82fn would_block<T>(result: Result<T, Error>) -> bool {
83 matches!(result, Err(Error::Io(error)) if error.kind() == io::ErrorKind::WouldBlock)
84}
85
86/// What the section answered for the edit committed under `fault` and the one after it,
87/// whether a failed write was reported, and the queue as the section saw it.
88struct Outcome {
89 replies: [Result<u64, String>; 2],
90 failed: bool,
91 pending: Vec<u64>,
92}
93
94/// Seeds the WAL with one edit, so the armed commit's sync is the only one it makes, then
95/// commits under `fault`, the cache failing to reopen `opens` times, and once more after it;
96/// with `crash`, leaves the cache open.
97fn scenario(root: &Path, fault: u8, opens: usize, crash: bool) -> Outcome {
98 install();
99 let replica = Replica::open(root.join("replica.sqlite")).unwrap();
100 let section = Section::resume(root.join("Fault.one"), replica, true, || {}).unwrap();
101 let space = section.pages().unwrap()[0].0;
102 let apply = |value| {
103 let (sender, receiver) = mpsc::channel();
104 section
105 .apply_with("Fixture", colour(space, value), move |result| {
106 sender.send(result).unwrap();
107 })
108 .unwrap();
109 receiver
110 .recv_timeout(Duration::from_secs(10))
111 .unwrap()
112 .map_err(|error| error.to_string())
113 };
114 apply(1).unwrap();
115 WAL_WRITES_AFTER_FAULT.store(0, Ordering::SeqCst);
116 FAILING_OPENS.store(opens, Ordering::SeqCst);
117 ARMED.store(fault, Ordering::SeqCst);
118 let first = apply(2);
119 assert_eq!(ARMED.load(Ordering::SeqCst), 0, "fault {fault} never fired");
120 assert_eq!(
121 FAILING_OPENS.load(Ordering::SeqCst),
122 0,
123 "the cache never reopened"
124 );
125 assert_eq!(
126 WAL_WRITES_AFTER_FAULT.load(Ordering::SeqCst),
127 0,
128 "the warm connection wrote frames after its commit failed"
129 );
130 let second = apply(3);
131 let failed = section
132 .events()
133 .iter()
134 .any(|event| matches!(event, Event::Failed(_)));
135 let pending = section
136 .pending()
137 .unwrap()
138 .iter()
139 .map(|edit| edit.id)
140 .collect();
141 if crash {
142 std::mem::forget(section);
143 } else {
144 section.close().unwrap();
145 }
146 Outcome {
147 replies: [first, second],
148 failed,
149 pending,
150 }
151}
152
153#[test]
154fn commit_fault_child() {
155 let (Some(root), Some(fault), Some(opens)) = (
156 std::env::var_os("COMMIT_FAULT_CHILD"),
157 std::env::var("COMMIT_FAULT").ok(),
158 std::env::var("COMMIT_FAULT_OPENS").ok(),
159 ) else {
160 return;
161 };
162 let root = Path::new(&root);
163 let outcome = scenario(root, fault.parse().unwrap(), opens.parse().unwrap(), true);
164 let ids = |result: &Result<u64, String>| result.as_ref().map_or(0, |id| *id);
165 std::fs::write(
166 root.join("outcome"),
167 format!(
168 "{} {} {}",
169 ids(&outcome.replies[0]),
170 ids(&outcome.replies[1]),
171 outcome.failed
172 ),
173 )
174 .unwrap();
175 // Ends as a crash would: no SQLite destructor checkpoints or removes the WAL.
176 std::process::exit(0);
177}
178
179#[test]
180fn a_failed_commit_is_decided_by_a_cold_read_and_never_replayed_twice() {
181 let _exclusive = exclusive();
182 // (fault, whether the edit reached the cache, how many reopens fail)
183 for (fault, landed, opens) in [
184 (SYNC_BEFORE, false, 0),
185 (SYNC_AFTER, false, 0),
186 (WRITE_COMMIT, false, 0),
187 (SYNC_AFTER | KEEP_WAL, true, 0),
188 (SYNC_BEFORE, false, 2),
189 (SYNC_AFTER | KEEP_WAL, true, 2),
190 ] {
191 for crash in [false, true] {
192 let directory = tempfile::tempdir().unwrap();
193 let root = directory.path();
194 let cache = seed(root);
195 let (first, second, failed) = if crash {
196 let output = std::process::Command::new(std::env::current_exe().unwrap())
197 .args(["--exact", "commit_fault_child", "--nocapture"])
198 .env("COMMIT_FAULT_CHILD", root)
199 .env("COMMIT_FAULT", fault.to_string())
200 .env("COMMIT_FAULT_OPENS", opens.to_string())
201 .output()
202 .unwrap();
203 assert!(output.status.success(), "{output:?}");
204 let outcome = std::fs::read_to_string(root.join("outcome")).unwrap();
205 let mut fields = outcome.split(' ');
206 let mut id = || fields.next().unwrap().parse::<u64>().unwrap();
207 let (first, second) = (id(), id());
208 (first, second, fields.next() == Some("true"))
209 } else {
210 let outcome = scenario(root, fault, opens, false);
211 let ids = outcome.replies.map(Result::unwrap);
212 assert_eq!(outcome.pending[1..], ids);
213 (ids[0], ids[1], outcome.failed)
214 };
215 // An edit waiting for the cache to reopen is reported, as one that did not land.
216 assert_eq!(failed, !landed || opens > 0, "fault {fault}, {opens} opens");
217 assert_eq!(
218 colours(&cache)[1..],
219 [(first, 2), (second, 3)],
220 "fault {fault}, {opens} opens"
221 );
222 }
223 }
224}
225
226#[test]
227fn edits_wait_while_the_cache_cannot_reopen_and_land_once() {
228 let _exclusive = exclusive();
229 for close in [false, true] {
230 let directory = tempfile::tempdir().unwrap();
231 let root = directory.path();
232 let cache = seed(root);
233 let section = Section::resume(
234 root.join("Fault.one"),
235 Replica::open(&cache).unwrap(),
236 true,
237 || {},
238 )
239 .unwrap();
240 let space = section.pages().unwrap()[0].0;
241 let (replied, replies) = mpsc::channel();
242 let apply = |value| {
243 let replied = replied.clone();
244 section
245 .apply_with("Fixture", colour(space, value), move |result| {
246 replied
247 .send(result.map_err(|error| error.to_string()))
248 .unwrap();
249 })
250 .unwrap();
251 };
252 let reply = || {
253 replies
254 .recv_timeout(Duration::from_secs(10))
255 .unwrap()
256 .unwrap()
257 };
258 apply(1);
259 let seeded = reply();
260 FAILING_OPENS.store(1000, Ordering::SeqCst);
261 ARMED.store(SYNC_BEFORE, Ordering::SeqCst);
262 apply(2);
263 until(&section, |event| matches!(event, Event::Failed(_)));
264 // The cache is tried again once a second.
265 let left = FAILING_OPENS.load(Ordering::SeqCst);
266 std::thread::sleep(Duration::from_secs(3));
267 let tries = left - FAILING_OPENS.load(Ordering::SeqCst);
268 // Never faster than once a second; as slow as the machine is busy.
269 assert!((1..=4).contains(&tries), "{tries} tries in 3 s");
270 apply(3);
271 // A caller blocking on its edit learns at once, and the waiting edits stay.
272 assert!(would_block(
273 section.replica().apply("Fixture", colour(space, 4))
274 ));
275 // A sync step meets the closed cache and the worker carries on.
276 section.wake();
277 until(
278 &section,
279 |event| matches!(event, Event::Failed(error) if error == "The cache reopens once it can"),
280 );
281 // Each second the cache is tried again, and the edits still wait.
282 until(&section, |event| matches!(event, Event::Failed(_)));
283 assert!(replies.try_recv().is_err());
284 assert_eq!(section.page(space).unwrap().color, Some(3));
285 assert!(would_block(section.written()));
286 assert!(would_block(section.pending()));
287 assert_eq!(notebook::unwritten(), 2);
288 let status = section.sync_status().unwrap();
289 // The seed, queued before the commit that failed, and the two waiting.
290 assert_eq!((status.state(), status.queued), (SyncState::Failed, 3));
291 // Nobody else takes the cache meanwhile, nor once the section closes.
292 assert!(Replica::open(&cache).is_err_and(|error| error.busy()));
293 let open = if close {
294 section.close().unwrap();
295 assert!(Replica::open(&cache).is_err_and(|error| error.busy()));
296 None
297 } else {
298 Some(section)
299 };
300 FAILING_OPENS.store(0, Ordering::SeqCst);
301 let ids = [reply(), reply()];
302 if let Some(section) = open {
303 section.wake();
304 until(&section, |event| {
305 matches!(event, Event::Attempt { id, status: EditStatus::Published { .. } }
306 if *id == ids[1])
307 });
308 section.close().unwrap();
309 let arena = onestore::Arena::default();
310 let stored =
311 onestore::Section::open(&arena, std::fs::read(root.join("Fault.one")).unwrap())
312 .unwrap();
313 assert_eq!(stored.page(space).unwrap().color, Some(3));
314 assert!(colours(&cache).is_empty());
315 } else {
316 // Its cache is released once the edits are written.
317 let deadline = Instant::now() + Duration::from_secs(10);
318 while Replica::open(&cache).is_err() {
319 assert!(Instant::now() < deadline);
320 std::thread::sleep(Duration::from_millis(10));
321 }
322 assert_eq!(colours(&cache), [(seeded, 1), (ids[0], 2), (ids[1], 3)]);
323 }
324 assert_eq!(notebook::unwritten(), 0);
325 }
326}
327
328#[test]
329fn a_lost_cache_refuses_the_edits_that_wait() {
330 let _exclusive = exclusive();
331 for corrupt in [false, true] {
332 let directory = tempfile::tempdir().unwrap();
333 let root = directory.path();
334 let cache = seed(root);
335 let section = Section::resume(
336 root.join("Fault.one"),
337 Replica::open(&cache).unwrap(),
338 true,
339 || {},
340 )
341 .unwrap();
342 let space = section.pages().unwrap()[0].0;
343 let (replied, replies) = mpsc::channel();
344 let apply = |value| {
345 let replied = replied.clone();
346 section
347 .apply_with("Fixture", colour(space, value), move |result| {
348 replied
349 .send(result.map_err(|error| error.to_string()))
350 .unwrap();
351 })
352 .unwrap();
353 };
354 let reply = || replies.recv_timeout(Duration::from_secs(10)).unwrap();
355 apply(1);
356 reply().unwrap();
357 FAILING_OPENS.store(usize::MAX, Ordering::SeqCst);
358 ARMED.store(SYNC_BEFORE, Ordering::SeqCst);
359 apply(2);
360 until(&section, |event| matches!(event, Event::Failed(_)));
361 for suffix in ["-wal", "-shm"] {
362 let mut file = cache.clone().into_os_string();
363 file.push(suffix);
364 let _ = std::fs::remove_file(file);
365 }
366 if corrupt {
367 std::fs::write(&cache, [0xab; 4096]).unwrap();
368 } else {
369 std::fs::remove_file(&cache).unwrap();
370 }
371 FAILING_OPENS.store(0, Ordering::SeqCst);
372 assert!(reply().is_err());
373 assert_eq!(notebook::unwritten(), 0);
374 let pending = section.pending();
375 assert!(pending.is_err() && !would_block(pending));
376 apply(3);
377 assert!(reply().is_err());
378 let status = section.sync_status().unwrap();
379 assert_eq!((status.state(), status.queued), (SyncState::Failed, 0));
380 // The sync worker meets the lost cache and stops.
381 std::thread::sleep(Duration::from_millis(100));
382 section.events();
383 section.wake();
384 until(&section, |event| matches!(event, Event::Failed(_)));
385 assert!(section.close().is_err());
386 }
387}
388
389#[test]
390fn a_closed_cache_stays_owned_and_its_landed_commit_counts_once() {
391 let _exclusive = exclusive();
392 for close in [false, true] {
393 let directory = tempfile::tempdir().unwrap();
394 let root = directory.path();
395 let cache = seed(root);
396 let section = Section::resume(
397 root.join("Fault.one"),
398 Replica::open(&cache).unwrap(),
399 true,
400 || {},
401 )
402 .unwrap();
403 let space = section.pages().unwrap()[0].0;
404 let (replied, replies) = mpsc::channel::<Result<u64, String>>();
405 let reply = || {
406 replies
407 .recv_timeout(Duration::from_secs(10))
408 .unwrap()
409 .unwrap()
410 };
411 let apply = |value| {
412 let replied = replied.clone();
413 section
414 .apply_with("Fixture", colour(space, value), move |result| {
415 replied
416 .send(result.map_err(|error| error.to_string()))
417 .unwrap();
418 })
419 .unwrap();
420 };
421 apply(1);
422 let seeded = reply();
423 FAILING_OPENS.store(usize::MAX, Ordering::SeqCst);
424 ARMED.store(SYNC_AFTER | KEEP_WAL, Ordering::SeqCst);
425 // A caller blocking on an edit whose commit fails hears at once, though it landed.
426 let asked = Instant::now();
427 assert!(
428 section
429 .replica()
430 .apply("Fixture", colour(space, 2))
431 .is_err()
432 );
433 assert!(asked.elapsed() < Duration::from_secs(5));
434 // Waits after the commit the closed cache holds, in the same batch.
435 apply(3);
436 until(&section, |event| matches!(event, Event::Failed(_)));
437 // Nobody else takes the cache to publish and acknowledge that commit meanwhile.
438 assert!(Replica::open(&cache).is_err_and(|error| error.busy()));
439 let open = if close {
440 section.close().unwrap();
441 assert!(Replica::open(&cache).is_err_and(|error| error.busy()));
442 None
443 } else {
444 Some(section)
445 };
446 FAILING_OPENS.store(0, Ordering::SeqCst);
447 let third = reply();
448 if let Some(section) = open {
449 // Reread once the cache reopens, the section is the queue.
450 let deadline = Instant::now() + Duration::from_secs(10);
451 while !section.pending().is_ok_and(|pending| pending.len() == 3) {
452 assert!(Instant::now() < deadline);
453 std::thread::sleep(Duration::from_millis(10));
454 }
455 assert_eq!(section.page(space).unwrap().color, Some(3));
456 section.close().unwrap();
457 } else {
458 let deadline = Instant::now() + Duration::from_secs(10);
459 while Replica::open(&cache).is_err() {
460 assert!(Instant::now() < deadline);
461 std::thread::sleep(Duration::from_millis(10));
462 }
463 }
464 let stored = colours(&cache);
465 assert_eq!(stored.len(), 3, "{stored:?}");
466 assert_eq!(
467 (stored[0], stored[1].1, stored[2]),
468 ((seeded, 1), 2, (third, 3))
469 );
470 }
471}
472
473#[test]
474fn closed_crash_child() {
475 let (Some(root), Some(fault)) = (
476 std::env::var_os("CLOSED_CRASH_CHILD"),
477 std::env::var("COMMIT_FAULT").ok(),
478 ) else {
479 return;
480 };
481 install();
482 let root = Path::new(&root);
483 let replica = Replica::open(root.join("replica.sqlite")).unwrap();
484 let section = Section::resume(root.join("Fault.one"), replica, true, || {}).unwrap();
485 let space = section.pages().unwrap()[0].0;
486 let (replied, replies) = mpsc::channel();
487 let replied = move |result: Result<u64, Error>| replied.send(result.is_ok()).unwrap();
488 let apply = |value, replied: Box<dyn FnOnce(Result<u64, Error>) + Send>| {
489 section
490 .apply_with("Fixture", colour(space, value), replied)
491 .unwrap();
492 };
493 apply(1, Box::new(replied));
494 assert!(replies.recv_timeout(Duration::from_secs(10)).unwrap());
495 FAILING_OPENS.store(usize::MAX, Ordering::SeqCst);
496 ARMED.store(fault.parse().unwrap(), Ordering::SeqCst);
497 apply(2, Box::new(drop));
498 until(&section, |event| matches!(event, Event::Failed(_)));
499 apply(3, Box::new(drop));
500 std::thread::sleep(Duration::from_millis(100));
501 // Ends as a crash would, the cache still closed.
502 std::process::exit(0);
503}
504
505#[test]
506fn a_crash_while_the_cache_is_closed_leaves_what_landed() {
507 for (fault, landed) in [(SYNC_BEFORE, false), (SYNC_AFTER | KEEP_WAL, true)] {
508 let directory = tempfile::tempdir().unwrap();
509 let root = directory.path();
510 let cache = seed(root);
511 let output = std::process::Command::new(std::env::current_exe().unwrap())
512 .args(["--exact", "closed_crash_child", "--nocapture"])
513 .env("CLOSED_CRASH_CHILD", root)
514 .env("COMMIT_FAULT", fault.to_string())
515 .output()
516 .unwrap();
517 assert!(output.status.success(), "{output:?}");
518 let colours: Vec<_> = colours(&cache)
519 .into_iter()
520 .map(|(_, colour)| colour)
521 .collect();
522 assert_eq!(
523 colours,
524 if landed { vec![1, 2] } else { vec![1] },
525 "fault {fault}"
526 );
527 }
528}
529
530/// A section, offline, over a seeded cache in `root`, with its first page's space.
531fn opened(root: &Path) -> (PathBuf, Section, onestore::ExGuid) {
532 let cache = seed(root);
533 let section = Section::resume(
534 root.join("Fault.one"),
535 Replica::open(&cache).unwrap(),
536 true,
537 || {},
538 )
539 .unwrap();
540 let space = section.pages().unwrap()[0].0;
541 (cache, section, space)
542}
543
544#[test]
545fn edits_wait_for_the_reread_a_refused_edit_needs_while_the_cache_is_closed() {
546 let _exclusive = exclusive();
547 for close in [false, true] {
548 let directory = tempfile::tempdir().unwrap();
549 let root = directory.path();
550 let (cache, section, space) = opened(root);
551 let (replied, replies) = mpsc::channel::<(u32, Result<u64, String>)>();
552 let apply = |space, value| {
553 let replied = replied.clone();
554 section
555 .apply_with("Fixture", colour(space, value), move |result| {
556 let result = result.map_err(|error| error.to_string());
557 replied.send((value, result)).unwrap();
558 })
559 .unwrap();
560 };
561 let reply = || replies.recv_timeout(Duration::from_secs(10)).unwrap();
562 apply(space, 1);
563 let seeded = reply().1.unwrap();
564 FAILING_OPENS.store(usize::MAX, Ordering::SeqCst);
565 ARMED.store(SYNC_BEFORE, Ordering::SeqCst);
566 // A page a caller blocks on is refused, its commit never landing.
567 let creation = PageCreation::new(None, Some("Refused"), "Fixture").unwrap();
568 let made = creation.space();
569 let create = Edit {
570 at: 133_000_000_000_000_100,
571 ops: vec![Op::Section(SectionOp::Create(creation))],
572 };
573 assert!(section.replica().apply("Fixture", create).is_err());
574 // Until the cache reopens to reread the section, edits wait to apply, one on the
575 // refused page among them.
576 apply(made, 2);
577 apply(space, 3);
578 std::thread::sleep(Duration::from_millis(1500));
579 assert!(replies.try_recv().is_err());
580 assert_eq!(section.page(space).unwrap().color, Some(1));
581 assert!(would_block(section.written()));
582 assert_eq!(notebook::unwritten(), 2);
583 let open = if close {
584 section.close().unwrap();
585 None
586 } else {
587 Some(section)
588 };
589 FAILING_OPENS.store(0, Ordering::SeqCst);
590 // A host reading the section steadily, as one typing does, keeps no retry back.
591 let deadline = Instant::now() + Duration::from_secs(10);
592 let mut answered = Vec::new();
593 while answered.len() < 2 {
594 assert!(Instant::now() < deadline);
595 if let Some(section) = &open {
596 let _ = section.page(space);
597 }
598 answered.extend(replies.recv_timeout(Duration::from_millis(300)));
599 }
600 let mut answered: [_; 2] = answered.try_into().unwrap();
601 answered.sort_by_key(|(value, _)| *value);
602 let [(_, dependent), (_, kept)] = answered;
603 assert!(dependent.is_err(), "{dependent:?}");
604 let kept = kept.unwrap();
605 if let Some(section) = open {
606 // The reread tells the host to reload what it shows.
607 until(
608 &section,
609 |event| matches!(event, Event::Changed(spaces) if spaces.contains(&made)),
610 );
611 assert!(
612 section
613 .pages()
614 .unwrap()
615 .iter()
616 .all(|(listed, ..)| *listed != made)
617 );
618 assert_eq!(section.page(space).unwrap().color, Some(3));
619 section.close().unwrap();
620 }
621 let deadline = Instant::now() + Duration::from_secs(10);
622 while Replica::open(&cache).is_err_and(|error| error.busy()) {
623 assert!(Instant::now() < deadline);
624 std::thread::sleep(Duration::from_millis(10));
625 }
626 assert_eq!(colours(&cache), [(seeded, 1), (kept, 3)]);
627 }
628}
629
630#[test]
631fn a_cache_whose_owner_lock_cannot_be_taken_stays_held_by_its_warm_connection() {
632 let _exclusive = exclusive();
633 let directory = tempfile::tempdir().unwrap();
634 let root = directory.path();
635 let (cache, section, space) = opened(root);
636 let (replied, replies) = mpsc::channel();
637 let apply = |value| {
638 let replied = replied.clone();
639 section
640 .apply_with("Fixture", colour(space, value), move |result| {
641 replied
642 .send(result.map_err(|error| error.to_string()))
643 .unwrap();
644 })
645 .unwrap();
646 };
647 let reply = || {
648 replies
649 .recv_timeout(Duration::from_secs(10))
650 .unwrap()
651 .unwrap()
652 };
653 apply(1);
654 let seeded = reply();
655 let mut lock = cache.clone().into_os_string();
656 lock.push("-lock");
657 std::fs::create_dir(&lock).unwrap();
658 ARMED.store(SYNC_BEFORE, Ordering::SeqCst);
659 apply(2);
660 until(&section, |event| matches!(event, Event::Failed(_)));
661 std::thread::sleep(Duration::from_millis(1500));
662 assert!(replies.try_recv().is_err());
663 assert!(Replica::open(&cache).is_err_and(|error| error.busy()));
664 std::fs::remove_dir(&lock).unwrap();
665 let second = reply();
666 section.close().unwrap();
667 assert_eq!(colours(&cache), [(seeded, 1), (second, 2)]);
668}