1use onestore::{
2 ExGuid, FileDataReference, ObjectData, PropertySets, RevisionIndex, Store,
3 document::{Document, Kind},
4 op::{PageOp, SectionOp},
5};
6use std::collections::BTreeSet;
7
8const SOURCE: &[u8] =
9 include_bytes!("../../../corpus/page-lifecycle/04-nested/notebook/Lifecycle.one");
10
11fn verify(original: &[u8], written: &[u8], native: &[u8], removed: &[ExGuid]) {
12 let stores = [original, written, native].map(|bytes| Store::parse(bytes).unwrap());
13 let indexes = stores.each_ref().map(|store| {
14 assert!(store.checksum_mismatches.is_empty());
15 let index = RevisionIndex::parse(store).unwrap();
16 index.validate_current().unwrap();
17 index
18 });
19 let documents = indexes
20 .each_ref()
21 .map(|index| Document::parse(index).unwrap());
22 assert_eq!(stores[0].header.file_id, stores[1].header.file_id);
23 assert_eq!(
24 stores[1].header.transaction_count,
25 stores[0].header.transaction_count + 1
26 );
27 assert_eq!(
28 indexes[0].spaces.keys().collect::<Vec<_>>(),
29 indexes[1].spaces.keys().collect::<Vec<_>>()
30 );
31 let expected: Vec<_> = documents[0]
32 .pages()
33 .unwrap()
34 .into_iter()
35 .filter(|(sid, _)| !removed.contains(sid))
36 .collect();
37 assert_eq!(documents[1].pages().unwrap(), expected);
38 assert_eq!(documents[2].pages().unwrap(), expected);
39 assert_eq!(
40 documents[1].spaces.keys().collect::<Vec<_>>(),
41 documents[2].spaces.keys().collect::<Vec<_>>()
42 );
43 for (sid, space) in &documents[1].spaces {
44 let expected = &documents[2].spaces[sid];
45 assert_eq!(
46 space.contexts.keys().collect::<Vec<_>>(),
47 expected.contexts.keys().collect::<Vec<_>>()
48 );
49 for (context, rid) in &space.contexts {
50 let actual = &space.revisions[rid];
51 let expected = &expected.revisions[&expected.contexts[context]];
52 assert_eq!(actual.roots, expected.roots);
53 assert!(
54 serde_json::to_value(&actual.nodes).unwrap()
55 == serde_json::to_value(&expected.nodes).unwrap(),
56 "native page graph {sid}/{context}"
57 );
58 }
59 }
60 let mut payloads = BTreeSet::new();
61 for (sid, space) in &indexes[0].spaces {
62 for rid in space.revisions.keys() {
63 let before = indexes[0].resolve(*sid, *rid).unwrap();
64 let after = indexes[1].resolve(*sid, *rid).unwrap();
65 assert_eq!(
66 format!("{before:?}"),
67 format!("{after:?}"),
68 "prior revision {sid}/{rid}"
69 );
70 for object in before.objects.values() {
71 if let Some(FileDataReference::Internal(guid)) = object.file_reference().unwrap() {
72 payloads.insert(guid);
73 }
74 }
75 }
76 if removed.contains(sid) {
77 assert!(!documents[1].spaces.contains_key(sid));
78 let before = indexes[0]
79 .resolve(*sid, space.labels[&(ExGuid::default(), 1)])
80 .unwrap();
81 let after = indexes[1]
82 .resolve(*sid, indexes[1].spaces[sid].labels[&(ExGuid::default(), 1)])
83 .unwrap();
84 assert_eq!(before.roots, after.roots);
85 let ObjectData::Properties(bytes) = after.objects[&after.roots[&1]].data else {
86 panic!()
87 };
88 assert!(PropertySets::parse(bytes).unwrap().sets[0].is_empty());
89 let ObjectData::Properties(bytes) = after.objects[&after.roots[&2]].data else {
90 panic!()
91 };
92 let metadata = PropertySets::parse(bytes).unwrap();
93 assert!(metadata.sets[0].iter().any(|p| p.id == 0x88001de9));
94 let native = indexes[2]
95 .resolve(*sid, indexes[2].spaces[sid].labels[&(ExGuid::default(), 1)])
96 .unwrap();
97 assert_eq!(after.reachable().unwrap(), native.reachable().unwrap());
98 for (label, rid) in &space.labels {
99 if *label != (ExGuid::default(), 1) {
100 assert_eq!(indexes[1].spaces[sid].labels[label], *rid);
101 }
102 }
103 }
104 }
105 for guid in payloads {
106 assert_eq!(
107 stores[0].file_data(guid).unwrap(),
108 stores[1].file_data(guid).unwrap()
109 );
110 }
111}
112
113#[test]
114fn removal_matches_all_native_cases_and_retains_every_old_revision() {
115 let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
116 .join("../../corpus/page-lifecycle/removal");
117 let cases: serde_json::Value =
118 serde_json::from_slice(&std::fs::read(root.join("provenance.json")).unwrap()).unwrap();
119 for (name, case) in cases.as_object().unwrap() {
120 let original =
121 std::fs::read(root.join(name).join("native/before/notebook/Lifecycle.one")).unwrap();
122 let native =
123 std::fs::read(root.join(name).join("native/after/notebook/Lifecycle.one")).unwrap();
124 let store = Store::parse(&original).unwrap();
125 let index = RevisionIndex::parse(&store).unwrap();
126 let document = Document::parse(&index).unwrap();
127 let pages = document.pages().unwrap();
128 let selected: Vec<usize> =
129 serde_json::from_value(case["request"]["selected"].clone()).unwrap();
130 let removed: Vec<_> = selected
131 .into_iter()
132 .map(|ordinal| pages[ordinal].0)
133 .collect();
134 let prepared = ops::section_op(&original, SectionOp::Delete(removed.to_vec())).unwrap();
135 if let Some(output) = std::env::var_os("ONESTORE_PAGE_REMOVAL_OUTPUT") {
136 let output = std::path::Path::new(&output);
137 assert!(output.is_absolute());
138 std::fs::create_dir_all(output).unwrap();
139 let case = output.join(name);
140 std::fs::create_dir(&case).unwrap();
141 std::fs::write(case.join("Lifecycle.one"), prepared.as_bytes()).unwrap();
142 }
143 verify(&original, prepared.as_bytes(), &native, &removed);
144 assert!(ops::section_op(prepared.as_bytes(), SectionOp::Delete(removed.to_vec())).is_err());
145 }
146}
147
148#[test]
149fn removal_rejects_duplicate_missing_and_non_page_spaces_without_publication() {
150 let store = Store::parse(SOURCE).unwrap();
151 let index = RevisionIndex::parse(&store).unwrap();
152 let document = Document::parse(&index).unwrap();
153 let pages = document.pages().unwrap();
154 for selected in [
155 vec![pages[0].0, pages[0].0],
156 vec![index.root],
157 vec![ExGuid::default()],
158 vec![ExGuid {
159 guid: [97; 16],
160 n: 1,
161 }],
162 vec![pages[0].0, index.root],
163 ] {
164 assert!(ops::section_op(SOURCE, SectionOp::Delete(selected.to_vec())).is_err());
165 }
166 assert_eq!(
167 ops::section_op(SOURCE, SectionOp::Delete([].to_vec()))
168 .unwrap()
169 .as_bytes(),
170 SOURCE
171 );
172 let written = ops::section_op(SOURCE, SectionOp::Delete([pages[0].0].to_vec())).unwrap();
173 assert!(
174 ops::section_op(
175 written.as_bytes(),
176 SectionOp::Delete([pages[1].0, pages[0].0].to_vec())
177 )
178 .is_err()
179 );
180}
181
182use page_schedule::{current, disk, ops};
183
184/// OneNote created a page in each section after Rust permanently removed pages from it;
185/// the surviving pages keep their revisions through the native save and a cold reopen.
186#[test]
187fn native_pages_created_after_rust_removal_reopen_cold() {
188 let root = std::path::Path::new(concat!(
189 env!("CARGO_MANIFEST_DIR"),
190 "/../../corpus/page-lifecycle/removal/rust-followup"
191 ));
192 for case in ["parent", "all", "features", "leading-parent"] {
193 let read = |phase: &str| {
194 std::fs::read(root.join(case).join(phase).join("notebook/Lifecycle.one")).unwrap()
195 };
196 let (candidate, followup, cold) =
197 (read("candidate"), read("followup"), read("followup-cold"));
198 let title =
199 std::fs::read_to_string(root.join(case).join("followup/native-title.txt")).unwrap();
200 let title = title.trim_start_matches('\u{feff}').trim_end();
201 let stores = [&candidate, &followup, &cold].map(|bytes| Store::parse(bytes).unwrap());
202 let indexes = stores.each_ref().map(|store| {
203 assert!(store.checksum_mismatches.is_empty());
204 let index = RevisionIndex::parse(store).unwrap();
205 index.validate_current().unwrap();
206 index
207 });
208 let documents = indexes
209 .each_ref()
210 .map(|index| Document::parse(index).unwrap());
211 let before = documents[0].pages().unwrap();
212 for (document, index) in documents[1..].iter().zip(&indexes[1..]) {
213 let pages = document.pages().unwrap();
214 assert_eq!(pages.len(), before.len() + 1, "{case}");
215 assert_eq!(
216 &pages[..before.len()],
217 before.as_slice(),
218 "{case}: surviving page order"
219 );
220 for (sid, _) in &before {
221 assert_eq!(
222 index.active(*sid).unwrap(),
223 indexes[0].active(*sid).unwrap(),
224 "{case}: {sid}"
225 );
226 }
227 let (native, page) = pages[before.len()];
228 let revision = document.active(native).unwrap();
229 let metadata = revision
230 .roots
231 .get(&2)
232 .and_then(|id| revision.nodes.get(id))
233 .unwrap();
234 assert!(
235 matches!(&metadata.kind, Kind::Metadata { title: Some(name), .. } if name == title),
236 "{case}"
237 );
238 let body = revision
239 .parents(&[page])
240 .unwrap()
241 .into_keys()
242 .any(|id| matches!(&revision.nodes[&id].kind, Kind::RichText { text, .. } if text == "Native body after Rust removal."));
243 assert!(body, "{case}: native body");
244 }
245 // The section root may gain OneNote's per-series navigation metadata copies on open.
246 for sid in indexes[1]
247 .spaces
248 .keys()
249 .filter(|sid| **sid != indexes[1].root)
250 {
251 assert_eq!(
252 indexes[1].active(*sid).unwrap(),
253 indexes[2].active(*sid).unwrap(),
254 "{case}: cold reopen changed {sid}"
255 );
256 }
257 }
258}
259
260#[test]
261fn tombstones_and_first_page_promotion_survive_each_storage_interruption() {
262 let leading = include_bytes!(
263 "../../../corpus/page-lifecycle/removal/leading-parent/native/before/notebook/Lifecycle.one"
264 );
265 for (source, all) in [(leading.as_slice(), false), (SOURCE, true)] {
266 let store = Store::parse(source).unwrap();
267 let index = RevisionIndex::parse(&store).unwrap();
268 let pages = Document::parse(&index).unwrap().pages().unwrap();
269 let removed: Vec<_> = pages
270 .iter()
271 .take(if all { pages.len() } else { 1 })
272 .map(|page| page.0)
273 .collect();
274 let prepared = ops::section_op(source, SectionOp::Delete(removed.to_vec())).unwrap();
275 let old = current::current(source);
276 let new = current::current(prepared.as_bytes());
277 assert_ne!(old, new);
278 assert_eq!(
279 old.keys().collect::<Vec<_>>(),
280 new.keys().collect::<Vec<_>>()
281 );
282 // Byte-at-a-time writes tear the commit at every byte, a full sweep's matrix.
283 let limits: &[usize] = if sweep::full().is_some() {
284 &[1, 17, 4096]
285 } else {
286 &[17, 4096]
287 };
288 for &write_limit in limits {
289 let mut complete = disk::Disk {
290 visible: source.to_vec(),
291 durable: source.to_vec(),
292 operation: 0,
293 fail_at: None,
294 write_limit,
295 random: 1997,
296 };
297 prepared.commit(&mut complete).unwrap();
298 assert_eq!(complete.durable, prepared.as_bytes());
299 for fail_at in 1..=complete.operation {
300 let mut interrupted = disk::Disk {
301 visible: source.to_vec(),
302 durable: source.to_vec(),
303 operation: 0,
304 fail_at: Some(fail_at),
305 write_limit,
306 random: 1997 + u64::try_from(fail_at).unwrap(),
307 };
308 let error = prepared.commit(&mut interrupted).unwrap_err();
309 let observed = current::current(&interrupted.durable);
310 assert!(
311 observed == old || observed == new,
312 "{all}:{write_limit}:{fail_at}"
313 );
314 match error.state {
315 onestore::CommitState::NotCommitted => assert_eq!(observed, old),
316 onestore::CommitState::Committed => assert_eq!(observed, new),
317 onestore::CommitState::Unknown => {}
318 }
319 }
320 }
321 }
322}
323
324#[test]
325fn stale_removal_cannot_delete_a_newer_page_edit() {
326 let store = Store::parse(SOURCE).unwrap();
327 let index = RevisionIndex::parse(&store).unwrap();
328 let document = Document::parse(&index).unwrap();
329 let sid = document.pages().unwrap()[3].0;
330 let space = &document.spaces[&sid];
331 let revision = &space.revisions[&space.contexts[&ExGuid::default()]];
332 let text = revision
333 .nodes
334 .iter()
335 .find_map(|(id, node)| match &node.kind {
336 onestore::document::Kind::RichText {
337 text,
338 boilerplate: false,
339 ..
340 } if !text.is_empty() => Some(*id),
341 _ => None,
342 })
343 .unwrap();
344 let removal = ops::section_op(SOURCE, SectionOp::Delete([sid].to_vec())).unwrap();
345 let remote = ops::page_edited(
346 SOURCE,
347 sid,
348 vec![PageOp::Text {
349 text,
350 range: 0..0,
351 with: "New remote content ".into(),
352 }],
353 )
354 .unwrap();
355 let mut disk = disk::Disk {
356 visible: remote.clone(),
357 durable: remote.clone(),
358 operation: 0,
359 fail_at: None,
360 write_limit: 17,
361 random: 1999,
362 };
363 let error = removal.commit(&mut disk).unwrap_err();
364 assert_eq!(error.state, onestore::CommitState::NotCommitted);
365 assert_eq!(error.error.kind(), std::io::ErrorKind::ResourceBusy);
366 assert_eq!(disk.visible, remote);
367 assert_eq!(disk.durable, remote);
368}
369
370#[path = "support/page_schedule.rs"]
371mod page_schedule;
372#[path = "support/sweep.rs"]
373mod sweep;
374
375#[test]
376fn twelve_client_page_schedules_include_removal_empty_sections_and_stale_edits() {
377 page_schedule::run(&[
378 0, 192, 0, 128, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 18, 0, 0, 0, 0, 0, 0, 3, 192, 0,
379 128, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0,
380 ]);
381 for seed in sweep::seeds(1..33, 2) {
382 let mut random = seed * 2003;
383 let mut input = [0; 192];
384 for byte in &mut input {
385 random ^= random << 13;
386 random ^= random >> 7;
387 random ^= random << 17;
388 *byte = random.to_le_bytes()[0];
389 }
390 page_schedule::run(&input);
391 }
392}
393
394#[test]
395fn external_payloads_and_their_historical_references_survive_file_removal() {
396 let fixture = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
397 .join("../../corpus/native-external-assets/notebook");
398 let original = std::fs::read(fixture.join("synthetic.one")).unwrap();
399 let store = Store::parse(&original).unwrap();
400 let index = RevisionIndex::parse(&store).unwrap();
401 let pages = Document::parse(&index).unwrap().pages().unwrap();
402 assert_eq!(pages.len(), 3);
403 let selected: Vec<_> = pages.iter().map(|page| page.0).collect();
404 let prepared = ops::section_op(&original, SectionOp::Delete(selected.to_vec())).unwrap();
405 let output = std::env::var_os("ONESTORE_PAGE_REMOVAL_EXTERNAL_OUTPUT");
406 let root = output
407 .as_ref()
408 .map(std::path::PathBuf::from)
409 .unwrap_or_else(|| {
410 std::env::temp_dir().join(format!(
411 "onestore-page-removal-{}-{}",
412 std::process::id(),
413 std::time::SystemTime::now()
414 .duration_since(std::time::UNIX_EPOCH)
415 .unwrap()
416 .as_nanos()
417 ))
418 });
419 assert!(root.is_absolute());
420 std::fs::create_dir(&root).unwrap();
421 let files = root.join("synthetic_onefiles");
422 std::fs::create_dir(&files).unwrap();
423 let mut payloads = std::collections::BTreeMap::new();
424 for entry in std::fs::read_dir(fixture.join("synthetic_onefiles")).unwrap() {
425 let entry = entry.unwrap();
426 let bytes = std::fs::read(entry.path()).unwrap();
427 std::fs::write(files.join(entry.file_name()), &bytes).unwrap();
428 payloads.insert(entry.file_name(), bytes);
429 }
430 assert_eq!(payloads.len(), 3);
431 let path = root.join("synthetic.one");
432 std::fs::write(&path, &original).unwrap();
433 prepared
434 .transaction
435 .as_ref()
436 .unwrap()
437 .commit_file(&path)
438 .unwrap();
439 let written = onestore::read_file(&path).unwrap();
440 assert_eq!(written, prepared.as_bytes());
441 let current_store = Store::parse(&written).unwrap();
442 let current_index = RevisionIndex::parse(&current_store).unwrap();
443 current_index.validate_current().unwrap();
444 assert!(
445 Document::parse(&current_index)
446 .unwrap()
447 .pages()
448 .unwrap()
449 .is_empty()
450 );
451 let mut external = BTreeSet::new();
452 for (sid, space) in &index.spaces {
453 for rid in space.revisions.keys() {
454 let before = index.resolve(*sid, *rid).unwrap();
455 let after = current_index.resolve(*sid, *rid).unwrap();
456 assert_eq!(format!("{before:?}"), format!("{after:?}"));
457 for object in before.objects.values() {
458 if let Some(FileDataReference::External(reference)) =
459 object.file_reference().unwrap()
460 {
461 external.insert(reference);
462 }
463 }
464 }
465 }
466 assert_eq!(external.len(), 3);
467 for (name, bytes) in payloads {
468 assert_eq!(std::fs::read(files.join(name)).unwrap(), bytes);
469 }
470 if output.is_none() {
471 std::fs::remove_dir_all(root).unwrap();
472 }
473}