1//! Every structural edit the editor makes saves as the ops the editor recorded and rereads
2//! as the model it saved: random Enter, Backspace, Delete, Tab,
3//! Shift+Tab, list and paste edits, alone and several to a save, on every outline of the probe
4//! section and of any sections named in `CANVAS_SWEEP_SECTIONS` (`:`-separated paths). The
5//! smoke slice takes the first pages of each section.
6
7use canvas::{
8 document::TextPosition,
9 editor::{CanvasEditor, Formatting},
10 layout::TextEngine,
11};
12use onestore::{
13 Arena, ExGuid, RevisionIndex, Section, Store,
14 document::Document,
15 page::{Outline, Page, PageObject, PageParagraph, ParagraphContent},
16};
17use std::{collections::BTreeMap, path::Path};
18
19#[path = "../../onestore/tests/support/sweep.rs"]
20mod sweep;
21
22#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
23enum Kind {
24 EnterStart,
25 EnterMiddle,
26 EnterEnd,
27 Backspace,
28 Delete,
29 Tab,
30 ShiftTab,
31 TabMiddle,
32 Paste,
33 Bullets,
34 Numbering,
35}
36
37const KINDS: [Kind; 11] = [
38 Kind::EnterStart,
39 Kind::EnterMiddle,
40 Kind::EnterEnd,
41 Kind::Backspace,
42 Kind::Delete,
43 Kind::Tab,
44 Kind::ShiftTab,
45 Kind::TabMiddle,
46 Kind::Paste,
47 Kind::Bullets,
48 Kind::Numbering,
49];
50
51struct Random(u64);
52
53impl Random {
54 fn below(&mut self, bound: usize) -> usize {
55 self.0 ^= self.0 << 13;
56 self.0 ^= self.0 >> 7;
57 self.0 ^= self.0 << 17;
58 (self.0 % bound as u64) as usize
59 }
60}
61
62fn pages(section: &[u8]) -> Vec<(ExGuid, Page)> {
63 let store = Store::parse(section).unwrap();
64 let index = RevisionIndex::parse(&store).unwrap();
65 let document = Document::parse(&index).unwrap();
66 let mut spaces: Vec<_> = document
67 .pages()
68 .unwrap()
69 .into_iter()
70 .map(|(space, _)| space)
71 .collect();
72 spaces.dedup();
73 spaces
74 .into_iter()
75 .filter_map(|space| Some((space, Page::from_space(&document, space).ok()?)))
76 .collect()
77}
78
79/// Applies `kind` at a random paragraph of the focused outline; false when the editor
80/// declines it.
81fn apply(
82 editor: &mut CanvasEditor,
83 engine: &mut TextEngine,
84 kind: Kind,
85 random: &mut Random,
86) -> bool {
87 let count = editor.active_outline().document().paragraphs().count();
88 let paragraph = random.below(count);
89 let text = editor
90 .active_outline()
91 .document()
92 .paragraphs()
93 .nth(paragraph)
94 .unwrap();
95 let length = text.text().encode_utf16().count() as u32;
96 let middle = || {
97 let chars: Vec<_> = text.text().char_indices().map(|(at, _)| at).collect();
98 let at = chars.get(chars.len() / 2).copied().unwrap_or(0);
99 text.utf16_offset(at).unwrap()
100 };
101 let offset = match kind {
102 Kind::EnterStart | Kind::Backspace | Kind::Tab | Kind::ShiftTab => 0,
103 Kind::EnterEnd | Kind::Delete => length,
104 Kind::EnterMiddle | Kind::TabMiddle | Kind::Paste | Kind::Bullets | Kind::Numbering => {
105 middle()
106 }
107 };
108 let position = TextPosition { paragraph, offset };
109 if editor.select([position; 2].into()).is_err() {
110 return false;
111 }
112 let result = match kind {
113 Kind::EnterStart | Kind::EnterMiddle | Kind::EnterEnd => editor.enter(engine, false),
114 Kind::Backspace => editor.delete(engine, true).map(|_| ()),
115 Kind::Delete => editor.delete(engine, false).map(|_| ()),
116 Kind::Tab | Kind::TabMiddle => editor.tab(engine, false),
117 Kind::ShiftTab => editor.tab(engine, true),
118 Kind::Paste => editor.paste(engine, "Line one\r\nLine two", 1033),
119 Kind::Bullets => editor.format(engine, Formatting::Bullets),
120 Kind::Numbering => editor.format(engine, Formatting::Numbering),
121 };
122 result.is_ok()
123}
124
125/// Every paragraph on the page, cells' included, by identity.
126/// `page` with its tags' places in their elements' property arenas, which a read assigns,
127/// cleared.
128fn settled(mut page: Page) -> Page {
129 fn walk(list: &mut [PageParagraph]) {
130 for paragraph in list {
131 for tag in &mut paragraph.tags {
132 tag.extra_set = 0;
133 }
134 match &mut paragraph.content {
135 ParagraphContent::Text(text) => {
136 for tag in &mut text.tags {
137 tag.extra_set = 0;
138 }
139 }
140 ParagraphContent::Table(table) => {
141 for cell in table.rows.iter_mut().flat_map(|row| &mut row.cells) {
142 walk(&mut cell.paragraphs);
143 }
144 }
145 _ => {}
146 }
147 }
148 }
149 for object in &mut page.objects {
150 match object {
151 PageObject::Outline(outline) => walk(&mut outline.paragraphs),
152 PageObject::Title(title) => {
153 for outline in &mut title.outlines {
154 walk(&mut outline.paragraphs);
155 }
156 }
157 _ => {}
158 }
159 }
160 page
161}
162
163fn paragraphs(page: &Page) -> BTreeMap<ExGuid, PageParagraph> {
164 fn walk(list: &[PageParagraph], out: &mut BTreeMap<ExGuid, PageParagraph>) {
165 for paragraph in list {
166 out.insert(paragraph.id, paragraph.clone());
167 if let ParagraphContent::Table(table) = &paragraph.content {
168 for cell in table.rows.iter().flat_map(|row| &row.cells) {
169 walk(&cell.paragraphs, out);
170 }
171 }
172 }
173 }
174 let mut out = BTreeMap::new();
175 for object in &page.objects {
176 match object {
177 PageObject::Outline(outline) => walk(&outline.paragraphs, &mut out),
178 PageObject::Title(title) => {
179 for outline in &title.outlines {
180 walk(&outline.paragraphs, &mut out);
181 }
182 }
183 _ => {}
184 }
185 }
186 out
187}
188
189/// The fields two values disagree on, as `name: saved -> reread`.
190fn fields<T: serde::Serialize>(saved: &T, reread: &T) -> String {
191 let (serde_json::Value::Object(a), serde_json::Value::Object(b)) = (
192 serde_json::to_value(saved).unwrap(),
193 serde_json::to_value(reread).unwrap(),
194 ) else {
195 return "differs".into();
196 };
197 a.iter()
198 .filter(|(key, value)| b.get(*key) != Some(value))
199 .map(|(key, value)| format!("{key}: {value} -> {}", b[key]))
200 .collect::<Vec<_>>()
201 .join(", ")
202}
203
204/// The first field a reread paragraph lost, as a signature without identities.
205fn difference(saved: &Page, reread: &Page) -> String {
206 let (a, b) = (paragraphs(saved), paragraphs(reread));
207 for (id, x) in &a {
208 let Some(y) = b.get(id) else {
209 return "a paragraph identity is missing".into();
210 };
211 macro_rules! field {
212 ($($name:ident),*) => {$(
213 if x.$name != y.$name {
214 return format!(
215 "{} {:?} reread as {:?}",
216 stringify!($name),
217 x.$name,
218 y.$name
219 );
220 }
221 )*};
222 }
223 if x.lists.len() == y.lists.len() && x.lists != y.lists {
224 let shared = a
225 .values()
226 .any(|other| other.id != x.id && other.lists.iter().any(|l| x.lists.contains(l)));
227 return format!("list identities (shared {shared})");
228 }
229 if x.format != y.format {
230 return format!("paragraph format {}", fields(&x.format, &y.format));
231 }
232 field!(parent, level, style, lists, tags, collapsed);
233 if let (Some(p), Some(q)) = (x.text(), y.text()) {
234 if p.id != q.id {
235 return "text identity".into();
236 }
237 if p.text.text() != q.text.text() {
238 return "text".into();
239 }
240 if p.text.spans().len() != q.text.spans().len() {
241 return format!(
242 "{} spans reread as {}",
243 p.text.spans().len(),
244 q.text.spans().len()
245 );
246 }
247 for (s, t) in p.text.spans().iter().zip(q.text.spans()) {
248 if s.format != t.format {
249 return format!("span {}", fields(&s.format, &t.format));
250 }
251 }
252 if p != q {
253 return "text object".into();
254 }
255 }
256 if x != y {
257 return "content".into();
258 }
259 }
260 if a.len() != b.len() {
261 return "extra paragraphs".into();
262 }
263 let kind = |definition: &onestore::page::Definition| match &definition.kind {
264 onestore::document::Kind::List { .. } => "list",
265 onestore::document::Kind::Style { .. } => "style",
266 _ => "tag",
267 };
268 for (id, definition) in &saved.definitions {
269 match reread.definitions.get(id) {
270 None => return format!("{} definition missing", kind(definition)),
271 Some(other) if other != definition => {
272 return format!(
273 "{} definition {}",
274 kind(definition),
275 fields(&definition.format, &other.format)
276 );
277 }
278 _ => {}
279 }
280 }
281 if let Some(extra) = reread
282 .definitions
283 .iter()
284 .find(|(id, _)| !saved.definitions.contains_key(id))
285 {
286 return format!("extra {} definition", kind(extra.1));
287 }
288 for (x, y) in saved.objects.iter().zip(&reread.objects) {
289 if x != y {
290 return match (x, y) {
291 (PageObject::Outline(x), PageObject::Outline(y)) => format!(
292 "outline {}",
293 fields(
294 &Outline {
295 paragraphs: Vec::new(),
296 ..x.clone()
297 },
298 &Outline {
299 paragraphs: Vec::new(),
300 ..y.clone()
301 }
302 )
303 ),
304 _ => format!("page object {x:?}\n reread as {y:?}"),
305 };
306 }
307 }
308 format!(
309 "page {} ({} objects reread as {})",
310 fields(
311 &Page {
312 objects: Vec::new(),
313 definitions: Default::default(),
314 ..saved.clone()
315 },
316 &Page {
317 objects: Vec::new(),
318 definitions: Default::default(),
319 ..reread.clone()
320 }
321 ),
322 saved.objects.len(),
323 reread.objects.len()
324 )
325}
326
327#[derive(Default)]
328struct Tally {
329 samples: usize,
330 failures: BTreeMap<String, Vec<String>>,
331}
332
333fn sweep(
334 path: &Path,
335 random: &mut Random,
336 engine: &mut TextEngine,
337 tally: &mut Tally,
338 count: usize,
339) {
340 let section = std::fs::read(path).unwrap();
341 for (space, page) in pages(&section).into_iter().take(count) {
342 let Ok(editor) = CanvasEditor::from_page(page.clone(), engine) else {
343 continue;
344 };
345 let Ok(before) = editor.page() else {
346 continue;
347 };
348 let outlines: Vec<ExGuid> = editor
349 .outlines()
350 .iter()
351 .filter(|outline| !outline.title)
352 .map(|outline| outline.id)
353 .collect();
354 for outline in outlines {
355 let plans: Vec<Vec<Kind>> = KINDS
356 .iter()
357 .map(|kind| vec![*kind])
358 .chain((0..3).map(|_| {
359 (0..2 + random.below(3))
360 .map(|_| KINDS[random.below(KINDS.len())])
361 .collect()
362 }))
363 .collect();
364 for plan in plans {
365 let mut editor = CanvasEditor::from_page(page.clone(), engine).unwrap();
366 editor.focus_outline(outline).unwrap();
367 let mut applied = 0;
368 for kind in &plan {
369 applied += usize::from(apply(&mut editor, engine, *kind, random));
370 }
371 let Ok(after) = editor.page() else { continue };
372 if applied == 0 || after == before {
373 continue;
374 }
375 tally.samples += 1;
376 if let Some(dir) = std::env::var_os("CANVAS_SWEEP_DUMP") {
377 let value = serde_json::json!({"section": path.canonicalize().unwrap(), "space": space, "page": after});
378 std::fs::write(
379 Path::new(&dir).join(format!("{}.json", tally.samples)),
380 serde_json::to_vec(&value).unwrap(),
381 )
382 .unwrap();
383 }
384 // An untitled page's title follows its first line as the writer stores it.
385 let rereads = |reread: &Page| {
386 settled(Page {
387 title: after.title.clone(),
388 ..reread.clone()
389 }) == settled(after.clone())
390 };
391 let arena = Arena::default();
392 let mut stored = Section::open(&arena, section.clone()).unwrap();
393 let edit = onestore::op::Edit {
394 at: 133_000_000_000_000_000,
395 ops: editor
396 .take_ops()
397 .unwrap()
398 .into_iter()
399 .map(|op| onestore::op::Op::Page { space, op })
400 .collect(),
401 };
402 let failure = match stored.apply("Sweep", &edit) {
403 Err(error) => Some(format!("ops: {error}")),
404 Ok(()) => {
405 let reread = stored.page(space).unwrap();
406 (!rereads(&reread)).then(|| format!("ops: {}", difference(&after, &reread)))
407 }
408 };
409 if let Some(failure) = failure {
410 tally.failures.entry(failure).or_default().push(format!(
411 "{plan:?} on {:?} in {}",
412 page.title,
413 path.display()
414 ));
415 }
416 }
417 }
418 }
419}
420
421#[test]
422fn structural_edits_reread_as_saved() {
423 let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
424 let mut paths = vec![root.join("corpus/structural-probe/probe.one")];
425 if let Some(extra) = std::env::var_os("CANVAS_SWEEP_SECTIONS") {
426 paths.extend(std::env::split_paths(&extra));
427 }
428 let full = sweep::full();
429 let count = if full.is_some() { usize::MAX } else { 24 };
430 let mut random = Random(1 + full.unwrap_or(0));
431 let mut engine = TextEngine::default();
432 let mut tally = Tally::default();
433 for path in &paths {
434 sweep(path, &mut random, &mut engine, &mut tally, count);
435 }
436 let failed: usize = tally.failures.values().map(Vec::len).sum();
437 let mut report = String::new();
438 for (failure, samples) in &tally.failures {
439 report += &format!("{} x {failure}\n e.g. {}\n", samples.len(), samples[0]);
440 if std::env::var_os("CANVAS_SWEEP_ALL").is_some() {
441 for sample in &samples[1..] {
442 report += &format!(" e.g. {sample}\n");
443 }
444 }
445 }
446 println!(
447 "{} of {} edits reread as saved\n{report}",
448 tally.samples - failed,
449 tally.samples
450 );
451 assert_eq!(failed, 0, "{report}");
452}
453
454/// Where a scripted edit puts the caret: in the first paragraph containing the text, before
455/// or after it.
456enum Caret {
457 Before(&'static str),
458 After(&'static str),
459}
460
461/// A key or command the scripted edits press.
462enum Press {
463 Enter,
464 Backspace,
465 Delete,
466 Tab,
467 ShiftTab,
468 Paste(&'static str),
469 Numbering,
470}
471
472/// Opens the first page titled `title`, presses each `(caret, key)` in one editing session and
473/// saves the page once, checking that it rereads as saved.
474fn edit(section: &mut Vec<u8>, title: &str, keys: &[(Caret, Press)], engine: &mut TextEngine) {
475 let (space, page) = pages(section)
476 .into_iter()
477 .find(|(_, page)| page.title == title)
478 .unwrap();
479 let mut editor = CanvasEditor::from_page(page, engine).unwrap();
480 for (caret, key) in keys {
481 let needle = match caret {
482 Caret::Before(needle) | Caret::After(needle) => *needle,
483 };
484 let outline = editor
485 .outlines()
486 .iter()
487 .find(|outline| {
488 outline
489 .document()
490 .paragraphs()
491 .any(|p| p.text().contains(needle))
492 })
493 .unwrap()
494 .id;
495 editor.focus_outline(outline).unwrap();
496 let (paragraph, text) = editor
497 .active_outline()
498 .document()
499 .paragraphs()
500 .enumerate()
501 .find(|(_, p)| p.text().contains(needle))
502 .unwrap();
503 let at = text.text().find(needle).unwrap()
504 + match caret {
505 Caret::Before(_) => 0,
506 Caret::After(_) => needle.len(),
507 };
508 let offset = text.utf16_offset(at).unwrap();
509 editor
510 .select([TextPosition { paragraph, offset }; 2].into())
511 .unwrap();
512 match key {
513 Press::Enter => editor.enter(engine, false).unwrap(),
514 Press::Backspace => assert!(editor.delete(engine, true).unwrap()),
515 Press::Delete => assert!(editor.delete(engine, false).unwrap()),
516 Press::Tab => editor.tab(engine, false).unwrap(),
517 Press::ShiftTab => editor.tab(engine, true).unwrap(),
518 Press::Paste(text) => editor.paste(engine, text, 1033).unwrap(),
519 Press::Numbering => editor.format(engine, Formatting::Numbering).unwrap(),
520 }
521 }
522 let after = editor.page().unwrap();
523 let arena = Arena::default();
524 let mut stored = Section::open(&arena, section.to_vec()).unwrap();
525 let edit = onestore::op::Edit {
526 at: 133_000_000_000_000_000,
527 ops: editor
528 .take_ops()
529 .unwrap()
530 .into_iter()
531 .map(|op| onestore::op::Op::Page { space, op })
532 .collect(),
533 };
534 stored.apply("Snowbound", &edit).unwrap();
535 stored.seal().unwrap();
536 let saved = stored.image();
537 let (_, reread) = pages(&saved)
538 .into_iter()
539 .find(|(id, _)| *id == space)
540 .unwrap();
541 assert_eq!(
542 settled(Page {
543 title: after.title.clone(),
544 ..reread.clone()
545 }),
546 settled(after),
547 "{title}"
548 );
549 *section = saved;
550}
551
552/// Group indents, pasted lines, splits and joins around note tags, headings, hyperlinks and
553/// a recording link, and several of them in one save, written into the probe section and two
554/// native feature sections. `CANVAS_STRUCTURAL_EXPORT` names a new directory receiving them
555/// for a cold OneNote reopen (`corpus/structural-edit`).
556#[test]
557fn structural_edits_write_what_onenote_reads() {
558 use Caret::{After, Before};
559 let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
560 let read = |path: &str| std::fs::read(root.join(path)).unwrap();
561 let mut engine = TextEngine::default();
562 let mut probe = read("corpus/structural-probe/probe.one");
563 for (title, keys) in [
564 ("c6-plain", vec![(Before("Above"), Press::Tab)]),
565 ("c6-first", vec![(Before("Target text"), Press::ShiftTab)]),
566 (
567 "c10-tab-parent",
568 vec![(Before("Omega"), Press::Tab), (Before("Child"), Press::Tab)],
569 ),
570 ("c1-todo", vec![(Before("Target text"), Press::Enter)]),
571 (
572 "c1-bullet",
573 vec![(After("Target "), Press::Paste("Line one\r\nLine two"))],
574 ),
575 ("c9-h1", vec![(After("ing"), Press::Enter)]),
576 (
577 "c2-q",
578 vec![
579 (Before("Target "), Press::Enter),
580 (Before("Target "), Press::Backspace),
581 (Before("Xtext"), Press::Backspace),
582 ],
583 ),
584 (
585 "c3-q",
586 vec![
587 (After("Above"), Press::Enter),
588 (After("Above"), Press::Delete),
589 ],
590 ),
591 (
592 "c4-imp",
593 vec![
594 (After("First li"), Press::Paste("Line one\r\nLine two")),
595 (Before("Line one"), Press::Backspace),
596 ],
597 ),
598 ("c1-q", vec![(After("Target "), Press::Tab)]),
599 (
600 "c1-rem",
601 vec![
602 (Before("Target text"), Press::Numbering),
603 (Before("Target text"), Press::Tab),
604 (Before("Target text"), Press::ShiftTab),
605 ],
606 ),
607 ] {
608 edit(&mut probe, title, &keys, &mut engine);
609 }
610 let mut features = read("corpus/m6/native-features-01/notebook/Features.one");
611 edit(
612 &mut features,
613 "Files and recording",
614 &[(After("Recording "), Press::Enter)],
615 &mut engine,
616 );
617 let mut links = read("corpus/m6/native-link-controls-01/notebook/Links.one");
618 edit(
619 &mut links,
620 "Hyperlink boundary controls",
621 &[
622 (
623 Before("\u{fddf}HYPERLINK \"https://example.invalid/label/0"),
624 Press::Enter,
625 ),
626 (After("label/4\"label"), Press::Delete),
627 ],
628 &mut engine,
629 );
630 if let Some(directory) = std::env::var_os("CANVAS_STRUCTURAL_EXPORT") {
631 let directory = Path::new(&directory);
632 std::fs::create_dir(directory).unwrap();
633 for (name, bytes) in [
634 ("probe.one", &probe),
635 ("Features.one", &features),
636 ("Links.one", &links),
637 ] {
638 std::fs::write(directory.join(name), bytes).unwrap();
639 }
640 }
641}