1#[path = "support/ops.rs"]
2mod ops;
3use onestore::{
4 RevisionIndex, Store,
5 document::Document,
6 page::{Math, Page, PageObject},
7};
8
9/// Three equations authored through OneNote 2010's equation editor, whose COM read exports
10/// MathML (`corpus/m6/native-math-01`).
11const NATIVE: &[u8] = include_bytes!("../../../corpus/m6/native-math-01/notebook/synthetic.one");
12const NATIVE_READ: &str = include_str!("../../../corpus/m6/native-math-01/read/page-000.xml");
13/// Five more expressions typed through the equation editor by `tools/native_math.py`:
14/// subscripts, a sub-superscript, parentheses, an integral and a sum with limits.
15const EDITOR: &[u8] = include_bytes!("../../../corpus/math-edit/native-editor/notebook/links.one");
16const EDITOR_READ: &str = include_str!("../../../corpus/math-edit/native-editor/read/page-000.xml");
17/// A second editor session: square and n-th roots, a plain fraction, a limit, a product,
18/// square brackets, an overbar and a hat.
19const EDITOR_2: &[u8] =
20 include_bytes!("../../../corpus/math-edit/native-editor-2/notebook/links.one");
21const EDITOR_2_READ: &str =
22 include_str!("../../../corpus/math-edit/native-editor-2/read/page-000.xml");
23/// A third session: a matrix, an equation array, upper and lower limit objects, boxes, a
24/// nested fraction and a sum with only an upper limit.
25const EDITOR_3: &[u8] =
26 include_bytes!("../../../corpus/math-edit/native-editor-3/notebook/links.one");
27const EDITOR_3_READ: &str =
28 include_str!("../../../corpus/math-edit/native-editor-3/read/page-000.xml");
29/// A fourth session repeating most of the others and adding an exponent, Greek letters, a
30/// fraction of sums, a Pythagorean sum, a limit applied as a function and braces.
31const EDITOR_4: &[u8] =
32 include_bytes!("../../../corpus/math-edit/native-editor-4/notebook/links.one");
33
34fn native_mathml(read: &str) -> Vec<String> {
35 read.split("<mml:math")
36 .skip(1)
37 .map(|rest| {
38 let body = &rest[rest.find('>').unwrap() + 1..rest.find("</mml:math>").unwrap()];
39 let mut decoded = String::new();
40 let mut remainder = body;
41 while let Some(start) = remainder.find("&#") {
42 decoded.push_str(&remainder[..start]);
43 let end = start + remainder[start..].find(';').unwrap();
44 let code: u32 = remainder[start + 2..end].parse().unwrap();
45 decoded.push(char::from_u32(code).unwrap());
46 remainder = &remainder[end + 1..];
47 }
48 decoded.push_str(remainder);
49 decoded
50 })
51 .collect()
52}
53
54fn rendered_equations(bytes: &[u8], title: &str) -> Vec<String> {
55 let store = Store::parse(bytes).unwrap();
56 let index = RevisionIndex::parse(&store).unwrap();
57 let document = Document::parse(&index).unwrap();
58 let page = document
59 .pages()
60 .unwrap()
61 .into_iter()
62 .map(|(space, _)| Page::from_space(&document, space).unwrap())
63 .find(|page| page.title == title)
64 .unwrap();
65 let mut rendered = Vec::new();
66 for object in &page.objects {
67 let PageObject::Outline(outline) = object else {
68 continue;
69 };
70 for paragraph in &outline.paragraphs {
71 let Some(text) = paragraph.text() else {
72 continue;
73 };
74 if !Math::is_equation(&text.text) {
75 continue;
76 }
77 // Inline math follows ordinary text in the same paragraph, and a line break
78 // divides an equation: each run of math is an equation of its own.
79 let mut at = 0;
80 let mut zones: Vec<Vec<(String, onestore::document::Format)>> = vec![Vec::new()];
81 for span in text.text.spans() {
82 let piece = text.text.text()[at..span.end].to_owned();
83 at = span.end;
84 if span.format.math == Some(true) {
85 zones.last_mut().unwrap().push((piece, span.format.clone()));
86 } else if !zones.last().unwrap().is_empty() {
87 zones.push(Vec::new());
88 }
89 }
90 for zone in zones.into_iter().filter(|zone| !zone.is_empty()) {
91 let equation = onestore::page::Paragraph::from_runs(zone);
92 rendered.push(Math::mathml(&Math::parse(&equation).unwrap()));
93 }
94 }
95 }
96 rendered
97}
98
99#[test]
100fn native_equations_render_to_the_mathml_onenote_exports() {
101 assert_eq!(
102 rendered_equations(NATIVE, "Equation controls"),
103 native_mathml(NATIVE_READ)
104 );
105}
106
107/// Enter inside equations (`tools/native_enter.py`): a line break dividing one, and an
108/// argument made an equation array.
109const ENTER: &[u8] = include_bytes!("../../../corpus/math-edit/native-enter/notebook/links.one");
110const ENTER_READ: &str = include_str!("../../../corpus/math-edit/native-enter/read/page-000.xml");
111
112#[test]
113fn equation_editor_expressions_render_to_the_mathml_onenote_exports() {
114 assert_eq!(
115 rendered_equations(ENTER, "Read about Rust the Rust site"),
116 native_mathml(ENTER_READ)
117 );
118 assert_eq!(
119 rendered_equations(EDITOR, "Read about Rust the Rust site"),
120 native_mathml(EDITOR_READ)
121 );
122 assert_eq!(
123 rendered_equations(EDITOR_2, "Read about Rust the Rust site"),
124 native_mathml(EDITOR_2_READ)
125 );
126 assert_eq!(
127 rendered_equations(EDITOR_3, "Read about Rust the Rust site"),
128 native_mathml(EDITOR_3_READ)
129 );
130}
131
132fn object(kind: u32, symbols: &str, arguments: Vec<Vec<Math>>) -> Math {
133 Math::Object {
134 kind,
135 symbols: symbols.chars().collect(),
136 columns: None,
137 arguments,
138 }
139}
140
141fn built_equations() -> Vec<(Vec<Math>, &'static str)> {
142 vec![
143 (
144 vec![
145 object(
146 31,
147 "^",
148 vec![vec![Math::Identifier('x')], vec![Math::Number("2".into())]],
149 ),
150 Math::Operator('+'),
151 Math::Number("1".into()),
152 ],
153 "<mml:msup><mml:mi>x</mml:mi><mml:mn>2</mml:mn></mml:msup><mml:mo>+</mml:mo><mml:mn>1</mml:mn>",
154 ),
155 (
156 vec![object(
157 16,
158 "/",
159 vec![
160 vec![
161 Math::Identifier('a'),
162 Math::Operator('+'),
163 Math::Identifier('b'),
164 ],
165 vec![object(
166 31,
167 "^",
168 vec![vec![Math::Identifier('c')], vec![Math::Number("2".into())]],
169 )],
170 ],
171 )],
172 "<mml:mfrac><mml:mrow><mml:mi>a</mml:mi><mml:mo>+</mml:mo><mml:mi>b</mml:mi></mml:mrow><mml:mrow><mml:msup><mml:mi>c</mml:mi><mml:mn>2</mml:mn></mml:msup></mml:mrow></mml:mfrac>",
173 ),
174 ]
175}
176
177#[test]
178fn equations_built_from_the_tree_store_the_native_linear_form_and_parse_back() {
179 for (nodes, mathml) in built_equations() {
180 let paragraph = Math::paragraph(&nodes, &onestore::document::Format::default());
181 assert!(Math::is_equation(&paragraph));
182 assert_eq!(Math::parse(&paragraph).unwrap(), nodes);
183 assert_eq!(Math::mathml(&nodes), mathml);
184 }
185 let paragraph = Math::paragraph(&built_equations()[0].0, &Default::default());
186 assert_eq!(paragraph.text(), "\u{fdd0}\u{1d465}\u{fdee}2\u{fdef}+1");
187 let kinds: Vec<(Option<u32>, Option<u32>)> = paragraph
188 .spans()
189 .iter()
190 .map(|s| {
191 let object = s.format.math_object.as_ref().unwrap();
192 (Some(object.kind), object.arguments)
193 })
194 .collect();
195 assert_eq!(
196 kinds,
197 [
198 (Some(31), Some(2)),
199 (Some(31), None),
200 (Some(31), Some(1)),
201 (Some(0x9000_0000), None)
202 ]
203 );
204}
205
206/// `ONESTORE_MATH_EXPORT` names a new directory receiving the candidate for a cold reopen:
207/// every native equation copied onto a fresh page, then two built from the tree.
208#[test]
209fn equations_are_written_and_read_back() {
210 use onestore::page::{ParagraphContent, TextObject, text::new_id};
211 let source = onestore::create_section("math.one", "Equations below", "Author").unwrap();
212 let store = Store::parse(&source).unwrap();
213 let index = RevisionIndex::parse(&store).unwrap();
214 let document = Document::parse(&index).unwrap();
215 let (space, _) = document.pages().unwrap()[0];
216 let before = Page::from_space(&document, space).unwrap();
217 let mut after = before.clone();
218 let mut expected_mathml = Vec::new();
219 let mut paragraphs = Vec::new();
220 for (bytes, title, read) in [
221 (NATIVE, "Equation controls", NATIVE_READ),
222 (EDITOR, "Read about Rust the Rust site", EDITOR_READ),
223 ] {
224 expected_mathml.extend(native_mathml(read));
225 let store = Store::parse(bytes).unwrap();
226 let index = RevisionIndex::parse(&store).unwrap();
227 let document = Document::parse(&index).unwrap();
228 let page = document
229 .pages()
230 .unwrap()
231 .into_iter()
232 .map(|(space, _)| Page::from_space(&document, space).unwrap())
233 .find(|page| page.title == title)
234 .unwrap();
235 for object in &page.objects {
236 let PageObject::Outline(outline) = object else {
237 continue;
238 };
239 for paragraph in &outline.paragraphs {
240 if paragraph.text().is_some_and(|t| Math::is_equation(&t.text)) {
241 paragraphs.push(paragraph.clone());
242 }
243 }
244 }
245 }
246 for (nodes, mathml) in built_equations() {
247 let mut paragraph = paragraphs[0].clone();
248 paragraph.content = ParagraphContent::Text(TextObject {
249 id: new_id().unwrap(),
250 date_field: None,
251 text: Math::paragraph(&nodes, &Default::default()),
252 tags: Vec::new(),
253 });
254 paragraphs.push(paragraph);
255 expected_mathml.push(mathml.to_owned());
256 }
257 let outline = after
258 .objects
259 .iter_mut()
260 .find_map(|object| match object {
261 PageObject::Outline(outline) if !outline.title => Some(outline),
262 _ => None,
263 })
264 .unwrap();
265 for mut paragraph in paragraphs {
266 paragraph.id = new_id().unwrap();
267 paragraph.parent = None;
268 paragraph.level = 1;
269 paragraph.lists.clear();
270 paragraph.tags.clear();
271 paragraph.style = None;
272 paragraph.collapsed = false;
273 if let ParagraphContent::Text(text) = &mut paragraph.content {
274 text.id = new_id().unwrap();
275 text.tags.clear();
276 // Paragraph-level spacing came from the source page's paragraph style.
277 let mut at = 0;
278 text.text =
279 onestore::page::Paragraph::from_runs(text.text.spans().iter().map(|span| {
280 let piece = text.text.text()[at..span.end].to_owned();
281 at = span.end;
282 let mut format = span.format.clone();
283 format.space_before = None;
284 format.space_after = None;
285 format.line_spacing = None;
286 (piece, format)
287 }));
288 }
289 outline.paragraphs.push(paragraph);
290 }
291 let written = ops::saved(&source, space, &after).unwrap();
292 let store = Store::parse(written.as_slice()).unwrap();
293 let index = RevisionIndex::parse(&store).unwrap();
294 let document = Document::parse(&index).unwrap();
295 let stored = Page::from_space(&document, space).unwrap();
296 let mut expected = after.clone();
297 expected.title = stored.title.clone();
298 let body = |page: &Page| {
299 page.objects
300 .iter()
301 .find_map(|object| match object {
302 PageObject::Outline(outline) if !outline.title => Some(outline.paragraphs.clone()),
303 _ => None,
304 })
305 .unwrap()
306 };
307 for (index, (a, b)) in body(&stored).iter().zip(body(&expected)).enumerate() {
308 if a != &b {
309 let (ta, tb) = (a.text().unwrap(), b.text().unwrap());
310 assert_eq!(ta.text.text(), tb.text.text(), "paragraph {index} text");
311 for (sa, sb) in ta.text.spans().iter().zip(tb.text.spans()) {
312 assert_eq!(sa, sb, "paragraph {index} span");
313 }
314 assert_eq!(
315 ta.text.spans().len(),
316 tb.text.spans().len(),
317 "paragraph {index} spans"
318 );
319 assert_eq!(a, &b, "paragraph {index}");
320 }
321 }
322 assert_eq!(stored, expected);
323 assert_eq!(
324 rendered_equations(written.as_slice(), &stored.title),
325 expected_mathml
326 );
327 if let Some(directory) = std::env::var_os("ONESTORE_MATH_EXPORT") {
328 let directory = std::path::PathBuf::from(directory);
329 std::fs::create_dir(&directory).unwrap();
330 std::fs::write(directory.join("math.one"), written.as_slice()).unwrap();
331 std::fs::write(
332 directory.join("Open Notebook.onetoc2"),
333 onestore::create_table_of_contents(
334 "Open Notebook.onetoc2",
335 &[("math.one", store.header.file_id)],
336 )
337 .unwrap(),
338 )
339 .unwrap();
340 std::fs::write(
341 directory.join("expected-mathml.json"),
342 serde_json::to_string_pretty(&expected_mathml).unwrap(),
343 )
344 .unwrap();
345 }
346}
347
348/// Backspacing through inline math leaves plain text in the model, but the stored text object
349/// still holds the math runs, which the ordinary text edit refuses.
350#[test]
351fn a_paragraph_emptied_of_its_inline_math_saves() {
352 let store = Store::parse(NATIVE).unwrap();
353 let index = RevisionIndex::parse(&store).unwrap();
354 let document = Document::parse(&index).unwrap();
355 let (space, before) = document
356 .pages()
357 .unwrap()
358 .into_iter()
359 .map(|(space, _)| (space, Page::from_space(&document, space).unwrap()))
360 .find(|(_, page)| page.title == "Equation controls")
361 .unwrap();
362 let mut after = before.clone();
363 let text = after
364 .objects
365 .iter_mut()
366 .filter_map(|object| match object {
367 PageObject::Outline(outline) => Some(outline),
368 _ => None,
369 })
370 .flat_map(|outline| &mut outline.paragraphs)
371 .filter_map(|paragraph| match &mut paragraph.content {
372 onestore::page::ParagraphContent::Text(text) => Some(text),
373 _ => None,
374 })
375 .find(|text| text.text.text() == "Inline 𝛼+𝛽")
376 .unwrap();
377 let plain = text.text.spans()[0].format.clone();
378 assert_ne!(plain.math, Some(true));
379 text.text = onestore::page::Paragraph::new(String::new(), plain);
380 let written = ops::saved(NATIVE, space, &after).unwrap();
381 let store = Store::parse(written.as_slice()).unwrap();
382 let index = RevisionIndex::parse(&store).unwrap();
383 let document = Document::parse(&index).unwrap();
384 assert_eq!(Page::from_space(&document, space).unwrap(), after);
385}
386
387type MathRun = (String, Option<onestore::document::MathObject>, Option<bool>);
388
389/// The math runs of each equation paragraph on the page titled `title`: text, run data and
390/// whether the run is italic, which is what the equation editor decides.
391fn stored_equations(bytes: &[u8], title: &str) -> Vec<Vec<MathRun>> {
392 let store = Store::parse(bytes).unwrap();
393 let index = RevisionIndex::parse(&store).unwrap();
394 let document = Document::parse(&index).unwrap();
395 let page = document
396 .pages()
397 .unwrap()
398 .into_iter()
399 .map(|(space, _)| Page::from_space(&document, space).unwrap())
400 .find(|page| page.title == title)
401 .unwrap();
402 let mut equations = Vec::new();
403 for object in &page.objects {
404 let PageObject::Outline(outline) = object else {
405 continue;
406 };
407 for paragraph in &outline.paragraphs {
408 if let Some(text) = paragraph
409 .text()
410 .filter(|text| Math::is_equation(&text.text))
411 {
412 equations.push(runs(&text.text));
413 }
414 }
415 }
416 equations
417}
418
419fn runs(paragraph: &onestore::page::Paragraph) -> Vec<MathRun> {
420 let mut at = 0;
421 paragraph
422 .spans()
423 .iter()
424 .filter_map(|span| {
425 let text = paragraph.text()[at..span.end].to_owned();
426 at = span.end;
427 (span.format.math == Some(true))
428 .then(|| (text, span.format.math_object.clone(), span.format.italic))
429 })
430 .collect()
431}
432
433/// What OneNote 2010's equation editor stored for text typed into it, each followed by the
434/// space that builds it up, in the order the equations appear; `None` skips an equation the
435/// editor built in a way this reading does not follow.
436const TYPED: &[(&[u8], &[Option<&str>])] = &[
437 (
438 EDITOR,
439 &[
440 Some("a_1+b_2"),
441 Some("x_i^2"),
442 Some("(a+b)"),
443 Some("\\int_0^1 x dx"),
444 Some("\\sum_(i=1)^n i"),
445 ],
446 ),
447 (
448 EDITOR_2,
449 &[
450 Some("\\sqrt x+1"),
451 Some("\\cbrt(x)"),
452 Some("\\sqrt(n&x)"),
453 Some("a/b"),
454 Some("\\lim_(x\\to 0) f(x)"),
455 Some("\\prod_(k=1)^n k"),
456 Some("[a+b]"),
457 Some("\\overline(x)"),
458 Some("x\\hat"),
459 ],
460 ),
461 (
462 EDITOR_3,
463 &[
464 Some("\\matrix(1&2@3&4)"),
465 Some("\\eqarray(x&=1@y&=2)"),
466 Some("x\\above 2"),
467 Some("x\\below 2"),
468 Some("\\box(x)"),
469 Some("\\rect(x)"),
470 // `\underline` became an unrelated symbol.
471 None,
472 Some("\\iint x dx dy"),
473 Some("f(x)/(x^2+1)"),
474 Some("\\sum^n x"),
475 ],
476 ),
477 (
478 EDITOR_4,
479 &[
480 Some("a_1+b_2"),
481 // The editor was still starting this one and kept its placeholder.
482 None,
483 Some("\\sum_(i=1)^n i"),
484 Some("\\int_0^1 x dx"),
485 Some("\\sqrt x+1"),
486 Some("\\cbrt(x)"),
487 Some("\\sqrt(n&x)"),
488 Some("a/b"),
489 Some("\\prod_(k=1)^n k"),
490 Some("[a+b]"),
491 Some("\\overline(x)"),
492 Some("x\\hat"),
493 Some("\\matrix(1&2@3&4)"),
494 Some("\\eqarray(x&=1@y&=2)"),
495 Some("x\\above 2"),
496 Some("x\\below 2"),
497 Some("\\box(x)"),
498 Some("\\rect(x)"),
499 Some("f(x)/(x^2+1)"),
500 Some("\\sum^n x"),
501 Some("e^(x+1)"),
502 Some("\\alpha+\\beta"),
503 Some("(a+b)/(c+d)"),
504 Some("x^2+y^2=z^2"),
505 // A function applied to a limit, an object this reading does not build.
506 None,
507 Some("{a+b}"),
508 ],
509 ),
510];
511
512#[test]
513fn typed_linear_text_builds_up_as_the_equation_editor_stores_it() {
514 for (bytes, typed) in TYPED {
515 let stored = stored_equations(bytes, "Read about Rust the Rust site");
516 assert_eq!(stored.len(), typed.len());
517 for (stored, typed) in stored.iter().zip(*typed) {
518 let Some(typed) = typed else {
519 continue;
520 };
521 let built = Math::paragraph(
522 &Math::from_linear(&format!("{typed} ")),
523 &onestore::document::Format::default(),
524 );
525 assert_eq!(&runs(&built), stored, "{typed}");
526 }
527 }
528}
529
530/// Every native equation reads back from the linear text written for it, except the two
531/// the linear format here cannot write: a limit applied as a function and an empty
532/// equation's placeholder (`EDITOR_4`).
533#[test]
534fn native_equations_survive_their_linear_form() {
535 let mut unwritten = 0;
536 for (bytes, title) in [
537 (NATIVE, "Equation controls"),
538 (EDITOR, "Read about Rust the Rust site"),
539 (EDITOR_2, "Read about Rust the Rust site"),
540 (EDITOR_3, "Read about Rust the Rust site"),
541 (EDITOR_4, "Read about Rust the Rust site"),
542 ] {
543 for stored in stored_equations(bytes, title) {
544 let paragraph = onestore::page::Paragraph::from_runs(stored.iter().map(
545 |(text, object, italic)| {
546 let format = onestore::document::Format {
547 math: Some(true),
548 math_object: object.clone(),
549 italic: *italic,
550 ..Default::default()
551 };
552 (text.clone(), format)
553 },
554 ));
555 let Some(linear) = Math::linear(&Math::parse(&paragraph).unwrap()) else {
556 unwritten += 1;
557 continue;
558 };
559 let rebuilt = Math::paragraph(&Math::from_linear(&linear), &Default::default());
560 // Linear text without objects stores no run data until it is built up.
561 if stored.iter().all(|(_, object, _)| object.is_none()) {
562 assert_eq!(rebuilt.text(), paragraph.text());
563 continue;
564 }
565 assert_eq!(runs(&rebuilt), stored, "{linear}");
566 }
567 }
568 assert_eq!(unwritten, 2);
569}
570
571/// OneNote 2010's Linear on equations typed through its editor (`corpus/math-edit/native-linear`):
572/// the tree built from the typed text writes as the linear text OneNote showed and stored,
573/// and that text builds up to the same tree.
574#[test]
575fn equations_write_the_linear_text_onenote_shows() {
576 const LINEAR: &[u8] =
577 include_bytes!("../../../corpus/math-edit/native-linear/notebook/links.one");
578 let typed = [
579 "x^2+1",
580 "(a+b)/(c+d)",
581 "a_1+b_2",
582 "x_i^2",
583 "\\sum_(i=1)^n i",
584 "\\int_0^1 x dx",
585 "\\sqrt x+1",
586 "\\cbrt(x)",
587 "\\sqrt(n&x)",
588 "a/b",
589 "\\prod_(k=1)^n k",
590 "[a+b]",
591 "\\overline(x)",
592 "x\\hat",
593 "\\matrix(1&2@3&4)",
594 "\\eqarray(x&=1@y&=2)",
595 "x\\above 2",
596 "x\\below 2",
597 "\\box(x)",
598 "\\rect(x)",
599 "f(x)/(x^2+1)",
600 "\\sum^n x",
601 "e^(x+1)",
602 "\\alpha+\\beta",
603 ];
604 let shown: Vec<String> = stored_equations(LINEAR, "Read about Rust the Rust siteR")
605 .into_iter()
606 .map(|runs| runs.into_iter().map(|(text, ..)| text).collect())
607 .collect();
608 assert_eq!(shown.len(), typed.len());
609 for (shown, typed) in shown.iter().zip(typed) {
610 let built = Math::from_linear(&format!("{typed} "));
611 assert_eq!(
612 Math::linear(&built).as_deref(),
613 Some(shown.as_str()),
614 "{typed}"
615 );
616 let base = onestore::document::Format::default();
617 assert_eq!(
618 Math::paragraph(&Math::from_linear(shown), &base),
619 Math::paragraph(&built, &base),
620 "{typed}"
621 );
622 }
623}