1//! Every place the canvas shows a page's content as a placeholder, draws it read-only, or
2//! refuses an ordinary edit as content it cannot edit, across the active pages of every
3//! section under the given directories, grouped by kind as Markdown. Identical sections
4//! count once.
5//!
6//! `cargo run --release -p canvas --features gpu --example unsupported_inventory -- corpus`
7
8use canvas::{
9 document::TextPosition,
10 editor::{CanvasEditor, EditorError, TextOutline},
11 layout::{LayoutError, TextEngine},
12 outline::Arrange,
13};
14use onestore::{
15 Arena, ExGuid, Section,
16 page::{
17 Image, Outline, Page, PageObject, PageParagraph, Paragraph, ParagraphContent,
18 text::{Affinity, EditError},
19 },
20};
21use std::{
22 collections::{BTreeMap, BTreeSet, hash_map::DefaultHasher},
23 hash::{Hash, Hasher},
24 path::{Path, PathBuf},
25};
26
27#[derive(Default)]
28struct Inventory {
29 /// Kind to the places it occurs, as `section — page`.
30 kinds: BTreeMap<(Stage, String), Vec<String>>,
31 sections: usize,
32 pages: usize,
33}
34
35#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
36enum Stage {
37 /// Drawn as an "Unsupported content" (or "Image unavailable") box.
38 Placeholder,
39 /// Drawn as stored, but the editor cannot hold it.
40 ReadOnly,
41 /// An ordinary edit the editor refuses as content it cannot edit.
42 Refused,
43 /// The page or section does not open at all.
44 Failure,
45}
46
47impl Inventory {
48 fn add(&mut self, stage: Stage, kind: impl Into<String>, place: &str) {
49 self.kinds
50 .entry((stage, kind.into()))
51 .or_default()
52 .push(place.to_owned());
53 }
54}
55
56/// MS-ONE's names for the object kinds it documents, and the observed undocumented ones.
57fn jcid_name(jcid: u32) -> String {
58 let name = match jcid {
59 0x60007 => "section",
60 0x60008 => "page series",
61 0x6000b => "page",
62 0x6000c => "outline",
63 0x6000d => "outline element",
64 0x6000e => "rich text",
65 0x60011 => "picture",
66 0x60012 => "number list",
67 0x60014 => "ink container",
68 0x60019 => "outline group",
69 0x60022 => "table",
70 0x60023 => "table row",
71 0x60024 => "table cell",
72 0x6002c => "title",
73 0x60035 => "embedded file",
74 0x60037 => "page manifest",
75 0x6003c => "version history",
76 0x6003d => "version proxy",
77 _ => "undocumented",
78 };
79 format!("jcid 0x{jcid:05x} ({name})")
80}
81
82fn unsupported_error(error: &EditorError) -> bool {
83 matches!(
84 error,
85 EditorError::Layout(LayoutError::UnsupportedContent)
86 | EditorError::Edit(EditError::UnsupportedContent)
87 )
88}
89
90/// What a text paragraph holds that bears on editing it.
91fn traits(text: &Paragraph) -> Vec<&'static str> {
92 let spans = text.spans();
93 let mut traits = Vec::new();
94 if text.text().contains('\u{fffc}')
95 || spans.iter().any(|s| s.format.embedded_object == Some(true))
96 {
97 traits.push("embedded object in text");
98 }
99 if spans.iter().any(|s| s.format.math == Some(true)) {
100 traits.push("equation");
101 }
102 if spans.iter().any(|s| s.format.hyperlink == Some(true)) {
103 traits.push("link");
104 }
105 if spans.iter().any(|s| s.format.hidden == Some(true)) {
106 traits.push("hidden text");
107 }
108 if text.text().is_empty() {
109 traits.push("empty");
110 }
111 traits
112}
113
114/// Why `node`, alone in an outline, does not lay out as content the canvas understands.
115fn culprit(
116 engine: &mut TextEngine,
117 outline: &Outline,
118 node: &PageParagraph,
119 page: &Page,
120) -> Option<String> {
121 let alone = Outline {
122 paragraphs: vec![PageParagraph {
123 parent: None,
124 level: 1,
125 ..node.clone()
126 }],
127 unsupported: Vec::new(),
128 ..outline.clone()
129 };
130 match alone.layout(engine, &page.definitions) {
131 Err(LayoutError::UnsupportedContent) => {}
132 _ => return None,
133 }
134 Some(match &node.content {
135 ParagraphContent::Unsupported(u) => format!("holds {}", jcid_name(u.jcid)),
136 ParagraphContent::Table(table) if !table.tags.is_empty() => "table with note tags".into(),
137 ParagraphContent::Table(table) => {
138 let cells = || table.rows.iter().flat_map(|row| &row.cells);
139 if let Some(cell) = cells().find(|cell| !cell.unsupported.is_empty()) {
140 format!(
141 "table cell holding {}",
142 cell.unsupported
143 .iter()
144 .map(|u| jcid_name(u.jcid))
145 .collect::<Vec<_>>()
146 .join(", ")
147 )
148 } else {
149 let inner = cells()
150 .flat_map(|cell| &cell.paragraphs)
151 .find_map(|inner| culprit(engine, outline, inner, page));
152 format!(
153 "table cell: {}",
154 inner.unwrap_or_else(|| "nested content".into())
155 )
156 }
157 }
158 ParagraphContent::Image(_) => "picture without a size".into(),
159 ParagraphContent::Ink(_) => "handwriting without strokes".into(),
160 ParagraphContent::Attachment(_) => "file".into(),
161 ParagraphContent::Text(text) => {
162 let lists: Vec<String> = node
163 .lists
164 .iter()
165 .filter_map(|id| page.definitions.get(id))
166 .map(|definition| format!("{:?}", definition.kind))
167 .collect();
168 if !lists.is_empty() {
169 format!("list marker {}", lists.join(", "))
170 } else if text
171 .text
172 .spans()
173 .iter()
174 .any(|s| s.format.math == Some(true))
175 {
176 "equation the math layout does not parse".into()
177 } else {
178 format!("text ({})", traits(&text.text).join(", "))
179 }
180 }
181 })
182}
183
184/// Why an outline the page shows is not editable.
185fn diagnose(engine: &mut TextEngine, outline: &Outline, page: &Page) -> Vec<String> {
186 let mut reasons: Vec<String> = outline
187 .unsupported
188 .iter()
189 .map(|u| format!("outline child {}", jcid_name(u.jcid)))
190 .collect();
191 reasons.extend(
192 outline
193 .paragraphs
194 .iter()
195 .filter_map(|node| culprit(engine, outline, node, page)),
196 );
197 if reasons.is_empty() {
198 let text = outline.paragraphs.iter().any(|node| {
199 node.text().is_some()
200 || matches!(&node.content, ParagraphContent::Table(t)
201 if t.rows.iter().flat_map(|r| &r.cells).any(|c| !c.paragraphs.is_empty()))
202 });
203 reasons.push(if text {
204 match TextOutline::from_outline(engine, outline, &page.definitions) {
205 Err(error) => format!("editor: {error}"),
206 Ok(_) => "outline on its own is editable".into(),
207 }
208 } else {
209 let kinds: BTreeSet<&str> = outline
210 .paragraphs
211 .iter()
212 .map(|node| match node.content {
213 ParagraphContent::Image(_) => "pictures",
214 ParagraphContent::Attachment(_) => "files",
215 ParagraphContent::Ink(_) => "handwriting",
216 _ => "other",
217 })
218 .collect();
219 format!(
220 "no paragraph of text (only {})",
221 kinds.into_iter().collect::<Vec<_>>().join(", ")
222 )
223 });
224 }
225 reasons
226}
227
228/// A picture that draws as "Image unavailable": no data, or (with `--features gpu`) data
229/// the renderer does not decode, named by its leading bytes.
230fn picture(image: &Image, at: &str, inventory: &mut Inventory, place: &str) {
231 let Some(bytes) = &image.bytes else {
232 return inventory.add(
233 Stage::Placeholder,
234 format!("{at} picture without data"),
235 place,
236 );
237 };
238 #[cfg(feature = "gpu")]
239 if [Some(bytes), image.display.as_ref()]
240 .into_iter()
241 .flatten()
242 .all(|bytes| draw::RasterImage::measure(bytes).is_err())
243 {
244 let format = match &bytes[..bytes.len().min(4)] {
245 [0x50, 0x4b, 3, 4] => "a zip package (XPS printout)",
246 [1, 0, 0, 0] => "EMF",
247 [0xd7, 0xcd, 0xc6, 0x9a] | [1, 0, 9, 0] => "WMF",
248 [b'B', b'M', ..] => "BMP",
249 _ => "an unknown format",
250 };
251 inventory.add(
252 Stage::Placeholder,
253 format!("{at} picture stored as {format}"),
254 place,
255 );
256 }
257 #[cfg(not(feature = "gpu"))]
258 let _ = bytes;
259}
260
261/// Pictures that draw as "Image unavailable" and objects the model keeps aside.
262fn scan_paragraphs(nodes: &[PageParagraph], at: &str, inventory: &mut Inventory, place: &str) {
263 for node in nodes {
264 match &node.content {
265 ParagraphContent::Unsupported(u) => inventory.add(
266 Stage::Placeholder,
267 format!("{at} paragraph holding {}", jcid_name(u.jcid)),
268 place,
269 ),
270 ParagraphContent::Image(image) => picture(image, at, inventory, place),
271 ParagraphContent::Table(table) => {
272 for cell in table.rows.iter().flat_map(|row| &row.cells) {
273 for u in &cell.unsupported {
274 inventory.add(
275 Stage::Placeholder,
276 format!("table cell child {}", jcid_name(u.jcid)),
277 place,
278 );
279 }
280 scan_paragraphs(&cell.paragraphs, "table cell", inventory, place);
281 }
282 }
283 _ => {}
284 }
285 }
286}
287
288/// Ordinary edits at every text paragraph of every editable outline: typing at its start,
289/// middle and end, Enter, Backspace at its start, Delete at its end, and deleting from its
290/// middle into the next.
291fn probe(
292 editor: &mut CanvasEditor,
293 engine: &mut TextEngine,
294 inventory: &mut Inventory,
295 place: &str,
296) {
297 let ids: Vec<ExGuid> = editor.outlines().iter().map(|o| o.id).collect();
298 for id in ids {
299 if editor.focus_outline(id).is_err() {
300 continue;
301 }
302 let document = editor.active_outline().document();
303 let containers = containers(document.nodes());
304 let leaves: Vec<(Option<ExGuid>, Paragraph)> = document
305 .text_nodes()
306 .map(|node| {
307 (
308 containers.get(&node.id).copied().flatten(),
309 node.text().unwrap().text.clone(),
310 )
311 })
312 .collect();
313 for (index, (container, text)) in leaves.iter().enumerate() {
314 let length = text.text().encode_utf16().count() as u32;
315 // The middle of what shows, where a click could put the caret.
316 let Ok(projection) = text.project() else {
317 continue;
318 };
319 let shown = projection.text();
320 let middle = shown
321 .text()
322 .char_indices()
323 .map(|(at, _)| at)
324 .nth(shown.text().chars().count() / 2)
325 .and_then(|at| shown.utf16_offset(at).ok())
326 .and_then(|at| projection.source_offset(at, Affinity::Downstream).ok())
327 .unwrap_or(length);
328 let at = |offset| TextPosition {
329 paragraph: index,
330 offset,
331 };
332 let next = leaves.get(index + 1);
333 let mut probes: Vec<(&str, [TextPosition; 2], Probe)> = vec![
334 ("typing at a paragraph's start", [at(0); 2], Probe::Type),
335 ("typing inside a paragraph", [at(middle); 2], Probe::Type),
336 ("typing at a paragraph's end", [at(length); 2], Probe::Type),
337 ("Enter inside a paragraph", [at(middle); 2], Probe::Enter),
338 (
339 "Delete at a paragraph's end",
340 [at(length); 2],
341 Probe::Delete,
342 ),
343 ];
344 if index > 0 {
345 probes.push((
346 "Backspace at a paragraph's start",
347 [at(0); 2],
348 Probe::Backspace,
349 ));
350 }
351 if let Some((_, next_text)) = next {
352 let end = next_text.text().encode_utf16().count() as u32 / 2;
353 probes.push((
354 "deleting a selection into the next paragraph",
355 [
356 at(middle),
357 TextPosition {
358 paragraph: index + 1,
359 offset: end,
360 },
361 ],
362 Probe::Backspace,
363 ));
364 }
365 for (name, positions, kind) in probes {
366 if editor.select(positions.into()).is_err() {
367 continue;
368 }
369 let result = match kind {
370 Probe::Type => editor.insert(engine, "x").map(|()| true),
371 Probe::Enter => editor.enter(engine, false).map(|()| true),
372 Probe::Delete => editor.delete(engine, false),
373 Probe::Backspace => editor.delete(engine, true),
374 };
375 match result {
376 Ok(true) => {
377 editor.undo(engine).ok();
378 }
379 Ok(false) => {}
380 Err(error) if unsupported_error(&error) => {
381 let crossing = positions[0].paragraph != positions[1].paragraph
382 || name.starts_with("Backspace")
383 || name.starts_with("Delete");
384 let neighbour = match name {
385 n if n.starts_with("Backspace") => leaves.get(index - 1),
386 _ if crossing => next,
387 _ => None,
388 };
389 let mut why: Vec<&str> = traits(text);
390 if let Some((other, other_text)) = neighbour {
391 if other != container {
392 why.push("across a table cell boundary");
393 }
394 why.extend(traits(other_text).into_iter().map(|t| match t {
395 "embedded object in text" => "next to an embedded object",
396 "equation" => "next to an equation",
397 "link" => "next to a link",
398 t => t,
399 }));
400 }
401 if container.is_some() {
402 why.push("in a table cell");
403 }
404 why.sort();
405 why.dedup();
406 inventory.add(
407 Stage::Refused,
408 format!("{name} [{}]", why.join(", ")),
409 place,
410 );
411 }
412 Err(_) => {}
413 }
414 }
415 }
416 }
417}
418
419#[derive(Clone, Copy)]
420enum Probe {
421 Type,
422 Enter,
423 Delete,
424 Backspace,
425}
426
427/// Each paragraph's table cell, `None` at the outline's top.
428fn containers(nodes: &[PageParagraph]) -> BTreeMap<ExGuid, Option<ExGuid>> {
429 let mut map = BTreeMap::new();
430 let mut pending = vec![(None, nodes)];
431 while let Some((container, nodes)) = pending.pop() {
432 for node in nodes {
433 map.insert(node.id, container);
434 if let ParagraphContent::Table(table) = &node.content {
435 for cell in table.rows.iter().flat_map(|row| &row.cells) {
436 pending.push((Some(cell.id), cell.paragraphs.as_slice()));
437 }
438 }
439 }
440 }
441 map
442}
443
444fn page(engine: &mut TextEngine, page: Page, place: &str, inventory: &mut Inventory) {
445 for object in &page.objects {
446 match object {
447 PageObject::Unsupported(u) => inventory.add(
448 Stage::Placeholder,
449 format!("page object {}", jcid_name(u.jcid)),
450 place,
451 ),
452 PageObject::Image(image) => picture(image, "page", inventory, place),
453 PageObject::Outline(outline) => {
454 for u in &outline.unsupported {
455 inventory.add(
456 Stage::Placeholder,
457 format!("outline child {}", jcid_name(u.jcid)),
458 place,
459 );
460 }
461 scan_paragraphs(&outline.paragraphs, "outline", inventory, place);
462 }
463 PageObject::Title(title) => {
464 for outline in &title.outlines {
465 scan_paragraphs(&outline.paragraphs, "title", inventory, place);
466 }
467 }
468 _ => {}
469 }
470 }
471 // Pictures with pixels the scene still shows as placeholders, for want of a size or the
472 // like; undecodable ones are counted above.
473 #[cfg(feature = "gpu")]
474 if let Ok((scene, shown)) = canvas::gpu::page::PageScene::from_page(page.clone(), engine) {
475 for object in scene.read_only(Some(&shown)) {
476 if let PageObject::Image(image) = &object.source
477 && [image.bytes.as_ref(), image.display.as_ref()]
478 .into_iter()
479 .flatten()
480 .any(|bytes| draw::RasterImage::measure(bytes).is_ok())
481 {
482 inventory.add(
483 Stage::Placeholder,
484 "page picture with pixels drawn as \"Image unavailable\"",
485 place,
486 );
487 }
488 }
489 }
490 let mut editor = match CanvasEditor::from_page(page.clone(), engine) {
491 Ok(editor) => editor,
492 Err(error) => {
493 inventory.add(
494 Stage::Failure,
495 format!("page does not open: {error}"),
496 place,
497 );
498 return;
499 }
500 };
501 for object in &page.objects {
502 let (outlines, title) = match object {
503 PageObject::Outline(outline) => (std::slice::from_ref(outline), None),
504 PageObject::Title(title) => (title.outlines.as_slice(), Some(title)),
505 _ => continue,
506 };
507 for outline in outlines {
508 if editor.has_page_outline(outline.id)
509 || editor.outlines().iter().any(|o| o.id == outline.id)
510 {
511 continue;
512 }
513 if let Some(title) = title {
514 if title.date == Some(outline.id) {
515 if editor.date().is_none() {
516 inventory.add(Stage::ReadOnly, "page date the date editor declines", place);
517 }
518 continue;
519 }
520 if !outline.title {
521 continue;
522 }
523 }
524 let placeholder = matches!(
525 outline.layout(engine, &page.definitions),
526 Err(LayoutError::UnsupportedContent)
527 );
528 let at = if title.is_some() { "title" } else { "outline" };
529 for reason in diagnose(engine, outline, &page) {
530 let stage = if placeholder {
531 Stage::Placeholder
532 } else {
533 Stage::ReadOnly
534 };
535 let whole = if placeholder {
536 format!("whole {at} as a box: {reason}")
537 } else {
538 format!("{at} read-only: {reason}")
539 };
540 inventory.add(stage, whole, place);
541 }
542 }
543 }
544 probe(&mut editor, engine, inventory, place);
545}
546
547/// The passwords the roots' `manifest.json` files record for their protected sections.
548fn passwords(directory: &Path, out: &mut Vec<String>) {
549 let Ok(entries) = std::fs::read_dir(directory) else {
550 return;
551 };
552 for path in entries.flatten().map(|entry| entry.path()) {
553 if path.is_dir() {
554 passwords(&path, out);
555 } else if path.file_name().is_some_and(|name| name == "manifest.json")
556 && let Ok(bytes) = std::fs::read(&path)
557 && let Ok(manifest) = serde_json::from_slice::<serde_json::Value>(&bytes)
558 {
559 let named = manifest["native"]
560 .as_object()
561 .into_iter()
562 .flat_map(|map| map.values());
563 out.extend(
564 std::iter::once(&manifest["password"])
565 .chain(named)
566 .filter_map(|password| password.as_str().map(str::to_owned)),
567 );
568 }
569 }
570}
571
572fn section(
573 engine: &mut TextEngine,
574 path: &Path,
575 root: &Path,
576 passwords: &[String],
577 inventory: &mut Inventory,
578) {
579 let Ok(image) = std::fs::read(path) else {
580 return;
581 };
582 let name = path
583 .strip_prefix(root)
584 .unwrap_or(path)
585 .display()
586 .to_string();
587 let arena = Arena::default();
588 // A protected section opens under the password its fixture records.
589 let key = passwords
590 .iter()
591 .find_map(|password| onestore::protected::Key::open(&image, password).ok());
592 let opened = match &key {
593 Some(key) => Section::unlock(&arena, image, key).map_err(onestore::Error::from),
594 None => Section::open(&arena, image),
595 };
596 let mut section = match opened {
597 Ok(section) => section,
598 Err(error) => {
599 inventory.add(
600 Stage::Failure,
601 format!("section does not open: {}", error.message),
602 &name,
603 );
604 return;
605 }
606 };
607 inventory.sections += 1;
608 let pages = match section.pages() {
609 Ok(pages) => pages,
610 Err(error) => {
611 inventory.add(
612 Stage::Failure,
613 format!("page list does not read: {}", error.message),
614 &name,
615 );
616 return;
617 }
618 };
619 for (space, title, _) in pages {
620 inventory.pages += 1;
621 let place = format!("{name} — {title}");
622 match section.page(space) {
623 Ok(model) => page(engine, model, &place, inventory),
624 Err(error) => inventory.add(
625 Stage::Failure,
626 format!("page does not read: {}", error.message),
627 &place,
628 ),
629 }
630 }
631}
632
633fn sections(directory: &Path, out: &mut Vec<PathBuf>) {
634 let Ok(entries) = std::fs::read_dir(directory) else {
635 return;
636 };
637 let mut entries: Vec<PathBuf> = entries.flatten().map(|entry| entry.path()).collect();
638 entries.sort();
639 for path in entries {
640 if path.is_dir() {
641 sections(&path, out);
642 } else if path.extension().is_some_and(|e| e == "one") {
643 out.push(path);
644 }
645 }
646}
647
648fn main() {
649 let roots: Vec<PathBuf> = std::env::args().skip(1).map(PathBuf::from).collect();
650 let mut engine = TextEngine::default();
651 let mut inventory = Inventory::default();
652 let mut seen = BTreeSet::new();
653 let mut duplicates = 0;
654 let mut known = Vec::new();
655 for root in &roots {
656 passwords(root, &mut known);
657 }
658 for root in &roots {
659 let mut paths = Vec::new();
660 sections(root, &mut paths);
661 for path in paths {
662 let Ok(bytes) = std::fs::read(&path) else {
663 continue;
664 };
665 let mut hasher = DefaultHasher::new();
666 bytes.hash(&mut hasher);
667 if !seen.insert(hasher.finish()) {
668 duplicates += 1;
669 continue;
670 }
671 section(&mut engine, &path, root, &known, &mut inventory);
672 }
673 }
674 println!("# Unsupported content inventory\n");
675 println!(
676 "{} distinct sections ({duplicates} identical copies skipped), {} pages, under {}.\n",
677 inventory.sections,
678 inventory.pages,
679 roots
680 .iter()
681 .map(|r| format!("`{}`", r.display()))
682 .collect::<Vec<_>>()
683 .join(", ")
684 );
685 let headings = [
686 (Stage::Placeholder, "Drawn as a placeholder box"),
687 (Stage::ReadOnly, "Drawn as stored, read-only"),
688 (
689 Stage::Refused,
690 "Edits refused as content the editor cannot edit",
691 ),
692 (Stage::Failure, "Does not open"),
693 ];
694 for (stage, heading) in headings {
695 let mut rows: Vec<(&String, &Vec<String>)> = inventory
696 .kinds
697 .iter()
698 .filter(|((s, _), _)| *s == stage)
699 .map(|((_, kind), places)| (kind, places))
700 .collect();
701 rows.sort_by(|a, b| b.1.len().cmp(&a.1.len()).then(a.0.cmp(b.0)));
702 println!("## {heading}\n");
703 if rows.is_empty() {
704 println!("None.\n");
705 continue;
706 }
707 println!("| Kind | Count | Pages | Examples |\n| --- | ---: | ---: | --- |");
708 for (kind, places) in rows {
709 let pages: BTreeSet<&String> = places.iter().collect();
710 let examples: Vec<String> = pages.iter().take(3).map(|p| format!("`{p}`")).collect();
711 println!(
712 "| {} | {} | {} | {} |",
713 kind.replace('|', "\\|"),
714 places.len(),
715 pages.len(),
716 examples.join("<br>").replace('|', "\\|")
717 );
718 }
719 println!();
720 }
721}