From 7f30bf34648c9ac7a69e74956b04546b9894a380 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Sat, 12 Sep 2026 17:09:11 -0700 Subject: [PATCH] feat: read ink drawings and handwriting as strokes in page coordinates The public MS-ONE text has no ink object; the layout matches what the onenote_parser crate (MPL-2.0, reimplemented here rather than copied) and Joplin's converter read: a container (jcid 0x60014) whose data node (0x2003b) lists strokes (0x20047), each packing its coordinates as ISF multi-byte first differences in HIMETRIC, one block per dimension of the style object's (0x120048) GUID table, with pen width, height, colour, transparency and tip on the style. The reader interprets those four kinds, and page::Ink decodes every stroke into a polyline of absolute page points, keeping nested containers as groups; ink is a page object or paragraph content, and the writer treats it as fixed. corpus/native-ink/cold-ui-ink is the oracle: two mouse-drawn drawings whose native positions and sizes the decoded extents reproduce (OneNote reports a size one HIMETRIC unit larger than the point extent). tests/page_ink.rs checks the Rust decoding and tools/test_ink_read.py an independent Python decoding of the same packets; verify-document.py now compares every ink drawing's extent, projected through the page origin on left-to-right pages, and reports differences as geometry entries. The canvas places its read-only ink placeholder at the stroke extent. Assisted-by: claude-fable-5.1 --- crates/canvas/src/bin/page_probe.rs | 2 + crates/canvas/src/document.rs | 1 + crates/canvas/src/editor.rs | 3 + crates/canvas/src/editor/page.rs | 19 +- crates/canvas/src/outline.rs | 1 + crates/onestore/README.md | 6 +- crates/onestore/src/document.rs | 71 ++++++- crates/onestore/src/page/ink.rs | 215 ++++++++++++++++++++ crates/onestore/src/page/mod.rs | 16 ++ crates/onestore/src/page/write.rs | 9 +- crates/onestore/tests/document.rs | 2 +- crates/onestore/tests/page_ink.rs | 76 +++++++ crates/onestore/tests/page_model.rs | 3 +- crates/onestore/tests/page_tables.rs | 1 + crates/onestore/tests/support/page_edits.rs | 2 + tools/test_ink_read.py | 42 ++++ tools/verify-document.py | 67 ++++++ 17 files changed, 521 insertions(+), 15 deletions(-) create mode 100644 crates/onestore/src/page/ink.rs create mode 100644 crates/onestore/tests/page_ink.rs create mode 100644 tools/test_ink_read.py diff --git a/crates/canvas/src/bin/page_probe.rs b/crates/canvas/src/bin/page_probe.rs index 4b5e0181cedc7615a3fa23464385d92b88790c70..f3fcad3abe3417881eb3de561e44957037526a05 100644 --- a/crates/canvas/src/bin/page_probe.rs +++ b/crates/canvas/src/bin/page_probe.rs @@ -65,6 +65,8 @@ fn main() -> Result<(), Box> { PageObject::Image(image) => objects.push(json!({"kind": "image", "id": image.id, "layout": image.layout, "bytes": image.bytes.as_ref().map(|b| b.len()), "alt": image.alt, "background": image.background})), + PageObject::Ink(ink) => objects.push(json!({"kind": "ink", "id": ink.id, + "bounds": ink.bounds(), "strokes": ink.strokes.len(), "groups": ink.groups.len()})), PageObject::Unsupported(object) => objects.push(json!({"kind": "unsupported", "id": object.id, "layout": object.layout, "jcid": object.jcid})), } diff --git a/crates/canvas/src/document.rs b/crates/canvas/src/document.rs index 54682ff839128bd574e3f89169fe638450fda89b..15d5c8f7be59bb006fe3261051f693c9a36b5284 100644 --- a/crates/canvas/src/document.rs +++ b/crates/canvas/src/document.rs @@ -155,6 +155,7 @@ pub(crate) fn validate_nodes( } ParagraphContent::Image(_) | ParagraphContent::Attachment(_) + | ParagraphContent::Ink(_) | ParagraphContent::Unsupported(_) => { return Err(EditError::UnsupportedContent); } diff --git a/crates/canvas/src/editor.rs b/crates/canvas/src/editor.rs index c5d23e9af6f7d47f4ec3e8718815f0018093e6d2..e53d2a5b71694e8d6de4a428605fed20ca792f64 100644 --- a/crates/canvas/src/editor.rs +++ b/crates/canvas/src/editor.rs @@ -458,6 +458,7 @@ impl TextOutline { } onestore::page::ParagraphContent::Image(_) | onestore::page::ParagraphContent::Attachment(_) + | onestore::page::ParagraphContent::Ink(_) | onestore::page::ParagraphContent::Unsupported(_) => { return Err(EditError::UnsupportedContent); } @@ -509,6 +510,7 @@ impl TextOutline { } onestore::page::ParagraphContent::Image(_) | onestore::page::ParagraphContent::Attachment(_) + | onestore::page::ParagraphContent::Ink(_) | onestore::page::ParagraphContent::Unsupported(_) => { return Err(EditError::UnsupportedContent); } @@ -1699,6 +1701,7 @@ impl CanvasEditor { } onestore::page::ParagraphContent::Image(_) | onestore::page::ParagraphContent::Attachment(_) + | onestore::page::ParagraphContent::Ink(_) | onestore::page::ParagraphContent::Unsupported(_) => { return Err(EditError::UnsupportedContent.into()); } diff --git a/crates/canvas/src/editor/page.rs b/crates/canvas/src/editor/page.rs index 84301fd840fe982b0f4f03d79de48a984d4eac8f..d6494f37fdeb1bc91049b295332ad40402616355 100644 --- a/crates/canvas/src/editor/page.rs +++ b/crates/canvas/src/editor/page.rs @@ -80,10 +80,15 @@ impl ReadOnlyObject { [0.0; 2] }; let layout = source.layout(); - let x = layout.x.unwrap_or(0.0) + offset[0]; - let y = layout.y.unwrap_or(0.0) + offset[1]; - let width = layout.max_width.unwrap_or(160.0); - let height = layout.max_height.unwrap_or(42.0); + // Ink carries no layout of its own; its strokes say where it is. + let extent = match &source { + PageObject::Ink(ink) => ink.bounds(), + _ => None, + }; + let x = extent.map_or(layout.x.unwrap_or(0.0), |e| e[0]) + offset[0]; + let y = extent.map_or(layout.y.unwrap_or(0.0), |e| e[1]) + offset[1]; + let width = extent.map_or(layout.max_width.unwrap_or(160.0), |e| e[2]); + let height = extent.map_or(layout.max_height.unwrap_or(42.0), |e| e[3]); if [x, y, width, height].iter().any(|v| !v.is_finite()) || width <= 0.0 || height <= 0.0 { return Err(EditorError::InvalidGeometry); } @@ -321,6 +326,12 @@ pub(crate) fn build( }; objects.push(Content::Image(source)); } + PageObject::Ink(_) => objects.push(Content::ReadOnly(ReadOnlyObject::new( + object, + page.margin_origin, + "Ink\nRead-only", + engine, + )?)), PageObject::Unsupported(_) => objects.push(Content::ReadOnly(ReadOnlyObject::new( object, page.margin_origin, diff --git a/crates/canvas/src/outline.rs b/crates/canvas/src/outline.rs index c7a989f8c440673f9a6c9ed0d2246b0303c405cf..fdaabd69a65e48ffebbd52877c71d5130d752593 100644 --- a/crates/canvas/src/outline.rs +++ b/crates/canvas/src/outline.rs @@ -376,6 +376,7 @@ impl OutlineLayout { } ParagraphContent::Image(_) | ParagraphContent::Attachment(_) + | ParagraphContent::Ink(_) | ParagraphContent::Unsupported(_) => { return Err(LayoutError::UnsupportedContent); } diff --git a/crates/onestore/README.md b/crates/onestore/README.md index c49355f2254a6d4a3ace5b93fd4254053f99e139..049734dcee99c9139261bfbbe627beea8d82d2cf 100644 --- a/crates/onestore/README.md +++ b/crates/onestore/README.md @@ -43,7 +43,7 @@ harness also accepts `--client-profile release`. | `Object::file_reference`, `Store::file_data` | Identify internal/external payloads and read internal payload bytes | | `document::Document`, `Revision::text_runs` | Interpret document objects and inherited text formatting while retaining unknown properties and revision identities | | `Document::active`, `Document::pages_in`, `Revision::parents`, `RevisionIndex::active` | Resolve the active revision, the pages of a space and parent links without repeating the lookups | -| `page::Page`, `page::Paragraph` | Build an editable page model (title, outlines, paragraphs with coalesced text spans, tables, images) with stored identities; content outside the model is retained as `Unsupported` | +| `page::Page`, `page::Paragraph`, `page::Ink` | Build an editable page model (title, outlines, paragraphs with coalesced text spans, tables, images, attachments, ink drawings and handwriting decoded to stroke polylines in page points) with stored identities; content outside the model is retained as `Unsupported` | | `protected::UnlockedSection` (optional feature) | Own decoded buffers for explicit known-password inspection; clear those buffers on drop; derived document strings/exports remain caller-owned | | `create_section` | Create one page containing one plain-text paragraph and an author, including Unicode | | `PageCreation`, `PreparedEdit::create_page` | Add an empty top-level page and its section entry atomically, retaining page identities across retries | @@ -174,8 +174,8 @@ cargo run --example random_edit -- /tmp/one-demo/synthetic.one /tmp/edited.one 4 The report contains readable pages, document JSON, assets, source identities and coordinates. It preserves paragraph nesting, lists, tables, links and tags. Historical contexts, recycle-bin pages and default templates are represented -separately. Native ink and structured equations retain their source data and -appear explicitly as opaque content. The report is a reading view; its native +separately. Native ink is decoded to strokes; structured equations retain their +source data and appear explicitly as opaque content. The report is a reading view; its native PDF references supply the original canvas layout. Read and validate a snapshot before interpreting its graph: diff --git a/crates/onestore/src/document.rs b/crates/onestore/src/document.rs index 9f1c193fbed44e228e23bc2e5fe466369f610d3a..ee7dd354c0cd1e9ea8cc777ca9876325ca4e7724 100644 --- a/crates/onestore/src/document.rs +++ b/crates/onestore/src/document.rs @@ -326,8 +326,41 @@ pub enum Kind<'a> { collapse_state: Option, }, OutlineGroup, - /// Native ink properties remain uninterpreted in the element's extra arena. - Ink, + /// An ink drawing or handwriting container: strokes hang off `data`, nested containers off + /// the element's content list. Scaling multiplies stroke coordinates. + Ink { + data: Option, + scale_x: Option, + scale_y: Option, + }, + InkData { + strokes: Vec, + bounds: Option<[i32; 4]>, + }, + /// One stroke: `path` holds ISF multi-byte first differences, one block per dimension of + /// the style's dimension table, in HIMETRIC (1/2540 inch) page coordinates. + InkStroke { + path: Vec, + style: Option, + bias: Option, + language: Option, + identity: Option<[u8; 16]>, + index: Option, + }, + /// Drawing attributes shared by strokes; `dimensions` holds 32-byte entries of GUID, lower + /// and upper limit, and resolution. + InkStyle { + dimensions: Vec, + width: Option, + height: Option, + color: Option, + transparency: Option, + pen_tip: Option, + raster_operation: Option, + antialiased: Option, + fit_to_curve: Option, + ignore_pressure: Option, + }, RichText { text: String, runs: Vec, @@ -993,7 +1026,39 @@ impl<'a> Element<'a> { level: f.u32(0x14001dff)?, author: f.text(0x1c001d9e)?, }, - 0x60014 => Kind::Ink, + 0x60014 => Kind::Ink { + data: f.one(0x20003415)?, + scale_x: f.float(0x14001c46, 1.0)?, + scale_y: f.float(0x14001c47, 1.0)?, + }, + 0x2003b => Kind::InkData { + strokes: f.refs(0x24003416, IdStream::Objects)?, + bounds: f.fixed::<16>(0x1c003418)?.map(|b| { + std::array::from_fn(|i| { + i32::from_le_bytes(b[i * 4..i * 4 + 4].try_into().unwrap()) + }) + }), + }, + 0x20047 => Kind::InkStroke { + path: f.bytes(0x1c00340b)?.unwrap_or_default().to_vec(), + style: f.one(0x20003409)?, + bias: f.u8(0x0c00341c)?, + language: f.u16(0x1000341b)?, + identity: f.fixed(0x1c00341a)?, + index: f.u32(0x14003419)?, + }, + 0x120048 => Kind::InkStyle { + dimensions: f.bytes(0x1c00340a)?.unwrap_or_default().to_vec(), + width: f.float(0x1400340d, 1.0)?, + height: f.float(0x1400340c, 1.0)?, + color: f.u32(0x1400340f)?, + transparency: f.u8(0x0c003414)?, + pen_tip: f.u8(0x0c003412)?, + raster_operation: f.u8(0x0c003413)?, + antialiased: f.boolean(0x0800340e)?, + fit_to_curve: f.boolean(0x08003410)?, + ignore_pressure: f.boolean(0x08003411)?, + }, 0x20031 => Kind::SectionMetadata { name: f.text(0x1c00349b)?, color: f.u32(0x14001cbe)?, diff --git a/crates/onestore/src/page/ink.rs b/crates/onestore/src/page/ink.rs new file mode 100644 index 0000000000000000000000000000000000000000..b787423b0869a4774243dd20081abaa5c1163ce1 --- /dev/null +++ b/crates/onestore/src/page/ink.rs @@ -0,0 +1,215 @@ +//! Ink drawings and handwriting: strokes as page-coordinate polylines in points, decoded from +//! the stroke packets OneNote stores (ISF multi-byte first differences in HIMETRIC). + +use super::Error; +use crate::{ + ExGuid, + document::{Kind, Layout, Revision}, +}; + +const HIMETRIC_PER_POINT: f32 = 2540.0 / 72.0; +const DIMENSION_X: [u8; 16] = [ + 0x8f, 0x6a, 0x8a, 0x59, 0xc0, 0x52, 0xa0, 0x4b, 0x93, 0xaf, 0xaf, 0x35, 0x74, 0x11, 0xa5, 0x61, +]; +const DIMENSION_Y: [u8; 16] = [ + 0x75, 0x9f, 0x3f, 0xb5, 0xe0, 0x04, 0x98, 0x44, 0xa7, 0xee, 0xc3, 0x0d, 0xbb, 0x5a, 0x90, 0x11, +]; + +#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct Ink { + pub id: ExGuid, + pub layout: Layout, + pub strokes: Vec, + /// Nested ink containers, as newer OneNote versions group handwriting. + pub groups: Vec, +} + +#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct InkStroke { + pub id: ExGuid, + /// Absolute page coordinates in points. + pub points: Vec<[f32; 2]>, + /// Pen width and height in points. + pub width: f32, + pub height: f32, + /// COLORREF; absent means the window text colour. + pub color: Option, + /// 0 is opaque, 255 fully transparent. + pub transparency: Option, + /// 0 is a round (ball) tip, 1 a rectangle. + pub pen_tip: Option, +} + +impl Ink { + pub(super) fn read( + revision: &Revision<'_>, + id: ExGuid, + node: &crate::document::Element<'_>, + ) -> Result { + let invalid = |message| Error { offset: 0, message }; + let Kind::Ink { + data, + scale_x, + scale_y, + } = &node.kind + else { + unreachable!() + }; + let scale = [scale_x.unwrap_or(1.0), scale_y.unwrap_or(1.0)]; + let mut strokes = Vec::new(); + if let Some(data) = data { + let data = revision + .nodes + .get(data) + .ok_or_else(|| invalid("Missing ink data"))?; + let Kind::InkData { strokes: ids, .. } = &data.kind else { + return Err(invalid("Ink data has the wrong type")); + }; + for stroke_id in ids { + let stroke = revision + .nodes + .get(stroke_id) + .ok_or_else(|| invalid("Missing ink stroke"))?; + strokes.push(InkStroke::read(revision, *stroke_id, stroke, scale)?); + } + } + let mut groups = Vec::new(); + for child in &node.content { + let element = revision + .nodes + .get(child) + .ok_or_else(|| invalid("Missing nested ink"))?; + if matches!(element.kind, Kind::Ink { .. }) { + groups.push(Ink::read(revision, *child, element)?); + } + } + Ok(Self { + id, + layout: node.layout.clone(), + strokes, + groups, + }) + } + + /// The extent of every stroke point, as `[left, top, width, height]` in points. + pub fn bounds(&self) -> Option<[f32; 4]> { + let mut extent: Option<[f32; 4]> = None; + for point in self + .strokes + .iter() + .flat_map(|stroke| stroke.points.iter()) + .copied() + .chain( + self.groups + .iter() + .filter_map(Ink::bounds) + .flat_map(|[x, y, w, h]| [[x, y], [x + w, y + h]]), + ) + { + extent = Some(match extent { + None => [point[0], point[1], point[0], point[1]], + Some([x0, y0, x1, y1]) => [ + x0.min(point[0]), + y0.min(point[1]), + x1.max(point[0]), + y1.max(point[1]), + ], + }); + } + extent.map(|[x0, y0, x1, y1]| [x0, y0, x1 - x0, y1 - y0]) + } +} + +impl InkStroke { + fn read( + revision: &Revision<'_>, + id: ExGuid, + node: &crate::document::Element<'_>, + scale: [f32; 2], + ) -> Result { + let invalid = |message| Error { offset: 0, message }; + let Kind::InkStroke { path, style, .. } = &node.kind else { + return Err(invalid("Ink stroke has the wrong type")); + }; + let style = style + .and_then(|style| revision.nodes.get(&style)) + .ok_or_else(|| invalid("Missing ink stroke style"))?; + let Kind::InkStyle { + dimensions, + width, + height, + color, + transparency, + pen_tip, + .. + } = &style.kind + else { + return Err(invalid("Ink stroke style has the wrong type")); + }; + if dimensions.len() % 32 != 0 || dimensions.is_empty() { + return Err(invalid("Ink dimension table has an invalid length")); + } + let guids: Vec<&[u8]> = dimensions.chunks_exact(32).map(|d| &d[..16]).collect(); + let axis = |guid: [u8; 16]| guids.iter().position(|g| **g == guid); + let (Some(x), Some(y)) = (axis(DIMENSION_X), axis(DIMENSION_Y)) else { + return Err(invalid("Ink dimension table lacks X and Y")); + }; + let values = multi_byte(path).ok_or_else(|| invalid("Ink stroke packet is malformed"))?; + if values.len() % guids.len() != 0 { + return Err(invalid("Ink stroke packet does not cover its dimensions")); + } + let per_dimension = values.len() / guids.len(); + let coordinates = |dimension: usize, factor: f32| { + let mut position = 0i64; + values[dimension * per_dimension..(dimension + 1) * per_dimension] + .iter() + .map(move |delta| { + position += delta; + position as f32 * factor / HIMETRIC_PER_POINT + }) + }; + let points = coordinates(x, scale[0]) + .zip(coordinates(y, scale[1])) + .map(|(x, y)| [x, y]) + .collect(); + Ok(Self { + id, + points, + width: width.unwrap_or(0.0) / HIMETRIC_PER_POINT, + height: height.unwrap_or(0.0) / HIMETRIC_PER_POINT, + color: *color, + transparency: *transparency, + pen_tip: *pen_tip, + }) + } +} + +/// ISF multi-byte decoding: a count, then that many 7-bit little-endian varints whose low bit +/// is the sign. +fn multi_byte(bytes: &[u8]) -> Option> { + let mut cursor = 0; + let mut next = || { + let mut value = 0u64; + let mut shift = 0; + loop { + let byte = *bytes.get(cursor)?; + cursor += 1; + if shift >= 64 { + return None; + } + value |= u64::from(byte & 0x7f) << shift; + shift += 7; + if byte & 0x80 == 0 { + return Some(value); + } + } + }; + let count = next()? >> 1; + let mut values = Vec::with_capacity(usize::try_from(count).ok()?.min(1 << 20)); + for _ in 0..count { + let raw = next()?; + let magnitude = i64::try_from(raw >> 1).ok()?; + values.push(if raw & 1 == 1 { -magnitude } else { magnitude }); + } + (cursor == bytes.len()).then_some(values) +} diff --git a/crates/onestore/src/page/mod.rs b/crates/onestore/src/page/mod.rs index 789da80149c8ed6d4967acc1095a850ca7d43ae3..169dffbaf48731a81cf251f5e39163e4e07ea105 100644 --- a/crates/onestore/src/page/mod.rs +++ b/crates/onestore/src/page/mod.rs @@ -10,8 +10,11 @@ use std::{ sync::Arc, }; +pub mod ink; pub mod link; pub mod text; + +pub use ink::{Ink, InkStroke}; pub(crate) mod write; pub use text::Paragraph; @@ -45,6 +48,7 @@ pub enum PageObject { Outline(Outline), Title(Title), Image(Image), + Ink(Ink), Unsupported(Unsupported), } @@ -54,6 +58,7 @@ impl PageObject { Self::Outline(value) => value.id, Self::Title(value) => value.id, Self::Image(value) => value.id, + Self::Ink(value) => value.id, Self::Unsupported(value) => value.id, } } @@ -62,6 +67,7 @@ impl PageObject { Self::Outline(value) => &value.layout, Self::Title(value) => &value.layout, Self::Image(value) => &value.layout, + Self::Ink(value) => &value.layout, Self::Unsupported(value) => &value.layout, } } @@ -71,6 +77,7 @@ impl PageObject { Self::Outline(value) => &mut value.layout, Self::Title(value) => &mut value.layout, Self::Image(value) => &mut value.layout, + Self::Ink(value) => &mut value.layout, Self::Unsupported(value) => &mut value.layout, } } @@ -116,6 +123,8 @@ pub enum ParagraphContent { Image(Image), /// An embedded file, as OneNote inserts attachments into outlines. Attachment(Attachment), + /// Handwriting held by the paragraph. + Ink(Ink), Unsupported(Unsupported), } @@ -471,6 +480,10 @@ impl Page { page.objects .push(PageObject::Image(Image::read(revision, id, node)?)); } + Kind::Ink { .. } => { + page.objects + .push(PageObject::Ink(Ink::read(revision, id, node)?)); + } _ => page.objects.push(PageObject::Unsupported(Unsupported { id, jcid: node.jcid, @@ -601,6 +614,8 @@ fn read_paragraphs( ParagraphContent::Image(Image::read(revision, *content_id, content)?) } else if matches!(content.kind, Kind::Attachment { .. }) { ParagraphContent::Attachment(Attachment::read(revision, *content_id, content)?) + } else if matches!(content.kind, Kind::Ink { .. }) { + ParagraphContent::Ink(Ink::read(revision, *content_id, content)?) } else if matches!(content.kind, Kind::Table { .. }) { ParagraphContent::Table(read_table( revision, @@ -625,6 +640,7 @@ fn read_paragraphs( ParagraphContent::Table(table) => table.tags.as_slice(), ParagraphContent::Image(_) | ParagraphContent::Attachment(_) + | ParagraphContent::Ink(_) | ParagraphContent::Unsupported(_) => &[], }) .filter_map(|tag| tag.definition.as_ref()) diff --git a/crates/onestore/src/page/write.rs b/crates/onestore/src/page/write.rs index 2435df64b5dee6f2b371230eba86850a3d2d0b3e..70c47fe6d5cf45bbdfc9b91e24d1fff287d1783d 100644 --- a/crates/onestore/src/page/write.rs +++ b/crates/onestore/src/page/write.rs @@ -203,7 +203,7 @@ impl<'a> View<'a> { view.outline(outline)?; } } - PageObject::Image(_) | PageObject::Unsupported(_) => {} + PageObject::Image(_) | PageObject::Ink(_) | PageObject::Unsupported(_) => {} } } Ok(view) @@ -366,6 +366,7 @@ impl Lowering<'_> { .iter() .filter_map(|object| match object { PageObject::Unsupported(unsupported) => Some(format!("{unsupported:?}")), + PageObject::Ink(ink) => Some(format!("{ink:?}")), PageObject::Title(title) => Some(format!( "{:?} {:?} {:?} {:?}", title.id, @@ -383,7 +384,7 @@ impl Lowering<'_> { after_fixed.sort(); if before_fixed != after_fixed { return Err(invalid( - "Titles and unsupported objects cannot be edited through the page model", + "Titles, ink and unsupported objects cannot be edited through the page model", )); } for (id, outline) in &new.outlines { @@ -672,7 +673,9 @@ impl Lowering<'_> { let outlines: Vec<&Outline> = match object { PageObject::Outline(outline) => vec![outline], PageObject::Title(title) => title.outlines.iter().collect(), - PageObject::Image(_) | PageObject::Unsupported(_) => Vec::new(), + PageObject::Image(_) | PageObject::Ink(_) | PageObject::Unsupported(_) => { + Vec::new() + } }; for outline in outlines { containers.push(outline.id); diff --git a/crates/onestore/tests/document.rs b/crates/onestore/tests/document.rs index 7618bc53e72e74b9593890cd753ddeabdd2ae781..20874875d3826865e851767aff8651715db7d7ca 100644 --- a/crates/onestore/tests/document.rs +++ b/crates/onestore/tests/document.rs @@ -860,7 +860,7 @@ fn native_conflict_relationship_and_opaque_ink_are_retained() { .values() .flat_map(|s| s.revisions.values()) .flat_map(|r| r.nodes.values()) - .filter(|node| matches!(node.kind, Kind::Ink)) + .filter(|node| matches!(node.kind, Kind::Ink { .. })) .collect(); assert_eq!(ink.len(), 2); assert!( diff --git a/crates/onestore/tests/page_ink.rs b/crates/onestore/tests/page_ink.rs new file mode 100644 index 0000000000000000000000000000000000000000..4b7f9935b1030c70363a25d85e3a00532f460cc2 --- /dev/null +++ b/crates/onestore/tests/page_ink.rs @@ -0,0 +1,76 @@ +use onestore::{ + ExGuid, RevisionIndex, Store, + document::Document, + page::{Ink, Page, PageObject}, +}; + +/// Two drawings made with the mouse in OneNote 2010 (`tools/native/ink.ahk`): a diamond and a +/// diagonal line, whose native read reports their positions and sizes. +const NATIVE: &[u8] = + include_bytes!("../../../corpus/native-ink/cold-ui-ink/notebook/synthetic.one"); + +fn first_page(bytes: &[u8]) -> (ExGuid, Page) { + let store = Store::parse(bytes).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let (space, _) = document.pages().unwrap()[0]; + (space, Page::from_space(&document, space).unwrap()) +} + +fn drawings(page: &Page) -> Vec<&Ink> { + page.objects + .iter() + .filter_map(|object| match object { + PageObject::Ink(ink) => Some(ink), + _ => None, + }) + .collect() +} + +fn close(actual: [f32; 4], expected: [f32; 4]) -> bool { + actual + .iter() + .zip(expected) + .all(|(a, e)| (a - e).abs() < 0.002) +} + +#[test] +fn native_ink_drawings_read_as_strokes_in_page_coordinates() { + let (_, page) = first_page(NATIVE); + let found = drawings(&page); + let [diamond, line] = found.as_slice() else { + panic!("{:?}", found.len()); + }; + assert_eq!(diamond.strokes.len(), 1); + assert_eq!(diamond.strokes[0].points.len(), 253); + // OneNote reports a drawing's size one HIMETRIC unit larger than its point extent. + let unit = 72.0 / 2540.0; + let native = |[x, y, w, h]: [f32; 4]| [x, y, w + unit, h + unit]; + let bounds = ( + native(diamond.bounds().unwrap()), + native(line.bounds().unwrap()), + ); + assert!( + close(bounds.0, [421.5118, 84.7559, 60.00945, 60.00944]) + && close(bounds.1, [504.0, 92.23936, 30.0189, 45.04252]), + "{bounds:?}" + ); + let stroke = &diamond.strokes[0]; + assert!(close( + [stroke.points[0][0], stroke.points[0][1], 0.0, 0.0], + [451.50235, 84.755905, 0.0, 0.0] + )); + assert!((stroke.width - 35.0 * 72.0 / 2540.0).abs() < 1e-5); + assert_eq!( + (stroke.color, stroke.transparency, stroke.pen_tip), + (None, None, None) + ); + assert!(diamond.groups.is_empty()); + for stroke in [&diamond.strokes[0], &line.strokes[0]] { + assert!(stroke.points.windows(2).all(|pair| { + let [dx, dy] = [pair[1][0] - pair[0][0], pair[1][1] - pair[0][1]]; + dx.abs() < 3.0 && dy.abs() < 3.0 + })); + } +} diff --git a/crates/onestore/tests/page_model.rs b/crates/onestore/tests/page_model.rs index 527b69bbdf60b17f68bb8235afd3d8db5632dca5..6977397bc55e99f12b245fa050e509cb8ac9b851 100644 --- a/crates/onestore/tests/page_model.rs +++ b/crates/onestore/tests/page_model.rs @@ -27,6 +27,7 @@ fn identities(page: &Page) -> Vec { ids.push(paragraph.id); match ¶graph.content { ParagraphContent::Text(text) => ids.push(text.id), + ParagraphContent::Ink(ink) => ids.push(ink.id), ParagraphContent::Table(table) => { ids.push(table.id); for row in &table.rows { @@ -59,7 +60,7 @@ fn identities(page: &Page) -> Vec { ids.extend(outline.unsupported.iter().map(|u| u.id)); } } - PageObject::Image(_) | PageObject::Unsupported(_) => {} + PageObject::Image(_) | PageObject::Ink(_) | PageObject::Unsupported(_) => {} } } ids diff --git a/crates/onestore/tests/page_tables.rs b/crates/onestore/tests/page_tables.rs index 8d4541993d6a4bba2c2f7534d6ff23fcacd5d5a9..b3072cda11f0ef358c7b07e7da9999a3ff7ed782 100644 --- a/crates/onestore/tests/page_tables.rs +++ b/crates/onestore/tests/page_tables.rs @@ -74,6 +74,7 @@ fn referenced(page: &Page) -> std::collections::BTreeSet { } ParagraphContent::Image(_) | ParagraphContent::Attachment(_) + | ParagraphContent::Ink(_) | ParagraphContent::Unsupported(_) => {} } } diff --git a/crates/onestore/tests/support/page_edits.rs b/crates/onestore/tests/support/page_edits.rs index 831af3719600e739f6ea25d014bb697d7607dfd9..3db5882ab0acb2d314129d091e815d9b9f97b8ad 100644 --- a/crates/onestore/tests/support/page_edits.rs +++ b/crates/onestore/tests/support/page_edits.rs @@ -129,6 +129,7 @@ fn projection(page: &Page) -> String { ParagraphContent::Attachment(attachment) => { out.push_str(&format!(" attachment {}", attachment.id)) } + ParagraphContent::Ink(ink) => out.push_str(&format!(" ink {}", ink.id)), ParagraphContent::Unsupported(u) => { out.push_str(&format!(" unsupported {}", u.id)) } @@ -138,6 +139,7 @@ fn projection(page: &Page) -> String { } PageObject::Title(title) => out.push_str(&format!("title {}\n", title.id)), PageObject::Image(image) => out.push_str(&format!("image {}\n", image.id)), + PageObject::Ink(ink) => out.push_str(&format!("ink {}\n", ink.id)), PageObject::Unsupported(u) => out.push_str(&format!("unsupported {}\n", u.id)), } } diff --git a/tools/test_ink_read.py b/tools/test_ink_read.py new file mode 100644 index 0000000000000000000000000000000000000000..63e55982b4153d9271eba74837bd1a15b31d8b69 --- /dev/null +++ b/tools/test_ink_read.py @@ -0,0 +1,42 @@ +import json +from pathlib import Path +import runpy +import subprocess +from tempfile import TemporaryDirectory +import unittest +import xml.etree.ElementTree as ET + +from document_model import EXPORTER, ordered_pages +from native_xml import ns + +ROOT = Path(__file__).resolve().parent.parent +FIXTURE = ROOT / 'corpus/native-ink/cold-ui-ink' +ink_extent = runpy.run_path(str(ROOT / 'tools/verify-document.py'))['ink_extent'] + + +class InkReadTest(unittest.TestCase): + def test_decoded_ink_strokes_span_the_native_drawing_extents(self): + """An independent decoding of the stored stroke packets reproduces the native positions and sizes.""" + with TemporaryDirectory() as temporary: + exported = Path(temporary) / 'document' + subprocess.run([EXPORTER, FIXTURE / 'notebook/synthetic.one', exported], check=True) + document = json.loads((exported / 'document.json').read_text()) + (_, _, revision, page_id), = ordered_pages(document) + source_page = revision['nodes'][page_id] + native, = (ET.parse(path).getroot() for path in sorted((FIXTURE / 'read').glob('page-*.xml'))) + drawings = native.findall('one:InkDrawing', ns) + self.assertEqual(len(drawings), 2) + for drawing in drawings: + z = int(drawing.find('one:Position', ns).get('z')) + source = revision['nodes'][source_page['children'][z]] + self.assertEqual(source['kind']['type'], 'Ink') + position, size = drawing.find('one:Position', ns), drawing.find('one:Size', ns) + expected = [float(position.get('x')), float(position.get('y')), + float(size.get('width')) - 72 / 2540, float(size.get('height')) - 72 / 2540] + extent = ink_extent(revision, source) + for actual, wanted in zip(extent, expected): + self.assertAlmostEqual(actual, wanted, delta=0.002) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/verify-document.py b/tools/verify-document.py index 6b775bf0cb8b31dab6824b42fda91fd573107583..7b723165a7cda25f6e585f5e54fe5cbf7ab25034 100644 --- a/tools/verify-document.py +++ b/tools/verify-document.py @@ -83,6 +83,58 @@ def visible_text(node, space): return "" if text == "\u00a0" else project_text(text.removesuffix('\r')) +ISF_X = bytes.fromhex('8f6a8a59c052a04b93afaf357411a561') +ISF_Y = bytes.fromhex('759f3fb5e0049844a7eec30dbb5a9011') + + +def multi_byte(data): + """ISF multi-byte signed integers: a count, then 7-bit little-endian varints with the sign in bit 0.""" + values = [] + index = 0 + while index < len(data): + value = shift = 0 + while True: + byte = data[index] + index += 1 + value |= (byte & 0x7f) << shift + shift += 7 + if not byte & 0x80: + break + values.append(-(value >> 1) if value & 1 else value >> 1) + count, values = values[0], values[1:] + assert count == len(values), 'Ink packet count differs' + return values + + +def ink_extent(space, container): + """Every stroke point of an ink container in points, as [left, top, width, height].""" + points = [] + scale = [container['kind']['scale_x'] or 1.0, container['kind']['scale_y'] or 1.0] + for stroke_id in space['nodes'][container['kind']['data']]['kind']['strokes']: + stroke = space['nodes'][stroke_id]['kind'] + style = space['nodes'][stroke['style']]['kind'] + dimensions = bytes(style['dimensions']) + guids = [dimensions[i:i + 16] for i in range(0, len(dimensions), 32)] + values = multi_byte(bytes(stroke['path'])) + per_dimension = len(values) // len(guids) + axes = [] + for guid, factor in ((ISF_X, scale[0]), (ISF_Y, scale[1])): + start = guids.index(guid) * per_dimension + position, coordinates = 0, [] + for delta in values[start:start + per_dimension]: + position += delta + coordinates.append(position * factor * 72 / 2540) + axes.append(coordinates) + points.extend(zip(*axes)) + for child in container['content']: + nested = space['nodes'][child] + if nested['kind']['type'] == 'Ink': + x, y, w, h = ink_extent(space, nested) + points.extend([(x, y), (x + w, y + h)]) + xs, ys = [p[0] for p in points], [p[1] for p in points] + return [min(xs), min(ys), max(xs) - min(xs), max(ys) - min(ys)] + + def compare_objects(space, roots, page, native_roots, assets, native_payloads, autofit): types = {'T': 'RichText', 'Image': 'Image', 'InsertedFile': 'Attachment', 'MediaFile': 'Attachment', 'Table': 'Table'} actual = [n for root in roots for _, n in walk(space, root) @@ -280,6 +332,21 @@ def compare(notebook, native, versions=None, password_file=None): z = int(position.attrib['z']) source = space['nodes'][source_page['children'][z]] if source['kind']['type'] == 'Ink': + if child.tag == '{%s}InkDrawing' % ns['one']: + # Stroke coordinates are absolute; left-to-right pages report them + # relative to the canonical margin origin, right-to-left pages as stored. + extent = ink_extent(space, source) + if not source_page['kind']['rtl']: + for index, (axis, canonical_origin) in enumerate((('x', 36.0), ('y', 14.4))): + origin = source_page['kind']['margin_origin_' + axis] + extent[index] += canonical_origin - origin if origin is not None else 0 + size = child.find('one:Size', ns) + # Native sizes are one HIMETRIC unit larger than the point extent. + reported = [float(position.attrib['x']), float(position.attrib['y']), float(size.get('width')) - 72 / 2540, float(size.get('height')) - 72 / 2540] + for index, key in enumerate(('x', 'y', 'width', 'height')): + if abs(extent[index] - reported[index]) > 0.01: + geometry.append({'file': str(relative), 'page': ordinal, 'z': z, 'axis': key, + 'stored': extent[index], 'native': reported[index]}) continue for axis in ('x', 'y'): if source['layout'][axis] is not None: -- 2.54.0