diff --git a/crates/canvas/src/bin/page_probe.rs b/crates/canvas/src/bin/page_probe.rs index 4b5e0181cedc7615a3fa23464385d92b88790c70..f3fcad3abe3417881eb3de561e44957037526a05 100644 --- a/crates/canvas/src/bin/page_probe.rs +++ b/crates/canvas/src/bin/page_probe.rs @@ -65,6 +65,8 @@ fn main() -> Result<(), Box> { PageObject::Image(image) => objects.push(json!({"kind": "image", "id": image.id, "layout": image.layout, "bytes": image.bytes.as_ref().map(|b| b.len()), "alt": image.alt, "background": image.background})), + PageObject::Ink(ink) => objects.push(json!({"kind": "ink", "id": ink.id, + "bounds": ink.bounds(), "strokes": ink.strokes.len(), "groups": ink.groups.len()})), PageObject::Unsupported(object) => objects.push(json!({"kind": "unsupported", "id": object.id, "layout": object.layout, "jcid": object.jcid})), } diff --git a/crates/canvas/src/document.rs b/crates/canvas/src/document.rs index 54682ff839128bd574e3f89169fe638450fda89b..15d5c8f7be59bb006fe3261051f693c9a36b5284 100644 --- a/crates/canvas/src/document.rs +++ b/crates/canvas/src/document.rs @@ -155,6 +155,7 @@ pub(crate) fn validate_nodes( } ParagraphContent::Image(_) | ParagraphContent::Attachment(_) + | ParagraphContent::Ink(_) | ParagraphContent::Unsupported(_) => { return Err(EditError::UnsupportedContent); } diff --git a/crates/canvas/src/editor.rs b/crates/canvas/src/editor.rs index c5d23e9af6f7d47f4ec3e8718815f0018093e6d2..e53d2a5b71694e8d6de4a428605fed20ca792f64 100644 --- a/crates/canvas/src/editor.rs +++ b/crates/canvas/src/editor.rs @@ -458,6 +458,7 @@ impl TextOutline { } onestore::page::ParagraphContent::Image(_) | onestore::page::ParagraphContent::Attachment(_) + | onestore::page::ParagraphContent::Ink(_) | onestore::page::ParagraphContent::Unsupported(_) => { return Err(EditError::UnsupportedContent); } @@ -509,6 +510,7 @@ impl TextOutline { } onestore::page::ParagraphContent::Image(_) | onestore::page::ParagraphContent::Attachment(_) + | onestore::page::ParagraphContent::Ink(_) | onestore::page::ParagraphContent::Unsupported(_) => { return Err(EditError::UnsupportedContent); } @@ -1699,6 +1701,7 @@ impl CanvasEditor { } onestore::page::ParagraphContent::Image(_) | onestore::page::ParagraphContent::Attachment(_) + | onestore::page::ParagraphContent::Ink(_) | onestore::page::ParagraphContent::Unsupported(_) => { return Err(EditError::UnsupportedContent.into()); } diff --git a/crates/canvas/src/editor/page.rs b/crates/canvas/src/editor/page.rs index 84301fd840fe982b0f4f03d79de48a984d4eac8f..d6494f37fdeb1bc91049b295332ad40402616355 100644 --- a/crates/canvas/src/editor/page.rs +++ b/crates/canvas/src/editor/page.rs @@ -80,10 +80,15 @@ impl ReadOnlyObject { [0.0; 2] }; let layout = source.layout(); - let x = layout.x.unwrap_or(0.0) + offset[0]; - let y = layout.y.unwrap_or(0.0) + offset[1]; - let width = layout.max_width.unwrap_or(160.0); - let height = layout.max_height.unwrap_or(42.0); + // Ink carries no layout of its own; its strokes say where it is. + let extent = match &source { + PageObject::Ink(ink) => ink.bounds(), + _ => None, + }; + let x = extent.map_or(layout.x.unwrap_or(0.0), |e| e[0]) + offset[0]; + let y = extent.map_or(layout.y.unwrap_or(0.0), |e| e[1]) + offset[1]; + let width = extent.map_or(layout.max_width.unwrap_or(160.0), |e| e[2]); + let height = extent.map_or(layout.max_height.unwrap_or(42.0), |e| e[3]); if [x, y, width, height].iter().any(|v| !v.is_finite()) || width <= 0.0 || height <= 0.0 { return Err(EditorError::InvalidGeometry); } @@ -321,6 +326,12 @@ pub(crate) fn build( }; objects.push(Content::Image(source)); } + PageObject::Ink(_) => objects.push(Content::ReadOnly(ReadOnlyObject::new( + object, + page.margin_origin, + "Ink\nRead-only", + engine, + )?)), PageObject::Unsupported(_) => objects.push(Content::ReadOnly(ReadOnlyObject::new( object, page.margin_origin, diff --git a/crates/canvas/src/outline.rs b/crates/canvas/src/outline.rs index c7a989f8c440673f9a6c9ed0d2246b0303c405cf..fdaabd69a65e48ffebbd52877c71d5130d752593 100644 --- a/crates/canvas/src/outline.rs +++ b/crates/canvas/src/outline.rs @@ -376,6 +376,7 @@ impl OutlineLayout { } ParagraphContent::Image(_) | ParagraphContent::Attachment(_) + | ParagraphContent::Ink(_) | ParagraphContent::Unsupported(_) => { return Err(LayoutError::UnsupportedContent); } diff --git a/crates/onestore/README.md b/crates/onestore/README.md index c49355f2254a6d4a3ace5b93fd4254053f99e139..049734dcee99c9139261bfbbe627beea8d82d2cf 100644 --- a/crates/onestore/README.md +++ b/crates/onestore/README.md @@ -43,7 +43,7 @@ harness also accepts `--client-profile release`. | `Object::file_reference`, `Store::file_data` | Identify internal/external payloads and read internal payload bytes | | `document::Document`, `Revision::text_runs` | Interpret document objects and inherited text formatting while retaining unknown properties and revision identities | | `Document::active`, `Document::pages_in`, `Revision::parents`, `RevisionIndex::active` | Resolve the active revision, the pages of a space and parent links without repeating the lookups | -| `page::Page`, `page::Paragraph` | Build an editable page model (title, outlines, paragraphs with coalesced text spans, tables, images) with stored identities; content outside the model is retained as `Unsupported` | +| `page::Page`, `page::Paragraph`, `page::Ink` | Build an editable page model (title, outlines, paragraphs with coalesced text spans, tables, images, attachments, ink drawings and handwriting decoded to stroke polylines in page points) with stored identities; content outside the model is retained as `Unsupported` | | `protected::UnlockedSection` (optional feature) | Own decoded buffers for explicit known-password inspection; clear those buffers on drop; derived document strings/exports remain caller-owned | | `create_section` | Create one page containing one plain-text paragraph and an author, including Unicode | | `PageCreation`, `PreparedEdit::create_page` | Add an empty top-level page and its section entry atomically, retaining page identities across retries | @@ -174,8 +174,8 @@ cargo run --example random_edit -- /tmp/one-demo/synthetic.one /tmp/edited.one 4 The report contains readable pages, document JSON, assets, source identities and coordinates. It preserves paragraph nesting, lists, tables, links and tags. Historical contexts, recycle-bin pages and default templates are represented -separately. Native ink and structured equations retain their source data and -appear explicitly as opaque content. The report is a reading view; its native +separately. Native ink is decoded to strokes; structured equations retain their +source data and appear explicitly as opaque content. The report is a reading view; its native PDF references supply the original canvas layout. Read and validate a snapshot before interpreting its graph: diff --git a/crates/onestore/src/document.rs b/crates/onestore/src/document.rs index 9f1c193fbed44e228e23bc2e5fe466369f610d3a..ee7dd354c0cd1e9ea8cc777ca9876325ca4e7724 100644 --- a/crates/onestore/src/document.rs +++ b/crates/onestore/src/document.rs @@ -326,8 +326,41 @@ pub enum Kind<'a> { collapse_state: Option, }, OutlineGroup, - /// Native ink properties remain uninterpreted in the element's extra arena. - Ink, + /// An ink drawing or handwriting container: strokes hang off `data`, nested containers off + /// the element's content list. Scaling multiplies stroke coordinates. + Ink { + data: Option, + scale_x: Option, + scale_y: Option, + }, + InkData { + strokes: Vec, + bounds: Option<[i32; 4]>, + }, + /// One stroke: `path` holds ISF multi-byte first differences, one block per dimension of + /// the style's dimension table, in HIMETRIC (1/2540 inch) page coordinates. + InkStroke { + path: Vec, + style: Option, + bias: Option, + language: Option, + identity: Option<[u8; 16]>, + index: Option, + }, + /// Drawing attributes shared by strokes; `dimensions` holds 32-byte entries of GUID, lower + /// and upper limit, and resolution. + InkStyle { + dimensions: Vec, + width: Option, + height: Option, + color: Option, + transparency: Option, + pen_tip: Option, + raster_operation: Option, + antialiased: Option, + fit_to_curve: Option, + ignore_pressure: Option, + }, RichText { text: String, runs: Vec, @@ -993,7 +1026,39 @@ impl<'a> Element<'a> { level: f.u32(0x14001dff)?, author: f.text(0x1c001d9e)?, }, - 0x60014 => Kind::Ink, + 0x60014 => Kind::Ink { + data: f.one(0x20003415)?, + scale_x: f.float(0x14001c46, 1.0)?, + scale_y: f.float(0x14001c47, 1.0)?, + }, + 0x2003b => Kind::InkData { + strokes: f.refs(0x24003416, IdStream::Objects)?, + bounds: f.fixed::<16>(0x1c003418)?.map(|b| { + std::array::from_fn(|i| { + i32::from_le_bytes(b[i * 4..i * 4 + 4].try_into().unwrap()) + }) + }), + }, + 0x20047 => Kind::InkStroke { + path: f.bytes(0x1c00340b)?.unwrap_or_default().to_vec(), + style: f.one(0x20003409)?, + bias: f.u8(0x0c00341c)?, + language: f.u16(0x1000341b)?, + identity: f.fixed(0x1c00341a)?, + index: f.u32(0x14003419)?, + }, + 0x120048 => Kind::InkStyle { + dimensions: f.bytes(0x1c00340a)?.unwrap_or_default().to_vec(), + width: f.float(0x1400340d, 1.0)?, + height: f.float(0x1400340c, 1.0)?, + color: f.u32(0x1400340f)?, + transparency: f.u8(0x0c003414)?, + pen_tip: f.u8(0x0c003412)?, + raster_operation: f.u8(0x0c003413)?, + antialiased: f.boolean(0x0800340e)?, + fit_to_curve: f.boolean(0x08003410)?, + ignore_pressure: f.boolean(0x08003411)?, + }, 0x20031 => Kind::SectionMetadata { name: f.text(0x1c00349b)?, color: f.u32(0x14001cbe)?, diff --git a/crates/onestore/src/page/ink.rs b/crates/onestore/src/page/ink.rs new file mode 100644 index 0000000000000000000000000000000000000000..b787423b0869a4774243dd20081abaa5c1163ce1 --- /dev/null +++ b/crates/onestore/src/page/ink.rs @@ -0,0 +1,215 @@ +//! Ink drawings and handwriting: strokes as page-coordinate polylines in points, decoded from +//! the stroke packets OneNote stores (ISF multi-byte first differences in HIMETRIC). + +use super::Error; +use crate::{ + ExGuid, + document::{Kind, Layout, Revision}, +}; + +const HIMETRIC_PER_POINT: f32 = 2540.0 / 72.0; +const DIMENSION_X: [u8; 16] = [ + 0x8f, 0x6a, 0x8a, 0x59, 0xc0, 0x52, 0xa0, 0x4b, 0x93, 0xaf, 0xaf, 0x35, 0x74, 0x11, 0xa5, 0x61, +]; +const DIMENSION_Y: [u8; 16] = [ + 0x75, 0x9f, 0x3f, 0xb5, 0xe0, 0x04, 0x98, 0x44, 0xa7, 0xee, 0xc3, 0x0d, 0xbb, 0x5a, 0x90, 0x11, +]; + +#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct Ink { + pub id: ExGuid, + pub layout: Layout, + pub strokes: Vec, + /// Nested ink containers, as newer OneNote versions group handwriting. + pub groups: Vec, +} + +#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct InkStroke { + pub id: ExGuid, + /// Absolute page coordinates in points. + pub points: Vec<[f32; 2]>, + /// Pen width and height in points. + pub width: f32, + pub height: f32, + /// COLORREF; absent means the window text colour. + pub color: Option, + /// 0 is opaque, 255 fully transparent. + pub transparency: Option, + /// 0 is a round (ball) tip, 1 a rectangle. + pub pen_tip: Option, +} + +impl Ink { + pub(super) fn read( + revision: &Revision<'_>, + id: ExGuid, + node: &crate::document::Element<'_>, + ) -> Result { + let invalid = |message| Error { offset: 0, message }; + let Kind::Ink { + data, + scale_x, + scale_y, + } = &node.kind + else { + unreachable!() + }; + let scale = [scale_x.unwrap_or(1.0), scale_y.unwrap_or(1.0)]; + let mut strokes = Vec::new(); + if let Some(data) = data { + let data = revision + .nodes + .get(data) + .ok_or_else(|| invalid("Missing ink data"))?; + let Kind::InkData { strokes: ids, .. } = &data.kind else { + return Err(invalid("Ink data has the wrong type")); + }; + for stroke_id in ids { + let stroke = revision + .nodes + .get(stroke_id) + .ok_or_else(|| invalid("Missing ink stroke"))?; + strokes.push(InkStroke::read(revision, *stroke_id, stroke, scale)?); + } + } + let mut groups = Vec::new(); + for child in &node.content { + let element = revision + .nodes + .get(child) + .ok_or_else(|| invalid("Missing nested ink"))?; + if matches!(element.kind, Kind::Ink { .. }) { + groups.push(Ink::read(revision, *child, element)?); + } + } + Ok(Self { + id, + layout: node.layout.clone(), + strokes, + groups, + }) + } + + /// The extent of every stroke point, as `[left, top, width, height]` in points. + pub fn bounds(&self) -> Option<[f32; 4]> { + let mut extent: Option<[f32; 4]> = None; + for point in self + .strokes + .iter() + .flat_map(|stroke| stroke.points.iter()) + .copied() + .chain( + self.groups + .iter() + .filter_map(Ink::bounds) + .flat_map(|[x, y, w, h]| [[x, y], [x + w, y + h]]), + ) + { + extent = Some(match extent { + None => [point[0], point[1], point[0], point[1]], + Some([x0, y0, x1, y1]) => [ + x0.min(point[0]), + y0.min(point[1]), + x1.max(point[0]), + y1.max(point[1]), + ], + }); + } + extent.map(|[x0, y0, x1, y1]| [x0, y0, x1 - x0, y1 - y0]) + } +} + +impl InkStroke { + fn read( + revision: &Revision<'_>, + id: ExGuid, + node: &crate::document::Element<'_>, + scale: [f32; 2], + ) -> Result { + let invalid = |message| Error { offset: 0, message }; + let Kind::InkStroke { path, style, .. } = &node.kind else { + return Err(invalid("Ink stroke has the wrong type")); + }; + let style = style + .and_then(|style| revision.nodes.get(&style)) + .ok_or_else(|| invalid("Missing ink stroke style"))?; + let Kind::InkStyle { + dimensions, + width, + height, + color, + transparency, + pen_tip, + .. + } = &style.kind + else { + return Err(invalid("Ink stroke style has the wrong type")); + }; + if dimensions.len() % 32 != 0 || dimensions.is_empty() { + return Err(invalid("Ink dimension table has an invalid length")); + } + let guids: Vec<&[u8]> = dimensions.chunks_exact(32).map(|d| &d[..16]).collect(); + let axis = |guid: [u8; 16]| guids.iter().position(|g| **g == guid); + let (Some(x), Some(y)) = (axis(DIMENSION_X), axis(DIMENSION_Y)) else { + return Err(invalid("Ink dimension table lacks X and Y")); + }; + let values = multi_byte(path).ok_or_else(|| invalid("Ink stroke packet is malformed"))?; + if values.len() % guids.len() != 0 { + return Err(invalid("Ink stroke packet does not cover its dimensions")); + } + let per_dimension = values.len() / guids.len(); + let coordinates = |dimension: usize, factor: f32| { + let mut position = 0i64; + values[dimension * per_dimension..(dimension + 1) * per_dimension] + .iter() + .map(move |delta| { + position += delta; + position as f32 * factor / HIMETRIC_PER_POINT + }) + }; + let points = coordinates(x, scale[0]) + .zip(coordinates(y, scale[1])) + .map(|(x, y)| [x, y]) + .collect(); + Ok(Self { + id, + points, + width: width.unwrap_or(0.0) / HIMETRIC_PER_POINT, + height: height.unwrap_or(0.0) / HIMETRIC_PER_POINT, + color: *color, + transparency: *transparency, + pen_tip: *pen_tip, + }) + } +} + +/// ISF multi-byte decoding: a count, then that many 7-bit little-endian varints whose low bit +/// is the sign. +fn multi_byte(bytes: &[u8]) -> Option> { + let mut cursor = 0; + let mut next = || { + let mut value = 0u64; + let mut shift = 0; + loop { + let byte = *bytes.get(cursor)?; + cursor += 1; + if shift >= 64 { + return None; + } + value |= u64::from(byte & 0x7f) << shift; + shift += 7; + if byte & 0x80 == 0 { + return Some(value); + } + } + }; + let count = next()? >> 1; + let mut values = Vec::with_capacity(usize::try_from(count).ok()?.min(1 << 20)); + for _ in 0..count { + let raw = next()?; + let magnitude = i64::try_from(raw >> 1).ok()?; + values.push(if raw & 1 == 1 { -magnitude } else { magnitude }); + } + (cursor == bytes.len()).then_some(values) +} diff --git a/crates/onestore/src/page/mod.rs b/crates/onestore/src/page/mod.rs index 789da80149c8ed6d4967acc1095a850ca7d43ae3..169dffbaf48731a81cf251f5e39163e4e07ea105 100644 --- a/crates/onestore/src/page/mod.rs +++ b/crates/onestore/src/page/mod.rs @@ -10,8 +10,11 @@ use std::{ sync::Arc, }; +pub mod ink; pub mod link; pub mod text; + +pub use ink::{Ink, InkStroke}; pub(crate) mod write; pub use text::Paragraph; @@ -45,6 +48,7 @@ pub enum PageObject { Outline(Outline), Title(Title), Image(Image), + Ink(Ink), Unsupported(Unsupported), } @@ -54,6 +58,7 @@ impl PageObject { Self::Outline(value) => value.id, Self::Title(value) => value.id, Self::Image(value) => value.id, + Self::Ink(value) => value.id, Self::Unsupported(value) => value.id, } } @@ -62,6 +67,7 @@ impl PageObject { Self::Outline(value) => &value.layout, Self::Title(value) => &value.layout, Self::Image(value) => &value.layout, + Self::Ink(value) => &value.layout, Self::Unsupported(value) => &value.layout, } } @@ -71,6 +77,7 @@ impl PageObject { Self::Outline(value) => &mut value.layout, Self::Title(value) => &mut value.layout, Self::Image(value) => &mut value.layout, + Self::Ink(value) => &mut value.layout, Self::Unsupported(value) => &mut value.layout, } } @@ -116,6 +123,8 @@ pub enum ParagraphContent { Image(Image), /// An embedded file, as OneNote inserts attachments into outlines. Attachment(Attachment), + /// Handwriting held by the paragraph. + Ink(Ink), Unsupported(Unsupported), } @@ -471,6 +480,10 @@ impl Page { page.objects .push(PageObject::Image(Image::read(revision, id, node)?)); } + Kind::Ink { .. } => { + page.objects + .push(PageObject::Ink(Ink::read(revision, id, node)?)); + } _ => page.objects.push(PageObject::Unsupported(Unsupported { id, jcid: node.jcid, @@ -601,6 +614,8 @@ fn read_paragraphs( ParagraphContent::Image(Image::read(revision, *content_id, content)?) } else if matches!(content.kind, Kind::Attachment { .. }) { ParagraphContent::Attachment(Attachment::read(revision, *content_id, content)?) + } else if matches!(content.kind, Kind::Ink { .. }) { + ParagraphContent::Ink(Ink::read(revision, *content_id, content)?) } else if matches!(content.kind, Kind::Table { .. }) { ParagraphContent::Table(read_table( revision, @@ -625,6 +640,7 @@ fn read_paragraphs( ParagraphContent::Table(table) => table.tags.as_slice(), ParagraphContent::Image(_) | ParagraphContent::Attachment(_) + | ParagraphContent::Ink(_) | ParagraphContent::Unsupported(_) => &[], }) .filter_map(|tag| tag.definition.as_ref()) diff --git a/crates/onestore/src/page/write.rs b/crates/onestore/src/page/write.rs index 2435df64b5dee6f2b371230eba86850a3d2d0b3e..70c47fe6d5cf45bbdfc9b91e24d1fff287d1783d 100644 --- a/crates/onestore/src/page/write.rs +++ b/crates/onestore/src/page/write.rs @@ -203,7 +203,7 @@ impl<'a> View<'a> { view.outline(outline)?; } } - PageObject::Image(_) | PageObject::Unsupported(_) => {} + PageObject::Image(_) | PageObject::Ink(_) | PageObject::Unsupported(_) => {} } } Ok(view) @@ -366,6 +366,7 @@ impl Lowering<'_> { .iter() .filter_map(|object| match object { PageObject::Unsupported(unsupported) => Some(format!("{unsupported:?}")), + PageObject::Ink(ink) => Some(format!("{ink:?}")), PageObject::Title(title) => Some(format!( "{:?} {:?} {:?} {:?}", title.id, @@ -383,7 +384,7 @@ impl Lowering<'_> { after_fixed.sort(); if before_fixed != after_fixed { return Err(invalid( - "Titles and unsupported objects cannot be edited through the page model", + "Titles, ink and unsupported objects cannot be edited through the page model", )); } for (id, outline) in &new.outlines { @@ -672,7 +673,9 @@ impl Lowering<'_> { let outlines: Vec<&Outline> = match object { PageObject::Outline(outline) => vec![outline], PageObject::Title(title) => title.outlines.iter().collect(), - PageObject::Image(_) | PageObject::Unsupported(_) => Vec::new(), + PageObject::Image(_) | PageObject::Ink(_) | PageObject::Unsupported(_) => { + Vec::new() + } }; for outline in outlines { containers.push(outline.id); diff --git a/crates/onestore/tests/document.rs b/crates/onestore/tests/document.rs index 7618bc53e72e74b9593890cd753ddeabdd2ae781..20874875d3826865e851767aff8651715db7d7ca 100644 --- a/crates/onestore/tests/document.rs +++ b/crates/onestore/tests/document.rs @@ -860,7 +860,7 @@ fn native_conflict_relationship_and_opaque_ink_are_retained() { .values() .flat_map(|s| s.revisions.values()) .flat_map(|r| r.nodes.values()) - .filter(|node| matches!(node.kind, Kind::Ink)) + .filter(|node| matches!(node.kind, Kind::Ink { .. })) .collect(); assert_eq!(ink.len(), 2); assert!( diff --git a/crates/onestore/tests/page_ink.rs b/crates/onestore/tests/page_ink.rs new file mode 100644 index 0000000000000000000000000000000000000000..4b7f9935b1030c70363a25d85e3a00532f460cc2 --- /dev/null +++ b/crates/onestore/tests/page_ink.rs @@ -0,0 +1,76 @@ +use onestore::{ + ExGuid, RevisionIndex, Store, + document::Document, + page::{Ink, Page, PageObject}, +}; + +/// Two drawings made with the mouse in OneNote 2010 (`tools/native/ink.ahk`): a diamond and a +/// diagonal line, whose native read reports their positions and sizes. +const NATIVE: &[u8] = + include_bytes!("../../../corpus/native-ink/cold-ui-ink/notebook/synthetic.one"); + +fn first_page(bytes: &[u8]) -> (ExGuid, Page) { + let store = Store::parse(bytes).unwrap(); + assert!(store.checksum_mismatches.is_empty()); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let (space, _) = document.pages().unwrap()[0]; + (space, Page::from_space(&document, space).unwrap()) +} + +fn drawings(page: &Page) -> Vec<&Ink> { + page.objects + .iter() + .filter_map(|object| match object { + PageObject::Ink(ink) => Some(ink), + _ => None, + }) + .collect() +} + +fn close(actual: [f32; 4], expected: [f32; 4]) -> bool { + actual + .iter() + .zip(expected) + .all(|(a, e)| (a - e).abs() < 0.002) +} + +#[test] +fn native_ink_drawings_read_as_strokes_in_page_coordinates() { + let (_, page) = first_page(NATIVE); + let found = drawings(&page); + let [diamond, line] = found.as_slice() else { + panic!("{:?}", found.len()); + }; + assert_eq!(diamond.strokes.len(), 1); + assert_eq!(diamond.strokes[0].points.len(), 253); + // OneNote reports a drawing's size one HIMETRIC unit larger than its point extent. + let unit = 72.0 / 2540.0; + let native = |[x, y, w, h]: [f32; 4]| [x, y, w + unit, h + unit]; + let bounds = ( + native(diamond.bounds().unwrap()), + native(line.bounds().unwrap()), + ); + assert!( + close(bounds.0, [421.5118, 84.7559, 60.00945, 60.00944]) + && close(bounds.1, [504.0, 92.23936, 30.0189, 45.04252]), + "{bounds:?}" + ); + let stroke = &diamond.strokes[0]; + assert!(close( + [stroke.points[0][0], stroke.points[0][1], 0.0, 0.0], + [451.50235, 84.755905, 0.0, 0.0] + )); + assert!((stroke.width - 35.0 * 72.0 / 2540.0).abs() < 1e-5); + assert_eq!( + (stroke.color, stroke.transparency, stroke.pen_tip), + (None, None, None) + ); + assert!(diamond.groups.is_empty()); + for stroke in [&diamond.strokes[0], &line.strokes[0]] { + assert!(stroke.points.windows(2).all(|pair| { + let [dx, dy] = [pair[1][0] - pair[0][0], pair[1][1] - pair[0][1]]; + dx.abs() < 3.0 && dy.abs() < 3.0 + })); + } +} diff --git a/crates/onestore/tests/page_model.rs b/crates/onestore/tests/page_model.rs index 527b69bbdf60b17f68bb8235afd3d8db5632dca5..6977397bc55e99f12b245fa050e509cb8ac9b851 100644 --- a/crates/onestore/tests/page_model.rs +++ b/crates/onestore/tests/page_model.rs @@ -27,6 +27,7 @@ fn identities(page: &Page) -> Vec { ids.push(paragraph.id); match ¶graph.content { ParagraphContent::Text(text) => ids.push(text.id), + ParagraphContent::Ink(ink) => ids.push(ink.id), ParagraphContent::Table(table) => { ids.push(table.id); for row in &table.rows { @@ -59,7 +60,7 @@ fn identities(page: &Page) -> Vec { ids.extend(outline.unsupported.iter().map(|u| u.id)); } } - PageObject::Image(_) | PageObject::Unsupported(_) => {} + PageObject::Image(_) | PageObject::Ink(_) | PageObject::Unsupported(_) => {} } } ids diff --git a/crates/onestore/tests/page_tables.rs b/crates/onestore/tests/page_tables.rs index 8d4541993d6a4bba2c2f7534d6ff23fcacd5d5a9..b3072cda11f0ef358c7b07e7da9999a3ff7ed782 100644 --- a/crates/onestore/tests/page_tables.rs +++ b/crates/onestore/tests/page_tables.rs @@ -74,6 +74,7 @@ fn referenced(page: &Page) -> std::collections::BTreeSet { } ParagraphContent::Image(_) | ParagraphContent::Attachment(_) + | ParagraphContent::Ink(_) | ParagraphContent::Unsupported(_) => {} } } diff --git a/crates/onestore/tests/support/page_edits.rs b/crates/onestore/tests/support/page_edits.rs index 831af3719600e739f6ea25d014bb697d7607dfd9..3db5882ab0acb2d314129d091e815d9b9f97b8ad 100644 --- a/crates/onestore/tests/support/page_edits.rs +++ b/crates/onestore/tests/support/page_edits.rs @@ -129,6 +129,7 @@ fn projection(page: &Page) -> String { ParagraphContent::Attachment(attachment) => { out.push_str(&format!(" attachment {}", attachment.id)) } + ParagraphContent::Ink(ink) => out.push_str(&format!(" ink {}", ink.id)), ParagraphContent::Unsupported(u) => { out.push_str(&format!(" unsupported {}", u.id)) } @@ -138,6 +139,7 @@ fn projection(page: &Page) -> String { } PageObject::Title(title) => out.push_str(&format!("title {}\n", title.id)), PageObject::Image(image) => out.push_str(&format!("image {}\n", image.id)), + PageObject::Ink(ink) => out.push_str(&format!("ink {}\n", ink.id)), PageObject::Unsupported(u) => out.push_str(&format!("unsupported {}\n", u.id)), } } diff --git a/tools/test_ink_read.py b/tools/test_ink_read.py new file mode 100644 index 0000000000000000000000000000000000000000..63e55982b4153d9271eba74837bd1a15b31d8b69 --- /dev/null +++ b/tools/test_ink_read.py @@ -0,0 +1,42 @@ +import json +from pathlib import Path +import runpy +import subprocess +from tempfile import TemporaryDirectory +import unittest +import xml.etree.ElementTree as ET + +from document_model import EXPORTER, ordered_pages +from native_xml import ns + +ROOT = Path(__file__).resolve().parent.parent +FIXTURE = ROOT / 'corpus/native-ink/cold-ui-ink' +ink_extent = runpy.run_path(str(ROOT / 'tools/verify-document.py'))['ink_extent'] + + +class InkReadTest(unittest.TestCase): + def test_decoded_ink_strokes_span_the_native_drawing_extents(self): + """An independent decoding of the stored stroke packets reproduces the native positions and sizes.""" + with TemporaryDirectory() as temporary: + exported = Path(temporary) / 'document' + subprocess.run([EXPORTER, FIXTURE / 'notebook/synthetic.one', exported], check=True) + document = json.loads((exported / 'document.json').read_text()) + (_, _, revision, page_id), = ordered_pages(document) + source_page = revision['nodes'][page_id] + native, = (ET.parse(path).getroot() for path in sorted((FIXTURE / 'read').glob('page-*.xml'))) + drawings = native.findall('one:InkDrawing', ns) + self.assertEqual(len(drawings), 2) + for drawing in drawings: + z = int(drawing.find('one:Position', ns).get('z')) + source = revision['nodes'][source_page['children'][z]] + self.assertEqual(source['kind']['type'], 'Ink') + position, size = drawing.find('one:Position', ns), drawing.find('one:Size', ns) + expected = [float(position.get('x')), float(position.get('y')), + float(size.get('width')) - 72 / 2540, float(size.get('height')) - 72 / 2540] + extent = ink_extent(revision, source) + for actual, wanted in zip(extent, expected): + self.assertAlmostEqual(actual, wanted, delta=0.002) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/verify-document.py b/tools/verify-document.py index 6b775bf0cb8b31dab6824b42fda91fd573107583..7b723165a7cda25f6e585f5e54fe5cbf7ab25034 100644 --- a/tools/verify-document.py +++ b/tools/verify-document.py @@ -83,6 +83,58 @@ def visible_text(node, space): return "" if text == "\u00a0" else project_text(text.removesuffix('\r')) +ISF_X = bytes.fromhex('8f6a8a59c052a04b93afaf357411a561') +ISF_Y = bytes.fromhex('759f3fb5e0049844a7eec30dbb5a9011') + + +def multi_byte(data): + """ISF multi-byte signed integers: a count, then 7-bit little-endian varints with the sign in bit 0.""" + values = [] + index = 0 + while index < len(data): + value = shift = 0 + while True: + byte = data[index] + index += 1 + value |= (byte & 0x7f) << shift + shift += 7 + if not byte & 0x80: + break + values.append(-(value >> 1) if value & 1 else value >> 1) + count, values = values[0], values[1:] + assert count == len(values), 'Ink packet count differs' + return values + + +def ink_extent(space, container): + """Every stroke point of an ink container in points, as [left, top, width, height].""" + points = [] + scale = [container['kind']['scale_x'] or 1.0, container['kind']['scale_y'] or 1.0] + for stroke_id in space['nodes'][container['kind']['data']]['kind']['strokes']: + stroke = space['nodes'][stroke_id]['kind'] + style = space['nodes'][stroke['style']]['kind'] + dimensions = bytes(style['dimensions']) + guids = [dimensions[i:i + 16] for i in range(0, len(dimensions), 32)] + values = multi_byte(bytes(stroke['path'])) + per_dimension = len(values) // len(guids) + axes = [] + for guid, factor in ((ISF_X, scale[0]), (ISF_Y, scale[1])): + start = guids.index(guid) * per_dimension + position, coordinates = 0, [] + for delta in values[start:start + per_dimension]: + position += delta + coordinates.append(position * factor * 72 / 2540) + axes.append(coordinates) + points.extend(zip(*axes)) + for child in container['content']: + nested = space['nodes'][child] + if nested['kind']['type'] == 'Ink': + x, y, w, h = ink_extent(space, nested) + points.extend([(x, y), (x + w, y + h)]) + xs, ys = [p[0] for p in points], [p[1] for p in points] + return [min(xs), min(ys), max(xs) - min(xs), max(ys) - min(ys)] + + def compare_objects(space, roots, page, native_roots, assets, native_payloads, autofit): types = {'T': 'RichText', 'Image': 'Image', 'InsertedFile': 'Attachment', 'MediaFile': 'Attachment', 'Table': 'Table'} actual = [n for root in roots for _, n in walk(space, root) @@ -280,6 +332,21 @@ def compare(notebook, native, versions=None, password_file=None): z = int(position.attrib['z']) source = space['nodes'][source_page['children'][z]] if source['kind']['type'] == 'Ink': + if child.tag == '{%s}InkDrawing' % ns['one']: + # Stroke coordinates are absolute; left-to-right pages report them + # relative to the canonical margin origin, right-to-left pages as stored. + extent = ink_extent(space, source) + if not source_page['kind']['rtl']: + for index, (axis, canonical_origin) in enumerate((('x', 36.0), ('y', 14.4))): + origin = source_page['kind']['margin_origin_' + axis] + extent[index] += canonical_origin - origin if origin is not None else 0 + size = child.find('one:Size', ns) + # Native sizes are one HIMETRIC unit larger than the point extent. + reported = [float(position.attrib['x']), float(position.attrib['y']), float(size.get('width')) - 72 / 2540, float(size.get('height')) - 72 / 2540] + for index, key in enumerate(('x', 'y', 'width', 'height')): + if abs(extent[index] - reported[index]) > 0.01: + geometry.append({'file': str(relative), 'page': ordinal, 'z': z, 'axis': key, + 'stored': extent[index], 'native': reported[index]}) continue for axis in ('x', 'y'): if source['layout'][axis] is not None: