authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-09-07 20:52:35-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-09-07 20:52:35-07:00
log3a32cbb2aad5a83965908734e6595ea252557f7f
tree75dda3f876940e0faaf92662e16a78f99b12622c
parent5bd5adf1ba1afbdc4eb6f3c20c23522d15945e16
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

feat: open native protected sections through an owned decode view

Add optional known-password OneNote 2010 AES-128/CBC and SHA-1 decoding with bounded work, native read-only hash and graph checks, and zeroizing owned buffers. Reuse document traversal and reference-stream parsing; preserve opaque reads and reject protected writes. Extend the diagnostic exporter with exact password-file input and private Unix output directories. Retain an independent cold native oracle. Correct table comparison to distinguish documented auto-fit from locked widths; fixed-width mismatches remain failures. Validation: 174 Rust tests, 120 Python tests, eight doctests and Clippy pass; 14 live/private tests remain explicitly ignored in the public lane. Two ASan fuzz passes complete 25,676 cases. Native comparisons cover 12 pages and 1,947 format checks with exact assets; device and simulator executables link. All owned clones removed. Assisted-by: gpt-6-astra

23 files changed, 1405 insertions(+), 49 deletions(-)

Cargo.lock+53
......@@ -21,6 +21,7 @@ dependencies = [
2121 "cipher",
2222 "cpubits",
2323 "cpufeatures",
24 "zeroize",
2425]
2526
2627[[package]]
......@@ -48,6 +49,12 @@ dependencies = [
4849 "syn 3.0.5",
4950]
5051
52[[package]]
53name = "base64"
54version = "0.22.1"
55source = "registry+https://github.com/rust-lang/crates.io-index"
56checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
57
5158[[package]]
5259name = "bitflags"
5360version = "2.13.1"
......@@ -59,6 +66,16 @@ name = "block-buffer"
5966version = "0.12.1"
6067source = "registry+https://github.com/rust-lang/crates.io-index"
6168checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa"
69dependencies = [
70 "hybrid-array",
71 "zeroize",
72]
73
74[[package]]
75name = "block-padding"
76version = "0.4.2"
77source = "registry+https://github.com/rust-lang/crates.io-index"
78checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b"
6279dependencies = [
6380 "hybrid-array",
6481]
......@@ -75,6 +92,15 @@ version = "1.12.1"
7592source = "registry+https://github.com/rust-lang/crates.io-index"
7693checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
7794
95[[package]]
96name = "cbc"
97version = "0.2.1"
98source = "registry+https://github.com/rust-lang/crates.io-index"
99checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896"
100dependencies = [
101 "cipher",
102]
103
78104[[package]]
79105name = "cc"
80106version = "1.4.5"
......@@ -118,6 +144,7 @@ dependencies = [
118144 "block-buffer",
119145 "crypto-common",
120146 "inout",
147 "zeroize",
121148]
122149
123150[[package]]
......@@ -206,6 +233,7 @@ dependencies = [
206233 "const-oid",
207234 "crypto-common",
208235 "ctutils",
236 "zeroize",
209237]
210238
211239[[package]]
......@@ -342,6 +370,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
342370checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b"
343371dependencies = [
344372 "typenum",
373 "zeroize",
345374]
346375
347376[[package]]
......@@ -360,6 +389,7 @@ version = "0.2.2"
360389source = "registry+https://github.com/rust-lang/crates.io-index"
361390checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7"
362391dependencies = [
392 "block-padding",
363393 "hybrid-array",
364394]
365395
......@@ -503,11 +533,18 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
503533name = "onestore"
504534version = "0.1.0"
505535dependencies = [
536 "aes",
537 "base64",
538 "cbc",
506539 "getrandom",
507540 "md5",
508541 "nix",
542 "roxmltree",
509543 "serde",
510544 "serde_json",
545 "sha1",
546 "subtle",
547 "zeroize",
511548]
512549
513550[[package]]
......@@ -530,6 +567,7 @@ dependencies = [
530567 "serde_json",
531568 "tempfile",
532569 "thiserror",
570 "zeroize",
533571]
534572
535573[[package]]
......@@ -629,6 +667,15 @@ version = "0.10.1"
629667source = "registry+https://github.com/rust-lang/crates.io-index"
630668checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
631669
670[[package]]
671name = "roxmltree"
672version = "0.21.1"
673source = "registry+https://github.com/rust-lang/crates.io-index"
674checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb"
675dependencies = [
676 "memchr",
677]
678
632679[[package]]
633680name = "rsqlite-vfs"
634681version = "0.1.1"
......@@ -1019,6 +1066,12 @@ dependencies = [
10191066 "memchr",
10201067]
10211068
1069[[package]]
1070name = "zeroize"
1071version = "1.9.0"
1072source = "registry+https://github.com/rust-lang/crates.io-index"
1073checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
1074
10221075[[package]]
10231076name = "zmij"
10241077version = "1.0.23"
corpus/native-encrypted/cold-2010-4763/manifest.json created+18
......@@ -0,0 +1,18 @@
1{
2 "source": "../encrypted-01/notebook/synthetic.one",
3 "source_sha256": "7a6e519cc0ef8de10357ffd52a32bd94c33ea09d8e420ff2d0b216ed141ff695",
4 "native_build": "14.0.4763.1000",
5 "cold_open": {
6 "pages": 1,
7 "hostname": "ONE-M6-31D693B8",
8 "cold": true
9 },
10 "driver": {
11 "exitCode": 0,
12 "exited": true
13 },
14 "teardown": {
15 "absent": true
16 },
17 "observation": "Independent cold unlock of the unchanged encrypted image. The earlier selection-split empty T element is absent. Native auto-fits the first unlocked table column from stored 38.61 points to 39.146141 points without changing the source bytes."
18}
corpus/native-encrypted/cold-2010-4763/read/af1af4b8987109ce1e335e538910fb3860b7c57eb3b6110a06aa2114e06d69a7.attachment created+1
......@@ -0,0 +1 @@
1../../cold-encrypted-02/read/af1af4b8987109ce1e335e538910fb3860b7c57eb3b6110a06aa2114e06d69a7.attachment
\ No newline at end of file
corpus/native-encrypted/cold-2010-4763/read/environment.json created+7
......@@ -0,0 +1,7 @@
1{
2 "powershell": "5.1.14409.1005",
3 "schema": "xs2010",
4 "hostname": "ONE-M6-31D693B8",
5 "cold": false,
6 "onenote": "14.0.4763.1000"
7}
corpus/native-encrypted/cold-2010-4763/read/hierarchy.xml created+2
......@@ -0,0 +1,2 @@
1<?xml version="1.0"?>
2<one:Notebook xmlns:one="http://schemas.microsoft.com/office/onenote/2010/onenote" name="notebook" nickname="notebook" ID="{8AF43B16-0C62-45ED-99DB-0DE53DE844E0}{1}{B0}" path="C:\one-tests\runs\capture\notebook" lastModifiedTime="2026-09-08T03:59:38.000Z" color="#9595AA" isCurrentlyViewed="true"><one:Section name="synthetic" ID="{FD3CB2AF-D9ED-0E6A-1F38-C58D5CD38C03}{1}{B0}" path="C:\one-tests\runs\capture\notebook\synthetic.one" lastModifiedTime="2026-09-08T03:59:38.000Z" color="#8AA8E4" encrypted="true" isCurrentlyViewed="true"><one:Page ID="{75216EDF-30ED-03E0-13CD-C2FB6030B01D}{14}{B0}" name="Fictitious: café, 東京, مرحبا" dateTime="2026-09-05T05:06:54.000Z" lastModifiedTime="2026-09-08T03:59:38.000Z" pageLevel="1" isCurrentlyViewed="true"/></one:Section></one:Notebook>
corpus/native-encrypted/cold-2010-4763/read/page-000.xml created+8
......@@ -0,0 +1,8 @@
1<?xml version="1.0"?>
2<one:Page xmlns:one="http://schemas.microsoft.com/office/onenote/2010/onenote" ID="{75216EDF-30ED-03E0-13CD-C2FB6030B01D}{14}{B0}" name="Fictitious: café, 東京, مرحبا" dateTime="2026-09-05T05:06:54.000Z" lastModifiedTime="2026-09-08T03:59:44.000Z" pageLevel="1" isCurrentlyViewed="true" lang="en-US"><one:QuickStyleDef index="0" name="p" fontColor="automatic" highlightColor="automatic" font="Calibri" fontSize="11.0" spaceBefore="0.0" spaceAfter="0.0"/><one:PageSettings RTL="false" color="automatic"><one:PageSize><one:Automatic/></one:PageSize><one:RuleLines visible="false"/></one:PageSettings><one:Outline author="snow" authorInitials="S" lastModifiedBy="snow" lastModifiedByInitials="S" lastModifiedTime="2026-09-05T05:06:56.000Z" objectID="{37B65EC3-5FE2-0133-022F-E64595E063A3}{20}{B0}"><one:Position x="36.0" y="14.40000057220459" z="0"/><one:Size width="127.5136947631836" height="14.30000114440918"/><one:OEChildren><one:OE creationTime="2026-09-05T05:06:55.000Z" lastModifiedTime="2026-09-05T05:06:56.000Z" objectID="{37B65EC3-5FE2-0133-022F-E64595E063A3}{19}{B0}" alignment="left" quickStyleIndex="0" style="font-size:11.0pt;color:#1F4E79"><one:T><![CDATA[<span
3style='font-weight:bold;font-family:Calibri' lang=en-US>Fictitious: café, </span><span
4style='font-weight:bold;font-family:SimSun' lang=en-US>東京</span><span
5style='font-weight:bold;font-family:Calibri' lang=en-US>, </span><span
6style='font-weight:bold;font-family:Arial;direction:rtl;unicode-bidi:embed'
7lang=ar-SA>مرحبا</span>]]></one:T></one:OE></one:OEChildren></one:Outline><one:Outline author="snow" authorInitials="S" lastModifiedBy="snow" lastModifiedByInitials="S" lastModifiedTime="2026-09-08T03:59:44.000Z" objectID="{37B65EC3-5FE2-0133-022F-E64595E063A3}{25}{B0}"><one:Position x="144.0" y="96.0" z="1"/><one:Size width="167.0449523925781" height="13.42771339416504"/><one:OEChildren><one:OE creationTime="2026-09-05T05:06:57.000Z" lastModifiedTime="2026-09-08T03:59:44.000Z" objectID="{37B65EC3-5FE2-0133-022F-E64595E063A3}{24}{B0}" alignment="left" quickStyleIndex="0"><one:T><![CDATA[Fictitious positioned outline.]]></one:T></one:OE></one:OEChildren></one:Outline><one:Outline author="snow" authorInitials="S" lastModifiedBy="snow" lastModifiedByInitials="S" lastModifiedTime="2026-09-05T05:06:58.000Z" objectID="{37B65EC3-5FE2-0133-022F-E64595E063A3}{34}{B0}"><one:Position x="144.0" y="192.0" z="2"/><one:Size width="72.0" height="0.750005722045898"/><one:OEChildren><one:OE creationTime="2026-09-05T05:06:58.000Z" lastModifiedTime="2026-09-05T05:06:58.000Z" objectID="{37B65EC3-5FE2-0133-022F-E64595E063A3}{27}{B0}" alignment="left"><one:Image format="png" originalPageNumber="0"><one:Data>iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9J1uoA
8AAAASUVORK5CYII=</one:Data></one:Image></one:OE></one:OEChildren></one:Outline><one:Outline author="snow" authorInitials="S" lastModifiedBy="snow" lastModifiedByInitials="S" lastModifiedTime="2026-09-05T05:06:59.000Z" objectID="{37B65EC3-5FE2-0133-022F-E64595E063A3}{33}{B0}"><one:Position x="264.0" y="192.0" z="3"/><one:Size width="72.00001525878906" height="63.0"/><one:OEChildren><one:OE creationTime="2026-09-05T05:06:59.000Z" lastModifiedTime="2026-09-05T05:06:59.000Z" objectID="{37B65EC3-5FE2-0133-022F-E64595E063A3}{32}{B0}" alignment="left"><one:InsertedFile pathCache="C:\Users\clover\AppData\Local\Temp\OneNote\14.0\DNT\8722b1ff-e9f5-47ac-a873-6d5fcbde718b.txt" pathSource="C:\one-tests\runs\20260905-05\assets\fictitious-attachment.txt" preferredName="fictitious-attachment.txt"/></one:OE></one:OEChildren></one:Outline><one:Outline author="snow" authorInitials="S" lastModifiedBy="snow" lastModifiedByInitials="S" lastModifiedTime="2026-09-08T03:59:44.000Z" objectID="{37B65EC3-5FE2-0133-022F-E64595E063A3}{44}{B0}"><one:Position x="144.0" y="312.0" z="4"/><one:Size width="92.42181396484374" height="21.94773101806641"/><one:OEChildren><one:OE creationTime="2026-09-05T05:07:00.000Z" lastModifiedTime="2026-09-08T03:59:44.000Z" objectID="{37B65EC3-5FE2-0133-022F-E64595E063A3}{43}{B0}" alignment="left"><one:Table bordersVisible="true" lastModifiedTime="2026-09-08T03:59:44.000Z" objectID="{37B65EC3-5FE2-0133-022F-E64595E063A3}{42}{B0}"><one:Columns><one:Column index="0" width="39.14614105224609"/><one:Column index="1" width="45.14567184448242"/></one:Columns><one:Row objectID="{37B65EC3-5FE2-0133-022F-E64595E063A3}{41}{B0}" lastModifiedTime="2026-09-08T03:59:44.000Z"><one:Cell lastModifiedTime="2026-09-08T03:59:44.000Z" objectID="{37B65EC3-5FE2-0133-022F-E64595E063A3}{40}{B0}" lastModifiedByInitials="S"><one:OEChildren><one:OE creationTime="2026-09-05T05:07:00.000Z" lastModifiedTime="2026-09-08T03:59:44.000Z" objectID="{37B65EC3-5FE2-0133-022F-E64595E063A3}{39}{B0}" alignment="left" quickStyleIndex="0"><one:T><![CDATA[Left cell]]></one:T></one:OE></one:OEChildren></one:Cell><one:Cell lastModifiedTime="2026-09-08T03:59:44.000Z" objectID="{37B65EC3-5FE2-0133-022F-E64595E063A3}{37}{B0}" lastModifiedByInitials="S"><one:OEChildren><one:OE creationTime="2026-09-05T05:07:00.000Z" lastModifiedTime="2026-09-08T03:59:44.000Z" objectID="{37B65EC3-5FE2-0133-022F-E64595E063A3}{36}{B0}" alignment="left" quickStyleIndex="0"><one:T><![CDATA[Right cell]]></one:T></one:OE></one:OEChildren></one:Cell></one:Row></one:Table></one:OE></one:OEChildren></one:Outline></one:Page>
corpus/native-encrypted/cold-2010-4763/read/payloads.json created+10
......@@ -0,0 +1,10 @@
1[
2 {
3 "sha256": "af1af4b8987109ce1e335e538910fb3860b7c57eb3b6110a06aa2114e06d69a7",
4 "name": "fictitious-attachment.txt",
5 "object": "{37B65EC3-5FE2-0133-022F-E64595E063A3}{32}{B0}",
6 "kind": "InsertedFile",
7 "page": "{75216EDF-30ED-03E0-13CD-C2FB6030B01D}{14}{B0}",
8 "bytes": 43
9 }
10]
\ No newline at end of file
crates/onestore-notebook/Cargo.toml+2
......@@ -6,12 +6,14 @@ publish = false
66
77[features]
88smb = ["dep:onestore-smb"]
9protected = ["onestore/protected", "dep:zeroize"]
910
1011[dependencies]
1112onestore = { path = "../onestore" }
1213onestore-smb = { path = "../onestore-smb", optional = true }
1314serde = { version = "1", features = ["derive"] }
1415thiserror = "2"
16zeroize = { version = "1.9.0", optional = true }
1517
1618[dev-dependencies]
1719serde_json = "1"
crates/onestore-notebook/examples/document.rs+59-4
......@@ -23,16 +23,67 @@ fn write_json(
2323fn main() -> Result<(), Box<dyn std::error::Error>> {
2424 let mut args = env::args_os().skip(1);
2525 let path = args.next().ok_or("Provide a .one or .onetoc2 file.")?;
26 let destination = args.next().map(PathBuf::from);
27 if args.next().is_some() {
28 return Err("Provide a source file and an optional new export directory.".into());
29 }
26 let next = args.next();
27 let (destination, option) = if next.as_deref() == Some(std::ffi::OsStr::new("--password-file"))
28 {
29 (None, next)
30 } else {
31 (next.map(PathBuf::from), args.next())
32 };
33 let password_file = match (option, args.next(), args.next()) {
34 (None, None, None) => None,
35 (Some(flag), Some(path), None) if flag == "--password-file" => Some(PathBuf::from(path)),
36 _ => return Err("Provide a source file, an optional new export directory, and optionally --password-file PATH.".into()),
37 };
3038 let source_path = PathBuf::from(path);
3139 let bytes = onestore::read_file(&source_path)?;
3240 let store = Store::parse(&bytes)?;
3341 let index = RevisionIndex::parse(&store)?;
42 #[cfg(feature = "protected")]
43 let unlocked = if let Some(path) = &password_file {
44 use std::io::Read;
45 let mut password = zeroize::Zeroizing::new(String::new());
46 fs::File::open(path)?
47 .take(65537)
48 .read_to_string(&mut password)?;
49 if password.len() > 65536 {
50 return Err("The password file exceeds 64 KiB.".into());
51 }
52 Some(onestore::protected::UnlockedSection::open(
53 &index,
54 &password,
55 Default::default(),
56 )?)
57 } else {
58 None
59 };
60 #[cfg(not(feature = "protected"))]
61 if password_file.is_some() {
62 return Err(
63 "Build this exporter with the protected feature to open a password-protected section."
64 .into(),
65 );
66 }
67 #[cfg(feature = "protected")]
68 let document = match &unlocked {
69 Some(section) => section.document()?,
70 None => Document::parse(&index)?,
71 };
72 #[cfg(not(feature = "protected"))]
3473 let document = Document::parse(&index)?;
3574 if let Some(destination) = destination {
75 #[cfg(unix)]
76 {
77 use std::os::unix::fs::DirBuilderExt;
78 fs::DirBuilder::new()
79 .mode(if password_file.is_some() {
80 0o700
81 } else {
82 0o777
83 })
84 .create(&destination)?;
85 }
86 #[cfg(not(unix))]
3687 fs::create_dir(&destination)?;
3788 fs::create_dir(destination.join("assets"))?;
3889 let parent = source_path
......@@ -62,6 +113,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
62113 Cow::Borrowed(payload.ok_or("Missing embedded file data")?)
63114 }
64115 FileDataReference::External(filename) => {
116 if password_file.is_some() {
117 assets.push(serde_json::json!({"reference":reference,"path":null,"error":{"kind":"Unsupported","message":"Protected external payload export is not supported"}}));
118 continue;
119 }
65120 match onestore_notebook::read_external_asset(
66121 &mut source,
67122 section,
crates/onestore/Cargo.toml+10
......@@ -4,10 +4,20 @@ version = "0.1.0"
44edition = "2024"
55publish = false
66
7[features]
8protected = ["dep:aes", "dep:base64", "dep:cbc", "dep:roxmltree", "dep:sha1", "dep:subtle", "dep:zeroize"]
9
710[dependencies]
811md5 = "0.8.1"
912getrandom = "0.4.3"
1013serde = { version = "1.0.229", features = ["derive"] }
14aes = { version = "0.9.3", features = ["zeroize"], optional = true }
15base64 = { version = "0.22.1", optional = true }
16cbc = { version = "0.2.1", features = ["zeroize"], optional = true }
17roxmltree = { version = "0.21.1", optional = true }
18sha1 = { version = "0.11.0", features = ["zeroize"], optional = true }
19subtle = { version = "2.6.1", optional = true }
20zeroize = { version = "1.9.0", optional = true }
1121
1222[target.'cfg(target_os = "macos")'.dependencies]
1323nix = { version = "0.31.3", default-features = false, features = ["fs"] }
crates/onestore/README.md+39-2
......@@ -43,6 +43,7 @@ harness also accepts `--client-profile release`.
4343| `PropertySets`, `Object::references` | Decode properties and ID streams while retaining raw values |
4444| `Object::file_reference`, `Store::file_data` | Identify internal/external payloads and read internal payload bytes |
4545| `document::Document`, `Revision::text_runs` | Interpret document objects and inherited text formatting while retaining unknown properties and revision identities |
46| `protected::UnlockedSection` (optional feature) | Own decoded buffers for explicit known-password inspection; clear those buffers on drop; derived document strings/exports remain caller-owned |
4647| `create_section` | Create one page containing one plain-text paragraph and an author, including Unicode |
4748| `create_table_of_contents` | Create ordered section entries from filenames and file identities |
4849| `replace_property_bytes` | Append one scalar-property revision; preserve prior revisions and unrelated property values and references |
......@@ -63,8 +64,11 @@ protected objects and split surrogate pairs are
6364rejected before writing. Appended snapshots cap revision dependency depth at 512
6465while retaining historical revisions. TOC snapshots can remap encoded CompactIDs
6566without changing their resolved references. Password-protected
66sections retain their encrypted structure and payloads; the library does not
67derive password keys or decrypt their pages.
67sections retain their encrypted structure and payloads. With the optional
68`protected` feature, `protected::UnlockedSection` opens native OneNote 2010
69AES-128/CBC, SHA-1 password wrappers into a borrowed document view. Incorrect
70passwords, unsupported protection profiles and work-limit failures remain distinct.
71The source stays encrypted; protected writes are rejected.
6872Insertions update child references, reference counts, modification times and automatic
6973titles atomically. Paragraphs can be nested or inserted into table cells; outline
7074coordinates use points. Retain the `Insertion` value for rebasing: creating another
......@@ -241,3 +245,36 @@ exact changes as well as retained image, ink, table, and attachment content.
241245a dedicated loopback test session; its control JSON selects the successful response
242246and occurrence to withhold. The captured trace and result files are the regression
243247oracle; replaying the native experiments requires the supplied Windows/share setup.
248
249## Protected inspection
250
251```rust,no_run
252# #[cfg(feature = "protected")]
253# fn example() {
254use onestore::{RevisionIndex, Store, protected::{Limits, UnlockedSection}};
255# fn inspect(bytes: &[u8], password: &str) -> Result<(), Box<dyn std::error::Error>> {
256let store = Store::parse(bytes)?;
257let index = RevisionIndex::parse(&store)?;
258let unlocked = UnlockedSection::open(&index, password, Limits::default())?;
259let document = unlocked.document()?;
260assert!(!document.pages()?.is_empty());
261drop(document);
262drop(unlocked);
263# Ok(()) }
264# }
265```
266
267This example requires `features = ["protected"]`. The owner retains no password or
268key after opening. Its source-buffer views cannot outlive it; copies of parsed
269strings, serialized models and exports have their own lifetimes. These copies are
270plaintext, and dropping the unlock owner does not clear them. CBC has no general
271ciphertext-authentication guarantee; native read-only hashes and model validation
272check the corresponding structure. Internal payloads are decoded; external payload
273references remain references, and their protected decoding is not implemented.
274
275For a deliberate plaintext diagnostic export, build the notebook exporter with
276`--features protected` and pass `--password-file PATH` after the source and optional
277new output directory. The file contains exact UTF-8 password bytes; no newline is
278removed or Unicode normalization applied. The exporter creates protected exports
279under an owner-only directory on Unix and reports protected external payloads as
280unsupported. It never rewrites the encrypted source.
crates/onestore/src/document.rs+27-3
......@@ -298,6 +298,7 @@ pub enum Kind<'a> {
298298 Table {
299299 rows: Option<u32>,
300300 columns: Option<u32>,
301 /// Stored widths in points; unlocked columns may fit their content in native layout.
301302 widths: Vec<f32>,
302303 locked: Vec<bool>,
303304 borders: Option<bool>,
......@@ -606,6 +607,18 @@ impl<'a> Document<'a> {
606607
607608 /// Rejects checksum damage and follows referenced contexts, retaining encrypted payloads opaquely.
608609 pub fn parse(index: &RevisionIndex<'a>) -> Result<Self> {
610 Self::parse_with(
611 index,
612 |space, revision| index.resolve(space, revision),
613 |id| index.store.file_data(id),
614 )
615 }
616
617 pub(crate) fn parse_with(
618 index: &RevisionIndex<'a>,
619 mut resolve: impl FnMut(ExGuid, ExGuid) -> Result<crate::ResolvedRevision<'a>>,
620 mut file_data: impl FnMut([u8; 16]) -> Result<&'a [u8]>,
621 ) -> Result<Self> {
609622 if !index.store.checksum_mismatches.is_empty() {
610623 return Err(invalid("Document transaction checksum mismatch"));
611624 }
......@@ -628,7 +641,7 @@ impl<'a> Document<'a> {
628641 if space.revisions.contains_key(&rid) {
629642 continue;
630643 }
631 let revision = index.resolve(id, rid)?;
644 let revision = resolve(id, rid)?;
632645 let mut reachable = BTreeSet::new();
633646 let mut objects: Vec<_> = revision.roots.values().copied().collect();
634647 let mut encrypted = false;
......@@ -661,7 +674,10 @@ impl<'a> Document<'a> {
661674 );
662675 pending.extend(refs.contexts.into_iter().map(|context| (id, context)));
663676 }
664 nodes.insert(oid, Element::parse(object, index.store)?);
677 nodes.insert(
678 oid,
679 Element::parse_with(object, index.store, &mut file_data)?,
680 );
665681 }
666682 for node in nodes.values() {
667683 if let Kind::RichText {
......@@ -748,6 +764,14 @@ impl<'a> Document<'a> {
748764
749765impl<'a> Element<'a> {
750766 pub(crate) fn parse(object: &Object<'a>, store: &Store<'a>) -> Result<Self> {
767 Self::parse_with(object, store, &mut |id| store.file_data(id))
768 }
769
770 fn parse_with(
771 object: &Object<'a>,
772 store: &Store<'a>,
773 file_data: &mut impl FnMut([u8; 16]) -> Result<&'a [u8]>,
774 ) -> Result<Self> {
751775 let empty = |kind| Self {
752776 jcid: object.jcid,
753777 children: vec![],
......@@ -775,7 +799,7 @@ impl<'a> Element<'a> {
775799 .file_reference()?
776800 .ok_or_else(|| invalid("File object has no reference"))?;
777801 let payload = if let FileDataReference::Internal(guid) = &reference {
778 Some(store.file_data(*guid)?)
802 Some(file_data(*guid)?)
779803 } else {
780804 None
781805 };
crates/onestore/src/lib.rs+2
......@@ -12,6 +12,8 @@ mod formatting;
1212mod insertion;
1313mod objects;
1414mod properties;
15#[cfg(feature = "protected")]
16pub mod protected;
1517mod revisions;
1618mod snapshot;
1719mod store;
crates/onestore/src/properties.rs+37-32
......@@ -41,38 +41,7 @@ enum Work<'a> {
4141impl<'a> PropertySets<'a> {
4242 pub fn parse(bytes: &'a [u8]) -> Result<Self, Error> {
4343 let mut c = Cursor { bytes, offset: 0 };
44 let mut streams = std::array::from_fn::<_, 3, _>(|_| Cursor {
45 bytes: &[],
46 offset: 0,
47 });
48 let mut extended = false;
49 for (index, stream) in streams.iter_mut().enumerate() {
50 let header = u32::from_le_bytes(c.read()?);
51 let count = usize::try_from(header & 0xffffff).unwrap();
52 if (index > 0 && header & 0x80000000 != 0)
53 || (index == 0 && header & 0xc0000000 == 0xc0000000)
54 || (index == 1 && (header & 0x40000000 != 0) != extended)
55 || (index == 2 && header & 0x40000000 != 0)
56 {
57 return Err(Error {
58 offset: c.offset - 4,
59 message: "Inconsistent property reference-stream flags",
60 });
61 }
62 let offset = c.offset;
63 *stream = Cursor {
64 bytes: c.take(count * 4)?,
65 offset,
66 };
67 if index == 0 {
68 extended = header & 0x40000000 != 0;
69 if header & 0x80000000 != 0 {
70 break;
71 }
72 } else if index == 1 && !extended {
73 break;
74 }
75 }
44 let mut streams = reference_streams(&mut c)?;
7645 let mut sets = vec![Vec::new()];
7746 let mut root_ids = &bytes[..0];
7847 let mut work = vec![Work::Set(0)];
......@@ -196,3 +165,39 @@ impl<'a> PropertySets<'a> {
196165 })
197166 }
198167}
168
169pub(crate) fn reference_streams<'a>(c: &mut Cursor<'a>) -> Result<[Cursor<'a>; 3], Error> {
170 let mut streams = std::array::from_fn::<_, 3, _>(|_| Cursor {
171 bytes: &[],
172 offset: 0,
173 });
174 let mut extended = false;
175 for (index, stream) in streams.iter_mut().enumerate() {
176 let header = u32::from_le_bytes(c.read()?);
177 let count = usize::try_from(header & 0xffffff).unwrap();
178 if (index > 0 && header & 0x80000000 != 0)
179 || (index == 0 && header & 0xc0000000 == 0xc0000000)
180 || (index == 1 && (header & 0x40000000 != 0) != extended)
181 || (index == 2 && header & 0x40000000 != 0)
182 {
183 return Err(Error {
184 offset: c.offset - 4,
185 message: "Inconsistent property reference-stream flags",
186 });
187 }
188 let offset = c.offset;
189 *stream = Cursor {
190 bytes: c.take(count * 4)?,
191 offset,
192 };
193 if index == 0 {
194 extended = header & 0x40000000 != 0;
195 if header & 0x80000000 != 0 {
196 break;
197 }
198 } else if index == 1 && !extended {
199 break;
200 }
201 }
202 Ok(streams)
203}
crates/onestore/src/protected/crypto.rs created+349
......@@ -0,0 +1,349 @@
1use super::{Error, Result, invalid};
2use aes::cipher::{BlockModeDecrypt, KeyIvInit, block_padding::NoPadding};
3use base64::{Engine, engine::general_purpose::STANDARD};
4use sha1::{Digest, Sha1};
5use subtle::ConstantTimeEq;
6use zeroize::{Zeroize, Zeroizing};
7
8const NS: &str = "http://schemas.microsoft.com/office/2006/encryption";
9const PASSWORD_NS: &str = "http://schemas.microsoft.com/office/2006/keyEncryptor/password";
10
11pub(super) struct Key {
12 value: Zeroizing<[u8; 16]>,
13 file_iv: [u8; 16],
14}
15
16fn child<'a, 'input>(
17 node: roxmltree::Node<'a, 'input>,
18 name: &str,
19 ns: &str,
20) -> Result<roxmltree::Node<'a, 'input>> {
21 let mut matches = node.children().filter(|n| n.has_tag_name((ns, name)));
22 let value = matches
23 .next()
24 .ok_or_else(|| invalid("Missing encryption metadata element"))?;
25 if matches.next().is_some() {
26 return Err(invalid("Repeated encryption metadata element"));
27 }
28 Ok(value)
29}
30
31fn decoded<const N: usize>(node: roxmltree::Node<'_, '_>, name: &str) -> Result<[u8; N]> {
32 let value = node
33 .attribute(name)
34 .ok_or_else(|| invalid("Missing encryption metadata attribute"))?;
35 let bytes = STANDARD
36 .decode(value.split_ascii_whitespace().collect::<String>())
37 .map_err(|_| invalid("Invalid encryption metadata base64"))?;
38 bytes
39 .try_into()
40 .map_err(|_| invalid("Invalid encryption metadata byte length"))
41}
42
43fn profile(node: roxmltree::Node<'_, '_>) -> Result<()> {
44 for (attribute, value) in [
45 ("saltSize", 16),
46 ("blockSize", 16),
47 ("keyBits", 128),
48 ("hashSize", 20),
49 ] {
50 if number(node, attribute)? != value {
51 return Err(Error::Unsupported);
52 }
53 }
54 for (attribute, value) in [
55 ("cipherAlgorithm", "AES"),
56 ("cipherChaining", "ChainingModeCBC"),
57 ("hashAlgorithm", "SHA1"),
58 ] {
59 let actual = node
60 .attribute(attribute)
61 .ok_or_else(|| invalid("Missing encryption algorithm attribute"))?;
62 if actual != value {
63 return Err(Error::Unsupported);
64 }
65 }
66 Ok(())
67}
68
69fn number(node: roxmltree::Node<'_, '_>, name: &str) -> Result<u32> {
70 node.attribute(name)
71 .ok_or_else(|| invalid("Missing encryption numeric attribute"))?
72 .trim()
73 .parse()
74 .map_err(|_| invalid("Invalid encryption numeric attribute"))
75}
76
77fn decrypt(key: &[u8; 16], iv: &[u8; 16], bytes: &mut [u8]) -> Result<()> {
78 cbc::Decryptor::<aes::Aes128>::new(key.into(), iv.into())
79 .decrypt_padded::<NoPadding>(bytes)
80 .map_err(|_| invalid("Encrypted payload is not block aligned"))?;
81 Ok(())
82}
83
84impl Key {
85 pub(super) fn open(data: &[u8], password: &str, rounds: &mut u64) -> Result<Self> {
86 if data.len() > 65536 || password.len() > 65536 {
87 return Err(Error::Limit);
88 }
89 let mut c = crate::bytes::Cursor {
90 bytes: data,
91 offset: 0,
92 };
93 if u32::from_le_bytes(c.read()?) != 3 {
94 return Err(Error::Unsupported);
95 }
96 let length = u32::from_le_bytes(c.read()?) as usize;
97 let offset = u32::from_le_bytes(c.read()?) as usize;
98 let inner = u32::from_le_bytes(c.read()?) as usize;
99 if length != data.len() || offset != 16 || inner != data.len() - 16 {
100 return Err(invalid("Inconsistent encryption metadata framing"));
101 }
102 if c.read::<8>()? != [4, 0, 4, 0, 64, 0, 0, 0] {
103 return Err(Error::Unsupported);
104 }
105 let text = std::str::from_utf8(c.bytes)
106 .map_err(|_| invalid("Encryption metadata is not UTF-8"))?;
107 let xml = roxmltree::Document::parse_with_options(
108 text,
109 roxmltree::ParsingOptions {
110 nodes_limit: 64,
111 ..Default::default()
112 },
113 )
114 .map_err(|_| invalid("Invalid encryption metadata XML"))?;
115 let root = xml.root_element();
116 if !root.has_tag_name((NS, "encryption")) {
117 return Err(Error::Unsupported);
118 }
119 let data_key = child(root, "keyData", NS)?;
120 let encryptors = child(root, "keyEncryptors", NS)?;
121 if root.children().filter(|n| n.is_element()).count() != 2
122 || encryptors.children().filter(|n| n.is_element()).count() != 1
123 {
124 return Err(Error::Unsupported);
125 }
126 let encryptor = child(encryptors, "keyEncryptor", NS)?;
127 if encryptor.attribute("uri") != Some(PASSWORD_NS)
128 || encryptor.children().filter(|n| n.is_element()).count() != 1
129 {
130 return Err(Error::Unsupported);
131 }
132 let wrapped = child(encryptor, "encryptedKey", PASSWORD_NS)?;
133 profile(data_key)?;
134 profile(wrapped)?;
135 let count = number(wrapped, "spinCount")?;
136 *rounds = rounds.checked_sub(u64::from(count)).ok_or(Error::Limit)?;
137 let salt = decoded::<16>(wrapped, "saltValue")?;
138 let mut verifier = Zeroizing::new(decoded::<16>(wrapped, "encryptedVerifierHashInput")?);
139 let mut expected = Zeroizing::new(decoded::<32>(wrapped, "encryptedVerifierHashValue")?);
140 let mut value = Zeroizing::new(decoded::<16>(wrapped, "encryptedKeyValue")?);
141 let mut hash = Sha1::new();
142 hash.update(salt);
143 for word in password.encode_utf16() {
144 hash.update(word.to_le_bytes());
145 }
146 let mut seed = Zeroizing::new(<[u8; 20]>::from(hash.finalize()));
147 for index in 0..count {
148 let mut hash = Sha1::new();
149 hash.update(index.to_le_bytes());
150 hash.update(seed.as_ref());
151 *seed = hash.finalize().into();
152 }
153 for (label, bytes) in [
154 (
155 [0xfe, 0xa7, 0xd2, 0x76, 0x3b, 0x4b, 0x9e, 0x79],
156 verifier.as_mut_slice(),
157 ),
158 (
159 [0xd7, 0xaa, 0x0f, 0x6d, 0x30, 0x61, 0x34, 0x4e],
160 expected.as_mut_slice(),
161 ),
162 (
163 [0x14, 0x6e, 0x0b, 0xe7, 0xab, 0xac, 0xd0, 0xd6],
164 value.as_mut_slice(),
165 ),
166 ] {
167 let mut hash = Sha1::new();
168 hash.update(seed.as_ref());
169 hash.update(label);
170 let derived = Zeroizing::new(<[u8; 20]>::from(hash.finalize()));
171 decrypt(derived[..16].try_into().unwrap(), &salt, bytes)?;
172 }
173 let actual = Zeroizing::new(<[u8; 20]>::from(Sha1::digest(verifier.as_slice())));
174 if !bool::from(actual.as_slice().ct_eq(&expected[..20])) {
175 return Err(Error::PasswordMismatch);
176 }
177 let mut hash = Sha1::new();
178 hash.update(decoded::<16>(data_key, "saltValue")?);
179 hash.update(0_u32.to_le_bytes());
180 let file_iv = hash.finalize()[..16].try_into().unwrap();
181 Ok(Self { value, file_iv })
182 }
183
184 pub(super) fn property(&self, input: &[u8]) -> Result<Zeroizing<Vec<u8>>> {
185 let mut c = crate::bytes::Cursor {
186 bytes: input,
187 offset: 0,
188 };
189 crate::properties::reference_streams(&mut c)?;
190 let prefix = c.offset;
191 let length = u32::from_le_bytes(c.read()?) as usize;
192 let encrypted = c.take(length)?;
193 if c.bytes.len() > 7 || c.bytes.iter().any(|b| *b != 0) {
194 return Err(invalid("Invalid encrypted property alignment"));
195 }
196 let (iv, body) = encrypted
197 .split_first_chunk::<16>()
198 .ok_or_else(|| invalid("Missing encrypted property IV"))?;
199 let mut clear = Zeroizing::new(body.to_vec());
200 decrypt(&self.value, iv, &mut clear)?;
201 let padding = clear
202 .first_chunk::<2>()
203 .map(|b| usize::from(u16::from_le_bytes(*b)))
204 .ok_or_else(|| invalid("Missing encrypted property padding count"))?;
205 if padding >= 16 || clear.len() < 2 + padding {
206 return Err(invalid("Invalid encrypted property padding count"));
207 }
208 let mut output = Zeroizing::new(Vec::with_capacity(prefix + clear.len() - 2 - padding));
209 output.extend_from_slice(&input[..prefix]);
210 output.extend_from_slice(&clear[2..clear.len() - padding]);
211 crate::PropertySets::parse(&output)?;
212 Ok(output)
213 }
214
215 pub(super) fn file(&self, input: &[u8]) -> Result<Zeroizing<Vec<u8>>> {
216 if input.is_empty() {
217 return Ok(Zeroizing::new(Vec::new()));
218 }
219 let mut clear = Zeroizing::new(input.to_vec());
220 decrypt(&self.value, &self.file_iv, &mut clear)?;
221 let length = clear
222 .first_chunk::<8>()
223 .map(|b| u64::from_le_bytes(*b))
224 .ok_or_else(|| invalid("Missing encrypted file length"))?;
225 let length = usize::try_from(length)
226 .map_err(|_| invalid("Encrypted file length exceeds address space"))?;
227 if length > clear.len() - 8 || clear.len() - 8 - length >= 16 {
228 return Err(invalid("Invalid encrypted file length"));
229 }
230 clear.copy_within(8..8 + length, 0);
231 clear[length..].zeroize();
232 clear.truncate(length);
233 Ok(clear)
234 }
235}
236
237#[cfg(test)]
238mod tests {
239 use super::*;
240 use crate::{ObjectData, Reference, RevisionIndex, Store};
241
242 #[test]
243 fn native_frames_and_bounded_malformed_inputs() {
244 let root = std::path::Path::new("../../corpus/native-encrypted");
245 let bytes = std::fs::read(root.join("encrypted-01/notebook/synthetic.one")).unwrap();
246 let manifest: serde_json::Value =
247 serde_json::from_slice(&std::fs::read(root.join("manifest.json")).unwrap()).unwrap();
248 let password = manifest["password"].as_str().unwrap();
249 let store = Store::parse(&bytes).unwrap();
250 let index = RevisionIndex::parse(&store).unwrap();
251 let node = index
252 .spaces
253 .values()
254 .next()
255 .unwrap()
256 .revisions
257 .values()
258 .next()
259 .unwrap()
260 .nodes
261 .first()
262 .unwrap();
263 let Some(Reference::Data(chunk)) = node.reference else {
264 panic!("Missing native key")
265 };
266 let metadata = store.encryption_key(chunk).unwrap();
267 for length in 0..metadata.len() {
268 assert!(Key::open(&metadata[..length], password, &mut 0).is_err());
269 }
270 let key = Key::open(metadata, password, &mut 100000).unwrap();
271 let xml = std::str::from_utf8(&metadata[24..]).unwrap();
272 let frame = |xml: &str| {
273 let mut value = metadata[..24].to_vec();
274 value.extend_from_slice(xml.as_bytes());
275 let length = u32::try_from(value.len()).unwrap();
276 value[4..8].copy_from_slice(&length.to_le_bytes());
277 value[12..16].copy_from_slice(&(length - 16).to_le_bytes());
278 value
279 };
280 assert!(matches!(
281 Key::open(
282 &frame(&xml.replace("keyBits=\"128\"", "keyBits=\"256\"")),
283 password,
284 &mut 100000
285 ),
286 Err(Error::Unsupported)
287 ));
288 assert!(matches!(
289 Key::open(
290 &frame(&xml.replace("keyBits=\"128\"", "")),
291 password,
292 &mut 100000
293 ),
294 Err(Error::Invalid(_))
295 ));
296 let tree = roxmltree::Document::parse(xml).unwrap();
297 let salt = child(tree.root_element(), "keyData", NS)
298 .unwrap()
299 .attribute("saltValue")
300 .unwrap();
301 let spaced = xml
302 .replace("keyBits=\"128\"", "keyBits=\" +0128 \"")
303 .replace("spinCount=\"100000\"", "spinCount=\" 0100000 \"")
304 .replace(salt, &format!(" {}\n{} ", &salt[..8], &salt[8..]));
305 let spaced = Key::open(&frame(&spaced), password, &mut 100000).unwrap();
306 for (sid, space) in &index.spaces {
307 for rid in space.labels.values() {
308 let revision = index.resolve(*sid, *rid).unwrap();
309 for object in revision.objects.values() {
310 if let ObjectData::Encrypted(bytes) = object.data {
311 let mut cursor = crate::bytes::Cursor { bytes, offset: 0 };
312 crate::properties::reference_streams(&mut cursor).unwrap();
313 let length = u32::from_le_bytes(cursor.read().unwrap()) as usize;
314 let end = cursor.offset + length;
315 assert!(key.property(bytes).is_ok());
316 assert_eq!(
317 *key.property(bytes).unwrap(),
318 *spaced.property(bytes).unwrap()
319 );
320 for cut in 0..end {
321 assert!(key.property(&bytes[..cut]).is_err());
322 }
323 let mut changed = bytes.to_vec();
324 changed.push(1);
325 assert!(key.property(&changed).is_err());
326 }
327 }
328 }
329 }
330 let mut state = 0x4851_e529_b30d_620f_u64;
331 for length in 0..1024 {
332 let mut bytes = vec![0; length];
333 for byte in &mut bytes {
334 state ^= state << 13;
335 state ^= state >> 7;
336 state ^= state << 17;
337 *byte = state.to_le_bytes()[0];
338 }
339 let _ = key.property(&bytes);
340 let _ = key.file(&bytes);
341 assert!(Key::open(&bytes, password, &mut 0).is_err());
342 }
343 for index in 0..metadata.len() {
344 let mut changed = metadata.to_vec();
345 changed[index] ^= 0x80;
346 assert!(Key::open(&changed, password, &mut 0).is_err());
347 }
348 }
349}
crates/onestore/src/protected/mod.rs created+275
......@@ -0,0 +1,275 @@
1//! Explicit, in-memory opening of native OneNote 2010 protected sections.
2//!
3//! AES-128/CBC and SHA-1 Agile password wrappers are supported. Unknown wrappers
4//! remain opaque through the ordinary storage/document APIs. CBC provides no
5//! general ciphertext authentication; native read-only hashes and model checks
6//! detect structural inconsistencies, not arbitrary changes to all content.
7
8mod crypto;
9
10use crate::{ExGuid, FileDataReference, ObjectData, Reference, RevisionIndex, document::Document};
11use std::{
12 collections::{BTreeMap, BTreeSet},
13 fmt,
14};
15use zeroize::Zeroizing;
16
17type Result<T> = std::result::Result<T, Error>;
18
19#[derive(Debug)]
20pub enum Error {
21 PasswordMismatch,
22 Unsupported,
23 Limit,
24 Invalid(crate::Error),
25}
26
27impl From<crate::Error> for Error {
28 fn from(value: crate::Error) -> Self {
29 Self::Invalid(value)
30 }
31}
32
33impl fmt::Display for Error {
34 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
35 match self {
36 Self::PasswordMismatch => {
37 f.write_str("The password did not match the section verifier")
38 }
39 Self::Unsupported => f.write_str("This protection format is not supported"),
40 Self::Limit => f.write_str("Opening the protected section exceeded its work limit"),
41 Self::Invalid(error) => error.fmt(f),
42 }
43 }
44}
45impl std::error::Error for Error {
46 fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
47 match self {
48 Self::Invalid(error) => Some(error),
49 _ => None,
50 }
51 }
52}
53
54fn invalid(message: &'static str) -> Error {
55 Error::Invalid(crate::Error { offset: 0, message })
56}
57
58#[derive(Debug, Clone, Copy)]
59pub struct Limits {
60 /// Total password iterations across distinct encryption metadata containers.
61 pub kdf_rounds: u64,
62 /// Total ciphertext/reference bytes materialized; temporary copies can double this.
63 pub decoded_bytes: usize,
64 /// Sum of object counts in distinct labeled revisions.
65 pub object_visits: usize,
66}
67
68impl Default for Limits {
69 fn default() -> Self {
70 Self {
71 kdf_rounds: 1_000_000,
72 decoded_bytes: 256 * 1024 * 1024,
73 object_visits: 1_000_000,
74 }
75 }
76}
77
78/// Owns decoded buffers until dropped; returned views cannot outlive this owner.
79///
80/// Passwords and derived keys are not retained. Dropping this owner clears its
81/// decoded buffers. Owned strings or exports made from a `Document` are separate
82/// caller-owned copies and must be disposed of by the caller when locking.
83/// Opening never rewrites the source image or changes its protection state.
84///
85/// ```compile_fail
86/// # use onestore::{RevisionIndex, protected::{Limits, UnlockedSection}};
87/// fn outlive<'a>(index: &'a RevisionIndex<'a>, password: &str) {
88/// let unlocked = UnlockedSection::open(index, password, Limits::default()).unwrap();
89/// let document = unlocked.document().unwrap();
90/// drop(unlocked);
91/// document.pages().unwrap();
92/// }
93/// ```
94pub struct UnlockedSection<'a> {
95 index: &'a RevisionIndex<'a>,
96 objects: BTreeMap<(ExGuid, usize), Zeroizing<Vec<u8>>>,
97 files: BTreeMap<[u8; 16], Zeroizing<Vec<u8>>>,
98}
99
100impl<'a> UnlockedSection<'a> {
101 /// Verifies the password and every labeled revision before exposing a view.
102 /// Metadata and password input are each bounded to 64 KiB.
103 pub fn open(index: &'a RevisionIndex<'a>, password: &str, mut limits: Limits) -> Result<Self> {
104 if index.store.header.file_type != crate::FileType::Section {
105 return Err(Error::Unsupported);
106 }
107 if !index.store.checksum_mismatches.is_empty() {
108 return Err(invalid("Protected document transaction checksum mismatch"));
109 }
110 let mut result = Self {
111 index,
112 objects: BTreeMap::new(),
113 files: BTreeMap::new(),
114 };
115 let mut keys = BTreeMap::new();
116 let mut file_keys = BTreeMap::new();
117 let mut hashes = BTreeMap::new();
118 for node in index.store.lists.values().flat_map(|list| &list.nodes) {
119 if !matches!(node.id, 0xc4 | 0xc5) {
120 continue;
121 }
122 let Some(Reference::Data(chunk)) = node.reference else {
123 return Err(invalid("Read-only object has no data reference"));
124 };
125 let bytes = index.store.chunk_data(chunk)?;
126 let expected: [u8; 16] = node
127 .payload
128 .last_chunk::<16>()
129 .copied()
130 .ok_or_else(|| invalid("Missing read-only object hash"))?;
131 if hashes
132 .insert(bytes.as_ptr().addr(), expected)
133 .is_some_and(|old| old != expected)
134 {
135 return Err(invalid("Inconsistent read-only hashes for one payload"));
136 }
137 }
138 for (space_id, space) in &index.spaces {
139 let mut metadata = None;
140 for revision in space.revisions.values() {
141 if !revision.encrypted {
142 return Err(Error::Unsupported);
143 }
144 let node = revision
145 .nodes
146 .first()
147 .ok_or_else(|| invalid("Missing encryption key node"))?;
148 let Some(Reference::Data(chunk)) = node.reference else {
149 return Err(invalid("Missing encryption key reference"));
150 };
151 let data = index.store.encryption_key(chunk)?;
152 if metadata.replace(data).is_some_and(|old| old != data) {
153 return Err(invalid("Object space changes its encryption metadata"));
154 }
155 }
156 let metadata =
157 metadata.ok_or_else(|| invalid("Protected object space has no revision"))?;
158 if !keys.contains_key(metadata) {
159 keys.insert(
160 metadata,
161 crypto::Key::open(metadata, password, &mut limits.kdf_rounds)?,
162 );
163 }
164 let key = &keys[metadata];
165 for rid in space.labels.values().copied().collect::<BTreeSet<_>>() {
166 let revision = index.resolve(*space_id, rid)?;
167 limits.object_visits = limits
168 .object_visits
169 .checked_sub(revision.objects.len())
170 .ok_or(Error::Limit)?;
171 for object in revision.objects.values() {
172 match object.data {
173 ObjectData::Encrypted(bytes) => {
174 let identity = (*space_id, bytes.as_ptr().addr());
175 let expected = hashes.get(&identity.1);
176 if object.jcid & 0x100000 != 0 && expected.is_none() {
177 return Err(invalid("Read-only encrypted object has no hash"));
178 }
179 if result.objects.contains_key(&identity) {
180 continue;
181 }
182 limits.decoded_bytes = limits
183 .decoded_bytes
184 .checked_sub(bytes.len())
185 .ok_or(Error::Limit)?;
186 let decoded = key.property(bytes)?;
187 if let Some(expected) = expected {
188 let mut hash = md5::Context::new();
189 hash.consume(&decoded);
190 hash.consume(&[0; 7][..(8 - decoded.len() % 8) % 8]);
191 if hash.finalize().0 != *expected {
192 return Err(invalid(
193 "Decrypted read-only object hash mismatch",
194 ));
195 }
196 }
197 result.objects.insert(identity, decoded);
198 }
199 ObjectData::File { .. } => {
200 if let Some(FileDataReference::Internal(guid)) =
201 object.file_reference()?
202 {
203 if file_keys
204 .insert(guid, metadata)
205 .is_some_and(|old| old != metadata)
206 {
207 return Err(invalid(
208 "File payload uses inconsistent encryption metadata",
209 ));
210 }
211 if result.files.contains_key(&guid) {
212 continue;
213 }
214 let bytes = index.store.file_data(guid)?;
215 limits.decoded_bytes = limits
216 .decoded_bytes
217 .checked_sub(bytes.len())
218 .ok_or(Error::Limit)?;
219 result.files.insert(guid, key.file(bytes)?);
220 }
221 }
222 ObjectData::Properties(_) => {
223 return Err(invalid("Protected revision contains clear properties"));
224 }
225 }
226 }
227 }
228 }
229 drop(keys);
230 for (space, info) in &index.spaces {
231 for rid in info.labels.values().copied().collect::<BTreeSet<_>>() {
232 result.resolve(*space, rid)?.reachable()?;
233 }
234 }
235 result.document()?.pages()?;
236 Ok(result)
237 }
238
239 fn resolve(
240 &self,
241 space: ExGuid,
242 rid: ExGuid,
243 ) -> std::result::Result<crate::ResolvedRevision<'_>, crate::Error> {
244 let mut revision = self.index.resolve(space, rid)?;
245 for object in revision.objects.values_mut() {
246 if let ObjectData::Encrypted(bytes) = object.data {
247 let decoded =
248 self.objects
249 .get(&(space, bytes.as_ptr().addr()))
250 .ok_or(crate::Error {
251 offset: 0,
252 message: "Protected object was not decoded",
253 })?;
254 object.data = ObjectData::Properties(decoded);
255 }
256 }
257 Ok(revision)
258 }
259
260 pub fn document(&self) -> std::result::Result<Document<'_>, crate::Error> {
261 Document::parse_with(
262 self.index,
263 |space, rid| self.resolve(space, rid),
264 |id| {
265 self.files
266 .get(&id)
267 .map(|bytes| bytes.as_slice())
268 .ok_or(crate::Error {
269 offset: 0,
270 message: "Protected file payload was not decoded",
271 })
272 },
273 )
274 }
275}
crates/onestore/tests/protected.rs created+110
......@@ -0,0 +1,110 @@
1#![cfg(feature = "protected")]
2
3use onestore::{
4 RevisionIndex, Store,
5 document::{Document, Kind},
6 protected::{Error, Limits, UnlockedSection},
7};
8use std::{fs, path::Path};
9
10#[test]
11fn known_passwords_open_independent_native_fixtures() {
12 for (root, notebook, pages) in [
13 ("native-encrypted", "encrypted-01/notebook/synthetic.one", 1),
14 ("native-protected-boundaries", "notebook/synthetic.one", 11),
15 ] {
16 let root = Path::new("../../corpus").join(root);
17 let manifest: serde_json::Value =
18 serde_json::from_slice(&fs::read(root.join("manifest.json")).unwrap()).unwrap();
19 let password = manifest["password"].as_str().unwrap();
20 let bytes = fs::read(root.join(notebook)).unwrap();
21 let before = bytes.clone();
22 let store = Store::parse(&bytes).unwrap();
23 let index = RevisionIndex::parse(&store).unwrap();
24 assert!(Document::parse(&index).unwrap().pages().unwrap().is_empty());
25 let unlocked = UnlockedSection::open(&index, password, Limits::default()).unwrap();
26 let document = unlocked.document().unwrap();
27 assert_eq!(document.pages().unwrap().len(), pages);
28 let (space, _) = document.pages().unwrap()[0];
29 let current = &document.spaces[&space];
30 let revision = &current.revisions[&current.contexts[&onestore::ExGuid::default()]];
31 let (object, _) = revision
32 .nodes
33 .iter()
34 .find(|(_, node)| matches!(node.kind, Kind::RichText { .. }))
35 .unwrap();
36 assert!(
37 onestore::PreparedEdit::text(&bytes, space, *object, 0..0, "Must remain protected")
38 .is_err()
39 );
40 assert!(
41 document
42 .spaces
43 .values()
44 .flat_map(|s| s.revisions.values())
45 .flat_map(|r| r.nodes.values())
46 .all(|n| !matches!(n.kind, Kind::Encrypted { .. }))
47 );
48 assert!(matches!(
49 UnlockedSection::open(
50 &index,
51 "deliberately incorrect fixture password",
52 Limits::default()
53 ),
54 Err(Error::PasswordMismatch)
55 ));
56 assert!(Document::parse(&index).unwrap().pages().unwrap().is_empty());
57 assert_eq!(bytes, before);
58 }
59}
60
61#[test]
62fn limits_and_password_bytes_are_explicit() {
63 let root = Path::new("../../corpus/native-protected-boundaries");
64 let manifest: serde_json::Value =
65 serde_json::from_slice(&fs::read(root.join("manifest.json")).unwrap()).unwrap();
66 let password = manifest["password"].as_str().unwrap();
67 let bytes = fs::read(root.join("notebook/synthetic.one")).unwrap();
68 let store = Store::parse(&bytes).unwrap();
69 let index = RevisionIndex::parse(&store).unwrap();
70 for limits in [
71 Limits {
72 kdf_rounds: 0,
73 ..Limits::default()
74 },
75 Limits {
76 decoded_bytes: 0,
77 ..Limits::default()
78 },
79 Limits {
80 object_visits: 0,
81 ..Limits::default()
82 },
83 ] {
84 assert!(matches!(
85 UnlockedSection::open(&index, password, limits),
86 Err(Error::Limit)
87 ));
88 }
89 for changed in [
90 password.replace("e\u{301}", "é"),
91 format!("{password}\n"),
92 password.to_uppercase(),
93 ] {
94 assert!(matches!(
95 UnlockedSection::open(&index, &changed, Limits::default()),
96 Err(Error::PasswordMismatch)
97 ));
98 }
99 assert!(matches!(
100 UnlockedSection::open(&index, &"x".repeat(65537), Limits::default()),
101 Err(Error::Limit)
102 ));
103 let ordinary = onestore::create_section("ordinary.one", "Fictitious", "Author").unwrap();
104 let store = Store::parse(&ordinary).unwrap();
105 let index = RevisionIndex::parse(&store).unwrap();
106 assert!(matches!(
107 UnlockedSection::open(&index, password, Limits::default()),
108 Err(Error::Unsupported)
109 ));
110}
fuzz/Cargo.lock+197
......@@ -2,18 +2,64 @@
22# It is not intended for manual editing.
33version = 4
44
5[[package]]
6name = "aes"
7version = "0.9.3"
8source = "registry+https://github.com/rust-lang/crates.io-index"
9checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32"
10dependencies = [
11 "cipher",
12 "cpubits",
13 "cpufeatures",
14 "zeroize",
15]
16
517[[package]]
618name = "arbitrary"
719version = "1.4.2"
820source = "registry+https://github.com/rust-lang/crates.io-index"
921checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1"
1022
23[[package]]
24name = "base64"
25version = "0.22.1"
26source = "registry+https://github.com/rust-lang/crates.io-index"
27checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
28
1129[[package]]
1230name = "bitflags"
1331version = "2.13.1"
1432source = "registry+https://github.com/rust-lang/crates.io-index"
1533checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
1634
35[[package]]
36name = "block-buffer"
37version = "0.12.1"
38source = "registry+https://github.com/rust-lang/crates.io-index"
39checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa"
40dependencies = [
41 "hybrid-array",
42 "zeroize",
43]
44
45[[package]]
46name = "block-padding"
47version = "0.4.2"
48source = "registry+https://github.com/rust-lang/crates.io-index"
49checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b"
50dependencies = [
51 "hybrid-array",
52]
53
54[[package]]
55name = "cbc"
56version = "0.2.1"
57source = "registry+https://github.com/rust-lang/crates.io-index"
58checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896"
59dependencies = [
60 "cipher",
61]
62
1763[[package]]
1864name = "cc"
1965version = "1.4.5"
......@@ -38,6 +84,60 @@ version = "0.2.2"
3884source = "registry+https://github.com/rust-lang/crates.io-index"
3985checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527"
4086
87[[package]]
88name = "cipher"
89version = "0.5.2"
90source = "registry+https://github.com/rust-lang/crates.io-index"
91checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c"
92dependencies = [
93 "block-buffer",
94 "crypto-common",
95 "inout",
96 "zeroize",
97]
98
99[[package]]
100name = "const-oid"
101version = "0.10.2"
102source = "registry+https://github.com/rust-lang/crates.io-index"
103checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c"
104
105[[package]]
106name = "cpubits"
107version = "0.1.1"
108source = "registry+https://github.com/rust-lang/crates.io-index"
109checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae"
110
111[[package]]
112name = "cpufeatures"
113version = "0.3.1"
114source = "registry+https://github.com/rust-lang/crates.io-index"
115checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566"
116dependencies = [
117 "libc",
118]
119
120[[package]]
121name = "crypto-common"
122version = "0.2.2"
123source = "registry+https://github.com/rust-lang/crates.io-index"
124checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453"
125dependencies = [
126 "hybrid-array",
127]
128
129[[package]]
130name = "digest"
131version = "0.11.3"
132source = "registry+https://github.com/rust-lang/crates.io-index"
133checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2"
134dependencies = [
135 "block-buffer",
136 "const-oid",
137 "crypto-common",
138 "zeroize",
139]
140
41141[[package]]
42142name = "find-msvc-tools"
43143version = "0.1.12"
......@@ -55,6 +155,32 @@ dependencies = [
55155 "r-efi",
56156]
57157
158[[package]]
159name = "hybrid-array"
160version = "0.4.14"
161source = "registry+https://github.com/rust-lang/crates.io-index"
162checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b"
163dependencies = [
164 "typenum",
165 "zeroize",
166]
167
168[[package]]
169name = "inout"
170version = "0.2.2"
171source = "registry+https://github.com/rust-lang/crates.io-index"
172checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7"
173dependencies = [
174 "block-padding",
175 "hybrid-array",
176]
177
178[[package]]
179name = "itoa"
180version = "1.0.18"
181source = "registry+https://github.com/rust-lang/crates.io-index"
182checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
183
58184[[package]]
59185name = "jobserver"
60186version = "0.1.35"
......@@ -87,6 +213,12 @@ version = "0.8.1"
87213source = "registry+https://github.com/rust-lang/crates.io-index"
88214checksum = "7ebb8d8732c6a6df3d8f032a82911cfc747e00efb95cc46e8d0acd5b5b88570c"
89215
216[[package]]
217name = "memchr"
218version = "2.8.3"
219source = "registry+https://github.com/rust-lang/crates.io-index"
220checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
221
90222[[package]]
91223name = "nix"
92224version = "0.31.3"
......@@ -103,10 +235,17 @@ dependencies = [
103235name = "onestore"
104236version = "0.1.0"
105237dependencies = [
238 "aes",
239 "base64",
240 "cbc",
106241 "getrandom",
107242 "md5",
108243 "nix",
244 "roxmltree",
109245 "serde",
246 "sha1",
247 "subtle",
248 "zeroize",
110249]
111250
112251[[package]]
......@@ -116,6 +255,7 @@ dependencies = [
116255 "libfuzzer-sys",
117256 "md5",
118257 "onestore",
258 "serde_json",
119259]
120260
121261[[package]]
......@@ -142,6 +282,15 @@ version = "6.0.0"
142282source = "registry+https://github.com/rust-lang/crates.io-index"
143283checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
144284
285[[package]]
286name = "roxmltree"
287version = "0.21.1"
288source = "registry+https://github.com/rust-lang/crates.io-index"
289checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb"
290dependencies = [
291 "memchr",
292]
293
145294[[package]]
146295name = "serde"
147296version = "1.0.229"
......@@ -172,12 +321,42 @@ dependencies = [
172321 "syn",
173322]
174323
324[[package]]
325name = "serde_json"
326version = "1.0.151"
327source = "registry+https://github.com/rust-lang/crates.io-index"
328checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
329dependencies = [
330 "itoa",
331 "memchr",
332 "serde",
333 "serde_core",
334 "zmij",
335]
336
337[[package]]
338name = "sha1"
339version = "0.11.0"
340source = "registry+https://github.com/rust-lang/crates.io-index"
341checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214"
342dependencies = [
343 "cfg-if",
344 "cpufeatures",
345 "digest",
346]
347
175348[[package]]
176349name = "shlex"
177350version = "2.0.1"
178351source = "registry+https://github.com/rust-lang/crates.io-index"
179352checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
180353
354[[package]]
355name = "subtle"
356version = "2.6.1"
357source = "registry+https://github.com/rust-lang/crates.io-index"
358checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
359
181360[[package]]
182361name = "syn"
183362version = "3.0.5"
......@@ -189,8 +368,26 @@ dependencies = [
189368 "unicode-ident",
190369]
191370
371[[package]]
372name = "typenum"
373version = "1.20.1"
374source = "registry+https://github.com/rust-lang/crates.io-index"
375checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
376
192377[[package]]
193378name = "unicode-ident"
194379version = "1.0.24"
195380source = "registry+https://github.com/rust-lang/crates.io-index"
196381checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
382
383[[package]]
384name = "zeroize"
385version = "1.9.0"
386source = "registry+https://github.com/rust-lang/crates.io-index"
387checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
388
389[[package]]
390name = "zmij"
391version = "1.0.23"
392source = "registry+https://github.com/rust-lang/crates.io-index"
393checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
fuzz/Cargo.toml+9-1
......@@ -9,8 +9,16 @@ cargo-fuzz = true
99
1010[dependencies]
1111libfuzzer-sys = "0.4"
12onestore = { path = "../crates/onestore" }
12onestore = { path = "../crates/onestore", features = ["protected"] }
1313md5 = "0.8.1"
14serde_json = "1"
15
16[[bin]]
17name = "protected"
18path = "fuzz_targets/protected.rs"
19test = false
20doc = false
21bench = false
1422
1523[[bin]]
1624name = "store"
fuzz/fuzz_targets/protected.rs created+78
......@@ -0,0 +1,78 @@
1#![no_main]
2use libfuzzer_sys::fuzz_target;
3use onestore::{
4 Reference, RevisionIndex, Store,
5 protected::{Limits, UnlockedSection},
6};
7use std::{ops::Range, sync::LazyLock};
8
9static SOURCES: LazyLock<Vec<(Vec<u8>, String, Vec<Range<usize>>)>> = LazyLock::new(|| {
10 [
11 (&include_bytes!("../../corpus/native-encrypted/encrypted-01/notebook/synthetic.one")[..], include_str!("../../corpus/native-encrypted/manifest.json")),
12 (&include_bytes!("../../corpus/native-protected-boundaries/notebook/synthetic.one")[..], include_str!("../../corpus/native-protected-boundaries/manifest.json")),
13 ].into_iter().map(|(bytes, manifest)| {
14 let manifest: serde_json::Value = serde_json::from_str(manifest).unwrap();
15 let store = Store::parse(bytes).unwrap();
16 let mut ranges: Vec<_> = store.lists.values().flat_map(|list| &list.nodes).filter_map(|node| {
17 let Reference::Data(chunk) = node.reference? else { return None; };
18 let start = usize::try_from(chunk.offset).unwrap();
19 let end = start + usize::try_from(chunk.length).unwrap();
20 (start < end).then_some(start..end)
21 }).collect();
22 ranges.sort_by_key(|range| (range.start, range.end));
23 ranges.dedup();
24 (bytes.to_vec(), manifest["password"].as_str().unwrap().to_owned(), ranges)
25 }).collect()
26});
27
28fuzz_target!(|data: &[u8]| {
29 if data.len() < 8 {
30 return;
31 }
32 let (source, password, ranges) = &SOURCES[usize::from(data[0]) % SOURCES.len()];
33 let mut bytes = source.clone();
34 let mut password = password.clone();
35 let mode = data[1] % 3;
36 if mode == 2 {
37 password.push_str(&String::from_utf8_lossy(&data[8..]));
38 } else {
39 let range = if mode == 0 {
40 0..bytes.len()
41 } else {
42 ranges[usize::from(u16::from_le_bytes([data[2], data[3]])) % ranges.len()].clone()
43 };
44 let offset = u32::from_le_bytes(data[4..8].try_into().unwrap()) as usize % range.len();
45 let count = (range.len() - offset).min(data.len() - 8);
46 bytes[range.start + offset..range.start + offset + count]
47 .copy_from_slice(&data[8..8 + count]);
48 }
49 let Ok(store) = Store::parse(&bytes) else {
50 return;
51 };
52 let Ok(index) = RevisionIndex::parse(&store) else {
53 return;
54 };
55 let result = UnlockedSection::open(
56 &index,
57 &password,
58 Limits {
59 kdf_rounds: 200000,
60 decoded_bytes: 4 * 1024 * 1024,
61 object_visits: 20000,
62 },
63 );
64 if mode == 2 && data.len() > 8 {
65 assert!(result.is_err());
66 }
67 if let Ok(unlocked) = result {
68 let document = unlocked.document().unwrap();
69 let _ = document.pages();
70 for revision in document.spaces.values().flat_map(|s| s.revisions.values()) {
71 for (id, node) in &revision.nodes {
72 if matches!(node.kind, onestore::document::Kind::RichText { .. }) {
73 let _ = revision.text_runs(*id);
74 }
75 }
76 }
77 }
78});
tools/test_document_oracle.py+15
......@@ -12,6 +12,21 @@ compare = runpy.run_path(str(ROOT / 'tools/verify-document.py'))['compare']
1212
1313
1414class DocumentOracleTest(unittest.TestCase):
15 def test_locked_table_width_requires_the_native_value(self):
16 fixture = ROOT / 'corpus/m6/native-structure-01'
17 with TemporaryDirectory() as temporary:
18 native = Path(temporary) / 'read'
19 shutil.copytree(fixture / 'read', native)
20 compare(fixture / 'notebook', native)
21 path = native / 'page-001.xml'
22 xml = ET.parse(path)
23 column = next(n for n in xml.getroot().iter()
24 if n.tag.endswith('}Column') and n.get('isLocked') == 'true')
25 column.set('width', str(float(column.get('width')) + 1))
26 xml.write(path, encoding='utf-8')
27 with self.assertRaisesRegex(AssertionError, 'Locked table column width differs'):
28 compare(fixture / 'notebook', native)
29
1530 def test_native_2010_does_not_render_the_documented_cell_shading_control(self):
1631 import pdfplumber
1732 fixture = ROOT / 'corpus/m6/cell-shading-control-01/read/page-001'
tools/test_protected.py created+77
......@@ -0,0 +1,77 @@
1import json
2import os
3from pathlib import Path
4import runpy
5import shutil
6import stat
7import subprocess
8from tempfile import TemporaryDirectory
9import unittest
10
11from document_model import EXPORTER
12
13ROOT = Path(__file__).resolve().parent.parent
14compare = runpy.run_path(str(ROOT / 'tools/verify-document.py'))['compare']
15
16
17class ProtectedDocumentTest(unittest.TestCase):
18 def test_native_autofit_is_distinct_from_a_fixed_column_width(self):
19 import xml.etree.ElementTree as ET
20 fixture = ROOT / 'corpus/native-encrypted'
21 manifest = json.loads((fixture / 'manifest.json').read_text())
22 with TemporaryDirectory() as temporary:
23 root = Path(temporary)
24 password = root / 'password'
25 password.write_text(manifest['password'])
26 native = root / 'read'
27 shutil.copytree(fixture / 'cold-2010-4763/read', native)
28 compare(fixture / 'encrypted-01/notebook', native, password_file=password)
29 differences = json.loads((root / 'table-autofit.json').read_text())
30 self.assertEqual(len(differences), 1)
31 self.assertAlmostEqual(differences[0]['stored'], 38.61, places=4)
32 self.assertAlmostEqual(differences[0]['native'], 39.146141, places=4)
33 xml_path = native / 'page-000.xml'
34 xml = ET.parse(xml_path)
35 column = next(n for n in xml.getroot().iter() if n.tag.endswith('}Column'))
36 column.set('isLocked', 'true')
37 xml.write(xml_path, encoding='utf-8')
38 with self.assertRaisesRegex(AssertionError, 'Table column lock state differs'):
39 compare(fixture / 'encrypted-01/notebook', native, password_file=password)
40 column.attrib.pop('isLocked')
41 column.set('width', 'NaN')
42 xml.write(xml_path, encoding='utf-8')
43 with self.assertRaisesRegex(AssertionError, 'Invalid native table column width'):
44 compare(fixture / 'encrypted-01/notebook', native, password_file=password)
45
46 def test_native_page_formatting_and_all_attachment_boundaries(self):
47 fixture = ROOT / 'corpus/native-protected-boundaries'
48 manifest = json.loads((fixture / 'manifest.json').read_text())
49 with TemporaryDirectory() as temporary:
50 root = Path(temporary)
51 password = root / 'password'
52 password.write_text(manifest['password'])
53 native = root / 'read'
54 shutil.copytree(fixture / 'read', native)
55 compare(fixture / 'notebook', native, password_file=password)
56 output = root / 'export'
57 subprocess.run([EXPORTER, fixture / 'notebook/synthetic.one', output,
58 '--password-file', password], check=True, capture_output=True)
59 if os.name == 'posix':
60 self.assertEqual(stat.S_IMODE(output.stat().st_mode), 0o700)
61 password.write_text(manifest['password'] + '\n')
62 failed = root / 'failed'
63 result = subprocess.run([EXPORTER, fixture / 'notebook/synthetic.one', failed,
64 '--password-file', password], capture_output=True)
65 self.assertNotEqual(result.returncode, 0)
66 self.assertEqual(result.stdout, b'')
67 self.assertFalse(failed.exists())
68 self.assertNotIn(manifest['password'].encode(), result.stderr)
69 password.write_text('x' * 65537)
70 result = subprocess.run([EXPORTER, fixture / 'notebook/synthetic.one',
71 '--password-file', password], capture_output=True)
72 self.assertNotEqual(result.returncode, 0)
73 self.assertEqual(result.stdout, b'')
74
75
76if __name__ == '__main__':
77 unittest.main()
tools/verify-document.py+20-7
......@@ -8,6 +8,7 @@ from tempfile import TemporaryDirectory
88from PIL import Image
99from datetime import datetime, timezone
1010import json
11import math
1112from pathlib import Path, PureWindowsPath
1213from zoneinfo import ZoneInfo
1314import subprocess
......@@ -82,7 +83,7 @@ def visible_text(node, space):
8283 return "" if text == "\u00a0" else project_text(text.removesuffix('\r'))
8384
8485
85def compare_objects(space, roots, page, native_roots, assets, native_payloads):
86def compare_objects(space, roots, page, native_roots, assets, native_payloads, autofit):
8687 types = {'T': 'RichText', 'Image': 'Image', 'InsertedFile': 'Attachment', 'MediaFile': 'Attachment', 'Table': 'Table'}
8788 actual = [n for root in roots for _, n in walk(space, root)
8889 if n['kind']['type'] in types.values() and not n['kind'].get('boilerplate')]
......@@ -132,9 +133,15 @@ def compare_objects(space, roots, page, native_roots, assets, native_payloads):
132133 rows = native.findall('one:Row', ns)
133134 columns = native.findall('one:Columns/one:Column', ns)
134135 assert len(rows) == kind['rows'] and len(columns) == kind['columns'], 'Table dimensions differ'
135 for column, width in zip(columns, kind['widths'], strict=True):
136 assert abs(float(column.get('width')) - width) < .002, 'Table column width differs'
137136 assert (kind['locked'] or [False] * len(columns)) == [c.get('isLocked') == 'true' for c in columns], 'Table column lock state differs'
137 for column, width in zip(columns, kind['widths'], strict=True):
138 measured = float(column.get('width'))
139 assert math.isfinite(measured) and measured >= 0, 'Invalid native table column width'
140 if abs(measured - width) >= .002:
141 if column.get('isLocked') == 'true':
142 raise AssertionError('Locked table column width differs')
143 autofit.append({'page': page.get('ID'), 'table': parents[native].get('objectID'),
144 'column': column.get('index'), 'stored': width, 'native': measured})
138145 assert len(node['children']) == len(rows), 'Table row count differs'
139146 for oid, row in zip(node['children'], rows, strict=True):
140147 assert len(space['nodes'][oid]['children']) == len(row.findall('one:Cell', ns)), 'Table cell count differs'
......@@ -186,7 +193,7 @@ def compare_objects(space, roots, page, native_roots, assets, native_payloads):
186193 return count
187194
188195
189def compare(notebook, native, versions=None):
196def compare(notebook, native, versions=None, password_file=None):
190197 notebook = notebook.resolve(strict=True)
191198 native = native.resolve(strict=True)
192199 sections = sorted(notebook.rglob('*.one'))
......@@ -199,13 +206,17 @@ def compare(notebook, native, versions=None):
199206 discrepancies = []
200207 pdf_checks = []
201208 geometry = []
209 autofit = []
202210 for path in sections:
203211 relative = path.relative_to(notebook)
204212 if versions is not None and relative.as_posix() != versions['section']:
205213 continue
206214 with TemporaryDirectory() as temporary:
207215 exported = Path(temporary) / 'document'
208 subprocess.run([EXPORTER, path, exported], check=True)
216 command = [EXPORTER, path, exported]
217 if password_file is not None:
218 command.extend(['--password-file', password_file])
219 subprocess.run(command, check=True)
209220 document = json.loads((exported / 'document.json').read_text())
210221 resolved_text = json.loads((exported / 'text.json').read_text())
211222 assets = {json.dumps(a['reference'], sort_keys=True): (exported / a['path']).read_bytes()
......@@ -308,7 +319,7 @@ def compare(notebook, native, versions=None):
308319 raise AssertionError(f'{relative}: page {ordinal}: ordered text differs; inspect {destination}')
309320 native_roots = [n for n in native_children if n.tag == '{' + ns['one'] + '}Title'] + content
310321 try:
311 tags += compare_objects(space, roots, page, native_roots, assets, native_payloads)
322 tags += compare_objects(space, roots, page, native_roots, assets, native_payloads, autofit)
312323 native_runs = native_characters(page, native_roots)
313324 text_ids = [oid for root in roots for oid, n in walk(space, root)
314325 if n['kind']['type'] == 'RichText' and not n['kind']['boilerplate']]
......@@ -339,6 +350,7 @@ def compare(notebook, native, versions=None):
339350 if versions is not None:
340351 assert compared == len(versions['pages']) > 0, 'No historical source section was compared'
341352 (native.parent / 'geometry-differences.json').write_text(json.dumps(geometry, indent=2))
353 (native.parent / 'table-autofit.json').write_text(json.dumps(autofit, indent=2))
342354 (native.parent / 'pdf-format-checks.json').write_text(json.dumps(pdf_checks, indent=2))
343355 destination = native.parent / 'format-differences.json'
344356 destination.write_text(json.dumps(discrepancies, indent=2))
......@@ -358,5 +370,6 @@ if __name__ == '__main__':
358370 parser.add_argument('notebook', type=Path)
359371 parser.add_argument('native', type=Path)
360372 parser.add_argument('--versions', type=Path, help='Native history UI date and copied-page associations.')
373 parser.add_argument('--password-file', type=Path, help='Exact UTF-8 password bytes; requires the protected exporter feature.')
361374 args = parser.parse_args()
362 compare(args.notebook.resolve(), args.native.resolve(), json.loads(args.versions.read_text()) if args.versions else None)
375 compare(args.notebook.resolve(), args.native.resolve(), json.loads(args.versions.read_text()) if args.versions else None, args.password_file)