diff --git a/corpus/native-external-assets/README.md b/corpus/native-external-assets/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3d270c33404862557f37d7cde0e293d2203ac816 --- /dev/null +++ b/corpus/native-external-assets/README.md @@ -0,0 +1,16 @@ +# External payload fixture + +OneNote 2010 authored an empty attachment and a 1 KiB attachment with +`tools/native/external-assets.ps1`. The retained native section is in `native/`. +The fixture generator converts all file-data declarations into the documented +external representation while retaining their identities and exact payloads. +The `read/` capture is an independent fresh-cache OneNote reopen; `payloads.json` +contains the native attachment hashes. `provenance.json` records source hashes. + +```sh +cargo run -p onestore-notebook --example externalize_fixture -- corpus/native-external-assets/native/synthetic.one /tmp/external-notebook +python3 tools/native_runner.py /tmp/external-notebook /tmp/external-native --expected-pages 3 --collect-notebook +``` + +The generator creates a new destination and requires reserved fragment space. +It is a fixture converter, not a concurrent publication API. diff --git a/corpus/native-external-assets/native/synthetic.one b/corpus/native-external-assets/native/synthetic.one new file mode 100644 index 0000000000000000000000000000000000000000..0a8fbe28a77ac1920ea125ff03f4a288a9150e8d Binary files /dev/null and b/corpus/native-external-assets/native/synthetic.one differ diff --git a/corpus/native-external-assets/notebook/Open Notebook.onetoc2 b/corpus/native-external-assets/notebook/Open Notebook.onetoc2 new file mode 100644 index 0000000000000000000000000000000000000000..cd845617ca678d9dbf92b9972c089bed85aaddc6 Binary files /dev/null and b/corpus/native-external-assets/notebook/Open Notebook.onetoc2 differ diff --git a/corpus/native-external-assets/notebook/synthetic.one b/corpus/native-external-assets/notebook/synthetic.one new file mode 100644 index 0000000000000000000000000000000000000000..40d836918ba1382aab74792c9a4eb6534d938c8b Binary files /dev/null and b/corpus/native-external-assets/notebook/synthetic.one differ diff --git a/corpus/native-external-assets/notebook/synthetic_onefiles/04DC5FD1-10FD-4404-A37C-D035399F9D9E.onebin b/corpus/native-external-assets/notebook/synthetic_onefiles/04DC5FD1-10FD-4404-A37C-D035399F9D9E.onebin new file mode 100644 index 0000000000000000000000000000000000000000..347c6da5c201021c037a9f706791666654a5c487 Binary files /dev/null and b/corpus/native-external-assets/notebook/synthetic_onefiles/04DC5FD1-10FD-4404-A37C-D035399F9D9E.onebin differ diff --git a/corpus/native-external-assets/notebook/synthetic_onefiles/15575137-1653-47BC-9C05-78420499168B.onebin b/corpus/native-external-assets/notebook/synthetic_onefiles/15575137-1653-47BC-9C05-78420499168B.onebin new file mode 100644 index 0000000000000000000000000000000000000000..819ba0c0d4700b72b60d2a9981bc50762423d015 Binary files /dev/null and b/corpus/native-external-assets/notebook/synthetic_onefiles/15575137-1653-47BC-9C05-78420499168B.onebin differ diff --git a/corpus/native-external-assets/notebook/synthetic_onefiles/C5EABC1E-5091-4685-BAC1-2C785D6046DB.onebin b/corpus/native-external-assets/notebook/synthetic_onefiles/C5EABC1E-5091-4685-BAC1-2C785D6046DB.onebin new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/corpus/native-external-assets/provenance.json b/corpus/native-external-assets/provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..f034307687407e275832f988352af3f6c512bac4 --- /dev/null +++ b/corpus/native-external-assets/provenance.json @@ -0,0 +1,23 @@ +{ + "native_author": "tools/native/external-assets.ps1", + "generator": "crates/onestore-notebook/examples/externalize_fixture.rs", + "native_capture": "evidence/m10/external-assets-small-native", + "cold_capture": "evidence/m10/external-assets-small-cold", + "source_sha256": { + "native/synthetic.one": "3a4dcb0b6f9e46044a3cdc663ae505878911c2a7255a7787c55ae2c71a7f4fd6", + "read/page-002.xml": "4104b7f1b2a8bcc98c5ce73c335d365ae67d565a72f9da2ba78304dd7226944f", + "read/page-001.xml": "8c1a503344f3d6893a3df1a3ff5d51d591c0a61e04101c360aa1546d386f5a15", + "read/page-000.xml": "6796df2c0594a3263416dfbd1210ec412e0694d30115869f3ceab768c95a3e32", + "read/hierarchy.xml": "4a1207f82678cd2020ee175979074601bd9f85c102bd68e5df34c54e37616574", + "read/payloads.json": "0b31ae5e576ba7ee5eab7c9a62d8f673659a76865ad07f9e75f45458c06a7e65", + "read/environment.json": "1cd7ab391d85fe504344cd63bd786f969770ce09aef0008623eaf0abb880e655", + "notebook/Open Notebook.onetoc2": "8fec3cc6f72a9e0fff45ce604f62c184b0bb63d077018f90086b31396c8500b0", + "notebook/synthetic.one": "7343061e2a44c67b5f306c747e707fc1b99d126c51db510cc010885c2b016ca2", + "notebook/synthetic_onefiles/15575137-1653-47BC-9C05-78420499168B.onebin": "38b0469c7f0cddba0fa7739b44061117b2621bfe477348150cfc98396f41daf1", + "notebook/synthetic_onefiles/C5EABC1E-5091-4685-BAC1-2C785D6046DB.onebin": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "notebook/synthetic_onefiles/04DC5FD1-10FD-4404-A37C-D035399F9D9E.onebin": "2bce1ba628720664be4b9fdd77aae0678e5f0f3f02fc6ff641ec879094f6a404" + }, + "cold_teardown": { + "absent": true + } +} \ No newline at end of file diff --git a/corpus/native-external-assets/read/environment.json b/corpus/native-external-assets/read/environment.json new file mode 100644 index 0000000000000000000000000000000000000000..3828ce28159e138bd5dfca0c697914720fabaa17 --- /dev/null +++ b/corpus/native-external-assets/read/environment.json @@ -0,0 +1,7 @@ +{ + "powershell": "5.1.14409.1005", + "schema": "xs2010", + "hostname": "ONE-M6-3BB259AD", + "cold": true, + "onenote": "14.0.4763.1000" +} diff --git a/corpus/native-external-assets/read/hierarchy.xml b/corpus/native-external-assets/read/hierarchy.xml new file mode 100644 index 0000000000000000000000000000000000000000..2b47d71e5f1225910958aeb323081083b28ce6f8 --- /dev/null +++ b/corpus/native-external-assets/read/hierarchy.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/native-external-assets/read/page-000.xml b/corpus/native-external-assets/read/page-000.xml new file mode 100644 index 0000000000000000000000000000000000000000..a07d06fa7a3b9cf4dfd829721e4e36b2ac55539d --- /dev/null +++ b/corpus/native-external-assets/read/page-000.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/native-external-assets/read/page-001.xml b/corpus/native-external-assets/read/page-001.xml new file mode 100644 index 0000000000000000000000000000000000000000..85e4bfe7ce8040a02128c8cec8a90820e837b6e4 --- /dev/null +++ b/corpus/native-external-assets/read/page-001.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/native-external-assets/read/page-002.xml b/corpus/native-external-assets/read/page-002.xml new file mode 100644 index 0000000000000000000000000000000000000000..602c1d33b3f91346145fd2ca59b93dcdc8b65209 --- /dev/null +++ b/corpus/native-external-assets/read/page-002.xml @@ -0,0 +1,2 @@ + + diff --git a/corpus/native-external-assets/read/payloads.json b/corpus/native-external-assets/read/payloads.json new file mode 100644 index 0000000000000000000000000000000000000000..4c4ad818954bdad635921da8f9f01879244901bf --- /dev/null +++ b/corpus/native-external-assets/read/payloads.json @@ -0,0 +1,18 @@ +[ + { + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "name": "0.bin", + "object": "{068F783A-8A63-4688-A2AC-32EB81A6EA31}{40}{B0}", + "kind": "InsertedFile", + "page": "{2001177A-A50B-470C-A3B4-C397629432DC}{1}{B0}", + "bytes": 0 + }, + { + "sha256": "2bce1ba628720664be4b9fdd77aae0678e5f0f3f02fc6ff641ec879094f6a404", + "name": "1024.bin", + "object": "{F876120D-4126-473B-9D5F-F961FAF66D65}{40}{B0}", + "kind": "InsertedFile", + "page": "{5B886F89-E2CC-4184-AF00-42A8CB7EEDBF}{1}{B0}", + "bytes": 1024 + } +] \ No newline at end of file diff --git a/crates/onestore-notebook/README.md b/crates/onestore-notebook/README.md index bd9669954a44cb94607938da8f68be9edf5ff6fd..f60e6b49b3a66c82955ed3545fb85400a3e385d0 100644 --- a/crates/onestore-notebook/README.md +++ b/crates/onestore-notebook/README.md @@ -7,6 +7,7 @@ Discovery returns ordered sections and nested groups with file identities and share-relative paths. Section display-name overrides remain distinct from file names. TOC references whose identities are absent from the directory remain inspectable; a cached filename never substitutes for an identity match. +Reserved `_onefiles` directories are excluded from section-group traversal. Encrypted sections and valid storage with an unreadable document graph retain their identity as `Locked` or `Unreadable`, without being presented as empty pages. Malformed storage, failed reads and ambiguous identities reject the discovery. @@ -16,3 +17,9 @@ observation across multiple files, not an atomic notebook transaction or authorization to publish an edit. Refresh rejects observed topology changes and duplicate physical files with the same logical identity. Retain the last accepted catalog if discovery fails; a connection failure does not mean files were deleted. + +`read_external_asset` resolves a validated UUID `.onebin` filename beneath the +selected section's sibling `_onefiles` folder and returns exact bounded bytes. +Missing files, permissions and size failures retain their I/O error kinds; an +empty payload is a successful empty buffer. Embedded payloads remain available +directly from the core document model. diff --git a/crates/onestore-notebook/examples/document.rs b/crates/onestore-notebook/examples/document.rs new file mode 100644 index 0000000000000000000000000000000000000000..f894eb62368d2b46d541df5e0a1793067dc8b7d2 --- /dev/null +++ b/crates/onestore-notebook/examples/document.rs @@ -0,0 +1,115 @@ +use onestore::{ + FileDataReference, RevisionIndex, Store, + document::{Document, Kind}, +}; +use std::{ + borrow::Cow, + collections::BTreeSet, + env, fs, + io::{self, BufWriter, Write}, + path::{Path, PathBuf}, +}; + +fn write_json( + path: impl AsRef, + value: &impl serde::Serialize, +) -> Result<(), Box> { + let mut output = BufWriter::new(fs::File::create(path)?); + serde_json::to_writer(&mut output, value)?; + output.flush()?; + Ok(()) +} + +fn main() -> Result<(), Box> { + let mut args = env::args_os().skip(1); + let path = args.next().ok_or("Provide a .one or .onetoc2 file.")?; + let destination = args.next().map(PathBuf::from); + if args.next().is_some() { + return Err("Provide a source file and an optional new export directory.".into()); + } + let source_path = PathBuf::from(path); + let bytes = onestore::read_file(&source_path)?; + let store = Store::parse(&bytes)?; + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + if let Some(destination) = destination { + fs::create_dir(&destination)?; + fs::create_dir(destination.join("assets"))?; + let parent = source_path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let mut source = onestore_notebook::Local::open(parent)?; + let section = source_path + .file_name() + .and_then(|name| name.to_str()) + .ok_or("Use a UTF-8 section filename")?; + let mut assets = Vec::new(); + let mut seen = BTreeSet::new(); + for node in document + .spaces + .values() + .flat_map(|s| s.revisions.values()) + .flat_map(|r| r.nodes.values()) + { + if let Kind::File { + reference, payload, .. + } = &node.kind + && seen.insert(serde_json::to_string(reference)?) + { + let data = match reference { + FileDataReference::Internal(_) => { + Cow::Borrowed(payload.ok_or("Missing embedded file data")?) + } + FileDataReference::External(filename) => { + match onestore_notebook::read_external_asset( + &mut source, + section, + filename, + 256 * 1024 * 1024, + ) { + Ok(bytes) => Cow::Owned(bytes), + Err(error) => { + let kind = match &error { + onestore_notebook::Error::Io { error, .. } => { + format!("{:?}", error.kind()) + } + _ => "InvalidData".into(), + }; + assets.push(serde_json::json!({"reference":reference,"path":null,"error":{"kind":kind,"message":error.to_string()}})); + continue; + } + } + } + FileDataReference::Invalid => { + assets.push(serde_json::json!({"reference":reference,"path":null,"error":{"kind":"InvalidData","message":"The document marks this payload as unavailable"}})); + continue; + } + }; + let path = format!("assets/{}.bin", assets.len()); + fs::write(destination.join(&path), data)?; + assets.push(serde_json::json!({"reference":reference,"path":path})); + } + } + write_json(destination.join("assets.json"), &assets)?; + let mut text = std::collections::BTreeMap::new(); + for (sid, space) in &document.spaces { + let mut revisions = std::collections::BTreeMap::new(); + for (rid, revision) in &space.revisions { + let mut objects = std::collections::BTreeMap::new(); + for (oid, node) in &revision.nodes { + if matches!(node.kind, Kind::RichText { .. }) { + objects.insert(*oid, revision.text_runs(*oid)?); + } + } + revisions.insert(*rid, objects); + } + text.insert(*sid, revisions); + } + write_json(destination.join("text.json"), &text)?; + write_json(destination.join("document.json"), &document)?; + } else { + serde_json::to_writer(io::stdout().lock(), &document)?; + } + Ok(()) +} diff --git a/crates/onestore-notebook/examples/externalize_fixture.rs b/crates/onestore-notebook/examples/externalize_fixture.rs new file mode 100644 index 0000000000000000000000000000000000000000..2b82cd3798d8c710e79c14cd3f60a308588661b0 --- /dev/null +++ b/crates/onestore-notebook/examples/externalize_fixture.rs @@ -0,0 +1,101 @@ +//! Builds a new external-payload test fixture from a native capture with reserved fragment space. +use onestore::{FileDataReference, RevisionIndex, Store}; +use std::{fs, path::PathBuf}; +fn main() -> Result<(), Box> { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + if args.len() != 2 { + return Err("Provide a native source section and a new fixture directory".into()); + } + let source = fs::read(&args[0])?; + let destination = PathBuf::from(&args[1]); + fs::create_dir(&destination)?; + fs::create_dir(destination.join("synthetic_onefiles"))?; + let store = Store::parse(&source)?; + assert!(store.checksum_mismatches.is_empty()); + RevisionIndex::parse(&store)?.validate_current()?; + let mut output = source.clone(); + let mut changed = 0; + for list in store.lists.values() { + for chunk in &list.fragments { + let start = chunk.offset as usize; + let end = start + chunk.length as usize - 20; + let mut body = Vec::new(); + let mut touched = false; + for node in list + .nodes + .iter() + .filter(|node| node.offset >= start + 16 && node.offset < end) + { + let raw = u32::from_le_bytes(source[node.offset..node.offset + 4].try_into()?); + let size = ((raw >> 10) & 0x1fff) as usize; + let mut encoded = source[node.offset..node.offset + size].to_vec(); + if matches!(node.id, 0x72 | 0x73) { + let offset = 4 + 4 + 4 + if node.id == 0x72 { 1 } else { 4 }; + let count = + u32::from_le_bytes(encoded[offset..offset + 4].try_into()?) as usize; + let text = String::from_utf16( + &encoded[offset + 4..offset + 4 + count * 2] + .chunks_exact(2) + .map(|v| u16::from_le_bytes(v.try_into().unwrap())) + .collect::>(), + )?; + if let FileDataReference::Internal(guid) = text.parse()? { + let filename = format!( + "{}.onebin", + text.strip_prefix("{") + .unwrap() + .strip_suffix('}') + .unwrap() + ); + let payload = store.file_data(guid)?; + let path = destination.join("synthetic_onefiles").join(&filename); + if path.exists() { + assert_eq!(fs::read(&path)?, payload); + } else { + fs::write(path, payload)?; + } + let reference = format!("{filename}"); + let data: Vec<_> = reference + .encode_utf16() + .flat_map(u16::to_le_bytes) + .collect(); + encoded.splice(offset + 4..offset + 4 + count * 2, data.iter().copied()); + encoded[offset..offset + 4] + .copy_from_slice(&(data.len() as u32 / 2).to_le_bytes()); + assert!(encoded.len() <= 0x1fff); + let header = (raw & !(0x1fff << 10)) | ((encoded.len() as u32) << 10); + encoded[..4].copy_from_slice(&header.to_le_bytes()); + changed += 1; + touched = true; + } + } + body.extend(encoded); + } + if touched { + assert!( + body.len() + 4 <= end - start - 16, + "Native fragment has insufficient reserved space" + ); + body.extend_from_slice(&(0xff_u32 | (4 << 10)).to_le_bytes()); + output[start + 16..end].fill(0); + output[start + 16..start + 16 + body.len()].copy_from_slice(&body); + } + } + } + assert!(changed > 0); + let converted = Store::parse(&output)?; + assert!(converted.checksum_mismatches.is_empty()); + RevisionIndex::parse(&converted)?.validate_current()?; + fs::write(destination.join("synthetic.one"), &output)?; + fs::write( + destination.join("Open Notebook.onetoc2"), + onestore::create_table_of_contents( + "Open Notebook.onetoc2", + &[("synthetic.one", converted.header.file_id)], + )?, + )?; + println!( + "Converted {changed} file-data declarations without changing payload identities or bytes" + ); + Ok(()) +} diff --git a/crates/onestore-notebook/src/lib.rs b/crates/onestore-notebook/src/lib.rs index ff473ee25aabf19a99bbd376f59915c4b779dc1d..07aeb6c94d12ffd829c54bdb5bbeb74f59eed3c5 100644 --- a/crates/onestore-notebook/src/lib.rs +++ b/crates/onestore-notebook/src/lib.rs @@ -77,6 +77,46 @@ pub trait Source { fn entries(&mut self, path: &str, limit: usize) -> io::Result>; /// Return a consistent file snapshot, rejecting images larger than the byte limit. fn read(&mut self, path: &str, limit: usize) -> io::Result>; + /// Reads an external payload with the same completeness and size guarantees. + fn read_asset(&mut self, path: &str, limit: usize) -> io::Result> { + self.read(path, limit) + } +} + +/// Reads an external file-data reference from the section's sibling `_onefiles` folder. +/// `NotFound` in `Error::Io` is distinct from a successfully read zero-byte payload. +pub fn read_external_asset( + source: &mut impl Source, + section: &str, + filename: &str, + limit: usize, +) -> Result, Error> { + let (stem, extension) = section.rsplit_once('.').ok_or_else(|| Error::Entry { + path: section.into(), + })?; + if !extension.eq_ignore_ascii_case("one") + || !section.split('/').all(component) + || stem.ends_with('/') + || stem.is_empty() + { + return Err(Error::Entry { + path: section.into(), + }); + } + format!("{filename}") + .parse::() + .map_err(|_| Error::Entry { + path: filename.into(), + })?; + let path = format!("{stem}_onefiles/{filename}"); + let bytes = source.read_asset(&path, limit).map_err(|error| Error::Io { + path: path.clone(), + error, + })?; + if bytes.len() > limit { + return Err(Error::Limit { path }); + } + Ok(bytes) } pub struct Limits { @@ -113,7 +153,8 @@ pub enum Error { }, } -/// Discovers the complete rooted directory within caller-specified work and size limits. +/// Discovers rooted notebook topology within caller-specified work and size limits. +/// Reserved `_onefiles` directories contain payloads, not section groups. pub fn discover(source: &mut impl Source, limits: Limits) -> Result { let mut remaining = limits.entries; let mut identities = BTreeMap::new(); @@ -158,6 +199,9 @@ fn scan( for entry in &listing { let child = join(path, &entry.name); if entry.kind == EntryKind::Directory { + if entry.name.to_ascii_lowercase().ends_with("_onefiles") { + continue; + } result.groups.push(scan( source, &child, diff --git a/crates/onestore-notebook/src/source.rs b/crates/onestore-notebook/src/source.rs index e925896012c4819179630949a648cb4a629b71d2..4bceacd8600fe84c2497dfc43faa7d8c69b2a308 100644 --- a/crates/onestore-notebook/src/source.rs +++ b/crates/onestore-notebook/src/source.rs @@ -114,4 +114,8 @@ impl Source for Smb<'_> { fn read(&mut self, path: &str, limit: usize) -> io::Result> { self.client.read_storage(&self.path(path)?, limit) } + + fn read_asset(&mut self, path: &str, limit: usize) -> io::Result> { + self.client.read_asset(&self.path(path)?, limit) + } } diff --git a/crates/onestore-notebook/tests/assets.rs b/crates/onestore-notebook/tests/assets.rs new file mode 100644 index 0000000000000000000000000000000000000000..47f50d2e80a1a5aeed909bbcfc8e99b850796987 --- /dev/null +++ b/crates/onestore-notebook/tests/assets.rs @@ -0,0 +1,162 @@ +use onestore_notebook::{Error, Local, read_external_asset}; +use std::{fs, io}; + +#[test] +fn nested_external_payloads_are_exact_and_empty_is_distinct_from_missing() { + let root = tempfile::tempdir().unwrap(); + let directory = root.path().join("Group 🦀/Section é_onefiles"); + fs::create_dir_all(&directory).unwrap(); + let filename = "2a83ae62-6754-4383-8e2b-4033ff3cfba1.onebin"; + let mut source = Local::open(root.path()).unwrap(); + let section = "Group 🦀/Section é.ONE"; + assert!( + matches!(read_external_asset(&mut source, section, filename, 0), + Err(Error::Io { error, .. }) if error.kind() == io::ErrorKind::NotFound) + ); + fs::write(directory.join(filename), []).unwrap(); + assert!( + read_external_asset(&mut source, section, filename, 0) + .unwrap() + .is_empty() + ); + let bytes: Vec<_> = (0..65537).map(|index| (index % 251) as u8).collect(); + fs::write(directory.join(filename), &bytes).unwrap(); + assert!( + matches!(read_external_asset(&mut source, section, filename, bytes.len()-1), + Err(Error::Io { error, .. }) if error.kind() == io::ErrorKind::FileTooLarge) + ); + assert_eq!( + read_external_asset(&mut source, section, filename, bytes.len()).unwrap(), + bytes + ); + assert_eq!(fs::read(directory.join(filename)).unwrap(), bytes); +} + +#[test] +fn invalid_asset_locations_fail_before_accessing_the_source() { + struct Unused; + impl onestore_notebook::Source for Unused { + fn entries(&mut self, _: &str, _: usize) -> io::Result> { + panic!("Unexpected enumeration") + } + fn read(&mut self, _: &str, _: usize) -> io::Result> { + panic!("Unexpected read") + } + } + let filename = "2a83ae62-6754-4383-8e2b-4033ff3cfba1.onebin"; + for section in [ + "", + ".one", + "Group/.one", + "/Section.one", + "../Section.one", + "Group/../Section.one", + "Group\\Section.one", + "Section.onetoc2", + ] { + assert!(matches!( + read_external_asset(&mut Unused, section, filename, 100), + Err(Error::Entry { .. }) + )); + } + for name in [ + "", + "attachment.png", + "../2a83ae62-6754-4383-8e2b-4033ff3cfba1.onebin", + "2a83ae62-6754-4383-8e2b-4033ff3cfba1.onebin:other", + "", + ] { + assert!(matches!( + read_external_asset(&mut Unused, "Section.one", name, 100), + Err(Error::Entry { .. }) + )); + } +} + +#[test] +fn reserved_payload_directories_are_not_notebook_groups() { + let root = tempfile::tempdir().unwrap(); + fs::write( + root.path().join("Section.ONE"), + onestore::create_section("Section.one", "Fixture", "Author").unwrap(), + ) + .unwrap(); + for name in ["section_onefiles", "orphan_onefiles", "ordinary"] { + fs::create_dir(root.path().join(name)).unwrap(); + } + fs::write( + root.path().join("section_onefiles/unfinished.onebin"), + b"payload", + ) + .unwrap(); + let catalog = onestore_notebook::discover( + &mut Local::open(root.path()).unwrap(), + onestore_notebook::Limits { + entries: 4, + bytes_per_file: 1 << 20, + depth: 1, + }, + ) + .unwrap(); + assert_eq!(catalog.sections.len(), 1); + assert_eq!( + catalog + .groups + .iter() + .map(|group| group.path.as_str()) + .collect::>(), + ["ordinary"] + ); +} + +#[test] +#[cfg(feature = "smb")] +#[ignore = "requires a disposable Samba mirror of corpus/native-external-assets/notebook at ONESTORE_SMB_NOTEBOOK"] +fn live_external_payloads() { + use onestore::{ + FileDataReference, RevisionIndex, Store, + document::{Document, Kind}, + }; + let root = std::path::Path::new("../../corpus/native-external-assets/notebook"); + let bytes = fs::read(root.join("synthetic.one")).unwrap(); + let store = Store::parse(&bytes).unwrap(); + let index = RevisionIndex::parse(&store).unwrap(); + let document = Document::parse(&index).unwrap(); + let client = onestore_smb::Client::connect( + &std::env::var("ONESTORE_SMB_LAB").unwrap(), + "agent", + onestore_smb::Credentials::default(), + std::time::Duration::from_secs(5), + ) + .unwrap(); + let mut remote = + onestore_notebook::Smb::new(&client, &std::env::var("ONESTORE_SMB_NOTEBOOK").unwrap()) + .unwrap(); + let mut local = Local::open(root).unwrap(); + let mut seen = std::collections::BTreeSet::new(); + for node in document + .spaces + .values() + .flat_map(|space| space.revisions.values()) + .flat_map(|revision| revision.nodes.values()) + { + if let Kind::File { + reference: FileDataReference::External(filename), + .. + } = &node.kind + && seen.insert(filename) + { + let expected = fs::read(root.join("synthetic_onefiles").join(filename)).unwrap(); + assert_eq!( + read_external_asset(&mut local, "synthetic.one", filename, expected.len()).unwrap(), + expected + ); + assert_eq!( + read_external_asset(&mut remote, "synthetic.one", filename, expected.len()) + .unwrap(), + expected + ); + } + } + assert_eq!(seen.len(), 3); +} diff --git a/crates/onestore-smb/README.md b/crates/onestore-smb/README.md index 9228557fd90324876b5b1e0592dc318846cbb0ac..f13eb815e9f14c23e83fa10a1e3a17b74b41a642 100644 --- a/crates/onestore-smb/README.md +++ b/crates/onestore-smb/README.md @@ -47,6 +47,11 @@ are rejected with `ResourceBusy`. Notebook identities come from the files, not directory names or sizes. Missing paths, denied access and non-directory paths have distinct I/O error kinds. +`Client::read_asset(path, byte_limit)` reads an external payload under a read-only +share handle that excludes writes and deletion. Empty files succeed; limits, +interrupted reads and failed close never return partial bytes. This payload read +does not parse a OneStore header or acquire its reader-coordination bytes. + `python3 tools/test_smb_directory.py VM OUTPUT` checks a caller-owned disposable Linux lab VM against its filesystem listing and interrupts directory requests, responses and close. It creates synthetic files in that VM; the caller retains diff --git a/crates/onestore-smb/src/lib.rs b/crates/onestore-smb/src/lib.rs index ab2d77dbc5d93f727d422b439682d327b9932dd3..7f0d3dc562aa53b9e0c7f60707e8355c247847ee 100644 --- a/crates/onestore-smb/src/lib.rs +++ b/crates/onestore-smb/src/lib.rs @@ -175,6 +175,10 @@ impl Client { } fn open(&self, path: &str, write: bool) -> io::Result> { + self.open_shared(path, write, if write { 5 } else { 7 }) + } + + fn open_shared(&self, path: &str, write: bool, sharing: u32) -> io::Result> { if path.is_empty() || path.contains('\0') || path.encode_utf16().count() > 32767 { return Err(io::ErrorKind::InvalidInput.into()); } @@ -185,7 +189,7 @@ impl Client { impersonation_level: ImpersonationLevel::Impersonation, desired_access: FileAccessMask::new(if write { 0xc0000000 } else { 0x80000000 }), file_attributes: 0, - share_access: ShareAccess(if write { 5 } else { 7 }), + share_access: ShareAccess(sharing), create_disposition: CreateDisposition::FileOpen, create_options: 0x42, name: smb2::encode_path(&path.replace('\\', "/")), @@ -198,6 +202,30 @@ impl Client { }) } + /// Reads a bounded external payload while denying concurrent writes and deletion. + /// Empty files succeed; limits, sharing contention and failed close return no payload. + pub fn read_asset(&self, path: &str, limit: usize) -> io::Result> { + let mut file = self.open_shared(path, false, 1)?; + let mut bytes = Vec::new(); + let mut block = [0; 65536]; + loop { + let count = (limit - bytes.len()).min(block.len() - 1) + 1; + let read = file.read_at( + u64::try_from(bytes.len()).map_err(|_| io::ErrorKind::InvalidInput)?, + &mut block[..count], + )?; + if read == 0 { + break; + } + bytes.extend_from_slice(&block[..read]); + if bytes.len() > limit { + return Err(io::ErrorKind::FileTooLarge.into()); + } + } + file.close()?; + Ok(bytes) + } + /// Reads one bounded, consistent snapshot; contention returns WouldBlock. pub fn read(&self, path: &str, limit: usize) -> io::Result> { self.read_with(path, |file| { diff --git a/crates/onestore-smb/src/tests.rs b/crates/onestore-smb/src/tests.rs index 40da741c045aa8558e2f16da42fa5d6aaabc10f9..d89f1f700cf10d5dce2faa89148f988358af06d9 100644 --- a/crates/onestore-smb/src/tests.rs +++ b/crates/onestore-smb/src/tests.rs @@ -381,6 +381,55 @@ fn live_storage_inspection() { } } +#[test] +#[ignore = "requires ONESTORE_SMB_LAB pointing to disposable Samba"] +fn live_assets() { + let reader = client(); + let writer = client(); + for size in [0, 1, 65535, 65536, 65537, 1048577] { + let bytes: Vec<_> = (0..size).map(|index| (index % 251) as u8).collect(); + let path = format!( + "asset-{size}-{}.onebin", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + create(&writer, &path, &bytes); + assert_eq!(reader.read_asset(&path, size).unwrap(), bytes); + if size != 0 { + assert_eq!( + reader.read_asset(&path, size - 1).unwrap_err().kind(), + io::ErrorKind::FileTooLarge + ); + assert_eq!(reader.read_asset(&path, size).unwrap(), bytes); + } + let writing = writer.open(&path, true).unwrap(); + assert_eq!( + reader.read_asset(&path, size).unwrap_err().kind(), + io::ErrorKind::WouldBlock + ); + writing.close().unwrap(); + let asset = reader.open_shared(&path, false, 1).unwrap(); + assert_eq!( + writer.open(&path, true).err().unwrap().kind(), + io::ErrorKind::WouldBlock + ); + let other = writer.open_shared(&path, false, 1).unwrap(); + other.close().unwrap(); + asset.close().unwrap(); + writer.open(&path, true).unwrap().close().unwrap(); + assert_eq!(reader.read_asset(&path, size).unwrap(), bytes); + } + assert_eq!( + reader + .read_asset("absent-payload.onebin", 0) + .unwrap_err() + .kind(), + io::ErrorKind::NotFound + ); +} + #[test] #[ignore = "requires an owned Samba fixture and maintenance controller"] fn live_reader_hold() { diff --git a/crates/onestore-smb/src/tests/faults.rs b/crates/onestore-smb/src/tests/faults.rs index fb4daed377dfb6f4382ec8e8474dcae0b61cab92..99641ee062cb06e9a4dad56f284ca954293b65fc 100644 --- a/crates/onestore-smb/src/tests/faults.rs +++ b/crates/onestore-smb/src/tests/faults.rs @@ -171,14 +171,7 @@ fn configure(output: &Path, state: Value) -> usize { } } -#[test] -#[ignore = "requires an owned Samba share and a new ONESTORE_SMB_EVIDENCE directory"] -fn live_message_loss() { - let output = std::path::PathBuf::from(std::env::var("ONESTORE_SMB_EVIDENCE").unwrap()); - fs::create_dir(&output).unwrap(); - fs::create_dir(output.join("interrupted")).unwrap(); - fs::create_dir(output.join("recovered")).unwrap(); - fs::create_dir(output.join("source")).unwrap(); +fn proxy(output: &Path) -> (Proxy, String) { let address = std::env::var("ONESTORE_SMB_LAB").unwrap(); let (host, port) = address.rsplit_once(':').unwrap(); let mut proxy = Proxy( @@ -193,7 +186,7 @@ fn live_message_loss() { ); let deadline = Instant::now() + Duration::from_secs(5); let port = loop { - if let Some(port) = records(&output) + if let Some(port) = records(output) .iter() .find_map(|event| event["listening"].as_u64()) { @@ -203,7 +196,81 @@ fn live_message_loss() { assert!(Instant::now() < deadline, "proxy did not start"); std::thread::sleep(Duration::from_millis(5)); }; - let proxied = format!("127.0.0.1:{port}"); + (proxy, format!("127.0.0.1:{port}")) +} + +#[test] +#[ignore = "requires an owned Samba share and a new ONESTORE_SMB_EVIDENCE directory"] +fn live_asset_loss() { + let output = std::path::PathBuf::from(std::env::var("ONESTORE_SMB_EVIDENCE").unwrap()); + fs::create_dir(&output).unwrap(); + let (_proxy, address) = proxy(&output); + let observer = client(); + let bytes: Vec<_> = (0..1048577).map(|index| (index % 251) as u8).collect(); + let path = format!( + "asset-loss-{}.onebin", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + create(&observer, &path, &bytes); + for (command, occurrence) in [(8, 1), (8, 9), (8, 17), (8, 18), (6, 1)] { + for direction in ["request", "response"] { + let reader = Client::connect( + &address, + "agent", + Credentials::default(), + Duration::from_secs(5), + ) + .unwrap(); + // Response occurrences count only replies with the selected status. + let begin = configure( + &output, + json!({"cut":command,"occurrence":if command == 8 && occurrence == 18 && direction == "response" { 1 } else { occurrence }, + "direction":direction,"status":if command == 8 && occurrence == 18 { "0xc0000011" } else { "0x0" }}), + ); + assert!( + reader.read_asset(&path, bytes.len()).is_err(), + "{command}/{occurrence}/{direction}" + ); + assert_eq!( + reader.read_asset(&path, bytes.len()).unwrap_err().kind(), + io::ErrorKind::NotConnected + ); + assert_eq!( + records(&output)[begin..] + .iter() + .filter(|row| row.get("cut").is_some()) + .count(), + 1 + ); + configure( + &output, + json!({"phase":format!("{command}-{occurrence}-{direction}-reconnected")}), + ); + let reconnected = Client::connect( + &address, + "agent", + Credentials::default(), + Duration::from_secs(5), + ) + .unwrap(); + assert_eq!(reconnected.read_asset(&path, bytes.len()).unwrap(), bytes); + } + } + assert_eq!(observer.read_asset(&path, bytes.len()).unwrap(), bytes); +} + +#[test] +#[ignore = "requires an owned Samba share and a new ONESTORE_SMB_EVIDENCE directory"] +fn live_message_loss() { + let output = std::path::PathBuf::from(std::env::var("ONESTORE_SMB_EVIDENCE").unwrap()); + fs::create_dir(&output).unwrap(); + fs::create_dir(output.join("interrupted")).unwrap(); + fs::create_dir(output.join("recovered")).unwrap(); + fs::create_dir(output.join("source")).unwrap(); + let (_proxy, proxied) = proxy(&output); let observer = client(); let prefix = SystemTime::now() .duration_since(UNIX_EPOCH) diff --git a/crates/onestore/examples/document.rs b/crates/onestore/examples/document.rs deleted file mode 100644 index 98c28e84ebdb61a778a9464d2ee86579893a9a1c..0000000000000000000000000000000000000000 --- a/crates/onestore/examples/document.rs +++ /dev/null @@ -1,77 +0,0 @@ -use onestore::{ - FileDataReference, RevisionIndex, Store, - document::{Document, Kind}, -}; -use std::{ - collections::BTreeSet, - env, fs, - io::{self, BufWriter, Write}, - path::{Path, PathBuf}, -}; - -fn write_json( - path: impl AsRef, - value: &impl serde::Serialize, -) -> Result<(), Box> { - let mut output = BufWriter::new(fs::File::create(path)?); - serde_json::to_writer(&mut output, value)?; - output.flush()?; - Ok(()) -} - -fn main() -> Result<(), Box> { - let mut args = env::args_os().skip(1); - let path = args.next().ok_or("Provide a .one or .onetoc2 file.")?; - let destination = args.next().map(PathBuf::from); - if args.next().is_some() { - return Err("Provide a source file and an optional new export directory.".into()); - } - let bytes = onestore::read_file(path)?; - let store = Store::parse(&bytes)?; - let index = RevisionIndex::parse(&store)?; - let document = Document::parse(&index)?; - if let Some(destination) = destination { - fs::create_dir(&destination)?; - fs::create_dir(destination.join("assets"))?; - let mut assets = Vec::new(); - let mut seen = BTreeSet::new(); - for node in document - .spaces - .values() - .flat_map(|s| s.revisions.values()) - .flat_map(|r| r.nodes.values()) - { - if let Kind::File { - reference: FileDataReference::Internal(guid), - payload: Some(data), - .. - } = &node.kind - && seen.insert(guid) - { - let path = format!("assets/{}.bin", assets.len()); - fs::write(destination.join(&path), data)?; - assets.push(serde_json::json!({"reference": FileDataReference::Internal(*guid), "path": path})); - } - } - write_json(destination.join("assets.json"), &assets)?; - let mut text = std::collections::BTreeMap::new(); - for (sid, space) in &document.spaces { - let mut revisions = std::collections::BTreeMap::new(); - for (rid, revision) in &space.revisions { - let mut objects = std::collections::BTreeMap::new(); - for (oid, node) in &revision.nodes { - if matches!(node.kind, Kind::RichText { .. }) { - objects.insert(*oid, revision.text_runs(*oid)?); - } - } - revisions.insert(*rid, objects); - } - text.insert(*sid, revisions); - } - write_json(destination.join("text.json"), &text)?; - write_json(destination.join("document.json"), &document)?; - } else { - serde_json::to_writer(io::stdout().lock(), &document)?; - } - Ok(()) -} diff --git a/crates/onestore/src/files.rs b/crates/onestore/src/files.rs index 038bd0ba92d772602a0d27b9c3980af372b39bb9..e481dffb48a8343ec32be05099f6703d04fb3d5e 100644 --- a/crates/onestore/src/files.rs +++ b/crates/onestore/src/files.rs @@ -49,6 +49,14 @@ impl Object<'_> { offset: 0, message: "Invalid UTF-16 file-data reference", })?; + text.parse().map(Some) + } +} + +impl std::str::FromStr for FileDataReference { + type Err = Error; + + fn from_str(text: &str) -> Result { let parsed = if let Some(name) = text.strip_prefix("") { name.strip_suffix(".onebin") .and_then(guid) @@ -63,7 +71,7 @@ impl Object<'_> { } else { None }; - parsed.map(Some).ok_or(Error { + parsed.ok_or(Error { offset: 0, message: "Invalid file-data reference syntax", }) diff --git a/tools/native/external-assets.ps1 b/tools/native/external-assets.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..f0645047843f4cc781a5a88731e0d0c197f271dd --- /dev/null +++ b/tools/native/external-assets.ps1 @@ -0,0 +1,34 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '', [int[]]$Lengths = @(0, 1024)) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +& "$PSScriptRoot\cold-current.ps1" -Root $Root -CloneHost $CloneHost +$app = New-Object -ComObject OneNote.Application +$notebook = '' +try { + $inputs = Join-Path $Root 'inputs' + New-Item -ItemType Directory -Path $inputs | Out-Null + $app.OpenHierarchy((Join-Path $Root 'notebook'), '', [ref]$notebook, 0) + $section = '' + $app.OpenHierarchy('synthetic.one', $notebook, [ref]$section, 0) + foreach ($length in $Lengths) { + $path = Join-Path $inputs ($length.ToString() + '.bin') + $stream = [IO.File]::Create($path) + try { + $block = New-Object byte[] 1024 + for ($i = 0; $i -lt $block.Length; $i++) { $block[$i] = [byte]($i % 251) } + for ($written = 0; $written -lt $length; $written += $block.Length) { $stream.Write($block, 0, [Math]::Min($block.Length, $length - $written)) } + } finally { $stream.Dispose() } + $page = '' + $app.CreateNewPage($section, [ref]$page, 0) + $xml = 'Attachment ' + $length + '' + $app.UpdatePageContent($xml, [DateTime]::MinValue, 1, $false) + } + $app.SyncHierarchy($notebook) + +} finally { + if ($notebook) { $app.CloseNotebook($notebook, $false) } + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() +} diff --git a/tools/notebook_editor.py b/tools/notebook_editor.py index 936e8da36863540d6ed361fa34d48f7e636c26d6..98d1c1c039bdc1b359df7d2c4769b3ea1142d362 100644 --- a/tools/notebook_editor.py +++ b/tools/notebook_editor.py @@ -61,7 +61,7 @@ class Session: source = pending / 'snapshot' source.mkdir(parents=True) for path in sorted((self.output / 'notebook').rglob('*')): - if path.suffix.lower() not in ('.one', '.onetoc2'): continue + if path.suffix.lower() not in ('.one', '.onetoc2', '.onebin'): continue saved = source / path.relative_to(self.output / 'notebook') saved.parent.mkdir(parents=True, exist_ok=True) result = bridge('snapshot', path, saved) diff --git a/tools/notebook_report.py b/tools/notebook_report.py index e35ce06e0e0ce25e399a71a27365f9b6a0abb122..19721f6712374c969cd2c2ccb0f707c8999047b5 100644 --- a/tools/notebook_report.py +++ b/tools/notebook_report.py @@ -298,6 +298,8 @@ def generate(source, destination, native=None, versions=(), zone=timezone.utc, e exported = destination / 'model' / str(index) reusable = cached.get(relative.as_posix()) reused = reusable is not None and reusable[0] == manifest[-1]['sha256'] + if reused and any('External' in asset['reference'] for asset in json.loads((reusable[1] / 'assets.json').read_text())): + reused = False if reused: exported.mkdir() for name in ('document.json', 'text.json', 'assets.json'): @@ -317,6 +319,7 @@ def generate(source, destination, native=None, versions=(), zone=timezone.utc, e } rows = json.loads((exported / 'assets.json').read_text()) for asset in rows: + if asset['path'] is None: continue reference = json.dumps(asset['reference'], sort_keys=True) if reused: name = Path(asset['path']).name diff --git a/tools/test_notebook_discovery.py b/tools/test_notebook_discovery.py index e7e68b9b5636645be13ca22c74d5dfd3c710bc17..7d89bf6fa719db477257613e51083e38a871c945 100644 --- a/tools/test_notebook_discovery.py +++ b/tools/test_notebook_discovery.py @@ -12,7 +12,7 @@ ROOT = Path(__file__).resolve().parent.parent class NativeDiscovery(unittest.TestCase): def test_native_hierarchies(self): fixtures = ['m6/native-features-01', 'native-encrypted/cold-encrypted-02', - 'native-delete/cold-deletion-05'] + 'native-delete/cold-deletion-05', 'native-external-assets'] if os.environ.get('ONESTORE_NOTEBOOK_NATIVE'): fixtures.append(str(Path(os.environ['ONESTORE_NOTEBOOK_NATIVE']).resolve())) for fixture in fixtures: diff --git a/tools/test_notebook_report.py b/tools/test_notebook_report.py index 607f59df3d786237c71415a3e20b5490ca26ad23..1e11563862d07eaa619b35be8472f4414f0cb91f 100644 --- a/tools/test_notebook_report.py +++ b/tools/test_notebook_report.py @@ -1,6 +1,7 @@ import base64 from io import BytesIO import json +import hashlib from pathlib import Path import re from tempfile import TemporaryDirectory @@ -16,6 +17,40 @@ from document_model import EXPORTER class NotebookReportTest(unittest.TestCase): + def test_external_assets_match_native_bytes_and_refresh_without_a_section_edit(self): + from notebook_editor import Session + fixture = Path(__file__).resolve().parent.parent / 'corpus/native-external-assets' + with TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / 'source' + shutil.copytree(fixture / 'notebook', source) + session = Session(source, root / 'session') + first = root / 'session/g/0/report' + section = next(p.parent for p in (first / 'model').glob('*/assets.json') + if json.loads(p.read_text())) + rows = json.loads((section / 'assets.json').read_text()) + expected = {row['sha256'] for row in json.loads((fixture / 'read/payloads.json').read_text(encoding='utf-8-sig'))} + actual = {hashlib.sha256((section / row['path']).read_bytes()).hexdigest() for row in rows} + self.assertTrue(expected <= actual) + for payload in source.rglob('*.onebin'): + self.assertEqual(payload.read_bytes(), (root / 'session/g/0/snapshot' / payload.relative_to(source)).read_bytes()) + payload, = [p for p in source.rglob('*.onebin') if p.stat().st_size == 1024] + reference = {'External': payload.name} + old, = [row for row in rows if row['reference'] == reference] + previous_bytes = (section / old['path']).read_bytes() + payload.unlink() + generate(source, root / 'missing', previous=first) + missing, = [row for p in (root / 'missing/model').glob('*/assets.json') + for row in json.loads(p.read_text()) if row['reference'] == reference] + self.assertIsNone(missing['path']) + self.assertEqual(missing['error']['kind'], 'NotFound') + payload.write_bytes(b'Changed external payload') + generate(source, root / 'changed', previous=root / 'missing') + changed, = [(p.parent, row) for p in (root / 'changed/model').glob('*/assets.json') + for row in json.loads(p.read_text()) if row['reference'] == reference] + self.assertEqual((changed[0] / changed[1]['path']).read_bytes(), payload.read_bytes()) + self.assertEqual((section / old['path']).read_bytes(), previous_bytes) + def test_locked_and_unreadable_sections_remain_visible_in_cached_reports(self): fixture = Path(__file__).resolve().parent.parent / 'corpus/native-encrypted/cold-encrypted-02/notebook' with TemporaryDirectory() as temporary: