diff --git a/Cargo.lock b/Cargo.lock index 83b5d064c58b1b4cc4ba9e45b3c9d91fa9334ba8..7aac8ee77b097d29ed8c4008224c767104b79f2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -515,10 +515,23 @@ name = "onestore-diagnostic" version = "0.1.0" dependencies = [ "onestore", + "onestore-notebook", "serde", "serde_json", ] +[[package]] +name = "onestore-notebook" +version = "0.1.0" +dependencies = [ + "onestore", + "onestore-smb", + "serde", + "serde_json", + "tempfile", + "thiserror", +] + [[package]] name = "onestore-offline" version = "0.1.0" diff --git a/crates/onestore-diagnostic/Cargo.toml b/crates/onestore-diagnostic/Cargo.toml index adfac86624601961da50aa32be0a0a653f2bfd2a..cc4b96f0e146f2cff6bab02ca13e8d29d920c1bb 100644 --- a/crates/onestore-diagnostic/Cargo.toml +++ b/crates/onestore-diagnostic/Cargo.toml @@ -6,5 +6,6 @@ publish = false [dependencies] onestore = { path = "../onestore" } +onestore-notebook = { path = "../onestore-notebook" } serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" diff --git a/crates/onestore-diagnostic/src/main.rs b/crates/onestore-diagnostic/src/main.rs index 6352ff5035a89cc69b0154056fc60c9aea774789..5b39a4f147a8221205d3a636b59d6d46934080bd 100644 --- a/crates/onestore-diagnostic/src/main.rs +++ b/crates/onestore-diagnostic/src/main.rs @@ -40,14 +40,26 @@ enum Action { }, } -fn run() -> Result> { - let args: Vec<_> = env::args_os().skip(1).collect(); +fn run(args: &[std::ffi::OsString]) -> Result> { + if let [mode, root] = args + && mode == "catalog" + { + let catalog = onestore_notebook::discover( + &mut onestore_notebook::Local::open(root)?, + onestore_notebook::Limits { + entries: 100_000, + bytes_per_file: 256 * 1024 * 1024, + depth: 64, + }, + )?; + return Ok(json!({"ok": true, "catalog": catalog})); + } if args.len() != 3 || !["snapshot", "check", "commit"] .iter() .any(|mode| args[0] == *mode) { - return Err("Usage: onestore-diagnostic snapshot FILE NEW_SNAPSHOT | check SNAPSHOT - | commit FILE SNAPSHOT; edits arrive as JSON on stdin".into()); + return Err("Usage: onestore-diagnostic catalog ROOT | snapshot FILE NEW_SNAPSHOT | check SNAPSHOT - | commit FILE SNAPSHOT; edits arrive as JSON on stdin".into()); } if args[0] == "snapshot" { let bytes = onestore::read_file(&args[1])?; @@ -102,6 +114,13 @@ fn run() -> Result> { } fn main() { - let result = run().unwrap_or_else(|error| json!({"ok": false, "state": "NotCommitted", "kind": "Input", "error": error.to_string()})); + let args: Vec<_> = env::args_os().skip(1).collect(); + let result = run(&args).unwrap_or_else(|error| { + let mut result = json!({"ok": false, "kind": "Input", "error": error.to_string()}); + if args.first().is_some_and(|mode| mode == "commit") { + result["state"] = json!("NotCommitted"); + } + result + }); println!("{result}"); } diff --git a/crates/onestore-notebook/Cargo.toml b/crates/onestore-notebook/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..156fb00be96d586ef87e1fda176e66dad846e04d --- /dev/null +++ b/crates/onestore-notebook/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "onestore-notebook" +version = "0.1.0" +edition = "2024" +publish = false + +[features] +smb = ["dep:onestore-smb"] + +[dependencies] +onestore = { path = "../onestore" } +onestore-smb = { path = "../onestore-smb", optional = true } +serde = { version = "1", features = ["derive"] } +thiserror = "2" + +[dev-dependencies] +serde_json = "1" +tempfile = "3" diff --git a/crates/onestore-notebook/README.md b/crates/onestore-notebook/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bd9669954a44cb94607938da8f68be9edf5ff6fd --- /dev/null +++ b/crates/onestore-notebook/README.md @@ -0,0 +1,18 @@ +# onestore-notebook + +Read-only notebook discovery over a caller-supplied root. This experimental +sibling crate keeps directory traversal out of the single-file storage parser. + +Discovery returns ordered sections and nested groups with file identities and +share-relative paths. Section display-name overrides remain distinct from file +names. TOC references whose identities are absent from the directory remain +inspectable; a cached filename never substitutes for an identity match. +Encrypted sections and valid storage with an unreadable document graph retain +their identity as `Locked` or `Unreadable`, without being presented as empty pages. +Malformed storage, failed reads and ambiguous identities reject the discovery. + +Each file read must be a consistent, bounded snapshot. The result is an +observation across multiple files, not an atomic notebook transaction or +authorization to publish an edit. Refresh rejects observed topology changes and +duplicate physical files with the same logical identity. Retain the last accepted +catalog if discovery fails; a connection failure does not mean files were deleted. diff --git a/crates/onestore-notebook/examples/discover.rs b/crates/onestore-notebook/examples/discover.rs new file mode 100644 index 0000000000000000000000000000000000000000..fb73310d27b998ff66f3dde3a4f2dfc364e0829b --- /dev/null +++ b/crates/onestore-notebook/examples/discover.rs @@ -0,0 +1,40 @@ +use onestore_notebook::{Limits, Local, discover}; + +fn main() -> Result<(), Box> { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + let limits = Limits { + entries: 100_000, + bytes_per_file: 256 * 1024 * 1024, + depth: 64, + }; + let catalog = match args.as_slice() { + [root] => discover(&mut Local::open(root)?, limits)?, + #[cfg(feature = "smb")] + [flag, address, share, root] if flag == "--smb" => { + use onestore_smb::{Client, Credentials}; + let username = std::env::var("ONESTORE_SMB_USERNAME").unwrap_or_default(); + let password = std::env::var("ONESTORE_SMB_PASSWORD").unwrap_or_default(); + let domain = std::env::var("ONESTORE_SMB_DOMAIN").unwrap_or_default(); + let client = Client::connect( + address.to_str().ok_or("Use a UTF-8 server address")?, + share.to_str().ok_or("Use a UTF-8 share name")?, + Credentials { + username: &username, + password: &password, + domain: &domain, + }, + std::time::Duration::from_secs(5), + )?; + discover( + &mut onestore_notebook::Smb::new( + &client, + root.to_str().ok_or("Use a UTF-8 notebook path")?, + )?, + limits, + )? + } + _ => return Err("Provide ROOT, or --smb ADDRESS SHARE ROOT with the smb feature".into()), + }; + serde_json::to_writer_pretty(std::io::stdout().lock(), &catalog)?; + Ok(()) +} diff --git a/crates/onestore-notebook/src/lib.rs b/crates/onestore-notebook/src/lib.rs new file mode 100644 index 0000000000000000000000000000000000000000..ff473ee25aabf19a99bbd376f59915c4b779dc1d --- /dev/null +++ b/crates/onestore-notebook/src/lib.rs @@ -0,0 +1,381 @@ +#![forbid(unsafe_code)] +#![doc = include_str!("../README.md")] + +use onestore::{ + ExGuid, FileType, RevisionIndex, Store, + document::{Document, Kind}, +}; +use serde::Serialize; +use std::{ + collections::{BTreeMap, BTreeSet}, + io, +}; + +mod source; +pub use source::Local; +#[cfg(feature = "smb")] +pub use source::Smb; + +#[derive(Debug, Serialize)] +pub struct Section { + pub path: String, + /// Header.guidFile, also used by FileIdentityGuid in a parent TOC. + pub file_id: [u8; 16], + pub state: SectionState, +} + +#[derive(Debug, Serialize)] +pub enum SectionState { + Readable { + /// An explicit SectionDisplayName; otherwise use the current filename without its extension. + name: Option, + }, + Locked, + Unreadable(onestore::Error), +} + +#[derive(Debug, Serialize)] +pub struct Folder { + pub path: String, + pub toc: Option, + pub sections: Vec
, + pub groups: Vec, +} + +#[derive(Debug, Serialize)] +pub struct Toc { + pub filename: String, + pub file_id: [u8; 16], + /// Stale or unavailable TOC references; these are not inferred active sections. + pub unresolved: Vec, +} + +#[derive(Debug, Serialize)] +pub struct TocReference { + /// Native TOCs can retain an identity after removing its cached filename. + pub filename: Option, + pub file: [u8; 16], + pub order: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum EntryKind { + File, + Directory, + Other, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct Entry { + pub name: String, + pub kind: EntryKind, +} + +/// Rooted file access. Paths are relative, UTF-8, and use `/` between components. +pub trait Source { + /// Return the complete immediate directory or an error, never a truncated success. + fn entries(&mut self, path: &str, limit: usize) -> io::Result>; + /// Return a consistent file snapshot, rejecting images larger than the byte limit. + fn read(&mut self, path: &str, limit: usize) -> io::Result>; +} + +pub struct Limits { + pub entries: usize, + pub bytes_per_file: usize, + pub depth: usize, +} + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("Cannot read {path}: {error}")] + Io { + path: String, + #[source] + error: io::Error, + }, + #[error("Invalid notebook file {path}: {error}")] + Document { + path: String, + #[source] + error: onestore::Error, + }, + #[error("Discovery limit exceeded at {path}")] + Limit { path: String }, + #[error("Directory changed during discovery: {path}")] + Changed { path: String }, + #[error("Invalid or unsupported directory entry: {path}")] + Entry { path: String }, + #[error("File identity occurs at both {first} and {second}")] + DuplicateIdentity { + file: [u8; 16], + first: String, + second: String, + }, +} + +/// Discovers the complete rooted directory within caller-specified work and size limits. +pub fn discover(source: &mut impl Source, limits: Limits) -> Result { + let mut remaining = limits.entries; + let mut identities = BTreeMap::new(); + scan(source, "", &limits, 0, &mut remaining, &mut identities) +} + +fn scan( + source: &mut impl Source, + path: &str, + limits: &Limits, + depth: usize, + remaining: &mut usize, + identities: &mut BTreeMap<[u8; 16], String>, +) -> Result { + if depth > limits.depth { + return Err(Error::Limit { path: path.into() }); + } + let mut listing = source + .entries(path, *remaining) + .map_err(|error| Error::Io { + path: path.into(), + error, + })?; + *remaining = remaining + .checked_sub(listing.len()) + .ok_or_else(|| Error::Limit { path: path.into() })?; + listing.sort(); + let mut names = BTreeSet::new(); + for entry in &listing { + if !component(&entry.name) || !names.insert(&entry.name) { + return Err(Error::Entry { + path: join(path, &entry.name), + }); + } + } + let mut result = Folder { + path: path.into(), + toc: None, + sections: Vec::new(), + groups: Vec::new(), + }; + for entry in &listing { + let child = join(path, &entry.name); + if entry.kind == EntryKind::Directory { + result.groups.push(scan( + source, + &child, + limits, + depth + 1, + remaining, + identities, + )?); + continue; + } + let lower = entry.name.to_ascii_lowercase(); + let expected = if lower.ends_with(".one") { + FileType::Section + } else if lower.ends_with(".onetoc2") { + FileType::TableOfContents + } else { + continue; + }; + if entry.kind != EntryKind::File + || (expected == FileType::TableOfContents && result.toc.is_some()) + { + return Err(Error::Entry { path: child }); + } + let bytes = source + .read(&child, limits.bytes_per_file) + .map_err(|error| Error::Io { + path: child.clone(), + error, + })?; + if bytes.len() > limits.bytes_per_file { + return Err(Error::Limit { path: child }); + } + let store = Store::parse(&bytes).map_err(|error| Error::Document { + path: child.clone(), + error, + })?; + if store.header.file_type != expected || !store.checksum_mismatches.is_empty() { + return Err(Error::Document { + path: child, + error: onestore::Error { + offset: 0, + message: "Unexpected file type or checksum mismatch", + }, + }); + } + let file_id = store.header.file_id; + let parsed = (|| { + let index = RevisionIndex::parse(&store)?; + let document = Document::parse(&index)?; + let revision = document + .spaces + .get(&document.root) + .and_then(|space| { + space + .contexts + .get(&ExGuid::default()) + .and_then(|id| space.revisions.get(id)) + }) + .ok_or(onestore::Error { + offset: 0, + message: "Missing active notebook root revision", + })?; + if expected == FileType::Section { + if revision + .roots + .get(&1) + .and_then(|id| revision.nodes.get(id)) + .is_some_and(|node| matches!(node.kind, Kind::Encrypted { .. })) + { + result.sections.push(Section { + path: child.clone(), + file_id, + state: SectionState::Locked, + }); + return Ok(()); + } + index.validate_current()?; + document.pages()?; + let name = revision + .roots + .get(&2) + .and_then(|id| revision.nodes.get(id)) + .and_then(|node| match &node.kind { + Kind::SectionMetadata { name, .. } => name.clone(), + _ => None, + }); + result.sections.push(Section { + path: child.clone(), + file_id, + state: SectionState::Readable { name }, + }); + } else { + index.validate_current()?; + let node = revision.roots.get(&1).and_then(|id| revision.nodes.get(id)); + let Some(Kind::Toc { entries, .. }) = node.map(|node| &node.kind) else { + return Err(onestore::Error { + offset: 0, + message: "Missing notebook TOC root", + }); + }; + let mut seen = BTreeSet::new(); + let mut unresolved = Vec::new(); + for id in entries { + let Some(Kind::Toc { + filename, + identity: Some(file), + order: Some(order), + .. + }) = revision.nodes.get(id).map(|node| &node.kind) + else { + return Err(onestore::Error { + offset: 0, + message: "Incomplete notebook TOC reference", + }); + }; + if filename.as_deref().is_some_and(|name| !component(name)) + || !seen.insert(*file) + { + return Err(onestore::Error { + offset: 0, + message: "Invalid or duplicated notebook TOC reference", + }); + } + unresolved.push(TocReference { + filename: filename.clone(), + file: *file, + order: *order, + }); + } + result.toc = Some(Toc { + filename: entry.name.clone(), + file_id, + unresolved, + }); + } + Ok(()) + })(); + if let Err(error) = parsed { + if expected == FileType::Section { + result.sections.push(Section { + path: child.clone(), + file_id, + state: SectionState::Unreadable(error), + }); + } else { + return Err(Error::Document { path: child, error }); + } + } + if let Some(first) = identities.insert(file_id, child.clone()) { + return Err(Error::DuplicateIdentity { + file: file_id, + first, + second: child, + }); + } + } + let order: BTreeMap<_, _> = result + .toc + .iter() + .flat_map(|toc| &toc.unresolved) + .map(|entry| (entry.file, entry.order)) + .collect(); + result.sections.sort_by(|a, b| { + (order.get(&a.file_id).copied().unwrap_or(u32::MAX), &a.path) + .cmp(&(order.get(&b.file_id).copied().unwrap_or(u32::MAX), &b.path)) + }); + result.groups.sort_by(|a, b| { + ( + a.toc + .as_ref() + .and_then(|toc| order.get(&toc.file_id).copied()) + .unwrap_or(u32::MAX), + &a.path, + ) + .cmp(&( + b.toc + .as_ref() + .and_then(|toc| order.get(&toc.file_id).copied()) + .unwrap_or(u32::MAX), + &b.path, + )) + }); + let present: BTreeSet<_> = result + .sections + .iter() + .map(|section| section.file_id) + .chain( + result + .groups + .iter() + .filter_map(|group| group.toc.as_ref().map(|toc| toc.file_id)), + ) + .collect(); + if let Some(toc) = &mut result.toc { + toc.unresolved + .retain(|entry| !present.contains(&entry.file)); + } + let mut observed = source + .entries(path, listing.len()) + .map_err(|error| Error::Io { + path: path.into(), + error, + })?; + observed.sort(); + if observed != listing { + return Err(Error::Changed { path: path.into() }); + } + Ok(result) +} + +fn component(name: &str) -> bool { + !name.is_empty() && name != "." && name != ".." && !name.contains(['/', '\\', '\0']) +} + +fn join(parent: &str, name: &str) -> String { + if parent.is_empty() { + name.into() + } else { + format!("{parent}/{name}") + } +} diff --git a/crates/onestore-notebook/src/source.rs b/crates/onestore-notebook/src/source.rs new file mode 100644 index 0000000000000000000000000000000000000000..e925896012c4819179630949a648cb4a629b71d2 --- /dev/null +++ b/crates/onestore-notebook/src/source.rs @@ -0,0 +1,117 @@ +use super::{Entry, EntryKind, Source, component}; +use std::{ + io, + path::{Path, PathBuf}, +}; + +/// A canonical local root; symbolic-link entries are not traversed. +pub struct Local { + root: PathBuf, +} + +impl Local { + pub fn open(root: impl AsRef) -> io::Result { + let root = root.as_ref().canonicalize()?; + if !root.is_dir() { + return Err(io::ErrorKind::NotADirectory.into()); + } + Ok(Self { root }) + } + + fn path(&self, relative: &str) -> io::Result { + if !relative.is_empty() && !relative.split('/').all(component) { + return Err(io::ErrorKind::InvalidInput.into()); + } + let path = self.root.join(relative).canonicalize()?; + if !path.starts_with(&self.root) { + return Err(io::ErrorKind::PermissionDenied.into()); + } + Ok(path) + } +} + +impl Source for Local { + fn entries(&mut self, path: &str, limit: usize) -> io::Result> { + let mut entries = Vec::new(); + for entry in std::fs::read_dir(self.path(path)?)? { + let entry = entry?; + if entries.len() == limit { + return Err(io::ErrorKind::FileTooLarge.into()); + } + let name = entry + .file_name() + .into_string() + .map_err(|_| io::ErrorKind::InvalidData)?; + let kind = entry.file_type()?; + entries.push(Entry { + name, + kind: if kind.is_dir() { + EntryKind::Directory + } else if kind.is_file() { + EntryKind::File + } else { + EntryKind::Other + }, + }); + } + Ok(entries) + } + + fn read(&mut self, path: &str, limit: usize) -> io::Result> { + onestore::read_file_limited(self.path(path)?, limit) + } +} + +#[cfg(feature = "smb")] +/// A share-relative root on an existing blocking SMB connection. +pub struct Smb<'a> { + client: &'a onestore_smb::Client, + root: String, +} + +#[cfg(feature = "smb")] +impl<'a> Smb<'a> { + pub fn new(client: &'a onestore_smb::Client, root: &str) -> io::Result { + let root = root.replace('\\', "/"); + if !root.is_empty() && !root.split('/').all(component) { + return Err(io::ErrorKind::InvalidInput.into()); + } + Ok(Self { client, root }) + } + + fn path(&self, relative: &str) -> io::Result { + if !relative.is_empty() && !relative.split('/').all(component) { + return Err(io::ErrorKind::InvalidInput.into()); + } + Ok(if relative.is_empty() { + self.root.clone() + } else { + super::join(&self.root, relative) + }) + } +} + +#[cfg(feature = "smb")] +impl Source for Smb<'_> { + fn entries(&mut self, path: &str, limit: usize) -> io::Result> { + Ok(self + .client + .read_dir(&self.path(path)?, limit)? + .into_iter() + .map(|entry| Entry { + name: entry.name, + kind: if entry.attributes & 0x400 != 0 { + EntryKind::Other + } else if entry.attributes & 0x10 != 0 { + EntryKind::Directory + } else { + EntryKind::File + }, + }) + .collect()) + } + + fn read(&mut self, path: &str, limit: usize) -> io::Result> { + self.client.read_storage(&self.path(path)?, limit) + } +} diff --git a/crates/onestore-notebook/tests/discovery.rs b/crates/onestore-notebook/tests/discovery.rs new file mode 100644 index 0000000000000000000000000000000000000000..89a6f21debf9b218b64f59dad76fae89c6262830 --- /dev/null +++ b/crates/onestore-notebook/tests/discovery.rs @@ -0,0 +1,280 @@ +use onestore_notebook::{Entry, Error, Limits, Local, Source, discover}; +use std::{fs, io, path::Path}; + +fn limits() -> Limits { + Limits { + entries: 1000, + bytes_per_file: 16 * 1024 * 1024, + depth: 16, + } +} + +fn fixture(root: &Path) -> Vec { + let section = onestore::create_section("one.one", "Retained 馃", "Fixture").unwrap(); + fs::write(root.join("one.one"), §ion).unwrap(); + let identity = onestore::Store::parse(§ion).unwrap().header.file_id; + let toc = onestore::create_table_of_contents("Open Notebook.onetoc2", &[("one.one", identity)]) + .unwrap(); + fs::write(root.join("Open Notebook.onetoc2"), toc).unwrap(); + section +} + +#[test] +fn rename_preserves_identity_and_does_not_trust_a_stale_cached_filename() { + let root = tempfile::tempdir().unwrap(); + let section = fixture(root.path()); + let mut source = Local::open(root.path()).unwrap(); + let before = discover(&mut source, limits()).unwrap(); + fs::rename( + root.path().join("one.one"), + root.path().join("Renamed 馃.ONE"), + ) + .unwrap(); + let after = discover(&mut source, limits()).unwrap(); + assert_eq!(before.sections[0].file_id, after.sections[0].file_id); + assert_eq!(after.sections[0].path, "Renamed 馃.ONE"); + assert!(after.toc.as_ref().unwrap().unresolved.is_empty()); + assert_eq!( + fs::read(root.path().join("Renamed 馃.ONE")).unwrap(), + section + ); + fs::write( + root.path().join("one.one"), + onestore::create_section("one.one", "Different", "Fixture").unwrap(), + ) + .unwrap(); + let with_replacement = discover(&mut source, limits()).unwrap(); + assert_eq!( + with_replacement.sections[0].file_id, + before.sections[0].file_id + ); + assert_eq!(with_replacement.sections[0].path, "Renamed 馃.ONE"); + assert_ne!( + with_replacement.sections[1].file_id, + before.sections[0].file_id + ); + fs::remove_file(root.path().join("Renamed 馃.ONE")).unwrap(); + let absent = discover(&mut source, limits()).unwrap(); + assert_eq!(absent.toc.as_ref().unwrap().unresolved.len(), 1); + assert_eq!( + absent.toc.as_ref().unwrap().unresolved[0].file, + before.sections[0].file_id + ); + assert_ne!( + absent.sections[0].file_id, + absent.toc.as_ref().unwrap().unresolved[0].file + ); +} + +#[test] +fn copied_identities_are_ambiguous_even_across_different_groups() { + let root = tempfile::tempdir().unwrap(); + fixture(root.path()); + fs::create_dir(root.path().join("group")).unwrap(); + fs::copy( + root.path().join("one.one"), + root.path().join("group/other.one"), + ) + .unwrap(); + assert!( + matches!(discover(&mut Local::open(root.path()).unwrap(), limits()), + Err(Error::DuplicateIdentity { first, second, .. }) + if [first.as_str(), second.as_str()].contains(&"one.one") + && [first.as_str(), second.as_str()].contains(&"group/other.one")) + ); +} + +#[test] +fn locked_and_unreadable_sections_retain_their_storage_identity() { + let root = tempfile::tempdir().unwrap(); + for (name, fixture) in [ + ( + "locked.one", + "native-encrypted/encrypted-01/notebook/synthetic.one", + ), + ( + "stub.one", + "native-encrypted/cold-encrypted-02/notebook/Open Notebook.one", + ), + ] { + fs::copy( + Path::new("../../corpus").join(fixture), + root.path().join(name), + ) + .unwrap(); + } + let catalog = discover(&mut Local::open(root.path()).unwrap(), limits()).unwrap(); + assert!(catalog.toc.is_none()); + assert!(matches!( + catalog.sections[0].state, + onestore_notebook::SectionState::Locked + )); + assert!(matches!( + catalog.sections[1].state, + onestore_notebook::SectionState::Unreadable(_) + )); + for section in catalog.sections { + let bytes = fs::read(root.path().join(section.path)).unwrap(); + assert_eq!( + section.file_id, + onestore::Store::parse(&bytes).unwrap().header.file_id + ); + } +} + +#[test] +fn a_group_rename_retains_toc_identity_and_parent_order() { + let root = tempfile::tempdir().unwrap(); + let native = Path::new("../../corpus/m6/native-features-01/notebook"); + for relative in [ + "Open Notebook.onetoc2", + "Group A/Open Notebook.onetoc2", + "Group A/Duplicate.one", + "Group B/Open Notebook.onetoc2", + "Group B/Nested/Open Notebook.onetoc2", + "Group B/Nested/Duplicate.one", + ] { + let target = root.path().join(relative); + fs::create_dir_all(target.parent().unwrap()).unwrap(); + fs::copy(native.join(relative), target).unwrap(); + } + let before = discover(&mut Local::open(root.path()).unwrap(), limits()).unwrap(); + assert_eq!(before.groups[0].path, "Group A"); + fs::rename(root.path().join("Group A"), root.path().join("Renamed 馃")).unwrap(); + let catalog = discover(&mut Local::open(root.path()).unwrap(), limits()).unwrap(); + assert_eq!(catalog.groups[0].path, "Renamed 馃"); + assert_eq!( + catalog.groups[0].toc.as_ref().unwrap().file_id, + before.groups[0].toc.as_ref().unwrap().file_id + ); + assert_eq!(catalog.groups[1].path, "Group B"); + assert_eq!( + catalog.groups[0].sections[0].path, + "Renamed 馃/Duplicate.one" + ); + assert_eq!( + serde_json::to_value(&catalog.toc.unwrap().unresolved).unwrap(), + serde_json::to_value(&before.toc.unwrap().unresolved).unwrap() + ); +} + +#[test] +fn limits_and_incomplete_files_do_not_produce_a_catalog() { + let root = tempfile::tempdir().unwrap(); + fixture(root.path()); + let mut source = Local::open(root.path()).unwrap(); + assert!( + discover( + &mut source, + Limits { + entries: 1, + ..limits() + } + ) + .is_err() + ); + assert!( + discover( + &mut source, + Limits { + bytes_per_file: 8, + ..limits() + } + ) + .is_err() + ); + fs::create_dir(root.path().join("group")).unwrap(); + assert!(matches!( + discover( + &mut source, + Limits { + depth: 0, + ..limits() + } + ), + Err(Error::Limit { .. }) + )); + fs::write(root.path().join("one.one"), b"unfinished").unwrap(); + assert!( + matches!(discover(&mut source, limits()), Err(Error::Document { path, .. }) if path == "one.one") + ); +} + +#[test] +fn a_failed_refresh_retains_the_previous_catalog_as_an_independent_value() { + struct Changing { + source: Local, + reads: usize, + fail: bool, + } + impl Source for Changing { + fn entries(&mut self, path: &str, limit: usize) -> io::Result> { + self.reads += 1; + let mut entries = self.source.entries(path, limit)?; + if self.reads == 2 { + if self.fail { + return Err(io::ErrorKind::ConnectionAborted.into()); + } + entries[0].name.push_str(".renamed"); + } + Ok(entries) + } + fn read(&mut self, path: &str, limit: usize) -> io::Result> { + self.source.read(path, limit) + } + } + let root = tempfile::tempdir().unwrap(); + fixture(root.path()); + let mut source = Local::open(root.path()).unwrap(); + let catalog = discover(&mut source, limits()).unwrap(); + let before = serde_json::to_value(&catalog).unwrap(); + for fail in [false, true] { + let mut changing = Changing { + source: Local::open(root.path()).unwrap(), + reads: 0, + fail, + }; + let error = discover(&mut changing, limits()).unwrap_err(); + if fail { + assert!( + matches!(error, Error::Io { error, .. } if error.kind() == io::ErrorKind::ConnectionAborted) + ); + } else { + assert!(matches!(error, Error::Changed { .. })); + } + assert_eq!(serde_json::to_value(&catalog).unwrap(), before); + } +} + +#[test] +fn local_access_stays_inside_the_selected_root() { + let root = tempfile::tempdir().unwrap(); + fixture(root.path()); + let mut source = Local::open(root.path()).unwrap(); + for path in [ + "../one.one", + "/one.one", + "x/../one.one", + "one.one\0", + "x\\one.one", + ] { + assert_eq!( + source.read(path, 100).unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + } + #[cfg(unix)] + { + let other = tempfile::tempdir().unwrap(); + fs::write(other.path().join("other.one"), b"outside").unwrap(); + std::os::unix::fs::symlink(other.path().join("other.one"), root.path().join("link.one")) + .unwrap(); + assert_eq!( + source.read("link.one", 100).unwrap_err().kind(), + io::ErrorKind::PermissionDenied + ); + assert!( + matches!(discover(&mut source, limits()), Err(Error::Entry { path }) if path == "link.one") + ); + } +} diff --git a/crates/onestore-smb/src/lib.rs b/crates/onestore-smb/src/lib.rs index 4deb6eace482a2b6a2ef8f22f3f58f84ef4a9482..ab2d77dbc5d93f727d422b439682d327b9932dd3 100644 --- a/crates/onestore-smb/src/lib.rs +++ b/crates/onestore-smb/src/lib.rs @@ -200,8 +200,26 @@ impl Client { /// Reads one bounded, consistent snapshot; contention returns WouldBlock. pub fn read(&self, path: &str, limit: usize) -> io::Result> { + self.read_with(path, |file| { + onestore::read_snapshot(|offset, output| file.read_at(offset, output), limit) + }) + } + + /// Reads consistent storage, including encrypted or incomplete document graphs. + /// Storage validation alone does not establish edit readiness. + pub fn read_storage(&self, path: &str, limit: usize) -> io::Result> { + self.read_with(path, |file| { + onestore::read_storage_snapshot(|offset, output| file.read_at(offset, output), limit) + }) + } + + fn read_with( + &self, + path: &str, + snapshot: impl FnOnce(&mut File<'_>) -> io::Result>>, + ) -> io::Result> { let mut file = self.open(path, false)?.coordinate(path, false)?; - let result = onestore::read_snapshot(|offset, output| file.read_at(offset, output), limit) + let result = snapshot(&mut file) .and_then(|snapshot| snapshot.ok_or_else(|| io::ErrorKind::WouldBlock.into())); let closed = file.close(); let snapshot = result?; diff --git a/crates/onestore-smb/src/tests.rs b/crates/onestore-smb/src/tests.rs index 8080ccc50e8173026f79313896148925b46e814d..40da741c045aa8558e2f16da42fa5d6aaabc10f9 100644 --- a/crates/onestore-smb/src/tests.rs +++ b/crates/onestore-smb/src/tests.rs @@ -149,8 +149,9 @@ fn text(bytes: &[u8]) -> (ExGuid, ExGuid, String) { .unwrap() } #[test] -#[ignore = "requires ONESTORE_SMB_LAB pointing to disposable Samba"] +#[ignore = "requires disposable Samba and an existing ONESTORE_SMB_EVIDENCE directory"] fn live_coordination() { + let output = std::env::var("ONESTORE_SMB_EVIDENCE").unwrap(); let writer = client(); let path = format!( "adapter-{}.one", @@ -330,7 +331,6 @@ fn live_coordination() { .block_on(async { drop(spare); }); - let output = std::env::var("ONESTORE_SMB_EVIDENCE").unwrap(); fs::write(std::path::Path::new(&output).join("source.one"), &source).unwrap(); fs::write(std::path::Path::new(&output).join("committed.one"), &after).unwrap(); fs::write( @@ -344,6 +344,43 @@ fn live_coordination() { ); } +#[test] +#[ignore = "requires ONESTORE_SMB_LAB pointing to disposable Samba"] +fn live_storage_inspection() { + let reader = client(); + for (index, fixture) in [ + "native-encrypted/encrypted-01/notebook/synthetic.one", + "native-encrypted/cold-encrypted-02/notebook/Open Notebook.one", + ] + .into_iter() + .enumerate() + { + let source = fs::read(format!("../../corpus/{fixture}")).unwrap(); + let path = format!( + "inspection-{index}-{}.one", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + create(&reader, &path, &source); + assert_eq!(reader.read_storage(&path, source.len()).unwrap(), source); + assert_eq!( + reader.read(&path, source.len()).unwrap_err().kind(), + io::ErrorKind::WouldBlock + ); + let maintenance = client(); + let guard = maintenance.open(&path, false).unwrap(); + guard.lock(0xfffffffb, 0x12).unwrap(); + assert_eq!( + reader.read_storage(&path, source.len()).unwrap_err().kind(), + io::ErrorKind::WouldBlock + ); + guard.close().unwrap(); + assert_eq!(reader.read_storage(&path, source.len()).unwrap(), source); + } +} + #[test] #[ignore = "requires an owned Samba fixture and maintenance controller"] fn live_reader_hold() { diff --git a/crates/onestore/README.md b/crates/onestore/README.md index 1652d89d5f8106550d2d19d37b7dd0a1e30a72db..d98f72a783fa9f1964e18a37d091251bac0f3f04 100644 --- a/crates/onestore/README.md +++ b/crates/onestore/README.md @@ -89,6 +89,7 @@ cargo run --example create_notebook -- /tmp/one-demo 'Hello from Rust.' 'Example cargo run --example inventory -- /tmp/one-demo/synthetic.one cargo run --example inspect -- /tmp/one-demo/synthetic.one cargo run --example document -- /tmp/one-demo/synthetic.one /tmp/one-model +cargo build -p onestore-diagnostic python3 tools/notebook_report.py /tmp/one-demo /tmp/one-report --timezone America/Los_Angeles ``` diff --git a/crates/onestore/src/commit.rs b/crates/onestore/src/commit.rs index 64fc73dcdc70bb751e278d18c87a5a6ed8ac5985..ccd041f819c516faa7c76dfc7740d4e69ef687cc 100644 --- a/crates/onestore/src/commit.rs +++ b/crates/onestore/src/commit.rs @@ -75,12 +75,25 @@ impl Drop for FileIo { /// Native writers can expose incomplete graphs to unlocked filesystem reads. #[cfg(any(unix, windows))] pub fn read_file(path: impl AsRef) -> io::Result> { + read_file_limited(path, usize::MAX) +} + +/// Reads under whole-file exclusion, rejecting a snapshot larger than the byte limit. +/// A size failure returns `FileTooLarge` without a partial snapshot. +#[cfg(any(unix, windows))] +pub fn read_file_limited(path: impl AsRef, limit: usize) -> io::Result> { + let count = u64::try_from(limit) + .map_err(|_| ErrorKind::InvalidInput)? + .saturating_add(1); let mut io = FileIo::open(path, false)?; let mut bytes = Vec::new(); - let result = io.file.read_to_end(&mut bytes); + let result = (&mut io.file).take(count).read_to_end(&mut bytes); let released = io.release(); result?; released?; + if bytes.len() > limit { + return Err(ErrorKind::FileTooLarge.into()); + } Ok(bytes) } diff --git a/crates/onestore/src/lib.rs b/crates/onestore/src/lib.rs index b2d3fba467a0702725a893080fbfda05d37c29cb..1f8d1e2452be4f161b179a3784abb590ef027c19 100644 --- a/crates/onestore/src/lib.rs +++ b/crates/onestore/src/lib.rs @@ -22,7 +22,7 @@ pub use commit::{ confirm_snapshot, }; #[cfg(any(unix, windows))] -pub use commit::{commit_file_property, commit_file_text, read_file}; +pub use commit::{commit_file_property, commit_file_text, read_file, read_file_limited}; pub use create::{create_section, create_table_of_contents}; pub use edit::replace_text; pub use files::FileDataReference; @@ -31,6 +31,6 @@ pub use insertion::Insertion; pub use objects::{Object, ObjectData, ObjectReferences, ResolvedRevision}; pub use properties::{IdStream, Property, PropertySets, Value}; pub use revisions::{ExGuid, ObjectSpace, Revision, RevisionIndex}; -pub use snapshot::read_snapshot; +pub use snapshot::{read_snapshot, read_storage_snapshot}; pub use store::{Chunk, Error, FileType, Header, Node, NodeList, Reference, Store}; pub use write::replace_property_bytes; diff --git a/crates/onestore/src/snapshot.rs b/crates/onestore/src/snapshot.rs index 3e9c3266a53459822c14280242d4508d7f896ae3..8e94529b98034aa99338a556bf53e6845058f4a6 100644 --- a/crates/onestore/src/snapshot.rs +++ b/crates/onestore/src/snapshot.rs @@ -24,8 +24,28 @@ fn read_exact( /// Reads a bounded snapshot; the caller must provide fresh I/O and exclude in-place maintenance. /// Includes unpublished trailing bytes so subsequent commits can validate the physical file. pub fn read_snapshot( + read: impl FnMut(u64, &mut [u8]) -> io::Result, + limit: usize, +) -> io::Result>> { + snapshot(read, limit, |store| { + RevisionIndex::parse(store)?.validate_current() + }) +} + +/// Reads stable storage and checksums without requiring traversable property references. +/// Used to inspect encrypted or incomplete documents; this does not establish edit readiness. +/// The caller must provide fresh I/O and exclude in-place maintenance, as for `read_snapshot`. +pub fn read_storage_snapshot( + read: impl FnMut(u64, &mut [u8]) -> io::Result, + limit: usize, +) -> io::Result>> { + snapshot(read, limit, |_| Ok(())) +} + +fn snapshot( mut read: impl FnMut(u64, &mut [u8]) -> io::Result, limit: usize, + validate: impl FnOnce(&Store<'_>) -> Result<(), crate::Error>, ) -> io::Result>> { let mut header = [0; 1024]; read_exact(&mut read, 0, &mut header)?; @@ -69,7 +89,7 @@ pub fn read_snapshot( if !store.checksum_mismatches.is_empty() { return Ok(false); } - RevisionIndex::parse(&store)?.validate_current()?; + validate(&store)?; Ok(true) }); Ok(matches!(parsed, Ok(true)).then_some(bytes)) diff --git a/crates/onestore/src/store.rs b/crates/onestore/src/store.rs index cd8cf84b559d309c61a983d3d3b52f4ea723995a..d8cf334be8f076d613f4abd1f4768cd61957343c 100644 --- a/crates/onestore/src/store.rs +++ b/crates/onestore/src/store.rs @@ -3,7 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fmt; use std::ops::Range; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] pub struct Error { /// Parser byte offset; zero also represents errors without a byte location. pub offset: usize, diff --git a/crates/onestore/tests/commit.rs b/crates/onestore/tests/commit.rs index 519e6cf9e46dcc753ea568a7ff3be79e198a5f50..0c21e0adae4f18bd9b4a672a1c5c974d51cfa5ce 100644 --- a/crates/onestore/tests/commit.rs +++ b/crates/onestore/tests/commit.rs @@ -222,3 +222,40 @@ fn check_crashes(source: &[u8], osid: ExGuid, oid: ExGuid, property: u32, value: } } } +#[test] +#[cfg(any(unix, windows))] +fn bounded_file_reads_reject_partial_images_and_release_the_owner() { + use std::io::Write; + let path = std::env::temp_dir().join(format!( + "onestore-bounded-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let mut file = fs::File::options() + .write(true) + .create_new(true) + .open(&path) + .unwrap(); + assert!(onestore::read_file_limited(&path, 0).unwrap().is_empty()); + let bytes: Vec<_> = (0..10_000).map(|i| (i % 251) as u8).collect(); + file.write_all(&bytes).unwrap(); + drop(file); + for limit in [0, 1, 100, bytes.len() - 1] { + assert_eq!( + onestore::read_file_limited(&path, limit) + .unwrap_err() + .kind(), + std::io::ErrorKind::FileTooLarge + ); + assert_eq!( + onestore::read_file_limited(&path, bytes.len()).unwrap(), + bytes + ); + } + assert_eq!(onestore::read_file(&path).unwrap(), bytes); + assert_eq!(fs::read(&path).unwrap(), bytes); + fs::remove_file(path).unwrap(); +} diff --git a/crates/onestore/tests/shared_snapshot.rs b/crates/onestore/tests/shared_snapshot.rs index 22e81b09eb4e6bbc7b44dfa120cd80ddf3bd07c1..2fa337a18d5e384068c67b8cb6fe9a2f345deed9 100644 --- a/crates/onestore/tests/shared_snapshot.rs +++ b/crates/onestore/tests/shared_snapshot.rs @@ -15,6 +15,67 @@ use onestore::{ use std::{fs, io}; use trace::{Event, Trace}; +#[test] +fn storage_inspection_preserves_opaque_images_without_claiming_edit_readiness() { + for path in [ + "native-encrypted/encrypted-01/notebook/synthetic.one", + "native-encrypted/cold-encrypted-02/notebook/Open Notebook.one", + "malformed/native-inflight.one", + ] { + let source = fs::read(format!("../../corpus/{path}")).unwrap(); + for block in [1, 17, 65536] { + let mut read = |offset: u64, output: &mut [u8]| { + let offset = usize::try_from(offset).unwrap(); + let count = output + .len() + .min(block) + .min(source.len().saturating_sub(offset)); + output[..count].copy_from_slice(&source[offset..offset + count]); + Ok(count) + }; + assert_eq!( + onestore::read_storage_snapshot(&mut read, source.len()).unwrap(), + Some(source.clone()) + ); + assert!(read_snapshot(&mut read, source.len()).unwrap().is_none()); + } + let mut headers = 0; + let changed = onestore::read_storage_snapshot( + |offset, output| { + let offset = usize::try_from(offset).unwrap(); + let count = output.len().min(source.len().saturating_sub(offset)); + output[..count].copy_from_slice(&source[offset..offset + count]); + if offset == 0 { + headers += 1; + if headers == 2 { + output[0] ^= 1; + } + } + Ok(count) + }, + source.len(), + ) + .unwrap(); + assert!(changed.is_none()); + let mut broken = source.clone(); + let offset = usize::try_from(Store::parse(&source).unwrap().header.root.offset).unwrap(); + broken[offset] ^= 1; + assert!( + onestore::read_storage_snapshot( + |offset, output| { + let offset = usize::try_from(offset).unwrap(); + let count = output.len().min(broken.len().saturating_sub(offset)); + output[..count].copy_from_slice(&broken[offset..offset + count]); + Ok(count) + }, + broken.len() + ) + .unwrap() + .is_none() + ); + } +} + fn target(bytes: &[u8]) -> (ExGuid, ExGuid, u32) { let store = Store::parse(bytes).unwrap(); let index = RevisionIndex::parse(&store).unwrap(); diff --git a/tools/document_model.py b/tools/document_model.py index 3e64de11844ddb31ae238858b6c16b1c77fffa3f..f497a67eec36569e411ec3a0ea8374b03a95ca84 100644 --- a/tools/document_model.py +++ b/tools/document_model.py @@ -3,6 +3,7 @@ import os from pathlib import Path EXPORTER = Path(os.environ.get('ONESTORE_DOCUMENT', Path(__file__).resolve().parent.parent / 'target/debug/examples/document')) +BRIDGE = Path(__file__).resolve().parent.parent / 'target/debug/onestore-diagnostic' DEFAULT_CONTEXT = '{00000000-0000-0000-0000-000000000000},0' diff --git a/tools/native/toc-controls.ps1 b/tools/native/toc-controls.ps1 new file mode 100644 index 0000000000000000000000000000000000000000..dc772a7e212eae077dedb8f2730380b4c50b8f72 --- /dev/null +++ b/tools/native/toc-controls.ps1 @@ -0,0 +1,33 @@ +param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '') +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +& "$PSScriptRoot\cold-current.ps1" -Root $Root -CloneHost $CloneHost +$app = New-Object -ComObject OneNote.Application +$notebook = '' +try { + $app.OpenHierarchy((Join-Path $Root 'notebook'), '', [ref]$notebook, 0) + foreach ($name in @('Removed.one', 'Retired.one')) { + $section = '' + $app.OpenHierarchy($name, $notebook, [ref]$section, 3) + $page = '' + $app.CreateNewPage($section, [ref]$page, 2) + $xml = '' + $name + ' fixture.' + $app.UpdatePageContent($xml, [DateTime]::MinValue, 1, $false) + } + $app.SyncHierarchy($notebook) + $app.CloseNotebook($notebook, $false) + Copy-Item (Join-Path $Root 'notebook') (Join-Path $Root 'before') -Recurse + $app.OpenHierarchy((Join-Path $Root 'notebook'), '', [ref]$notebook, 0) + foreach ($name in @('Removed.one', 'Retired.one')) { + $section = '' + $app.OpenHierarchy($name, $notebook, [ref]$section, 0) + $app.DeleteHierarchy($section, [DateTime]::MinValue, ($name -eq 'Retired.one')) + } + $app.SyncHierarchy($notebook) +} finally { + if ($notebook) { $app.CloseNotebook($notebook, $false) } + [void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app) + $app = $null + [GC]::Collect() + [GC]::WaitForPendingFinalizers() +} diff --git a/tools/native_runner.py b/tools/native_runner.py index 412ac22d8be57414a5c8158eca2c542a2b0dd60c..4eaacdbf651ab106b67a6e51b73ed3588b50db32 100644 --- a/tools/native_runner.py +++ b/tools/native_runner.py @@ -116,7 +116,7 @@ def clone(output): (output / 'teardown.json').write_text(json.dumps({'absent': not vm.instance_path(name).exists()}) + '\n') -def capture(notebook, output, expected_pages=-1, author=None, screenshots=False, pdf=False, author_timeout=600, inspect=False): +def capture(notebook, output, expected_pages=-1, author=None, screenshots=False, pdf=False, author_timeout=600, inspect=False, collect_notebook=False): notebook = notebook.resolve(strict=True) if not notebook.is_dir(): raise ValueError('Choose a notebook directory.') @@ -133,6 +133,7 @@ def capture(notebook, output, expected_pages=-1, author=None, screenshots=False, 'notebook': str(notebook), 'expected_pages': expected_pages, 'author': str(author) if author else None, 'author_timeout_seconds': author_timeout, 'inspect': inspect, + 'collect_notebook': collect_notebook, 'base': json.loads(vm.BASE_MANIFEST.read_text()), 'scripts': {name: hashlib.sha256((scripts / name).read_bytes()).hexdigest() for name in sorted(p.name for p in scripts.iterdir())}, @@ -165,7 +166,7 @@ def capture(notebook, output, expected_pages=-1, author=None, screenshots=False, raise RuntimeError(result['error']) command(name, 'powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\author.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-%s' % name.upper(), output, author_timeout * 1000) command(name, 'powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-%s -ExpectedPages %d%s' % (name.upper(), expected_pages, (' -UseCurrentCache' if author else '') + (' -Pdf' if pdf else '') + (' -KeepOpen' if screenshots or inspect else '')), output, 600000) - collect_artifacts(name, output, '*' if author else 'read') + collect_artifacts(name, output, '*' if author or collect_notebook else 'read') if screenshots: for page in sorted((output / 'read').glob('page-*.xml')): page_id = ET.parse(page).getroot().attrib['ID'] @@ -208,6 +209,7 @@ if __name__ == '__main__': parser.add_argument('--pdf', action='store_true') parser.add_argument('--screenshots', action='store_true') parser.add_argument('--inspect', action='store_true', help='Keep the clone open for inspection until an output/finish file appears, up to 30 minutes.') + parser.add_argument('--collect-notebook', action='store_true', help='Retain the VM notebook alongside its native read capture.') parser.add_argument('--author', type=Path, help='Run a fixture-authoring PowerShell script in the clone before capture.') parser.add_argument('--author-timeout', type=int, default=600, help='Authoring deadline in seconds.') args = parser.parse_args() @@ -216,5 +218,5 @@ if __name__ == '__main__': signal.signal(signal.SIGTERM, interrupted) if args.author_timeout <= 0: parser.error('The authoring deadline must be positive.') - capture(args.notebook, args.output, args.expected_pages, args.author, args.screenshots, args.pdf, args.author_timeout, args.inspect) + capture(args.notebook, args.output, args.expected_pages, args.author, args.screenshots, args.pdf, args.author_timeout, args.inspect, args.collect_notebook) print('Captured native evidence in', args.output) diff --git a/tools/notebook_editor.py b/tools/notebook_editor.py index fa45ab62b3a8b6723d43f2825580674583bfaab3..936e8da36863540d6ed361fa34d48f7e636c26d6 100644 --- a/tools/notebook_editor.py +++ b/tools/notebook_editor.py @@ -15,11 +15,10 @@ import time from urllib.parse import parse_qs, urlsplit import uuid -from document_model import walk +from document_model import BRIDGE, walk from notebook_report import generate ROOT = Path(__file__).resolve().parent.parent -BRIDGE = ROOT / 'target/debug/onestore-diagnostic' def bridge(mode, source, destination, edit=None): diff --git a/tools/notebook_report.py b/tools/notebook_report.py index b61522470926a4288c2aaed6d1e75ef8e9b47eea..e35ce06e0e0ce25e399a71a27365f9b6a0abb122 100644 --- a/tools/notebook_report.py +++ b/tools/notebook_report.py @@ -16,7 +16,7 @@ import subprocess from urllib.parse import urlsplit from zoneinfo import ZoneInfo -from document_model import DEFAULT_CONTEXT, EXPORTER, ordered_pages, version_pages, view, walk +from document_model import BRIDGE, DEFAULT_CONTEXT, EXPORTER, ordered_pages, version_pages, view, walk ROOT = Path(__file__).resolve().parent.parent @@ -274,11 +274,27 @@ def generate(source, destination, native=None, versions=(), zone=timezone.utc, e (destination / 'model').mkdir(); (destination / 'assets').mkdir() cached = {row['path']: (row['sha256'], previous / 'model' / str(index)) for index, row in enumerate(json.loads((previous / 'source.json').read_text()))} if previous else {} - sections = []; tocs = {}; manifest = [] - for index, path in enumerate(sorted(p for p in source.rglob('*') if p.suffix.lower() in ('.one', '.onetoc2'))): - relative = path.relative_to(source) + result = json.loads(subprocess.check_output([BRIDGE, 'catalog', source], timeout=120)) + if not result['ok']: raise ValueError(result['error']) + catalog = result['catalog'] + (destination / 'catalog.json').write_text(json.dumps(catalog, indent=2, ensure_ascii=False)) + files = []; catalog_sections = {} + def collect(folder): + if folder['toc'] is not None: files.append(Path(folder['path']) / folder['toc']['filename']) + for section in folder['sections']: + relative = Path(section['path']) + files.append(relative); catalog_sections[relative] = section + for group in folder['groups']: collect(group) + collect(catalog) + sections = []; unavailable = []; manifest = [] + for index, relative in enumerate(sorted(files)): + path = source / relative before = path.read_bytes() manifest.append({'path': relative.as_posix(), 'sha256': hashlib.sha256(before).hexdigest(), 'bytes': len(before)}) + state = catalog_sections.get(relative, {}).get('state', {}) + if isinstance(state, dict) and 'Unreadable' in state: + unavailable.append((relative, state['Unreadable'])) + continue exported = destination / 'model' / str(index) reusable = cached.get(relative.as_posix()) reused = reusable is not None and reusable[0] == manifest[-1]['sha256'] @@ -291,9 +307,6 @@ def generate(source, destination, native=None, versions=(), zone=timezone.utc, e document = json.loads((exported / 'document.json').read_text()) if path.read_bytes() != before: raise ValueError('A source file changed during export.') - if path.suffix.lower() == '.onetoc2': - _, root = view(document, document['root']) - tocs[relative.parent] = [root['nodes'][oid]['kind'] for oid in root['nodes'][root['roots']['1']]['kind']['entries']] assets = {} previews = {} image_references = { @@ -339,20 +352,13 @@ def generate(source, destination, native=None, versions=(), zone=timezone.utc, e if path.suffix.lower() == '.one': sections.append({'path': relative, 'export': exported.relative_to(destination), 'document': document, 'text': json.loads((exported / 'text.json').read_text()), 'assets': assets, 'previews': previews}) - def order(section): - path = section['path']; entries = tocs.get(path.parent, []) - entry = next((entry for entry in entries if entry['identity'] == section['document']['file_id']), None) - result = []; parent = Path('.') - for part in path.parts[:-1]: - group = next((item for item in tocs.get(parent, []) if item['filename'] == part), None) - result.append((group['order'] if group and group['order'] is not None else 0xffffffff, part)) - parent /= part - result.append((entry['order'] if entry and entry['order'] is not None else 0xffffffff, path.name)) - return result - sections.sort(key=order) + order = {path: index for index, path in enumerate(catalog_sections)} + sections.sort(key=lambda section: order[section['path']]) pages = []; locked_sections = []; histories = {}; nav = 'Notebook review' for section in sections: - name = section['path'].stem + state = catalog_sections[section['path']]['state'] + name = state.get('Readable', {}).get('name') if isinstance(state, dict) else None + if name is None: name = section['path'].stem nav += '

' + esc(str(section['path'].parent) + ' / ' + name) + '

' _, root_space = view(section['document'], section['document']['root']) if root_space['nodes'][root_space['roots']['1']]['kind']['type'] == 'Encrypted': @@ -472,9 +478,11 @@ def generate(source, destination, native=None, versions=(), zone=timezone.utc, e accounting.append({'section': str(section['path']), 'ordinal': ordinal, 'space': sid, 'revision': rid, 'object': oid, 'title': title, 'report': filename, 'category': category, 'context': context, 'version_modified': version['modified'] if version else None, 'source_report': source_page, 'native_reference': reference.as_posix() if reference else None, 'rendered': dict(page.counts)}) (destination / 'source.json').write_text(json.dumps(manifest, indent=2)) (destination / 'pages.json').write_text(json.dumps(accounting, indent=2, ensure_ascii=False)) - intro = '

Notebook review

' + str(len(sections)) + ' sections 路 ' + str(len(pages)) + ' stored pages

Readable content follows the stored object order. Outline positions are shown in points. The document structure retains properties and identities that the readable view does not interpret.

Source hashes 路 Page inventory

' + intro = '

Notebook review

' + str(len(sections) + len(unavailable)) + ' sections 路 ' + str(len(pages)) + ' stored pages

Readable content follows the stored object order. Outline positions are shown in points. The document structure retains properties and identities that the readable view does not interpret.

Source hashes 路 Page inventory 路 Notebook catalog

' if locked_sections: intro += '

Page counts are unavailable for locked sections: ' + esc(', '.join(locked_sections)) + '.

' + for path, error in unavailable: + intro += '

Cannot read ' + esc(str(path)) + ': ' + esc(error['message']) + ' (offset ' + str(error['offset']) + ').

' (destination / 'index.html').write_text(html_page('Notebook review', nav, intro, editable)) diff --git a/tools/test_notebook_discovery.py b/tools/test_notebook_discovery.py new file mode 100644 index 0000000000000000000000000000000000000000..e7e68b9b5636645be13ca22c74d5dfd3c710bc17 --- /dev/null +++ b/tools/test_notebook_discovery.py @@ -0,0 +1,67 @@ +"""Compare library discovery with retained native OneNote hierarchy captures.""" +import json +import os +from pathlib import Path +import subprocess +import unittest +import xml.etree.ElementTree as ET + +ROOT = Path(__file__).resolve().parent.parent + + +class NativeDiscovery(unittest.TestCase): + def test_native_hierarchies(self): + fixtures = ['m6/native-features-01', 'native-encrypted/cold-encrypted-02', + 'native-delete/cold-deletion-05'] + if os.environ.get('ONESTORE_NOTEBOOK_NATIVE'): + fixtures.append(str(Path(os.environ['ONESTORE_NOTEBOOK_NATIVE']).resolve())) + for fixture in fixtures: + with self.subTest(fixture=fixture): + source = ROOT / 'corpus' / fixture + catalog = json.loads(subprocess.check_output( + [str(ROOT / 'target/debug/examples/discover'), str(source / 'notebook')])) + hierarchy = ET.parse(source / 'read/hierarchy.xml').getroot() + notebook = next(node for node in hierarchy.iter() if node.tag.endswith('}Notebook')) + + def path(value): + return '/'.join(part for part in value.replace('\\', '/').split('/') if part) + + prefix = path(notebook.get('path')) + + def native(node): + sections = [] + groups = [] + for child in node: + if child.tag.endswith('}Section'): + relative = path(child.get('path'))[len(prefix) + 1:] + name = child.get('name') + if Path(relative).parts[-2:] == ('OneNote_RecycleBin', 'OneNote_DeletedPages.one'): + self.assertEqual(name, 'Deleted Pages') + name = 'OneNote_DeletedPages' + sections.append((relative, name)) + elif child.tag.endswith('}SectionGroup'): + groups.append(native(child)) + return {'path': path(node.get('path'))[len(prefix) + 1:], 'sections': sections, 'groups': groups} + + def actual(folder): + sections = [] + for section in folder['sections']: + state = section['state'] + name = state.get('Readable', {}).get('name') if isinstance(state, dict) else None + sections.append((section['path'], Path(section['path']).stem if name is None else name)) + return {'path': folder['path'], 'sections': sections, + 'groups': [actual(group) for group in folder['groups']]} + + self.assertEqual(actual(catalog), native(notebook)) + if fixture == 'm6/native-features-01': + self.assertEqual(len(catalog['toc']['unresolved']), 1) + self.assertEqual(catalog['toc']['unresolved'][0]['filename'], 'synthetic.one') + elif fixture == 'native-encrypted/cold-encrypted-02': + states = {section['path']: section['state'] for section in catalog['sections']} + self.assertEqual(states['synthetic.one'], 'Locked') + self.assertEqual(states['Open Notebook.one']['Unreadable']['message'], + 'Object space has no revision manifest list') + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/test_notebook_report.py b/tools/test_notebook_report.py index f08c11cb927597ef52d81c7574cbf6b42646c912..607f59df3d786237c71415a3e20b5490ca26ad23 100644 --- a/tools/test_notebook_report.py +++ b/tools/test_notebook_report.py @@ -5,15 +5,31 @@ from pathlib import Path import re from tempfile import TemporaryDirectory import shutil +import subprocess import unittest from unittest.mock import patch import xml.etree.ElementTree as ET from PIL import Image from notebook_report import generate +from document_model import EXPORTER class NotebookReportTest(unittest.TestCase): + def test_locked_and_unreadable_sections_remain_visible_in_cached_reports(self): + fixture = Path(__file__).resolve().parent.parent / 'corpus/native-encrypted/cold-encrypted-02/notebook' + with TemporaryDirectory() as temporary: + root = Path(temporary) + generate(fixture, root / 'first') + generate(fixture, root / 'cached', previous=root / 'first') + index = (root / 'first/index.html').read_text() + self.assertIn('2 sections', index) + self.assertIn('Locked section', index) + self.assertIn('Cannot read Open Notebook.one', index) + self.assertIn('Object space has no revision manifest list', index) + self.assertEqual(json.loads((root / 'first/pages.json').read_text()), []) + self.assertEqual(index, (root / 'cached/index.html').read_text()) + def test_tiff_preview_matches_native_pixels_and_retains_original_bytes(self): fixture = Path(__file__).resolve().parent.parent / 'corpus/m6/native-features-01' with TemporaryDirectory() as temporary: @@ -42,15 +58,18 @@ class NotebookReportTest(unittest.TestCase): shutil.copytree(fixture, source) generate(source, root / 'first') before = {p.relative_to(root / 'first'): p.read_bytes() for p in (root / 'first').rglob('*') if p.is_file()} - with patch('notebook_report.subprocess.run', side_effect=AssertionError('Unchanged model exported again')): + def cached_run(command, *args, **kwargs): + self.assertNotEqual(command[0], EXPORTER, 'Unchanged model exported again') + return original(command, *args, **kwargs) + + original = subprocess.run + with patch('notebook_report.subprocess.run', side_effect=cached_run): generate(source, root / 'same', previous=root / 'first') self.assertEqual(before, {p.relative_to(root / 'same'): p.read_bytes() for p in (root / 'same').rglob('*') if p.is_file()}) - shutil.copyfile(source / 'synthetic.one', source / 'a.one') - import notebook_report - original = notebook_report.subprocess.run + subprocess.run([EXPORTER.parent / 'create_section', source / 'a.one', 'Independent section', 'Fixture'], check=True) with patch('notebook_report.subprocess.run', wraps=original) as run: generate(source, root / 'changed', previous=root / 'first') - self.assertEqual([Path(call.args[0][1]).name for call in run.call_args_list], ['a.one']) + self.assertEqual([Path(call.args[0][1]).name for call in run.call_args_list if call.args[0][0] == EXPORTER], ['a.one']) generate(source, root / 'fresh') self.assertEqual({p.relative_to(root / 'fresh'): p.read_bytes() for p in (root / 'fresh').rglob('*') if p.is_file()}, {p.relative_to(root / 'changed'): p.read_bytes() for p in (root / 'changed').rglob('*') if p.is_file()}) @@ -61,7 +80,7 @@ class NotebookReportTest(unittest.TestCase): self.assertEqual((root / f'first/model/{old_index}/document.json').stat().st_ino, (root / f'changed/model/{new_index}/document.json').stat().st_ino) (source / 'a.one').unlink() - with patch('notebook_report.subprocess.run', side_effect=AssertionError('Unchanged model exported again')): + with patch('notebook_report.subprocess.run', side_effect=cached_run): generate(source, root / 'deleted', previous=root / 'changed') self.assertEqual(before, {p.relative_to(root / 'deleted'): p.read_bytes() for p in (root / 'deleted').rglob('*') if p.is_file()})