Commit bca7120d8a96e56b4ae7c6c76d1e82ccef0c5d21 | snowbound

Cargo.lock+13

...	...	@@ -515,10 +515,23 @@ name = "onestore-diagnostic"
515	515	version = "0.1.0"
516	516	dependencies = [
517	517	"onestore",
	518	"onestore-notebook",
518	519	"serde",
519	520	"serde_json",
520	521	]
521	522
	523	[[package]]
	524	name = "onestore-notebook"
	525	version = "0.1.0"
	526	dependencies = [
	527	"onestore",
	528	"onestore-smb",
	529	"serde",
	530	"serde_json",
	531	"tempfile",
	532	"thiserror",
	533	]
	534
522	535	[[package]]
523	536	name = "onestore-offline"
524	537	version = "0.1.0"

crates/onestore-diagnostic/Cargo.toml+1

...	...	@@ -6,5 +6,6 @@ publish = false
6	6
7	7	[dependencies]
8	8	onestore = { path = "../onestore" }
	9	onestore-notebook = { path = "../onestore-notebook" }
9	10	serde = { version = "1.0.229", features = ["derive"] }
10	11	serde_json = "1.0.151"

crates/onestore-diagnostic/src/main.rs+23-4

...	...	@@ -40,14 +40,26 @@ enum Action {
40	40	},
41	41	}
42	42
43		fn run() -> Result<Value, Box<dyn std::error::Error>> {
44		let args: Vec<_> = env::args_os().skip(1).collect();
	43	fn run(args: &[std::ffi::OsString]) -> Result<Value, Box<dyn std::error::Error>> {
	44	if let [mode, root] = args
	45	&& mode == "catalog"
	46	{
	47	let catalog = onestore_notebook::discover(
	48	&mut onestore_notebook::Local::open(root)?,
	49	onestore_notebook::Limits {
	50	entries: 100_000,
	51	bytes_per_file: 256 * 1024 * 1024,
	52	depth: 64,
	53	},
	54	)?;
	55	return Ok(json!({"ok": true, "catalog": catalog}));
	56	}
45	57	if args.len() != 3
46	58	\|\| !["snapshot", "check", "commit"]
47	59	.iter()
48	60	.any(\|mode\| args[0] == *mode)
49	61	{
50		return Err("Usage: onestore-diagnostic snapshot FILE NEW_SNAPSHOT \| check SNAPSHOT - \| commit FILE SNAPSHOT; edits arrive as JSON on stdin".into());
	62	return Err("Usage: onestore-diagnostic catalog ROOT \| snapshot FILE NEW_SNAPSHOT \| check SNAPSHOT - \| commit FILE SNAPSHOT; edits arrive as JSON on stdin".into());
51	63	}
52	64	if args[0] == "snapshot" {
53	65	let bytes = onestore::read_file(&args[1])?;
...	...	@@ -102,6 +114,13 @@ fn run() -> Result<Value, Box<dyn std::error::Error>> {
102	114	}
103	115
104	116	fn main() {
105		let result = run().unwrap_or_else(\|error\| json!({"ok": false, "state": "NotCommitted", "kind": "Input", "error": error.to_string()}));
	117	let args: Vec<_> = env::args_os().skip(1).collect();
	118	let result = run(&args).unwrap_or_else(\|error\| {
	119	let mut result = json!({"ok": false, "kind": "Input", "error": error.to_string()});
	120	if args.first().is_some_and(\|mode\| mode == "commit") {
	121	result["state"] = json!("NotCommitted");
	122	}
	123	result
	124	});
106	125	println!("{result}");
107	126	}

crates/onestore-notebook/Cargo.toml created+18

...	...	@@ -0,0 +1,18 @@
	1	[package]
	2	name = "onestore-notebook"
	3	version = "0.1.0"
	4	edition = "2024"
	5	publish = false
	6
	7	[features]
	8	smb = ["dep:onestore-smb"]
	9
	10	[dependencies]
	11	onestore = { path = "../onestore" }
	12	onestore-smb = { path = "../onestore-smb", optional = true }
	13	serde = { version = "1", features = ["derive"] }
	14	thiserror = "2"
	15
	16	[dev-dependencies]
	17	serde_json = "1"
	18	tempfile = "3"

crates/onestore-notebook/README.md created+18

...	...	@@ -0,0 +1,18 @@
	1	# onestore-notebook
	2
	3	Read-only notebook discovery over a caller-supplied root. This experimental
	4	sibling crate keeps directory traversal out of the single-file storage parser.
	5
	6	Discovery returns ordered sections and nested groups with file identities and
	7	share-relative paths. Section display-name overrides remain distinct from file
	8	names. TOC references whose identities are absent from the directory remain
	9	inspectable; a cached filename never substitutes for an identity match.
	10	Encrypted sections and valid storage with an unreadable document graph retain
	11	their identity as `Locked` or `Unreadable`, without being presented as empty pages.
	12	Malformed storage, failed reads and ambiguous identities reject the discovery.
	13
	14	Each file read must be a consistent, bounded snapshot. The result is an
	15	observation across multiple files, not an atomic notebook transaction or
	16	authorization to publish an edit. Refresh rejects observed topology changes and
	17	duplicate physical files with the same logical identity. Retain the last accepted
	18	catalog if discovery fails; a connection failure does not mean files were deleted.

crates/onestore-notebook/examples/discover.rs created+40

...	...	@@ -0,0 +1,40 @@
	1	use onestore_notebook::{Limits, Local, discover};
	2
	3	fn main() -> Result<(), Box<dyn std::error::Error>> {
	4	let args: Vec<_> = std::env::args_os().skip(1).collect();
	5	let limits = Limits {
	6	entries: 100_000,
	7	bytes_per_file: 256 * 1024 * 1024,
	8	depth: 64,
	9	};
	10	let catalog = match args.as_slice() {
	11	[root] => discover(&mut Local::open(root)?, limits)?,
	12	#[cfg(feature = "smb")]
	13	[flag, address, share, root] if flag == "--smb" => {
	14	use onestore_smb::{Client, Credentials};
	15	let username = std::env::var("ONESTORE_SMB_USERNAME").unwrap_or_default();
	16	let password = std::env::var("ONESTORE_SMB_PASSWORD").unwrap_or_default();
	17	let domain = std::env::var("ONESTORE_SMB_DOMAIN").unwrap_or_default();
	18	let client = Client::connect(
	19	address.to_str().ok_or("Use a UTF-8 server address")?,
	20	share.to_str().ok_or("Use a UTF-8 share name")?,
	21	Credentials {
	22	username: &username,
	23	password: &password,
	24	domain: &domain,
	25	},
	26	std::time::Duration::from_secs(5),
	27	)?;
	28	discover(
	29	&mut onestore_notebook::Smb::new(
	30	&client,
	31	root.to_str().ok_or("Use a UTF-8 notebook path")?,
	32	)?,
	33	limits,
	34	)?
	35	}
	36	_ => return Err("Provide ROOT, or --smb ADDRESS SHARE ROOT with the smb feature".into()),
	37	};
	38	serde_json::to_writer_pretty(std::io::stdout().lock(), &catalog)?;
	39	Ok(())
	40	}

crates/onestore-notebook/src/lib.rs created+381

...	...	@@ -0,0 +1,381 @@
	1	#![forbid(unsafe_code)]
	2	#![doc = include_str!("../README.md")]
	3
	4	use onestore::{
	5	ExGuid, FileType, RevisionIndex, Store,
	6	document::{Document, Kind},
	7	};
	8	use serde::Serialize;
	9	use std::{
	10	collections::{BTreeMap, BTreeSet},
	11	io,
	12	};
	13
	14	mod source;
	15	pub use source::Local;
	16	#[cfg(feature = "smb")]
	17	pub use source::Smb;
	18
	19	#[derive(Debug, Serialize)]
	20	pub struct Section {
	21	pub path: String,
	22	/// Header.guidFile, also used by FileIdentityGuid in a parent TOC.
	23	pub file_id: [u8; 16],
	24	pub state: SectionState,
	25	}
	26
	27	#[derive(Debug, Serialize)]
	28	pub enum SectionState {
	29	Readable {
	30	/// An explicit SectionDisplayName; otherwise use the current filename without its extension.
	31	name: Option<String>,
	32	},
	33	Locked,
	34	Unreadable(onestore::Error),
	35	}
	36
	37	#[derive(Debug, Serialize)]
	38	pub struct Folder {
	39	pub path: String,
	40	pub toc: Option<Toc>,
	41	pub sections: Vec<Section>,
	42	pub groups: Vec<Folder>,
	43	}
	44
	45	#[derive(Debug, Serialize)]
	46	pub struct Toc {
	47	pub filename: String,
	48	pub file_id: [u8; 16],
	49	/// Stale or unavailable TOC references; these are not inferred active sections.
	50	pub unresolved: Vec<TocReference>,
	51	}
	52
	53	#[derive(Debug, Serialize)]
	54	pub struct TocReference {
	55	/// Native TOCs can retain an identity after removing its cached filename.
	56	pub filename: Option<String>,
	57	pub file: [u8; 16],
	58	pub order: u32,
	59	}
	60
	61	#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
	62	pub enum EntryKind {
	63	File,
	64	Directory,
	65	Other,
	66	}
	67
	68	#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
	69	pub struct Entry {
	70	pub name: String,
	71	pub kind: EntryKind,
	72	}
	73
	74	/// Rooted file access. Paths are relative, UTF-8, and use `/` between components.
	75	pub trait Source {
	76	/// Return the complete immediate directory or an error, never a truncated success.
	77	fn entries(&mut self, path: &str, limit: usize) -> io::Result<Vec<Entry>>;
	78	/// Return a consistent file snapshot, rejecting images larger than the byte limit.
	79	fn read(&mut self, path: &str, limit: usize) -> io::Result<Vec<u8>>;
	80	}
	81
	82	pub struct Limits {
	83	pub entries: usize,
	84	pub bytes_per_file: usize,
	85	pub depth: usize,
	86	}
	87
	88	#[derive(Debug, thiserror::Error)]
	89	pub enum Error {
	90	#[error("Cannot read {path}: {error}")]
	91	Io {
	92	path: String,
	93	#[source]
	94	error: io::Error,
	95	},
	96	#[error("Invalid notebook file {path}: {error}")]
	97	Document {
	98	path: String,
	99	#[source]
	100	error: onestore::Error,
	101	},
	102	#[error("Discovery limit exceeded at {path}")]
	103	Limit { path: String },
	104	#[error("Directory changed during discovery: {path}")]
	105	Changed { path: String },
	106	#[error("Invalid or unsupported directory entry: {path}")]
	107	Entry { path: String },
	108	#[error("File identity occurs at both {first} and {second}")]
	109	DuplicateIdentity {
	110	file: [u8; 16],
	111	first: String,
	112	second: String,
	113	},
	114	}
	115
	116	/// Discovers the complete rooted directory within caller-specified work and size limits.
	117	pub fn discover(source: &mut impl Source, limits: Limits) -> Result<Folder, Error> {
	118	let mut remaining = limits.entries;
	119	let mut identities = BTreeMap::new();
	120	scan(source, "", &limits, 0, &mut remaining, &mut identities)
	121	}
	122
	123	fn scan(
	124	source: &mut impl Source,
	125	path: &str,
	126	limits: &Limits,
	127	depth: usize,
	128	remaining: &mut usize,
	129	identities: &mut BTreeMap<[u8; 16], String>,
	130	) -> Result<Folder, Error> {
	131	if depth > limits.depth {
	132	return Err(Error::Limit { path: path.into() });
	133	}
	134	let mut listing = source
	135	.entries(path, *remaining)
	136	.map_err(\|error\| Error::Io {
	137	path: path.into(),
	138	error,
	139	})?;
	140	*remaining = remaining
	141	.checked_sub(listing.len())
	142	.ok_or_else(\|\| Error::Limit { path: path.into() })?;
	143	listing.sort();
	144	let mut names = BTreeSet::new();
	145	for entry in &listing {
	146	if !component(&entry.name) \|\| !names.insert(&entry.name) {
	147	return Err(Error::Entry {
	148	path: join(path, &entry.name),
	149	});
	150	}
	151	}
	152	let mut result = Folder {
	153	path: path.into(),
	154	toc: None,
	155	sections: Vec::new(),
	156	groups: Vec::new(),
	157	};
	158	for entry in &listing {
	159	let child = join(path, &entry.name);
	160	if entry.kind == EntryKind::Directory {
	161	result.groups.push(scan(
	162	source,
	163	&child,
	164	limits,
	165	depth + 1,
	166	remaining,
	167	identities,
	168	)?);
	169	continue;
	170	}
	171	let lower = entry.name.to_ascii_lowercase();
	172	let expected = if lower.ends_with(".one") {
	173	FileType::Section
	174	} else if lower.ends_with(".onetoc2") {
	175	FileType::TableOfContents
	176	} else {
	177	continue;
	178	};
	179	if entry.kind != EntryKind::File
	180	\|\| (expected == FileType::TableOfContents && result.toc.is_some())
	181	{
	182	return Err(Error::Entry { path: child });
	183	}
	184	let bytes = source
	185	.read(&child, limits.bytes_per_file)
	186	.map_err(\|error\| Error::Io {
	187	path: child.clone(),
	188	error,
	189	})?;
	190	if bytes.len() > limits.bytes_per_file {
	191	return Err(Error::Limit { path: child });
	192	}
	193	let store = Store::parse(&bytes).map_err(\|error\| Error::Document {
	194	path: child.clone(),
	195	error,
	196	})?;
	197	if store.header.file_type != expected \|\| !store.checksum_mismatches.is_empty() {
	198	return Err(Error::Document {
	199	path: child,
	200	error: onestore::Error {
	201	offset: 0,
	202	message: "Unexpected file type or checksum mismatch",
	203	},
	204	});
	205	}
	206	let file_id = store.header.file_id;
	207	let parsed = (\|\| {
	208	let index = RevisionIndex::parse(&store)?;
	209	let document = Document::parse(&index)?;
	210	let revision = document
	211	.spaces
	212	.get(&document.root)
	213	.and_then(\|space\| {
	214	space
	215	.contexts
	216	.get(&ExGuid::default())
	217	.and_then(\|id\| space.revisions.get(id))
	218	})
	219	.ok_or(onestore::Error {
	220	offset: 0,
	221	message: "Missing active notebook root revision",
	222	})?;
	223	if expected == FileType::Section {
	224	if revision
	225	.roots
	226	.get(&1)
	227	.and_then(\|id\| revision.nodes.get(id))
	228	.is_some_and(\|node\| matches!(node.kind, Kind::Encrypted { .. }))
	229	{
	230	result.sections.push(Section {
	231	path: child.clone(),
	232	file_id,
	233	state: SectionState::Locked,
	234	});
	235	return Ok(());
	236	}
	237	index.validate_current()?;
	238	document.pages()?;
	239	let name = revision
	240	.roots
	241	.get(&2)
	242	.and_then(\|id\| revision.nodes.get(id))
	243	.and_then(\|node\| match &node.kind {
	244	Kind::SectionMetadata { name, .. } => name.clone(),
	245	_ => None,
	246	});
	247	result.sections.push(Section {
	248	path: child.clone(),
	249	file_id,
	250	state: SectionState::Readable { name },
	251	});
	252	} else {
	253	index.validate_current()?;
	254	let node = revision.roots.get(&1).and_then(\|id\| revision.nodes.get(id));
	255	let Some(Kind::Toc { entries, .. }) = node.map(\|node\| &node.kind) else {
	256	return Err(onestore::Error {
	257	offset: 0,
	258	message: "Missing notebook TOC root",
	259	});
	260	};
	261	let mut seen = BTreeSet::new();
	262	let mut unresolved = Vec::new();
	263	for id in entries {
	264	let Some(Kind::Toc {
	265	filename,
	266	identity: Some(file),
	267	order: Some(order),
	268	..
	269	}) = revision.nodes.get(id).map(\|node\| &node.kind)
	270	else {
	271	return Err(onestore::Error {
	272	offset: 0,
	273	message: "Incomplete notebook TOC reference",
	274	});
	275	};
	276	if filename.as_deref().is_some_and(\|name\| !component(name))
	277	\|\| !seen.insert(*file)
	278	{
	279	return Err(onestore::Error {
	280	offset: 0,
	281	message: "Invalid or duplicated notebook TOC reference",
	282	});
	283	}
	284	unresolved.push(TocReference {
	285	filename: filename.clone(),
	286	file: *file,
	287	order: *order,
	288	});
	289	}
	290	result.toc = Some(Toc {
	291	filename: entry.name.clone(),
	292	file_id,
	293	unresolved,
	294	});
	295	}
	296	Ok(())
	297	})();
	298	if let Err(error) = parsed {
	299	if expected == FileType::Section {
	300	result.sections.push(Section {
	301	path: child.clone(),
	302	file_id,
	303	state: SectionState::Unreadable(error),
	304	});
	305	} else {
	306	return Err(Error::Document { path: child, error });
	307	}
	308	}
	309	if let Some(first) = identities.insert(file_id, child.clone()) {
	310	return Err(Error::DuplicateIdentity {
	311	file: file_id,
	312	first,
	313	second: child,
	314	});
	315	}
	316	}
	317	let order: BTreeMap<_, _> = result
	318	.toc
	319	.iter()
	320	.flat_map(\|toc\| &toc.unresolved)
	321	.map(\|entry\| (entry.file, entry.order))
	322	.collect();
	323	result.sections.sort_by(\|a, b\| {
	324	(order.get(&a.file_id).copied().unwrap_or(u32::MAX), &a.path)
	325	.cmp(&(order.get(&b.file_id).copied().unwrap_or(u32::MAX), &b.path))
	326	});
	327	result.groups.sort_by(\|a, b\| {
	328	(
	329	a.toc
	330	.as_ref()
	331	.and_then(\|toc\| order.get(&toc.file_id).copied())
	332	.unwrap_or(u32::MAX),
	333	&a.path,
	334	)
	335	.cmp(&(
	336	b.toc
	337	.as_ref()
	338	.and_then(\|toc\| order.get(&toc.file_id).copied())
	339	.unwrap_or(u32::MAX),
	340	&b.path,
	341	))
	342	});
	343	let present: BTreeSet<_> = result
	344	.sections
	345	.iter()
	346	.map(\|section\| section.file_id)
	347	.chain(
	348	result
	349	.groups
	350	.iter()
	351	.filter_map(\|group\| group.toc.as_ref().map(\|toc\| toc.file_id)),
	352	)
	353	.collect();
	354	if let Some(toc) = &mut result.toc {
	355	toc.unresolved
	356	.retain(\|entry\| !present.contains(&entry.file));
	357	}
	358	let mut observed = source
	359	.entries(path, listing.len())
	360	.map_err(\|error\| Error::Io {
	361	path: path.into(),
	362	error,
	363	})?;
	364	observed.sort();
	365	if observed != listing {
	366	return Err(Error::Changed { path: path.into() });
	367	}
	368	Ok(result)
	369	}
	370
	371	fn component(name: &str) -> bool {
	372	!name.is_empty() && name != "." && name != ".." && !name.contains(['/', '\\', '\0'])
	373	}
	374
	375	fn join(parent: &str, name: &str) -> String {
	376	if parent.is_empty() {
	377	name.into()
	378	} else {
	379	format!("{parent}/{name}")
	380	}
	381	}

crates/onestore-notebook/src/source.rs created+117

...	...	@@ -0,0 +1,117 @@
	1	use super::{Entry, EntryKind, Source, component};
	2	use std::{
	3	io,
	4	path::{Path, PathBuf},
	5	};
	6
	7	/// A canonical local root; symbolic-link entries are not traversed.
	8	pub struct Local {
	9	root: PathBuf,
	10	}
	11
	12	impl Local {
	13	pub fn open(root: impl AsRef<Path>) -> io::Result<Self> {
	14	let root = root.as_ref().canonicalize()?;
	15	if !root.is_dir() {
	16	return Err(io::ErrorKind::NotADirectory.into());
	17	}
	18	Ok(Self { root })
	19	}
	20
	21	fn path(&self, relative: &str) -> io::Result<PathBuf> {
	22	if !relative.is_empty() && !relative.split('/').all(component) {
	23	return Err(io::ErrorKind::InvalidInput.into());
	24	}
	25	let path = self.root.join(relative).canonicalize()?;
	26	if !path.starts_with(&self.root) {
	27	return Err(io::ErrorKind::PermissionDenied.into());
	28	}
	29	Ok(path)
	30	}
	31	}
	32
	33	impl Source for Local {
	34	fn entries(&mut self, path: &str, limit: usize) -> io::Result<Vec<Entry>> {
	35	let mut entries = Vec::new();
	36	for entry in std::fs::read_dir(self.path(path)?)? {
	37	let entry = entry?;
	38	if entries.len() == limit {
	39	return Err(io::ErrorKind::FileTooLarge.into());
	40	}
	41	let name = entry
	42	.file_name()
	43	.into_string()
	44	.map_err(\|_\| io::ErrorKind::InvalidData)?;
	45	let kind = entry.file_type()?;
	46	entries.push(Entry {
	47	name,
	48	kind: if kind.is_dir() {
	49	EntryKind::Directory
	50	} else if kind.is_file() {
	51	EntryKind::File
	52	} else {
	53	EntryKind::Other
	54	},
	55	});
	56	}
	57	Ok(entries)
	58	}
	59
	60	fn read(&mut self, path: &str, limit: usize) -> io::Result<Vec<u8>> {
	61	onestore::read_file_limited(self.path(path)?, limit)
	62	}
	63	}
	64
	65	#[cfg(feature = "smb")]
	66	/// A share-relative root on an existing blocking SMB connection.
	67	pub struct Smb<'a> {
	68	client: &'a onestore_smb::Client,
	69	root: String,
	70	}
	71
	72	#[cfg(feature = "smb")]
	73	impl<'a> Smb<'a> {
	74	pub fn new(client: &'a onestore_smb::Client, root: &str) -> io::Result<Self> {
	75	let root = root.replace('\\', "/");
	76	if !root.is_empty() && !root.split('/').all(component) {
	77	return Err(io::ErrorKind::InvalidInput.into());
	78	}
	79	Ok(Self { client, root })
	80	}
	81
	82	fn path(&self, relative: &str) -> io::Result<String> {
	83	if !relative.is_empty() && !relative.split('/').all(component) {
	84	return Err(io::ErrorKind::InvalidInput.into());
	85	}
	86	Ok(if relative.is_empty() {
	87	self.root.clone()
	88	} else {
	89	super::join(&self.root, relative)
	90	})
	91	}
	92	}
	93
	94	#[cfg(feature = "smb")]
	95	impl Source for Smb<'_> {
	96	fn entries(&mut self, path: &str, limit: usize) -> io::Result<Vec<Entry>> {
	97	Ok(self
	98	.client
	99	.read_dir(&self.path(path)?, limit)?
	100	.into_iter()
	101	.map(\|entry\| Entry {
	102	name: entry.name,
	103	kind: if entry.attributes & 0x400 != 0 {
	104	EntryKind::Other
	105	} else if entry.attributes & 0x10 != 0 {
	106	EntryKind::Directory
	107	} else {
	108	EntryKind::File
	109	},
	110	})
	111	.collect())
	112	}
	113
	114	fn read(&mut self, path: &str, limit: usize) -> io::Result<Vec<u8>> {
	115	self.client.read_storage(&self.path(path)?, limit)
	116	}
	117	}

crates/onestore-notebook/tests/discovery.rs created+280

...	...	@@ -0,0 +1,280 @@
	1	use onestore_notebook::{Entry, Error, Limits, Local, Source, discover};
	2	use std::{fs, io, path::Path};
	3
	4	fn limits() -> Limits {
	5	Limits {
	6	entries: 1000,
	7	bytes_per_file: 16 * 1024 * 1024,
	8	depth: 16,
	9	}
	10	}
	11
	12	fn fixture(root: &Path) -> Vec<u8> {
	13	let section = onestore::create_section("one.one", "Retained 🦀", "Fixture").unwrap();
	14	fs::write(root.join("one.one"), &section).unwrap();
	15	let identity = onestore::Store::parse(&section).unwrap().header.file_id;
	16	let toc = onestore::create_table_of_contents("Open Notebook.onetoc2", &[("one.one", identity)])
	17	.unwrap();
	18	fs::write(root.join("Open Notebook.onetoc2"), toc).unwrap();
	19	section
	20	}
	21
	22	#[test]
	23	fn rename_preserves_identity_and_does_not_trust_a_stale_cached_filename() {
	24	let root = tempfile::tempdir().unwrap();
	25	let section = fixture(root.path());
	26	let mut source = Local::open(root.path()).unwrap();
	27	let before = discover(&mut source, limits()).unwrap();
	28	fs::rename(
	29	root.path().join("one.one"),
	30	root.path().join("Renamed 🦀.ONE"),
	31	)
	32	.unwrap();
	33	let after = discover(&mut source, limits()).unwrap();
	34	assert_eq!(before.sections[0].file_id, after.sections[0].file_id);
	35	assert_eq!(after.sections[0].path, "Renamed 🦀.ONE");
	36	assert!(after.toc.as_ref().unwrap().unresolved.is_empty());
	37	assert_eq!(
	38	fs::read(root.path().join("Renamed 🦀.ONE")).unwrap(),
	39	section
	40	);
	41	fs::write(
	42	root.path().join("one.one"),
	43	onestore::create_section("one.one", "Different", "Fixture").unwrap(),
	44	)
	45	.unwrap();
	46	let with_replacement = discover(&mut source, limits()).unwrap();
	47	assert_eq!(
	48	with_replacement.sections[0].file_id,
	49	before.sections[0].file_id
	50	);
	51	assert_eq!(with_replacement.sections[0].path, "Renamed 🦀.ONE");
	52	assert_ne!(
	53	with_replacement.sections[1].file_id,
	54	before.sections[0].file_id
	55	);
	56	fs::remove_file(root.path().join("Renamed 🦀.ONE")).unwrap();
	57	let absent = discover(&mut source, limits()).unwrap();
	58	assert_eq!(absent.toc.as_ref().unwrap().unresolved.len(), 1);
	59	assert_eq!(
	60	absent.toc.as_ref().unwrap().unresolved[0].file,
	61	before.sections[0].file_id
	62	);
	63	assert_ne!(
	64	absent.sections[0].file_id,
	65	absent.toc.as_ref().unwrap().unresolved[0].file
	66	);
	67	}
	68
	69	#[test]
	70	fn copied_identities_are_ambiguous_even_across_different_groups() {
	71	let root = tempfile::tempdir().unwrap();
	72	fixture(root.path());
	73	fs::create_dir(root.path().join("group")).unwrap();
	74	fs::copy(
	75	root.path().join("one.one"),
	76	root.path().join("group/other.one"),
	77	)
	78	.unwrap();
	79	assert!(
	80	matches!(discover(&mut Local::open(root.path()).unwrap(), limits()),
	81	Err(Error::DuplicateIdentity { first, second, .. })
	82	if [first.as_str(), second.as_str()].contains(&"one.one")
	83	&& [first.as_str(), second.as_str()].contains(&"group/other.one"))
	84	);
	85	}
	86
	87	#[test]
	88	fn locked_and_unreadable_sections_retain_their_storage_identity() {
	89	let root = tempfile::tempdir().unwrap();
	90	for (name, fixture) in [
	91	(
	92	"locked.one",
	93	"native-encrypted/encrypted-01/notebook/synthetic.one",
	94	),
	95	(
	96	"stub.one",
	97	"native-encrypted/cold-encrypted-02/notebook/Open Notebook.one",
	98	),
	99	] {
	100	fs::copy(
	101	Path::new("../../corpus").join(fixture),
	102	root.path().join(name),
	103	)
	104	.unwrap();
	105	}
	106	let catalog = discover(&mut Local::open(root.path()).unwrap(), limits()).unwrap();
	107	assert!(catalog.toc.is_none());
	108	assert!(matches!(
	109	catalog.sections[0].state,
	110	onestore_notebook::SectionState::Locked
	111	));
	112	assert!(matches!(
	113	catalog.sections[1].state,
	114	onestore_notebook::SectionState::Unreadable(_)
	115	));
	116	for section in catalog.sections {
	117	let bytes = fs::read(root.path().join(section.path)).unwrap();
	118	assert_eq!(
	119	section.file_id,
	120	onestore::Store::parse(&bytes).unwrap().header.file_id
	121	);
	122	}
	123	}
	124
	125	#[test]
	126	fn a_group_rename_retains_toc_identity_and_parent_order() {
	127	let root = tempfile::tempdir().unwrap();
	128	let native = Path::new("../../corpus/m6/native-features-01/notebook");
	129	for relative in [
	130	"Open Notebook.onetoc2",
	131	"Group A/Open Notebook.onetoc2",
	132	"Group A/Duplicate.one",
	133	"Group B/Open Notebook.onetoc2",
	134	"Group B/Nested/Open Notebook.onetoc2",
	135	"Group B/Nested/Duplicate.one",
	136	] {
	137	let target = root.path().join(relative);
	138	fs::create_dir_all(target.parent().unwrap()).unwrap();
	139	fs::copy(native.join(relative), target).unwrap();
	140	}
	141	let before = discover(&mut Local::open(root.path()).unwrap(), limits()).unwrap();
	142	assert_eq!(before.groups[0].path, "Group A");
	143	fs::rename(root.path().join("Group A"), root.path().join("Renamed 🦀")).unwrap();
	144	let catalog = discover(&mut Local::open(root.path()).unwrap(), limits()).unwrap();
	145	assert_eq!(catalog.groups[0].path, "Renamed 🦀");
	146	assert_eq!(
	147	catalog.groups[0].toc.as_ref().unwrap().file_id,
	148	before.groups[0].toc.as_ref().unwrap().file_id
	149	);
	150	assert_eq!(catalog.groups[1].path, "Group B");
	151	assert_eq!(
	152	catalog.groups[0].sections[0].path,
	153	"Renamed 🦀/Duplicate.one"
	154	);
	155	assert_eq!(
	156	serde_json::to_value(&catalog.toc.unwrap().unresolved).unwrap(),
	157	serde_json::to_value(&before.toc.unwrap().unresolved).unwrap()
	158	);
	159	}
	160
	161	#[test]
	162	fn limits_and_incomplete_files_do_not_produce_a_catalog() {
	163	let root = tempfile::tempdir().unwrap();
	164	fixture(root.path());
	165	let mut source = Local::open(root.path()).unwrap();
	166	assert!(
	167	discover(
	168	&mut source,
	169	Limits {
	170	entries: 1,
	171	..limits()
	172	}
	173	)
	174	.is_err()
	175	);
	176	assert!(
	177	discover(
	178	&mut source,
	179	Limits {
	180	bytes_per_file: 8,
	181	..limits()
	182	}
	183	)
	184	.is_err()
	185	);
	186	fs::create_dir(root.path().join("group")).unwrap();
	187	assert!(matches!(
	188	discover(
	189	&mut source,
	190	Limits {
	191	depth: 0,
	192	..limits()
	193	}
	194	),
	195	Err(Error::Limit { .. })
	196	));
	197	fs::write(root.path().join("one.one"), b"unfinished").unwrap();
	198	assert!(
	199	matches!(discover(&mut source, limits()), Err(Error::Document { path, .. }) if path == "one.one")
	200	);
	201	}
	202
	203	#[test]
	204	fn a_failed_refresh_retains_the_previous_catalog_as_an_independent_value() {
	205	struct Changing {
	206	source: Local,
	207	reads: usize,
	208	fail: bool,
	209	}
	210	impl Source for Changing {
	211	fn entries(&mut self, path: &str, limit: usize) -> io::Result<Vec<Entry>> {
	212	self.reads += 1;
	213	let mut entries = self.source.entries(path, limit)?;
	214	if self.reads == 2 {
	215	if self.fail {
	216	return Err(io::ErrorKind::ConnectionAborted.into());
	217	}
	218	entries[0].name.push_str(".renamed");
	219	}
	220	Ok(entries)
	221	}
	222	fn read(&mut self, path: &str, limit: usize) -> io::Result<Vec<u8>> {
	223	self.source.read(path, limit)
	224	}
	225	}
	226	let root = tempfile::tempdir().unwrap();
	227	fixture(root.path());
	228	let mut source = Local::open(root.path()).unwrap();
	229	let catalog = discover(&mut source, limits()).unwrap();
	230	let before = serde_json::to_value(&catalog).unwrap();
	231	for fail in [false, true] {
	232	let mut changing = Changing {
	233	source: Local::open(root.path()).unwrap(),
	234	reads: 0,
	235	fail,
	236	};
	237	let error = discover(&mut changing, limits()).unwrap_err();
	238	if fail {
	239	assert!(
	240	matches!(error, Error::Io { error, .. } if error.kind() == io::ErrorKind::ConnectionAborted)
	241	);
	242	} else {
	243	assert!(matches!(error, Error::Changed { .. }));
	244	}
	245	assert_eq!(serde_json::to_value(&catalog).unwrap(), before);
	246	}
	247	}
	248
	249	#[test]
	250	fn local_access_stays_inside_the_selected_root() {
	251	let root = tempfile::tempdir().unwrap();
	252	fixture(root.path());
	253	let mut source = Local::open(root.path()).unwrap();
	254	for path in [
	255	"../one.one",
	256	"/one.one",
	257	"x/../one.one",
	258	"one.one\0",
	259	"x\\one.one",
	260	] {
	261	assert_eq!(
	262	source.read(path, 100).unwrap_err().kind(),
	263	io::ErrorKind::InvalidInput
	264	);
	265	}
	266	#[cfg(unix)]
	267	{
	268	let other = tempfile::tempdir().unwrap();
	269	fs::write(other.path().join("other.one"), b"outside").unwrap();
	270	std::os::unix::fs::symlink(other.path().join("other.one"), root.path().join("link.one"))
	271	.unwrap();
	272	assert_eq!(
	273	source.read("link.one", 100).unwrap_err().kind(),
	274	io::ErrorKind::PermissionDenied
	275	);
	276	assert!(
	277	matches!(discover(&mut source, limits()), Err(Error::Entry { path }) if path == "link.one")
	278	);
	279	}
	280	}

crates/onestore-smb/src/lib.rs+19-1

...	...	@@ -200,8 +200,26 @@ impl Client {
200	200
201	201	/// Reads one bounded, consistent snapshot; contention returns WouldBlock.
202	202	pub fn read(&self, path: &str, limit: usize) -> io::Result<Vec<u8>> {
	203	self.read_with(path, \|file\| {
	204	onestore::read_snapshot(\|offset, output\| file.read_at(offset, output), limit)
	205	})
	206	}
	207
	208	/// Reads consistent storage, including encrypted or incomplete document graphs.
	209	/// Storage validation alone does not establish edit readiness.
	210	pub fn read_storage(&self, path: &str, limit: usize) -> io::Result<Vec<u8>> {
	211	self.read_with(path, \|file\| {
	212	onestore::read_storage_snapshot(\|offset, output\| file.read_at(offset, output), limit)
	213	})
	214	}
	215
	216	fn read_with(
	217	&self,
	218	path: &str,
	219	snapshot: impl FnOnce(&mut File<'_>) -> io::Result<Option<Vec<u8>>>,
	220	) -> io::Result<Vec<u8>> {
203	221	let mut file = self.open(path, false)?.coordinate(path, false)?;
204		let result = onestore::read_snapshot(\|offset, output\| file.read_at(offset, output), limit)
	222	let result = snapshot(&mut file)
205	223	.and_then(\|snapshot\| snapshot.ok_or_else(\|\| io::ErrorKind::WouldBlock.into()));
206	224	let closed = file.close();
207	225	let snapshot = result?;

crates/onestore-smb/src/tests.rs+39-2

...	...	@@ -149,8 +149,9 @@ fn text(bytes: &[u8]) -> (ExGuid, ExGuid, String) {
149	149	.unwrap()
150	150	}
151	151	#[test]
152		#[ignore = "requires ONESTORE_SMB_LAB pointing to disposable Samba"]
	152	#[ignore = "requires disposable Samba and an existing ONESTORE_SMB_EVIDENCE directory"]
153	153	fn live_coordination() {
	154	let output = std::env::var("ONESTORE_SMB_EVIDENCE").unwrap();
154	155	let writer = client();
155	156	let path = format!(
156	157	"adapter-{}.one",
...	...	@@ -330,7 +331,6 @@ fn live_coordination() {
330	331	.block_on(async {
331	332	drop(spare);
332	333	});
333		let output = std::env::var("ONESTORE_SMB_EVIDENCE").unwrap();
334	334	fs::write(std::path::Path::new(&output).join("source.one"), &source).unwrap();
335	335	fs::write(std::path::Path::new(&output).join("committed.one"), &after).unwrap();
336	336	fs::write(
...	...	@@ -344,6 +344,43 @@ fn live_coordination() {
344	344	);
345	345	}
346	346
	347	#[test]
	348	#[ignore = "requires ONESTORE_SMB_LAB pointing to disposable Samba"]
	349	fn live_storage_inspection() {
	350	let reader = client();
	351	for (index, fixture) in [
	352	"native-encrypted/encrypted-01/notebook/synthetic.one",
	353	"native-encrypted/cold-encrypted-02/notebook/Open Notebook.one",
	354	]
	355	.into_iter()
	356	.enumerate()
	357	{
	358	let source = fs::read(format!("../../corpus/{fixture}")).unwrap();
	359	let path = format!(
	360	"inspection-{index}-{}.one",
	361	SystemTime::now()
	362	.duration_since(UNIX_EPOCH)
	363	.unwrap()
	364	.as_nanos()
	365	);
	366	create(&reader, &path, &source);
	367	assert_eq!(reader.read_storage(&path, source.len()).unwrap(), source);
	368	assert_eq!(
	369	reader.read(&path, source.len()).unwrap_err().kind(),
	370	io::ErrorKind::WouldBlock
	371	);
	372	let maintenance = client();
	373	let guard = maintenance.open(&path, false).unwrap();
	374	guard.lock(0xfffffffb, 0x12).unwrap();
	375	assert_eq!(
	376	reader.read_storage(&path, source.len()).unwrap_err().kind(),
	377	io::ErrorKind::WouldBlock
	378	);
	379	guard.close().unwrap();
	380	assert_eq!(reader.read_storage(&path, source.len()).unwrap(), source);
	381	}
	382	}
	383
347	384	#[test]
348	385	#[ignore = "requires an owned Samba fixture and maintenance controller"]
349	386	fn live_reader_hold() {

crates/onestore/README.md+1

...	...	@@ -89,6 +89,7 @@ cargo run --example create_notebook -- /tmp/one-demo 'Hello from Rust.' 'Example
89	89	cargo run --example inventory -- /tmp/one-demo/synthetic.one
90	90	cargo run --example inspect -- /tmp/one-demo/synthetic.one
91	91	cargo run --example document -- /tmp/one-demo/synthetic.one /tmp/one-model
	92	cargo build -p onestore-diagnostic
92	93	python3 tools/notebook_report.py /tmp/one-demo /tmp/one-report --timezone America/Los_Angeles
93	94	```
94	95

crates/onestore/src/commit.rs+14-1

...	...	@@ -75,12 +75,25 @@ impl Drop for FileIo {
75	75	/// Native writers can expose incomplete graphs to unlocked filesystem reads.
76	76	#[cfg(any(unix, windows))]
77	77	pub fn read_file(path: impl AsRef<Path>) -> io::Result<Vec<u8>> {
	78	read_file_limited(path, usize::MAX)
	79	}
	80
	81	/// Reads under whole-file exclusion, rejecting a snapshot larger than the byte limit.
	82	/// A size failure returns `FileTooLarge` without a partial snapshot.
	83	#[cfg(any(unix, windows))]
	84	pub fn read_file_limited(path: impl AsRef<Path>, limit: usize) -> io::Result<Vec<u8>> {
	85	let count = u64::try_from(limit)
	86	.map_err(\|_\| ErrorKind::InvalidInput)?
	87	.saturating_add(1);
78	88	let mut io = FileIo::open(path, false)?;
79	89	let mut bytes = Vec::new();
80		let result = io.file.read_to_end(&mut bytes);
	90	let result = (&mut io.file).take(count).read_to_end(&mut bytes);
81	91	let released = io.release();
82	92	result?;
83	93	released?;
	94	if bytes.len() > limit {
	95	return Err(ErrorKind::FileTooLarge.into());
	96	}
84	97	Ok(bytes)
85	98	}
86	99

crates/onestore/src/lib.rs+2-2

...	...	@@ -22,7 +22,7 @@ pub use commit::{
22	22	confirm_snapshot,
23	23	};
24	24	#[cfg(any(unix, windows))]
25		pub use commit::{commit_file_property, commit_file_text, read_file};
	25	pub use commit::{commit_file_property, commit_file_text, read_file, read_file_limited};
26	26	pub use create::{create_section, create_table_of_contents};
27	27	pub use edit::replace_text;
28	28	pub use files::FileDataReference;
...	...	@@ -31,6 +31,6 @@ pub use insertion::Insertion;
31	31	pub use objects::{Object, ObjectData, ObjectReferences, ResolvedRevision};
32	32	pub use properties::{IdStream, Property, PropertySets, Value};
33	33	pub use revisions::{ExGuid, ObjectSpace, Revision, RevisionIndex};
34		pub use snapshot::read_snapshot;
	34	pub use snapshot::{read_snapshot, read_storage_snapshot};
35	35	pub use store::{Chunk, Error, FileType, Header, Node, NodeList, Reference, Store};
36	36	pub use write::replace_property_bytes;

crates/onestore/src/snapshot.rs+21-1

...	...	@@ -24,8 +24,28 @@ fn read_exact(
24	24	/// Reads a bounded snapshot; the caller must provide fresh I/O and exclude in-place maintenance.
25	25	/// Includes unpublished trailing bytes so subsequent commits can validate the physical file.
26	26	pub fn read_snapshot(
	27	read: impl FnMut(u64, &mut [u8]) -> io::Result<usize>,
	28	limit: usize,
	29	) -> io::Result<Option<Vec<u8>>> {
	30	snapshot(read, limit, \|store\| {
	31	RevisionIndex::parse(store)?.validate_current()
	32	})
	33	}
	34
	35	/// Reads stable storage and checksums without requiring traversable property references.
	36	/// Used to inspect encrypted or incomplete documents; this does not establish edit readiness.
	37	/// The caller must provide fresh I/O and exclude in-place maintenance, as for `read_snapshot`.
	38	pub fn read_storage_snapshot(
	39	read: impl FnMut(u64, &mut [u8]) -> io::Result<usize>,
	40	limit: usize,
	41	) -> io::Result<Option<Vec<u8>>> {
	42	snapshot(read, limit, \|_\| Ok(()))
	43	}
	44
	45	fn snapshot(
27	46	mut read: impl FnMut(u64, &mut [u8]) -> io::Result<usize>,
28	47	limit: usize,
	48	validate: impl FnOnce(&Store<'_>) -> Result<(), crate::Error>,
29	49	) -> io::Result<Option<Vec<u8>>> {
30	50	let mut header = [0; 1024];
31	51	read_exact(&mut read, 0, &mut header)?;
...	...	@@ -69,7 +89,7 @@ pub fn read_snapshot(
69	89	if !store.checksum_mismatches.is_empty() {
70	90	return Ok(false);
71	91	}
72		RevisionIndex::parse(&store)?.validate_current()?;
	92	validate(&store)?;
73	93	Ok(true)
74	94	});
75	95	Ok(matches!(parsed, Ok(true)).then_some(bytes))

crates/onestore/src/store.rs+1-1

...	...	@@ -3,7 +3,7 @@ use std::collections::{BTreeMap, BTreeSet};
3	3	use std::fmt;
4	4	use std::ops::Range;
5	5
6		#[derive(Debug, Clone, PartialEq, Eq)]
	6	#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
7	7	pub struct Error {
8	8	/// Parser byte offset; zero also represents errors without a byte location.
9	9	pub offset: usize,

crates/onestore/tests/commit.rs+37

...	...	@@ -222,3 +222,40 @@ fn check_crashes(source: &[u8], osid: ExGuid, oid: ExGuid, property: u32, value:
222	222	}
223	223	}
224	224	}
	225	#[test]
	226	#[cfg(any(unix, windows))]
	227	fn bounded_file_reads_reject_partial_images_and_release_the_owner() {
	228	use std::io::Write;
	229	let path = std::env::temp_dir().join(format!(
	230	"onestore-bounded-{}-{}",
	231	std::process::id(),
	232	std::time::SystemTime::now()
	233	.duration_since(std::time::UNIX_EPOCH)
	234	.unwrap()
	235	.as_nanos()
	236	));
	237	let mut file = fs::File::options()
	238	.write(true)
	239	.create_new(true)
	240	.open(&path)
	241	.unwrap();
	242	assert!(onestore::read_file_limited(&path, 0).unwrap().is_empty());
	243	let bytes: Vec<_> = (0..10_000).map(\|i\| (i % 251) as u8).collect();
	244	file.write_all(&bytes).unwrap();
	245	drop(file);
	246	for limit in [0, 1, 100, bytes.len() - 1] {
	247	assert_eq!(
	248	onestore::read_file_limited(&path, limit)
	249	.unwrap_err()
	250	.kind(),
	251	std::io::ErrorKind::FileTooLarge
	252	);
	253	assert_eq!(
	254	onestore::read_file_limited(&path, bytes.len()).unwrap(),
	255	bytes
	256	);
	257	}
	258	assert_eq!(onestore::read_file(&path).unwrap(), bytes);
	259	assert_eq!(fs::read(&path).unwrap(), bytes);
	260	fs::remove_file(path).unwrap();
	261	}

crates/onestore/tests/shared_snapshot.rs+61

...	...	@@ -15,6 +15,67 @@ use onestore::{
15	15	use std::{fs, io};
16	16	use trace::{Event, Trace};
17	17
	18	#[test]
	19	fn storage_inspection_preserves_opaque_images_without_claiming_edit_readiness() {
	20	for path in [
	21	"native-encrypted/encrypted-01/notebook/synthetic.one",
	22	"native-encrypted/cold-encrypted-02/notebook/Open Notebook.one",
	23	"malformed/native-inflight.one",
	24	] {
	25	let source = fs::read(format!("../../corpus/{path}")).unwrap();
	26	for block in [1, 17, 65536] {
	27	let mut read = \|offset: u64, output: &mut [u8]\| {
	28	let offset = usize::try_from(offset).unwrap();
	29	let count = output
	30	.len()
	31	.min(block)
	32	.min(source.len().saturating_sub(offset));
	33	output[..count].copy_from_slice(&source[offset..offset + count]);
	34	Ok(count)
	35	};
	36	assert_eq!(
	37	onestore::read_storage_snapshot(&mut read, source.len()).unwrap(),
	38	Some(source.clone())
	39	);
	40	assert!(read_snapshot(&mut read, source.len()).unwrap().is_none());
	41	}
	42	let mut headers = 0;
	43	let changed = onestore::read_storage_snapshot(
	44	\|offset, output\| {
	45	let offset = usize::try_from(offset).unwrap();
	46	let count = output.len().min(source.len().saturating_sub(offset));
	47	output[..count].copy_from_slice(&source[offset..offset + count]);
	48	if offset == 0 {
	49	headers += 1;
	50	if headers == 2 {
	51	output[0] ^= 1;
	52	}
	53	}
	54	Ok(count)
	55	},
	56	source.len(),
	57	)
	58	.unwrap();
	59	assert!(changed.is_none());
	60	let mut broken = source.clone();
	61	let offset = usize::try_from(Store::parse(&source).unwrap().header.root.offset).unwrap();
	62	broken[offset] ^= 1;
	63	assert!(
	64	onestore::read_storage_snapshot(
	65	\|offset, output\| {
	66	let offset = usize::try_from(offset).unwrap();
	67	let count = output.len().min(broken.len().saturating_sub(offset));
	68	output[..count].copy_from_slice(&broken[offset..offset + count]);
	69	Ok(count)
	70	},
	71	broken.len()
	72	)
	73	.unwrap()
	74	.is_none()
	75	);
	76	}
	77	}
	78
18	79	fn target(bytes: &[u8]) -> (ExGuid, ExGuid, u32) {
19	80	let store = Store::parse(bytes).unwrap();
20	81	let index = RevisionIndex::parse(&store).unwrap();

tools/document_model.py+1

...	...	@@ -3,6 +3,7 @@ import os
3	3	from pathlib import Path
4	4
5	5	EXPORTER = Path(os.environ.get('ONESTORE_DOCUMENT', Path(__file__).resolve().parent.parent / 'target/debug/examples/document'))
	6	BRIDGE = Path(__file__).resolve().parent.parent / 'target/debug/onestore-diagnostic'
6	7
7	8	DEFAULT_CONTEXT = '{00000000-0000-0000-0000-000000000000},0'
8	9

tools/native/toc-controls.ps1 created+33

...	...	@@ -0,0 +1,33 @@
	1	param([Parameter(Mandatory=$true)][string]$Root, [string]$CloneHost = '')
	2	Set-StrictMode -Version Latest
	3	$ErrorActionPreference = 'Stop'
	4	& "$PSScriptRoot\cold-current.ps1" -Root $Root -CloneHost $CloneHost
	5	$app = New-Object -ComObject OneNote.Application
	6	$notebook = ''
	7	try {
	8	$app.OpenHierarchy((Join-Path $Root 'notebook'), '', [ref]$notebook, 0)
	9	foreach ($name in @('Removed.one', 'Retired.one')) {
	10	$section = ''
	11	$app.OpenHierarchy($name, $notebook, [ref]$section, 3)
	12	$page = ''
	13	$app.CreateNewPage($section, [ref]$page, 2)
	14	$xml = '<one:Page xmlns:one="http://schemas.microsoft.com/office/onenote/2010/onenote" ID="' + $page + '"><one:Outline><one:OEChildren><one:OE><one:T>' + $name + ' fixture.</one:T></one:OE></one:OEChildren></one:Outline></one:Page>'
	15	$app.UpdatePageContent($xml, [DateTime]::MinValue, 1, $false)
	16	}
	17	$app.SyncHierarchy($notebook)
	18	$app.CloseNotebook($notebook, $false)
	19	Copy-Item (Join-Path $Root 'notebook') (Join-Path $Root 'before') -Recurse
	20	$app.OpenHierarchy((Join-Path $Root 'notebook'), '', [ref]$notebook, 0)
	21	foreach ($name in @('Removed.one', 'Retired.one')) {
	22	$section = ''
	23	$app.OpenHierarchy($name, $notebook, [ref]$section, 0)
	24	$app.DeleteHierarchy($section, [DateTime]::MinValue, ($name -eq 'Retired.one'))
	25	}
	26	$app.SyncHierarchy($notebook)
	27	} finally {
	28	if ($notebook) { $app.CloseNotebook($notebook, $false) }
	29	[void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($app)
	30	$app = $null
	31	[GC]::Collect()
	32	[GC]::WaitForPendingFinalizers()
	33	}

tools/native_runner.py+5-3

...	...	@@ -116,7 +116,7 @@ def clone(output):
116	116	(output / 'teardown.json').write_text(json.dumps({'absent': not vm.instance_path(name).exists()}) + '\n')
117	117
118	118
119		def capture(notebook, output, expected_pages=-1, author=None, screenshots=False, pdf=False, author_timeout=600, inspect=False):
	119	def capture(notebook, output, expected_pages=-1, author=None, screenshots=False, pdf=False, author_timeout=600, inspect=False, collect_notebook=False):
120	120	notebook = notebook.resolve(strict=True)
121	121	if not notebook.is_dir():
122	122	raise ValueError('Choose a notebook directory.')
...	...	@@ -133,6 +133,7 @@ def capture(notebook, output, expected_pages=-1, author=None, screenshots=False,
133	133	'notebook': str(notebook), 'expected_pages': expected_pages, 'author': str(author) if author else None,
134	134	'author_timeout_seconds': author_timeout,
135	135	'inspect': inspect,
	136	'collect_notebook': collect_notebook,
136	137	'base': json.loads(vm.BASE_MANIFEST.read_text()),
137	138	'scripts': {name: hashlib.sha256((scripts / name).read_bytes()).hexdigest()
138	139	for name in sorted(p.name for p in scripts.iterdir())},
...	...	@@ -165,7 +166,7 @@ def capture(notebook, output, expected_pages=-1, author=None, screenshots=False,
165	166	raise RuntimeError(result['error'])
166	167	command(name, 'powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\author.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-%s' % name.upper(), output, author_timeout * 1000)
167	168	command(name, 'powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\\one-tests\\read-current.ps1 -Root C:\\one-tests\\runs\\capture -CloneHost ONE-%s -ExpectedPages %d%s' % (name.upper(), expected_pages, (' -UseCurrentCache' if author else '') + (' -Pdf' if pdf else '') + (' -KeepOpen' if screenshots or inspect else '')), output, 600000)
168		collect_artifacts(name, output, '*' if author else 'read')
	169	collect_artifacts(name, output, '*' if author or collect_notebook else 'read')
169	170	if screenshots:
170	171	for page in sorted((output / 'read').glob('page-*.xml')):
171	172	page_id = ET.parse(page).getroot().attrib['ID']
...	...	@@ -208,6 +209,7 @@ if __name__ == '__main__':
208	209	parser.add_argument('--pdf', action='store_true')
209	210	parser.add_argument('--screenshots', action='store_true')
210	211	parser.add_argument('--inspect', action='store_true', help='Keep the clone open for inspection until an output/finish file appears, up to 30 minutes.')
	212	parser.add_argument('--collect-notebook', action='store_true', help='Retain the VM notebook alongside its native read capture.')
211	213	parser.add_argument('--author', type=Path, help='Run a fixture-authoring PowerShell script in the clone before capture.')
212	214	parser.add_argument('--author-timeout', type=int, default=600, help='Authoring deadline in seconds.')
213	215	args = parser.parse_args()
...	...	@@ -216,5 +218,5 @@ if __name__ == '__main__':
216	218	signal.signal(signal.SIGTERM, interrupted)
217	219	if args.author_timeout <= 0:
218	220	parser.error('The authoring deadline must be positive.')
219		capture(args.notebook, args.output, args.expected_pages, args.author, args.screenshots, args.pdf, args.author_timeout, args.inspect)
	221	capture(args.notebook, args.output, args.expected_pages, args.author, args.screenshots, args.pdf, args.author_timeout, args.inspect, args.collect_notebook)
220	222	print('Captured native evidence in', args.output)

tools/notebook_editor.py+1-2

...	...	@@ -15,11 +15,10 @@ import time
15	15	from urllib.parse import parse_qs, urlsplit
16	16	import uuid
17	17
18		from document_model import walk
	18	from document_model import BRIDGE, walk
19	19	from notebook_report import generate
20	20
21	21	ROOT = Path(__file__).resolve().parent.parent
22		BRIDGE = ROOT / 'target/debug/onestore-diagnostic'
23	22
24	23
25	24	def bridge(mode, source, destination, edit=None):

tools/notebook_report.py+28-20

...	...	@@ -16,7 +16,7 @@ import subprocess
16	16	from urllib.parse import urlsplit
17	17	from zoneinfo import ZoneInfo
18	18
19		from document_model import DEFAULT_CONTEXT, EXPORTER, ordered_pages, version_pages, view, walk
	19	from document_model import BRIDGE, DEFAULT_CONTEXT, EXPORTER, ordered_pages, version_pages, view, walk
20	20
21	21	ROOT = Path(__file__).resolve().parent.parent
22	22
...	...	@@ -274,11 +274,27 @@ def generate(source, destination, native=None, versions=(), zone=timezone.utc, e
274	274	(destination / 'model').mkdir(); (destination / 'assets').mkdir()
275	275	cached = {row['path']: (row['sha256'], previous / 'model' / str(index))
276	276	for index, row in enumerate(json.loads((previous / 'source.json').read_text()))} if previous else {}
277		sections = []; tocs = {}; manifest = []
278		for index, path in enumerate(sorted(p for p in source.rglob('*') if p.suffix.lower() in ('.one', '.onetoc2'))):
279		relative = path.relative_to(source)
	277	result = json.loads(subprocess.check_output([BRIDGE, 'catalog', source], timeout=120))
	278	if not result['ok']: raise ValueError(result['error'])
	279	catalog = result['catalog']
	280	(destination / 'catalog.json').write_text(json.dumps(catalog, indent=2, ensure_ascii=False))
	281	files = []; catalog_sections = {}
	282	def collect(folder):
	283	if folder['toc'] is not None: files.append(Path(folder['path']) / folder['toc']['filename'])
	284	for section in folder['sections']:
	285	relative = Path(section['path'])
	286	files.append(relative); catalog_sections[relative] = section
	287	for group in folder['groups']: collect(group)
	288	collect(catalog)
	289	sections = []; unavailable = []; manifest = []
	290	for index, relative in enumerate(sorted(files)):
	291	path = source / relative
280	292	before = path.read_bytes()
281	293	manifest.append({'path': relative.as_posix(), 'sha256': hashlib.sha256(before).hexdigest(), 'bytes': len(before)})
	294	state = catalog_sections.get(relative, {}).get('state', {})
	295	if isinstance(state, dict) and 'Unreadable' in state:
	296	unavailable.append((relative, state['Unreadable']))
	297	continue
282	298	exported = destination / 'model' / str(index)
283	299	reusable = cached.get(relative.as_posix())
284	300	reused = reusable is not None and reusable[0] == manifest[-1]['sha256']
...	...	@@ -291,9 +307,6 @@ def generate(source, destination, native=None, versions=(), zone=timezone.utc, e
291	307	document = json.loads((exported / 'document.json').read_text())
292	308	if path.read_bytes() != before:
293	309	raise ValueError('A source file changed during export.')
294		if path.suffix.lower() == '.onetoc2':
295		_, root = view(document, document['root'])
296		tocs[relative.parent] = [root['nodes'][oid]['kind'] for oid in root['nodes'][root['roots']['1']]['kind']['entries']]
297	310	assets = {}
298	311	previews = {}
299	312	image_references = {
...	...	@@ -339,20 +352,13 @@ def generate(source, destination, native=None, versions=(), zone=timezone.utc, e
339	352	if path.suffix.lower() == '.one':
340	353	sections.append({'path': relative, 'export': exported.relative_to(destination), 'document': document,
341	354	'text': json.loads((exported / 'text.json').read_text()), 'assets': assets, 'previews': previews})
342		def order(section):
343		path = section['path']; entries = tocs.get(path.parent, [])
344		entry = next((entry for entry in entries if entry['identity'] == section['document']['file_id']), None)
345		result = []; parent = Path('.')
346		for part in path.parts[:-1]:
347		group = next((item for item in tocs.get(parent, []) if item['filename'] == part), None)
348		result.append((group['order'] if group and group['order'] is not None else 0xffffffff, part))
349		parent /= part
350		result.append((entry['order'] if entry and entry['order'] is not None else 0xffffffff, path.name))
351		return result
352		sections.sort(key=order)
	355	order = {path: index for index, path in enumerate(catalog_sections)}
	356	sections.sort(key=lambda section: order[section['path']])
353	357	pages = []; locked_sections = []; histories = {}; nav = '<a href="index.html">Notebook review</a>'
354	358	for section in sections:
355		name = section['path'].stem
	359	state = catalog_sections[section['path']]['state']
	360	name = state.get('Readable', {}).get('name') if isinstance(state, dict) else None
	361	if name is None: name = section['path'].stem
356	362	nav += '<h2>' + esc(str(section['path'].parent) + ' / ' + name) + '</h2>'
357	363	_, root_space = view(section['document'], section['document']['root'])
358	364	if root_space['nodes'][root_space['roots']['1']]['kind']['type'] == 'Encrypted':
...	...	@@ -472,9 +478,11 @@ def generate(source, destination, native=None, versions=(), zone=timezone.utc, e
472	478	accounting.append({'section': str(section['path']), 'ordinal': ordinal, 'space': sid, 'revision': rid, 'object': oid, 'title': title, 'report': filename, 'category': category, 'context': context, 'version_modified': version['modified'] if version else None, 'source_report': source_page, 'native_reference': reference.as_posix() if reference else None, 'rendered': dict(page.counts)})
473	479	(destination / 'source.json').write_text(json.dumps(manifest, indent=2))
474	480	(destination / 'pages.json').write_text(json.dumps(accounting, indent=2, ensure_ascii=False))
475		intro = '<h1>Notebook review</h1><p>' + str(len(sections)) + ' sections · ' + str(len(pages)) + ' stored pages</p><p>Readable content follows the stored object order. Outline positions are shown in points. The document structure retains properties and identities that the readable view does not interpret.</p><p><a href="source.json">Source hashes</a> · <a href="pages.json">Page inventory</a></p>'
	481	intro = '<h1>Notebook review</h1><p>' + str(len(sections) + len(unavailable)) + ' sections · ' + str(len(pages)) + ' stored pages</p><p>Readable content follows the stored object order. Outline positions are shown in points. The document structure retains properties and identities that the readable view does not interpret.</p><p><a href="source.json">Source hashes</a> · <a href="pages.json">Page inventory</a> · <a href="catalog.json">Notebook catalog</a></p>'
476	482	if locked_sections:
477	483	intro += '<p>Page counts are unavailable for locked sections: ' + esc(', '.join(locked_sections)) + '.</p>'
	484	for path, error in unavailable:
	485	intro += '<p>Cannot read ' + esc(str(path)) + ': ' + esc(error['message']) + ' (offset ' + str(error['offset']) + ').</p>'
478	486	(destination / 'index.html').write_text(html_page('Notebook review', nav, intro, editable))
479	487
480	488

tools/test_notebook_discovery.py created+67

...	...	@@ -0,0 +1,67 @@
	1	"""Compare library discovery with retained native OneNote hierarchy captures."""
	2	import json
	3	import os
	4	from pathlib import Path
	5	import subprocess
	6	import unittest
	7	import xml.etree.ElementTree as ET
	8
	9	ROOT = Path(__file__).resolve().parent.parent
	10
	11
	12	class NativeDiscovery(unittest.TestCase):
	13	def test_native_hierarchies(self):
	14	fixtures = ['m6/native-features-01', 'native-encrypted/cold-encrypted-02',
	15	'native-delete/cold-deletion-05']
	16	if os.environ.get('ONESTORE_NOTEBOOK_NATIVE'):
	17	fixtures.append(str(Path(os.environ['ONESTORE_NOTEBOOK_NATIVE']).resolve()))
	18	for fixture in fixtures:
	19	with self.subTest(fixture=fixture):
	20	source = ROOT / 'corpus' / fixture
	21	catalog = json.loads(subprocess.check_output(
	22	[str(ROOT / 'target/debug/examples/discover'), str(source / 'notebook')]))
	23	hierarchy = ET.parse(source / 'read/hierarchy.xml').getroot()
	24	notebook = next(node for node in hierarchy.iter() if node.tag.endswith('}Notebook'))
	25
	26	def path(value):
	27	return '/'.join(part for part in value.replace('\\', '/').split('/') if part)
	28
	29	prefix = path(notebook.get('path'))
	30
	31	def native(node):
	32	sections = []
	33	groups = []
	34	for child in node:
	35	if child.tag.endswith('}Section'):
	36	relative = path(child.get('path'))[len(prefix) + 1:]
	37	name = child.get('name')
	38	if Path(relative).parts[-2:] == ('OneNote_RecycleBin', 'OneNote_DeletedPages.one'):
	39	self.assertEqual(name, 'Deleted Pages')
	40	name = 'OneNote_DeletedPages'
	41	sections.append((relative, name))
	42	elif child.tag.endswith('}SectionGroup'):
	43	groups.append(native(child))
	44	return {'path': path(node.get('path'))[len(prefix) + 1:], 'sections': sections, 'groups': groups}
	45
	46	def actual(folder):
	47	sections = []
	48	for section in folder['sections']:
	49	state = section['state']
	50	name = state.get('Readable', {}).get('name') if isinstance(state, dict) else None
	51	sections.append((section['path'], Path(section['path']).stem if name is None else name))
	52	return {'path': folder['path'], 'sections': sections,
	53	'groups': [actual(group) for group in folder['groups']]}
	54
	55	self.assertEqual(actual(catalog), native(notebook))
	56	if fixture == 'm6/native-features-01':
	57	self.assertEqual(len(catalog['toc']['unresolved']), 1)
	58	self.assertEqual(catalog['toc']['unresolved'][0]['filename'], 'synthetic.one')
	59	elif fixture == 'native-encrypted/cold-encrypted-02':
	60	states = {section['path']: section['state'] for section in catalog['sections']}
	61	self.assertEqual(states['synthetic.one'], 'Locked')
	62	self.assertEqual(states['Open Notebook.one']['Unreadable']['message'],
	63	'Object space has no revision manifest list')
	64
	65
	66	if __name__ == '__main__':
	67	unittest.main()

tools/test_notebook_report.py+25-6

...	...	@@ -5,15 +5,31 @@ from pathlib import Path
5	5	import re
6	6	from tempfile import TemporaryDirectory
7	7	import shutil
	8	import subprocess
8	9	import unittest
9	10	from unittest.mock import patch
10	11	import xml.etree.ElementTree as ET
11	12
12	13	from PIL import Image
13	14	from notebook_report import generate
	15	from document_model import EXPORTER
14	16
15	17
16	18	class NotebookReportTest(unittest.TestCase):
	19	def test_locked_and_unreadable_sections_remain_visible_in_cached_reports(self):
	20	fixture = Path(__file__).resolve().parent.parent / 'corpus/native-encrypted/cold-encrypted-02/notebook'
	21	with TemporaryDirectory() as temporary:
	22	root = Path(temporary)
	23	generate(fixture, root / 'first')
	24	generate(fixture, root / 'cached', previous=root / 'first')
	25	index = (root / 'first/index.html').read_text()
	26	self.assertIn('2 sections', index)
	27	self.assertIn('Locked section', index)
	28	self.assertIn('Cannot read Open Notebook.one', index)
	29	self.assertIn('Object space has no revision manifest list', index)
	30	self.assertEqual(json.loads((root / 'first/pages.json').read_text()), [])
	31	self.assertEqual(index, (root / 'cached/index.html').read_text())
	32
17	33	def test_tiff_preview_matches_native_pixels_and_retains_original_bytes(self):
18	34	fixture = Path(__file__).resolve().parent.parent / 'corpus/m6/native-features-01'
19	35	with TemporaryDirectory() as temporary:
...	...	@@ -42,15 +58,18 @@ class NotebookReportTest(unittest.TestCase):
42	58	shutil.copytree(fixture, source)
43	59	generate(source, root / 'first')
44	60	before = {p.relative_to(root / 'first'): p.read_bytes() for p in (root / 'first').rglob('*') if p.is_file()}
45		with patch('notebook_report.subprocess.run', side_effect=AssertionError('Unchanged model exported again')):
	61	def cached_run(command, args, *kwargs):
	62	self.assertNotEqual(command[0], EXPORTER, 'Unchanged model exported again')
	63	return original(command, args, *kwargs)
	64
	65	original = subprocess.run
	66	with patch('notebook_report.subprocess.run', side_effect=cached_run):
46	67	generate(source, root / 'same', previous=root / 'first')
47	68	self.assertEqual(before, {p.relative_to(root / 'same'): p.read_bytes() for p in (root / 'same').rglob('*') if p.is_file()})
48		shutil.copyfile(source / 'synthetic.one', source / 'a.one')
49		import notebook_report
50		original = notebook_report.subprocess.run
	69	subprocess.run([EXPORTER.parent / 'create_section', source / 'a.one', 'Independent section', 'Fixture'], check=True)
51	70	with patch('notebook_report.subprocess.run', wraps=original) as run:
52	71	generate(source, root / 'changed', previous=root / 'first')
53		self.assertEqual([Path(call.args[0][1]).name for call in run.call_args_list], ['a.one'])
	72	self.assertEqual([Path(call.args[0][1]).name for call in run.call_args_list if call.args[0][0] == EXPORTER], ['a.one'])
54	73	generate(source, root / 'fresh')
55	74	self.assertEqual({p.relative_to(root / 'fresh'): p.read_bytes() for p in (root / 'fresh').rglob('*') if p.is_file()},
56	75	{p.relative_to(root / 'changed'): p.read_bytes() for p in (root / 'changed').rglob('*') if p.is_file()})
...	...	@@ -61,7 +80,7 @@ class NotebookReportTest(unittest.TestCase):
61	80	self.assertEqual((root / f'first/model/{old_index}/document.json').stat().st_ino,
62	81	(root / f'changed/model/{new_index}/document.json').stat().st_ino)
63	82	(source / 'a.one').unlink()
64		with patch('notebook_report.subprocess.run', side_effect=AssertionError('Unchanged model exported again')):
	83	with patch('notebook_report.subprocess.run', side_effect=cached_run):
65	84	generate(source, root / 'deleted', previous=root / 'changed')
66	85	self.assertEqual(before, {p.relative_to(root / 'deleted'): p.read_bytes() for p in (root / 'deleted').rglob('*') if p.is_file()})
67	86

author	clover caruso <git@paperclover.net> 2026-09-07 17:20:27-07:00
committer	clover caruso <git@paperclover.net> 2026-09-07 17:20:27-07:00
log	bca7120d8a96e56b4ae7c6c76d1e82ccef0c5d21
tree	a25ecdab015dd5946ed165c9a8e53179e074c8f8
parent	2c75dd41e5584f3d01c9253e50bdebec6623740f
signature	Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

feat: discover notebook identities and order over local and SMB roots