authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-09-19 21:50:05-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-09-25 20:26:20-07:00
logbe6b82da9a9e19b3a47c24655949fd2d66db0800
treeafe4c81369cdf10faf1f1ddb8aff5d220f73bacb
parent3d0dd98f39482111b7e2e32734d71bc806d826c8
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

perf: parse a section once per side when saving; store the working image as a patch

A page save parsed the section about twenty times with up to four parsed copies alive: the lowering re-read the unchanged page before every writer pass, the write path parsed and validated the source its caller had just parsed and validated, and squash kept both parsed images alive while checking the result. The lowered page is now cached until a writer changes the image, typed writers hand their validated index to the write layer, parsed sources are released before results are checked, GUID tables are built once from a sorted list and node lists drop their slack. The cache stored base and working whole in one row, so every save rewrote twice the section and every idle poll parsed, validated and rewrote both. working is now the byte ranges where it differs from base (schema 14) and an unchanged remote is recognised by comparison alone. Owner 12 MiB section: 141 ms / +65 MiB to 65 ms / +29 MiB per save. 24 MiB soak section: 780 ms / +506 MiB to 310 ms / +154 MiB; launch with two publications 7.3 s to 2.7 s; cache half the size. Assisted-by: claude-fable-5.1

20 files changed, 394 insertions(+), 147 deletions(-)

crates/notebook/README.md+3-2
...@@ -2,8 +2,9 @@...@@ -2,8 +2,9 @@
22
3The application-facing crate: notebook discovery, a durable local replica with3The application-facing crate: notebook discovery, a durable local replica with
4reconnect reconciliation, external-asset caching, recovery export and optional4reconnect reconciliation, external-asset caching, recovery export and optional
5embedded SMB access. The replica stores a complete working image of one section5embedded SMB access. The replica stores the last observed remote image of one section,
6and the editor's intents in a local SQLite database. `sync_once` provides a6the working image as its difference from that, and the editor's intents in a
7local SQLite database; a save writes the revision it appended, not the section. `sync_once` provides a
7reconciliation step and `start_sync` owns automatic polling and reconnects. Local8reconciliation step and `start_sync` owns automatic polling and reconnects. Local
8success does not acknowledge publication to a shared notebook.9success does not acknowledge publication to a shared notebook.
910
crates/notebook/src/assets.rs+2-6
...@@ -79,12 +79,8 @@ pub(crate) fn key(filename: &str) -> Result<String> {...@@ -79,12 +79,8 @@ pub(crate) fn key(filename: &str) -> Result<String> {
79}79}
8080
81fn referenced(connection: &Connection, key: &str) -> Result<bool> {81fn referenced(connection: &Connection, key: &str) -> Result<bool> {
82 for column in ["working", "base"] {82 let (base, working) = crate::images::both(connection)?;
83 let image: Vec<u8> = connection.query_row(83 for image in [working, base] {
84 &format!("SELECT {column} FROM replica WHERE id=1"),
85 [],
86 |row| row.get(0),
87 )?;
88 let store = Store::parse(&image)?;84 let store = Store::parse(&image)?;
89 let index = RevisionIndex::parse(&store)?;85 let index = RevisionIndex::parse(&store)?;
90 let document = Document::parse(&index)?;86 let document = Document::parse(&index)?;
crates/notebook/src/images.rs created+151
...@@ -0,0 +1,151 @@
1//! The cache's two section images. `base` is stored whole; `working` is stored as the byte
2//! ranges where it differs from `base`, so saving an edit writes the revision it appended
3//! instead of the section.
4
5use crate::Result;
6use rusqlite::{Connection, params};
7use std::io;
8
9const BLOCK: usize = 4096;
10
11/// `image` as its length and the `(offset, bytes)` runs of blocks that differ from `base`;
12/// empty when the images are equal.
13fn difference(base: &[u8], image: &[u8]) -> Vec<u8> {
14 if base == image {
15 return Vec::new();
16 }
17 let mut patch = (image.len() as u64).to_le_bytes().to_vec();
18 let mut run: Option<usize> = None;
19 let close = |patch: &mut Vec<u8>, start: usize, end: usize| {
20 patch.extend_from_slice(&(start as u64).to_le_bytes());
21 patch.extend_from_slice(&((end - start) as u64).to_le_bytes());
22 patch.extend_from_slice(&image[start..end]);
23 };
24 for start in (0..image.len()).step_by(BLOCK) {
25 let end = (start + BLOCK).min(image.len());
26 if base.get(start..end) == Some(&image[start..end]) {
27 if let Some(from) = run.take() {
28 close(&mut patch, from, start);
29 }
30 } else {
31 run.get_or_insert(start);
32 }
33 }
34 if let Some(from) = run {
35 close(&mut patch, from, image.len());
36 }
37 patch
38}
39
40fn restore(mut base: Vec<u8>, patch: &[u8]) -> Result<Vec<u8>> {
41 let damaged = || io::Error::new(io::ErrorKind::InvalidData, "Damaged working image");
42 let Some((length, mut runs)) = patch.split_first_chunk::<8>() else {
43 return if patch.is_empty() {
44 Ok(base)
45 } else {
46 Err(damaged().into())
47 };
48 };
49 base.resize(
50 usize::try_from(u64::from_le_bytes(*length)).map_err(|_| damaged())?,
51 0,
52 );
53 while let Some((header, rest)) = runs.split_first_chunk::<16>() {
54 let offset = usize::try_from(u64::from_le_bytes(header[..8].try_into().unwrap()))
55 .map_err(|_| damaged())?;
56 let size = usize::try_from(u64::from_le_bytes(header[8..].try_into().unwrap()))
57 .map_err(|_| damaged())?;
58 let (bytes, rest) = rest.split_at_checked(size).ok_or_else(damaged)?;
59 base.get_mut(offset..)
60 .and_then(|target| target.get_mut(..size))
61 .ok_or_else(damaged)?
62 .copy_from_slice(bytes);
63 runs = rest;
64 }
65 if runs.is_empty() {
66 Ok(base)
67 } else {
68 Err(damaged().into())
69 }
70}
71
72pub(crate) fn base(connection: &Connection) -> Result<Vec<u8>> {
73 Ok(connection.query_row("SELECT base FROM replica WHERE id=1", [], |row| row.get(0))?)
74}
75
76/// `(base, working)`.
77pub(crate) fn both(connection: &Connection) -> Result<(Vec<u8>, Vec<u8>)> {
78 let (base, patch): (Vec<u8>, Vec<u8>) =
79 connection.query_row("SELECT base, working FROM replica WHERE id=1", [], |row| {
80 Ok((row.get(0)?, row.get(1)?))
81 })?;
82 let working = restore(base.clone(), &patch)?;
83 Ok((base, working))
84}
85
86pub(crate) fn working(connection: &Connection) -> Result<Vec<u8>> {
87 let (base, patch): (Vec<u8>, Vec<u8>) =
88 connection.query_row("SELECT base, working FROM replica WHERE id=1", [], |row| {
89 Ok((row.get(0)?, row.get(1)?))
90 })?;
91 restore(base, &patch)
92}
93
94/// Replaces the working image; `base` is the stored base image.
95pub(crate) fn set_working(connection: &Connection, base: &[u8], working: &[u8]) -> Result<()> {
96 connection.execute(
97 "UPDATE replica SET working=?1 WHERE id=1",
98 [difference(base, working)],
99 )?;
100 Ok(())
101}
102
103/// Replaces the base image, keeping `working` as the working image. An unchanged base
104/// leaves its stored bytes alone.
105pub(crate) fn set_base(
106 connection: &Connection,
107 old: &[u8],
108 base: &[u8],
109 working: &[u8],
110) -> Result<()> {
111 if old == base {
112 return set_working(connection, base, working);
113 }
114 connection.execute(
115 "UPDATE replica SET base=?1, working=?2 WHERE id=1",
116 params![base, difference(base, working)],
117 )?;
118 Ok(())
119}
120
121#[cfg(test)]
122mod tests {
123 use super::*;
124
125 #[test]
126 fn a_working_image_survives_as_its_difference_from_any_base() {
127 let base: Vec<u8> = (0..3 * BLOCK + 17).map(|i| i as u8).collect();
128 let mut edited = base.clone();
129 edited[5] ^= 1;
130 edited[2 * BLOCK + 1] ^= 1;
131 edited.extend_from_slice(b"appended revision");
132 for image in [
133 base.clone(),
134 edited.clone(),
135 base[..BLOCK + 3].to_vec(),
136 Vec::new(),
137 vec![7; 5 * BLOCK],
138 ] {
139 let patch = difference(&base, &image);
140 assert_eq!(restore(base.clone(), &patch).unwrap(), image);
141 }
142 assert!(difference(&base, &base).is_empty());
143 assert!(difference(&base, &edited).len() < 3 * BLOCK);
144 let patch = difference(&base, &edited);
145 for cut in 1..patch.len() {
146 if let Ok(image) = restore(base.clone(), &patch[..cut]) {
147 assert_ne!(image, edited);
148 }
149 }
150 }
151}
crates/notebook/src/lib.rs+9-27
...@@ -20,6 +20,7 @@ use std::{...@@ -20,6 +20,7 @@ use std::{
20};20};
2121
22mod assets;22mod assets;
23mod images;
23mod merge;24mod merge;
24mod pages;25mod pages;
25mod rebase;26mod rebase;
...@@ -59,7 +60,7 @@ pub enum Error {...@@ -59,7 +60,7 @@ pub enum Error {
59type Result<T> = std::result::Result<T, Error>;60type Result<T> = std::result::Result<T, Error>;
6061
61const APPLICATION_ID: u32 = 0x4f4e454f;62const APPLICATION_ID: u32 = 0x4f4e454f;
62const SCHEMA_VERSION: u32 = 13;63const SCHEMA_VERSION: u32 = 14;
6364
64/// An edited page model together with the stored model it was edited from.65/// An edited page model together with the stored model it was edited from.
65/// `before` is the precondition reconciliation checks against the remote page.66/// `before` is the precondition reconciliation checks against the remote page.
...@@ -217,7 +218,7 @@ impl Replica {...@@ -217,7 +218,7 @@ impl Replica {
217 ",218 ",
218 )?;219 )?;
219 schema::create(&transaction)?;220 schema::create(&transaction)?;
220 transaction.execute("INSERT INTO replica VALUES (1, ?1, ?1)", [source])?;221 transaction.execute("INSERT INTO replica VALUES (1, ?1, x'')", [source])?;
221 } else {222 } else {
222 if application != APPLICATION_ID {223 if application != APPLICATION_ID {
223 return Err(224 return Err(
...@@ -251,11 +252,7 @@ impl Replica {...@@ -251,11 +252,7 @@ impl Replica {
251 .connection252 .connection
252 .lock()253 .lock()
253 .map_err(|_| io::Error::other("Cache owner panicked"))?;254 .map_err(|_| io::Error::other("Cache owner panicked"))?;
254 Ok(255 images::working(&connection)
255 connection.query_row("SELECT working FROM replica WHERE id=1", [], |row| {
256 row.get(0)
257 })?,
258 )
259 }256 }
260257
261 pub fn pending(&self) -> Result<Vec<PendingEdit>> {258 pub fn pending(&self) -> Result<Vec<PendingEdit>> {
...@@ -294,10 +291,7 @@ impl Replica {...@@ -294,10 +291,7 @@ impl Replica {
294 .map_err(|_| io::Error::other("Cache owner panicked"))?;291 .map_err(|_| io::Error::other("Cache owner panicked"))?;
295 let transaction =292 let transaction =
296 connection.transaction_with_behavior(TransactionBehavior::Immediate)?;293 connection.transaction_with_behavior(TransactionBehavior::Immediate)?;
297 let current: Vec<u8> =294 let (base, current) = images::both(&transaction)?;
298 transaction.query_row("SELECT working FROM replica WHERE id=1", [], |row| {
299 row.get(0)
300 })?;
301 if current != source {295 if current != source {
302 return Err(io::Error::new(296 return Err(io::Error::new(
303 io::ErrorKind::ResourceBusy,297 io::ErrorKind::ResourceBusy,
...@@ -334,10 +328,7 @@ impl Replica {...@@ -334,10 +328,7 @@ impl Replica {
334 id328 id
335 ],329 ],
336 )?;330 )?;
337 transaction.execute(331 images::set_working(&transaction, &base, prepared.as_bytes())?;
338 "UPDATE replica SET working=?1 WHERE id=1",
339 [prepared.as_bytes()],
340 )?;
341 transaction.commit()?;332 transaction.commit()?;
342 drop(connection);333 drop(connection);
343 self.wake_sync();334 self.wake_sync();
...@@ -388,10 +379,7 @@ impl Replica {...@@ -388,10 +379,7 @@ impl Replica {
388 .lock()379 .lock()
389 .map_err(|_| io::Error::other("Cache owner panicked"))?;380 .map_err(|_| io::Error::other("Cache owner panicked"))?;
390 let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?;381 let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?;
391 let current: Vec<u8> =382 let (base, current) = images::both(&transaction)?;
392 transaction.query_row("SELECT working FROM replica WHERE id=1", [], |row| {
393 row.get(0)
394 })?;
395 if current != source {383 if current != source {
396 return Err(io::Error::new(384 return Err(io::Error::new(
397 io::ErrorKind::ResourceBusy,385 io::ErrorKind::ResourceBusy,
...@@ -410,10 +398,7 @@ impl Replica {...@@ -410,10 +398,7 @@ impl Replica {
410 ],398 ],
411 )?;399 )?;
412 let id = u64::try_from(transaction.last_insert_rowid()).map_err(io::Error::other)?;400 let id = u64::try_from(transaction.last_insert_rowid()).map_err(io::Error::other)?;
413 transaction.execute(401 images::set_working(&transaction, &base, edit.as_bytes())?;
414 "UPDATE replica SET working=?1 WHERE id=1",
415 [edit.as_bytes()],
416 )?;
417 transaction.commit()?;402 transaction.commit()?;
418 drop(connection);403 drop(connection);
419 self.wake_sync();404 self.wake_sync();
...@@ -514,10 +499,7 @@ fn validate_images(connection: &Connection) -> Result<()> {...@@ -514,10 +499,7 @@ fn validate_images(connection: &Connection) -> Result<()> {
514 io::Error::new(io::ErrorKind::InvalidData, "Cache integrity check failed").into(),499 io::Error::new(io::ErrorKind::InvalidData, "Cache integrity check failed").into(),
515 );500 );
516 }501 }
517 let (base, working): (Vec<u8>, Vec<u8>) =502 let (base, working) = images::both(connection)?;
518 connection.query_row("SELECT base, working FROM replica WHERE id=1", [], |row| {
519 Ok((row.get(0)?, row.get(1)?))
520 })?;
521 if validate(&base)? != validate(&working)? {503 if validate(&base)? != validate(&working)? {
522 return Err(io::Error::new(504 return Err(io::Error::new(
523 io::ErrorKind::InvalidData,505 io::ErrorKind::InvalidData,
crates/notebook/src/recovery.rs+6-11
...@@ -58,17 +58,11 @@ impl Recovery {...@@ -58,17 +58,11 @@ impl Recovery {
58 }58 }
5959
60 pub fn snapshot(&self) -> Result<Vec<u8>> {60 pub fn snapshot(&self) -> Result<Vec<u8>> {
61 Ok(self61 crate::images::working(&self.connection)
62 .connection
63 .query_row("SELECT working FROM replica WHERE id=1", [], |row| {
64 row.get(0)
65 })?)
66 }62 }
6763
68 pub fn remote_snapshot(&self) -> Result<Vec<u8>> {64 pub fn remote_snapshot(&self) -> Result<Vec<u8>> {
69 Ok(self65 crate::images::base(&self.connection)
70 .connection
71 .query_row("SELECT base FROM replica WHERE id=1", [], |row| row.get(0))?)
72 }66 }
7367
74 pub fn pending(&self) -> Result<Vec<PendingEdit>> {68 pub fn pending(&self) -> Result<Vec<PendingEdit>> {
...@@ -150,10 +144,11 @@ fn summary(connection: &Connection) -> Result<RecoverySummary> {...@@ -150,10 +144,11 @@ fn summary(connection: &Connection) -> Result<RecoverySummary> {
150 [],144 [],
151 |row| Ok((unsigned(row, 0)?, unsigned(row, 1)?)),145 |row| Ok((unsigned(row, 0)?, unsigned(row, 1)?)),
152 )?;146 )?;
147 let working_bytes = crate::images::working(connection)?.len() as u64;
153 Ok(connection.query_row(148 Ok(connection.query_row(
154 "SELECT (SELECT count(*) FROM edits), (SELECT count(*) FROM conflicts),149 "SELECT (SELECT count(*) FROM edits), (SELECT count(*) FROM conflicts),
155 (SELECT count(*) FROM attempt), (SELECT count(*) FROM receipts),150 (SELECT count(*) FROM attempt), (SELECT count(*) FROM receipts),
156 length(working), length(base) FROM replica WHERE id=1",151 length(base) FROM replica WHERE id=1",
157 [],152 [],
158 |row| {153 |row| {
159 Ok(RecoverySummary {154 Ok(RecoverySummary {
...@@ -161,8 +156,8 @@ fn summary(connection: &Connection) -> Result<RecoverySummary> {...@@ -161,8 +156,8 @@ fn summary(connection: &Connection) -> Result<RecoverySummary> {
161 conflicts: unsigned(row, 1)?,156 conflicts: unsigned(row, 1)?,
162 uncertain_edits: unsigned(row, 2)?,157 uncertain_edits: unsigned(row, 2)?,
163 published_receipts: unsigned(row, 3)?,158 published_receipts: unsigned(row, 3)?,
164 working_bytes: unsigned(row, 4)?,159 working_bytes,
165 remote_bytes: unsigned(row, 5)?,160 remote_bytes: unsigned(row, 4)?,
166 cached_assets,161 cached_assets,
167 cached_asset_bytes,162 cached_asset_bytes,
168 })163 })
crates/notebook/src/sync.rs+25-22
...@@ -63,7 +63,7 @@ impl Replica {...@@ -63,7 +63,7 @@ impl Replica {
63 .connection63 .connection
64 .lock()64 .lock()
65 .map_err(|_| io::Error::other("Cache owner panicked"))?;65 .map_err(|_| io::Error::other("Cache owner panicked"))?;
66 Ok(connection.query_row("SELECT base FROM replica WHERE id=1", [], |row| row.get(0))?)66 images::base(&connection)
67 }67 }
6868
69 /// Reconciles one pending edit, or refreshes the working image when the queue is empty.69 /// Reconciles one pending edit, or refreshes the working image when the queue is empty.
...@@ -79,7 +79,12 @@ impl Replica {...@@ -79,7 +79,12 @@ impl Replica {
79 }79 }
80 let _step = Step(self);80 let _step = Step(self);
81 let snapshot = remote.read().map_err(Error::RemoteIo)?;81 let snapshot = remote.read().map_err(Error::RemoteIo)?;
82 let identity = validate(&snapshot)?;82 // An unchanged remote is the base image, validated when it was stored.
83 let identity = if snapshot == self.remote_snapshot()? {
84 None
85 } else {
86 Some(validate(&snapshot)?)
87 };
83 let (intent, attempted) = {88 let (intent, attempted) = {
84 let mut connection = self89 let mut connection = self
85 .connection90 .connection
...@@ -87,11 +92,10 @@ impl Replica {...@@ -87,11 +92,10 @@ impl Replica {
87 .map_err(|_| io::Error::other("Cache owner panicked"))?;92 .map_err(|_| io::Error::other("Cache owner panicked"))?;
88 let transaction =93 let transaction =
89 connection.transaction_with_behavior(TransactionBehavior::Immediate)?;94 connection.transaction_with_behavior(TransactionBehavior::Immediate)?;
90 let base: Vec<u8> =95 let (base, working) = images::both(&transaction)?;
91 transaction96 if let Some(identity) = identity
92 .query_row("SELECT base FROM replica WHERE id=1", [], |row| row.get(0))?;97 && RevisionIndex::parse(&Store::parse(&base)?)?.root != identity
93 let base_store = Store::parse(&base)?;98 {
94 if RevisionIndex::parse(&base_store)?.root != identity {
95 return Err(io::Error::new(99 return Err(io::Error::new(
96 io::ErrorKind::InvalidInput,100 io::ErrorKind::InvalidInput,
97 "Remote snapshot belongs to another document",101 "Remote snapshot belongs to another document",
...@@ -105,10 +109,7 @@ impl Replica {...@@ -105,10 +109,7 @@ impl Replica {
105 rows.next()?.map(pending_edit).transpose()?109 rows.next()?.map(pending_edit).transpose()?
106 };110 };
107 let Some(intent) = intent else {111 let Some(intent) = intent else {
108 transaction.execute(112 images::set_base(&transaction, &base, &snapshot, &snapshot)?;
109 "UPDATE replica SET base=?1, working=?1 WHERE id=1",
110 [&snapshot],
111 )?;
112 transaction.commit()?;113 transaction.commit()?;
113 return Ok(None);114 return Ok(None);
114 };115 };
...@@ -123,7 +124,7 @@ impl Replica {...@@ -123,7 +124,7 @@ impl Replica {
123 |row| row.get::<_, String>(0),124 |row| row.get::<_, String>(0),
124 )125 )
125 .optional()?;126 .optional()?;
126 transaction.execute("UPDATE replica SET base=?1 WHERE id=1", [&snapshot])?;127 images::set_base(&transaction, &base, &snapshot, &working)?;
127 transaction.commit()?;128 transaction.commit()?;
128 (intent, attempted)129 (intent, attempted)
129 };130 };
...@@ -415,10 +416,7 @@ impl Replica {...@@ -415,10 +416,7 @@ impl Replica {
415 .lock()416 .lock()
416 .map_err(|_| io::Error::other("Cache owner panicked"))?;417 .map_err(|_| io::Error::other("Cache owner panicked"))?;
417 let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?;418 let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?;
418 let (base, working): (Vec<u8>, Vec<u8>) =419 let (base, working) = images::both(&transaction)?;
419 transaction.query_row("SELECT base, working FROM replica WHERE id=1", [], |row| {
420 Ok((row.get(0)?, row.get(1)?))
421 })?;
422 if working != local || base != remote {420 if working != local || base != remote {
423 return Err(io::Error::new(421 return Err(io::Error::new(
424 io::ErrorKind::ResourceBusy,422 io::ErrorKind::ResourceBusy,
...@@ -464,7 +462,7 @@ impl Replica {...@@ -464,7 +462,7 @@ impl Replica {
464 [archive.to_string_lossy().into_owned()],462 [archive.to_string_lossy().into_owned()],
465 )?;463 )?;
466 transaction.execute("DELETE FROM edits", [])?;464 transaction.execute("DELETE FROM edits", [])?;
467 transaction.execute("UPDATE replica SET working=base WHERE id=1", [])?;465 transaction.execute("UPDATE replica SET working=x'' WHERE id=1", [])?;
468 }466 }
469 }467 }
470 transaction.commit()?;468 transaction.commit()?;
...@@ -499,10 +497,7 @@ impl Replica {...@@ -499,10 +497,7 @@ impl Replica {
499 .lock()497 .lock()
500 .map_err(|_| io::Error::other("Cache owner panicked"))?;498 .map_err(|_| io::Error::other("Cache owner panicked"))?;
501 let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?;499 let transaction = connection.transaction_with_behavior(TransactionBehavior::Immediate)?;
502 let (base, working): (Vec<u8>, Vec<u8>) =500 let (base, working) = images::both(&transaction)?;
503 transaction.query_row("SELECT base, working FROM replica WHERE id=1", [], |row| {
504 Ok((row.get(0)?, row.get(1)?))
505 })?;
506 if working != local || base != remote {501 if working != local || base != remote {
507 return Err(io::Error::new(502 return Err(io::Error::new(
508 io::ErrorKind::ResourceBusy,503 io::ErrorKind::ResourceBusy,
...@@ -561,7 +556,15 @@ impl Replica {...@@ -561,7 +556,15 @@ impl Replica {
561 params![id, revision.to_string()],556 params![id, revision.to_string()],
562 )?;557 )?;
563 transaction.execute("DELETE FROM edits WHERE id=?1", [id])?;558 transaction.execute("DELETE FROM edits WHERE id=?1", [id])?;
564 transaction.execute("UPDATE replica SET base=?1, working=CASE WHEN EXISTS(SELECT 1 FROM edits) THEN working ELSE ?1 END WHERE id=1", [snapshot])?;559 let (base, working) = images::both(&transaction)?;
560 let pending: bool =
561 transaction.query_row("SELECT EXISTS(SELECT 1 FROM edits)", [], |row| row.get(0))?;
562 images::set_base(
563 &transaction,
564 &base,
565 snapshot,
566 if pending { &working } else { snapshot },
567 )?;
565 transaction.commit()?;568 transaction.commit()?;
566 Ok(())569 Ok(())
567 }570 }
crates/onestore/src/edit.rs+2-2
...@@ -237,7 +237,7 @@ pub fn replace_text(...@@ -237,7 +237,7 @@ pub fn replace_text(
237 let Some((page, automatic, title_text)) =237 let Some((page, automatic, title_text)) =
238 page_title(revision, &pages, Some((object, &changed)))?238 page_title(revision, &pages, Some((object, &changed)))?
239 else {239 else {
240 return crate::write::replace_objects(source, space, &edits);240 return crate::write::replace_objects(&index, space, &edits);
241 };241 };
242 let Kind::Page {242 let Kind::Page {
243 alternate_title, ..243 alternate_title, ..
...@@ -296,7 +296,7 @@ pub fn replace_text(...@@ -296,7 +296,7 @@ pub fn replace_text(
296 &[]296 &[]
297 },297 },
298 });298 });
299 crate::write::replace_objects(source, space, &edits)299 crate::write::replace_objects(&index, space, &edits)
300}300}
301301
302pub(crate) fn editable_parents(302pub(crate) fn editable_parents(
crates/onestore/src/formatting.rs+2-2
...@@ -3,7 +3,7 @@ use crate::{...@@ -3,7 +3,7 @@ use crate::{
3 create::{current_timestamps, properties, string},3 create::{current_timestamps, properties, string},
4 document::{Document, Kind},4 document::{Document, Kind},
5 edit::editable_parents,5 edit::editable_parents,
6 write::{PropertyObject, fresh_guid, write_revision},6 write::{PropertyObject, fresh_guid, write_revision_on},
7};7};
8use serde::{Deserialize, Serialize};8use serde::{Deserialize, Serialize};
9use std::{9use std::{
...@@ -172,7 +172,7 @@ pub(crate) fn format_text(...@@ -172,7 +172,7 @@ pub(crate) fn format_text(
172 }172 }
173 }173 }
174 let modified = current_timestamps()?.0.to_le_bytes();174 let modified = current_timestamps()?.0.to_le_bytes();
175 write_revision(source, space, |raw| {175 write_revision_on(&index, space, |raw| {
176 let mut target = PropertyObject::from_object(&raw.objects[&object])?;176 let mut target = PropertyObject::from_object(&raw.objects[&object])?;
177 let fields = PropertySets::parse(&target.bytes)?;177 let fields = PropertySets::parse(&target.bytes)?;
178 if fields.sets[0].iter().any(|p| p.id == 0x24003458) {178 if fields.sets[0].iter().any(|p| p.id == 0x24003458) {
crates/onestore/src/insertion.rs+2-2
...@@ -3,7 +3,7 @@ use crate::{...@@ -3,7 +3,7 @@ use crate::{
3 create::{current_timestamps, default_text_style, properties, string},3 create::{current_timestamps, default_text_style, properties, string},
4 document::{Document, Element, Kind},4 document::{Document, Element, Kind},
5 edit::{editable_parents, page_title},5 edit::{editable_parents, page_title},
6 write::{PropertyObject, fresh_guid, write_revision},6 write::{PropertyObject, fresh_guid, write_revision_on},
7};7};
8use serde::{Deserialize, Serialize};8use serde::{Deserialize, Serialize};
9use std::{9use std::{
...@@ -394,7 +394,7 @@ impl Insertion {...@@ -394,7 +394,7 @@ impl Insertion {
394 drop(view);394 drop(view);
395 title395 title
396 };396 };
397 write_revision(source, space, |raw| {397 write_revision_on(&index, space, |raw| {
398 let mut changed = new;398 let mut changed = new;
399 for id in &ancestors {399 for id in &ancestors {
400 let mut object = PropertyObject::from_object(&raw.objects[id])?;400 let mut object = PropertyObject::from_object(&raw.objects[id])?;
crates/onestore/src/objects.rs+32-21
...@@ -277,6 +277,8 @@ impl<'a> RevisionIndex<'a> {...@@ -277,6 +277,8 @@ impl<'a> RevisionIndex<'a> {
277 let mut pending: Vec<&Node<'a>> = revision.nodes.iter().rev().collect();277 let mut pending: Vec<&Node<'a>> = revision.nodes.iter().rev().collect();
278 let mut groups = BTreeSet::new();278 let mut groups = BTreeSet::new();
279 let mut defining_table = false;279 let mut defining_table = false;
280 // Entries of the table being defined; the map is built once, at its end.
281 let mut defining: Vec<(u32, [u8; 16])> = Vec::new();
280 let initial_crc = if self.store.header.file_type == crate::FileType::Section {282 let initial_crc = if self.store.header.file_type == crate::FileType::Section {
281 u32::MAX283 u32::MAX
282 } else {284 } else {
...@@ -333,6 +335,7 @@ impl<'a> RevisionIndex<'a> {...@@ -333,6 +335,7 @@ impl<'a> RevisionIndex<'a> {
333 });335 });
334 }336 }
335 table = Arc::new(GlobalIds::new());337 table = Arc::new(GlobalIds::new());
338 defining.clear();
336 defining_table = true;339 defining_table = true;
337 }340 }
338 0x24..=0x26 => {341 0x24..=0x26 => {
...@@ -343,8 +346,9 @@ impl<'a> RevisionIndex<'a> {...@@ -343,8 +346,9 @@ impl<'a> RevisionIndex<'a> {
343 });346 });
344 }347 }
345 let first = u32::from_le_bytes(c.read()?);348 let first = u32::from_le_bytes(c.read()?);
346 let entries: Vec<_> = match node.id {349 let start = defining.len();
347 0x24 => vec![(first, c.read()?)],350 match node.id {
351 0x24 => defining.push((first, c.read()?)),
348 0x25 | 0x26 => {352 0x25 | 0x26 => {
349 let count = if node.id == 0x26 {353 let count = if node.id == 0x26 {
350 u32::from_le_bytes(c.read()?)354 u32::from_le_bytes(c.read()?)
...@@ -364,30 +368,28 @@ impl<'a> RevisionIndex<'a> {...@@ -364,30 +368,28 @@ impl<'a> RevisionIndex<'a> {
364 message: "Global ID import range exceeds its table",368 message: "Global ID import range exceeds its table",
365 });369 });
366 }370 }
367 let entries: Vec<_> = dependency_table371 defining.extend(
368 .range(first..end)372 dependency_table
369 .map(|(from, guid)| (to + from - first, *guid))373 .range(first..end)
370 .collect();374 .map(|(from, guid)| (to + from - first, *guid)),
371 if entries.len() as u64 != u64::from(count) {375 );
376 if (defining.len() - start) as u64 != u64::from(count) {
372 return Err(Error {377 return Err(Error {
373 offset: node.offset,378 offset: node.offset,
374 message: "Global ID import refers to a missing entry",379 message: "Global ID import refers to a missing entry",
375 });380 });
376 }381 }
377 entries
378 }382 }
379 _ => unreachable!(),383 _ => unreachable!(),
380 };384 }
381 for (index, guid) in entries {385 if defining[start..]
382 if index >= 0xffffff386 .iter()
383 || guid == [0; 16]387 .any(|(index, guid)| *index >= 0xffffff || *guid == [0; 16])
384 || Arc::make_mut(&mut table).insert(index, guid).is_some()388 {
385 {389 return Err(Error {
386 return Err(Error {390 offset: node.offset,
387 offset: node.offset,391 message: "Invalid or repeated global ID entry",
388 message: "Invalid or repeated global ID entry",392 });
389 });
390 }
391 }393 }
392 }394 }
393 0x28 => {395 0x28 => {
...@@ -398,13 +400,22 @@ impl<'a> RevisionIndex<'a> {...@@ -398,13 +400,22 @@ impl<'a> RevisionIndex<'a> {
398 });400 });
399 }401 }
400 defining_table = false;402 defining_table = false;
401 let unique: BTreeSet<_> = table.values().collect();403 defining.sort_unstable();
402 if unique.len() != table.len() {404 if defining.windows(2).any(|pair| pair[0].0 == pair[1].0) {
405 return Err(Error {
406 offset: node.offset,
407 message: "Invalid or repeated global ID entry",
408 });
409 }
410 let mut guids: Vec<_> = defining.iter().map(|(_, guid)| *guid).collect();
411 guids.sort_unstable();
412 if guids.windows(2).any(|pair| pair[0] == pair[1]) {
403 return Err(Error {413 return Err(Error {
404 offset: node.offset,414 offset: node.offset,
405 message: "Global ID table repeats a GUID",415 message: "Global ID table repeats a GUID",
406 });416 });
407 }417 }
418 table = Arc::new(defining.drain(..).collect());
408 }419 }
409 0x59 | 0x5a => {420 0x59 | 0x5a => {
410 let id = if node.id == 0x5a {421 let id = if node.id == 0x5a {
crates/onestore/src/outline.rs+3-2
...@@ -3,7 +3,7 @@ use crate::{...@@ -3,7 +3,7 @@ use crate::{
3 create::current_timestamps,3 create::current_timestamps,
4 document::{Document, Kind},4 document::{Document, Kind},
5 edit::{editable_parents, update_title},5 edit::{editable_parents, update_title},
6 write::{PropertyObject, write_revision},6 write::{PropertyObject, write_revision_on},
7};7};
8use serde::{Deserialize, Serialize};8use serde::{Deserialize, Serialize};
9use std::collections::{BTreeMap, BTreeSet};9use std::collections::{BTreeMap, BTreeSet};
...@@ -105,7 +105,7 @@ impl OutlineEdit {...@@ -105,7 +105,7 @@ impl OutlineEdit {
105 pending.extend(parents.get(&id).into_iter().flatten().copied());105 pending.extend(parents.get(&id).into_iter().flatten().copied());
106 }106 }
107 let modified = current_timestamps()?.0.to_le_bytes();107 let modified = current_timestamps()?.0.to_le_bytes();
108 write_revision(source, space, |raw| {108 write_revision_on(&index, space, |raw| {
109 let mut target = PropertyObject::from_object(&raw.objects[&object])?;109 let mut target = PropertyObject::from_object(&raw.objects[&object])?;
110 target.set(110 target.set(
111 &values111 &values
...@@ -139,6 +139,7 @@ impl OutlineEdit {...@@ -139,6 +139,7 @@ impl OutlineEdit {
139#[cfg(test)]139#[cfg(test)]
140mod tests {140mod tests {
141 use super::*;141 use super::*;
142 use crate::write::write_revision;
142143
143 #[test]144 #[test]
144 fn protection_on_the_target_or_ancestor_prevents_layout_edits() {145 fn protection_on_the_target_or_ancestor_prevents_layout_edits() {
crates/onestore/src/page/write.rs+46-26
...@@ -339,25 +339,34 @@ pub(crate) fn write_page(...@@ -339,25 +339,34 @@ pub(crate) fn write_page(
339 if author.contains('\0') {339 if author.contains('\0') {
340 return Err(invalid("Choose an author name without NUL"));340 return Err(invalid("Choose an author name without NUL"));
341 }341 }
342 let store = Store::parse(source)?;342 // The parsed source is released before the writers parse their own images.
343 let index = RevisionIndex::parse(&store)?;343 let (page, before, existing) = {
344 index.validate_current()?;344 let store = Store::parse(source)?;
345 let document = Document::parse(&index)?;345 let index = RevisionIndex::parse(&store)?;
346 let pages = document.pages_in(space)?;346 index.validate_current()?;
347 let [page] = pages.as_slice() else {347 let document = Document::parse(&index)?;
348 return Err(invalid("Choose an object space containing one active page"));348 let pages = document.pages_in(space)?;
349 let [page] = pages.as_slice() else {
350 return Err(invalid("Choose an object space containing one active page"));
351 };
352 let existing = index
353 .resolve_active(space)?
354 .objects
355 .keys()
356 .copied()
357 .collect();
358 (*page, Page::from_space(&document, space)?, existing)
349 };359 };
350 let before = Page::from_space(&document, space)?;
351 let raw = index.resolve_active(space)?;
352 let mut lowering = Lowering {360 let mut lowering = Lowering {
353 image: source.to_vec(),361 image: source.to_vec(),
362 current: Some(before.clone()),
354 space,363 space,
355 page: *page,364 page,
356 author,365 author,
357 alias: BTreeMap::new(),366 alias: BTreeMap::new(),
358 built: BTreeSet::new(),367 built: BTreeSet::new(),
359 };368 };
360 lowering.run(&before, after, &raw.objects.keys().copied().collect())?;369 lowering.run(&before, after, &existing)?;
361 if lowering.image == source {370 if lowering.image == source {
362 return Ok(lowering.image);371 return Ok(lowering.image);
363 }372 }
...@@ -453,6 +462,8 @@ impl<'a> View<'a> {...@@ -453,6 +462,8 @@ impl<'a> View<'a> {
453462
454struct Lowering<'a> {463struct Lowering<'a> {
455 image: Vec<u8>,464 image: Vec<u8>,
465 /// The page as `image` stores it, until a writer changes the image.
466 current: Option<Page>,
456 space: ExGuid,467 space: ExGuid,
457 page: ExGuid,468 page: ExGuid,
458 author: &'a str,469 author: &'a str,
...@@ -478,15 +489,22 @@ impl Lowering<'_> {...@@ -478,15 +489,22 @@ impl Lowering<'_> {
478 }489 }
479490
480 fn apply(&mut self, edit: impl FnOnce(&[u8]) -> Result<Vec<u8>, Error>) -> Result<(), Error> {491 fn apply(&mut self, edit: impl FnOnce(&[u8]) -> Result<Vec<u8>, Error>) -> Result<(), Error> {
481 self.image = edit(&self.image)?;492 let image = edit(&self.image)?;
493 if image != self.image {
494 self.image = image;
495 self.current = None;
496 }
482 Ok(())497 Ok(())
483 }498 }
484499
485 fn current(&self) -> Result<Page, Error> {500 fn current(&mut self) -> Result<Page, Error> {
486 let store = Store::parse(&self.image)?;501 if self.current.is_none() {
487 let index = RevisionIndex::parse(&store)?;502 let store = Store::parse(&self.image)?;
488 let document = Document::parse(&index)?;503 let index = RevisionIndex::parse(&store)?;
489 Page::from_space(&document, self.space)504 let document = Document::parse(&index)?;
505 self.current = Some(Page::from_space(&document, self.space)?);
506 }
507 Ok(self.current.clone().unwrap())
490 }508 }
491509
492 fn run(510 fn run(
...@@ -2921,10 +2939,6 @@ pub(crate) fn squash(...@@ -2921,10 +2939,6 @@ pub(crate) fn squash(
2921 .iter()2939 .iter()
2922 .map(|(model, image)| (*image, *model))2940 .map(|(model, image)| (*image, *model))
2923 .collect();2941 .collect();
2924 let applied_store = Store::parse(applied)?;
2925 let applied_index = RevisionIndex::parse(&applied_store)?;
2926 // Payloads the typed edits embedded travel into the squashed transaction as well.
2927 let source_store = Store::parse(source)?;
2928 let declared = |store: &Store<'_>| -> Vec<[u8; 16]> {2942 let declared = |store: &Store<'_>| -> Vec<[u8; 16]> {
2929 store2943 store
2930 .lists2944 .lists
...@@ -2934,7 +2948,10 @@ pub(crate) fn squash(...@@ -2934,7 +2948,10 @@ pub(crate) fn squash(
2934 .filter_map(|node| node.payload.get(..16).and_then(|g| g.try_into().ok()))2948 .filter_map(|node| node.payload.get(..16).and_then(|g| g.try_into().ok()))
2935 .collect()2949 .collect()
2936 };2950 };
2937 let existing = declared(&source_store);2951 let existing = declared(&Store::parse(source)?);
2952 let applied_store = Store::parse(applied)?;
2953 let applied_index = RevisionIndex::parse(&applied_store)?;
2954 // Payloads the typed edits embedded travel into the squashed transaction as well.
2938 let mut payloads = Vec::new();2955 let mut payloads = Vec::new();
2939 for guid in declared(&applied_store) {2956 for guid in declared(&applied_store) {
2940 if !existing.contains(&guid) {2957 if !existing.contains(&guid) {
...@@ -3018,10 +3035,13 @@ pub(crate) fn squash(...@@ -3018,10 +3035,13 @@ pub(crate) fn squash(
3018 }3035 }
3019 Ok(changes)3036 Ok(changes)
3020 };3037 };
3021 match protection {3038 let validate = protection.is_none();
3022 Some(_) => crate::write::append_revisions(source, &payloads, protection, edit),3039 let output = crate::write::build(source, &payloads, protection, validate, edit)?;
3023 None => crate::write::write_revisions_with_payloads(source, &payloads, edit),3040 // The parsed images are released before the result is parsed.
3024 }3041 drop(applied_index);
3042 drop(applied_store);
3043 crate::write::check(&output, validate)?;
3044 Ok(output)
3025}3045}
30263046
3027fn remap(object: &mut PropertyObject, rename: &BTreeMap<ExGuid, ExGuid>) -> Result<(), Error> {3047fn remap(object: &mut PropertyObject, rename: &BTreeMap<ExGuid, ExGuid>) -> Result<(), Error> {
crates/onestore/src/paragraph.rs+3-3
...@@ -3,7 +3,7 @@ use crate::{...@@ -3,7 +3,7 @@ use crate::{
3 create::{current_timestamps, properties, string},3 create::{current_timestamps, properties, string},
4 document::{Document, Element, Kind},4 document::{Document, Element, Kind},
5 edit::{editable_parents, update_title},5 edit::{editable_parents, update_title},
6 write::{PropertyObject, fresh_guid, write_revision},6 write::{PropertyObject, fresh_guid, write_revision_on},
7};7};
8use serde::{Deserialize, Serialize};8use serde::{Deserialize, Serialize};
9use std::{9use std::{
...@@ -252,7 +252,7 @@ impl ParagraphSplit {...@@ -252,7 +252,7 @@ impl ParagraphSplit {
252 object.set(&[(0x14001d7a, &modified)])?;252 object.set(&[(0x14001d7a, &modified)])?;
253 }253 }
254 update_title(&store, &raw, view, &pages, &mut changed)?;254 update_title(&store, &raw, view, &pages, &mut changed)?;
255 write_revision(source, space, |_| Ok(changed))255 write_revision_on(&index, space, |_| Ok(changed))
256 }256 }
257}257}
258258
...@@ -682,7 +682,7 @@ impl ParagraphJoin {...@@ -682,7 +682,7 @@ impl ParagraphJoin {
682 pending.extend(parents.get(&id).into_iter().flatten().copied());682 pending.extend(parents.get(&id).into_iter().flatten().copied());
683 }683 }
684 update_title(&store, &raw, view, &pages, &mut changed)?;684 update_title(&store, &raw, view, &pages, &mut changed)?;
685 write_revision(source, space, |_| Ok(changed))685 write_revision_on(&index, space, |_| Ok(changed))
686 }686 }
687}687}
688688
crates/onestore/src/store.rs+1
...@@ -505,6 +505,7 @@ impl<'a> Store<'a> {...@@ -505,6 +505,7 @@ impl<'a> Store<'a> {
505 pending.push(reference);505 pending.push(reference);
506 }506 }
507 }507 }
508 nodes.shrink_to_fit();
508 lists.insert(list_id.unwrap(), NodeList { fragments, nodes });509 lists.insert(list_id.unwrap(), NodeList { fragments, nodes });
509 }510 }
510 Ok(Self {511 Ok(Self {
crates/onestore/src/tree.rs+3-2
...@@ -3,7 +3,7 @@ use crate::{...@@ -3,7 +3,7 @@ use crate::{
3 create::{current_timestamps, properties, string},3 create::{current_timestamps, properties, string},
4 document::{Document, Kind, Revision},4 document::{Document, Kind, Revision},
5 edit::{editable_parents, update_title},5 edit::{editable_parents, update_title},
6 write::{PropertyObject, fresh_guid, write_revision},6 write::{PropertyObject, fresh_guid, write_revision_on},
7};7};
8use serde::{Deserialize, Serialize};8use serde::{Deserialize, Serialize};
9use std::{9use std::{
...@@ -371,7 +371,7 @@ impl TreeEdit {...@@ -371,7 +371,7 @@ impl TreeEdit {
371 changed.insert(self.object, object);371 changed.insert(self.object, object);
372 }372 }
373 update_title(&store, &raw, view, &pages, &mut changed)?;373 update_title(&store, &raw, view, &pages, &mut changed)?;
374 write_revision(source, space, |_| Ok(changed))374 write_revision_on(&index, space, |_| Ok(changed))
375 }375 }
376}376}
377377
...@@ -416,6 +416,7 @@ fn checked_path(...@@ -416,6 +416,7 @@ fn checked_path(
416#[cfg(test)]416#[cfg(test)]
417mod tests {417mod tests {
418 use super::*;418 use super::*;
419 use crate::write::write_revision;
419 use crate::{Insertion, PreparedEdit};420 use crate::{Insertion, PreparedEdit};
420421
421 #[test]422 #[test]
crates/onestore/src/write.rs+82-15
...@@ -305,8 +305,11 @@ pub fn replace_property_bytes(...@@ -305,8 +305,11 @@ pub fn replace_property_bytes(
305 message: "Property does not contain scalar bytes",305 message: "Property does not contain scalar bytes",
306 });306 });
307 }307 }
308 let store = Store::parse(source)?;
309 let index = RevisionIndex::parse(&store)?;
310 index.validate_current()?;
308 replace_objects(311 replace_objects(
309 source,312 &index,
310 space,313 space,
311 &[ObjectEdit {314 &[ObjectEdit {
312 object: object_id,315 object: object_id,
...@@ -316,12 +319,13 @@ pub fn replace_property_bytes(...@@ -316,12 +319,13 @@ pub fn replace_property_bytes(
316 )319 )
317}320}
318321
322/// Patches objects of a source the caller has parsed and validated.
319pub(crate) fn replace_objects(323pub(crate) fn replace_objects(
320 source: &[u8],324 index: &RevisionIndex<'_>,
321 space: ExGuid,325 space: ExGuid,
322 edits: &[ObjectEdit<'_>],326 edits: &[ObjectEdit<'_>],
323) -> Result<Vec<u8>> {327) -> Result<Vec<u8>> {
324 write_revision(source, space, |revision| {328 write_revision_on(index, space, |revision| {
325 let mut changed = BTreeMap::new();329 let mut changed = BTreeMap::new();
326 for edit in edits {330 for edit in edits {
327 if changed.contains_key(&edit.object) {331 if changed.contains_key(&edit.object) {
...@@ -788,23 +792,86 @@ pub(crate) fn write_revisions_with_payloads(...@@ -788,23 +792,86 @@ pub(crate) fn write_revisions_with_payloads(
788 payloads: &[([u8; 16], &[u8])],792 payloads: &[([u8; 16], &[u8])],
789 edit: impl FnOnce(&RevisionIndex<'_>) -> Result<BTreeMap<ExGuid, RevisionEdit>>,793 edit: impl FnOnce(&RevisionIndex<'_>) -> Result<BTreeMap<ExGuid, RevisionEdit>>,
790) -> Result<Vec<u8>> {794) -> Result<Vec<u8>> {
791 let store = Store::parse(source)?;795 publish(source, payloads, None, true, edit)
792 RevisionIndex::parse(&store)?.validate_current()?;
793 let output = append_revisions(source, payloads, None, edit)?;
794 let store = Store::parse(&output)?;
795 RevisionIndex::parse(&store)?.validate_current()?;
796 Ok(output)
797}796}
798797
799/// `write_revisions_with_payloads` without validating that current revisions are798/// `write_revisions_with_payloads` without validating that current revisions are
800/// complete: a protected section is validated by unlocking it, through `protection`.799/// complete: a protected section is validated by unlocking it, through `protection`.
800#[cfg(feature = "protected")]
801pub(crate) fn append_revisions(801pub(crate) fn append_revisions(
802 source: &[u8],802 source: &[u8],
803 payloads: &[([u8; 16], &[u8])],803 payloads: &[([u8; 16], &[u8])],
804 protection: Option<&dyn Protection>,804 protection: Option<&dyn Protection>,
805 edit: impl FnOnce(&RevisionIndex<'_>) -> Result<BTreeMap<ExGuid, RevisionEdit>>,805 edit: impl FnOnce(&RevisionIndex<'_>) -> Result<BTreeMap<ExGuid, RevisionEdit>>,
806) -> Result<Vec<u8>> {
807 publish(source, payloads, protection, false, edit)
808}
809
810fn publish(
811 source: &[u8],
812 payloads: &[([u8; 16], &[u8])],
813 protection: Option<&dyn Protection>,
814 validate: bool,
815 edit: impl FnOnce(&RevisionIndex<'_>) -> Result<BTreeMap<ExGuid, RevisionEdit>>,
816) -> Result<Vec<u8>> {
817 // The parsed source is released before the result is parsed.
818 let output = build(source, payloads, protection, validate, edit)?;
819 check(&output, validate)?;
820 Ok(output)
821}
822
823/// Parses a written image, and with `validate` requires its current revisions complete.
824pub(crate) fn check(output: &[u8], validate: bool) -> Result<()> {
825 let store = Store::parse(output)?;
826 let index = RevisionIndex::parse(&store)?;
827 if validate {
828 index.validate_current()?;
829 }
830 Ok(())
831}
832
833/// The written image, unchecked: `check` follows once the caller has released whatever
834/// `edit` borrowed.
835pub(crate) fn build(
836 source: &[u8],
837 payloads: &[([u8; 16], &[u8])],
838 protection: Option<&dyn Protection>,
839 validate: bool,
840 edit: impl FnOnce(&RevisionIndex<'_>) -> Result<BTreeMap<ExGuid, RevisionEdit>>,
806) -> Result<Vec<u8>> {841) -> Result<Vec<u8>> {
807 let store = Store::parse(source)?;842 let store = Store::parse(source)?;
843 let index = RevisionIndex::parse(&store)?;
844 if validate {
845 index.validate_current()?;
846 }
847 build_on(&index, payloads, protection, edit)
848}
849
850/// `write_revision` on a source the caller has parsed and validated.
851pub(crate) fn write_revision_on(
852 index: &RevisionIndex<'_>,
853 space: ExGuid,
854 edit: impl FnOnce(&crate::ResolvedRevision<'_>) -> Result<BTreeMap<ExGuid, PropertyObject>>,
855) -> Result<Vec<u8>> {
856 let output = build_on(index, &[], None, |index| {
857 let revision = index.resolve(space, index.active(space)?)?;
858 Ok(BTreeMap::from([(
859 space,
860 RevisionEdit::Update(edit(&revision)?),
861 )]))
862 })?;
863 check(&output, true)?;
864 Ok(output)
865}
866
867fn build_on(
868 index: &RevisionIndex<'_>,
869 payloads: &[([u8; 16], &[u8])],
870 protection: Option<&dyn Protection>,
871 edit: impl FnOnce(&RevisionIndex<'_>) -> Result<BTreeMap<ExGuid, RevisionEdit>>,
872) -> Result<Vec<u8>> {
873 let store = index.store;
874 let source = store.data;
808 let is_section = store.header.file_type == FileType::Section;875 let is_section = store.header.file_type == FileType::Section;
809 if !store.checksum_mismatches.is_empty() {876 if !store.checksum_mismatches.is_empty() {
810 return Err(Error {877 return Err(Error {
...@@ -812,13 +879,14 @@ pub(crate) fn append_revisions(...@@ -812,13 +879,14 @@ pub(crate) fn append_revisions(
812 message: "Cannot write a file with transaction checksum damage",879 message: "Cannot write a file with transaction checksum damage",
813 });880 });
814 }881 }
815 let index = RevisionIndex::parse(&store)?;
816 let resolve = |space, rid| match protection {882 let resolve = |space, rid| match protection {
817 Some(protection) => protection.resolve(space, rid),883 Some(protection) => protection.resolve(space, rid),
818 None => index.resolve(space, rid),884 None => index.resolve(space, rid),
819 };885 };
820 let changes = edit(&index)?;886 let changes = edit(index)?;
821 let mut output = source.to_vec();887 // Room for the revision, so appending does not double the image.
888 let mut output = Vec::with_capacity(source.len() + source.len() / 16 + (1 << 16));
889 output.extend_from_slice(source);
822 // Native files reserve 1 KiB per transaction-log fragment; a fragment that ends the file890 // Native files reserve 1 KiB per transaction-log fragment; a fragment that ends the file
823 // keeps that room before the first new chunk.891 // keeps that room before the first new chunk.
824 let tail = store.transaction_fragments.last().unwrap().chunk;892 let tail = store.transaction_fragments.last().unwrap().chunk;
...@@ -1348,12 +1416,12 @@ pub(crate) fn append_revisions(...@@ -1348,12 +1416,12 @@ pub(crate) fn append_revisions(
1348 let space_node = root1416 let space_node = root
1349 .nodes1417 .nodes
1350 .iter()1418 .iter()
1351 .find(|node| node.id == 8 && node.fields(&store).exguid() == Ok(space))1419 .find(|node| node.id == 8 && node.fields(store).exguid() == Ok(space))
1352 .ok_or(Error {1420 .ok_or(Error {
1353 offset: 0,1421 offset: 0,
1354 message: "Object space is absent from the root list",1422 message: "Object space is absent from the root list",
1355 })?;1423 })?;
1356 let space_list = space_node.referenced_list(&store)?;1424 let space_list = space_node.referenced_list(store)?;
1357 let revision_node = space_list.iter().rfind(|node| node.id == 0x10).unwrap();1425 let revision_node = space_list.iter().rfind(|node| node.id == 0x10).unwrap();
1358 let Some(Reference::NodeList(manifest_reference)) = revision_node.reference else {1426 let Some(Reference::NodeList(manifest_reference)) = revision_node.reference else {
1359 unreachable!()1427 unreachable!()
...@@ -1522,6 +1590,5 @@ pub(crate) fn append_revisions(...@@ -1522,6 +1590,5 @@ pub(crate) fn append_revisions(
1522 message: "File generation counter is exhausted",1590 message: "File generation counter is exhausted",
1523 })?;1591 })?;
1524 output[228..236].copy_from_slice(&generation.to_le_bytes());1592 output[228..236].copy_from_slice(&generation.to_le_bytes());
1525 RevisionIndex::parse(&Store::parse(&output)?)?;
1526 Ok(output)1593 Ok(output)
1527}1594}
tools/cache_images.py created+17
...@@ -0,0 +1,17 @@
1"""The cache's working image, stored as the byte ranges where it differs from the base."""
2import struct
3
4
5def working(connection):
6 base, patch = connection.execute('SELECT base, working FROM replica WHERE id=1').fetchone()
7 if not patch: return base
8 length, = struct.unpack_from('<Q', patch)
9 image = bytearray(base[:length].ljust(length, b'\0'))
10 at = 8
11 while at < len(patch):
12 offset, size = struct.unpack_from('<QQ', patch, at)
13 at += 16
14 assert at + size <= len(patch) and offset + size <= length, 'Damaged working image'
15 image[offset:offset + size] = patch[at:at + size]
16 at += size
17 return bytes(image)
tools/offline_publication_crash.py+2-1
...@@ -1,6 +1,7 @@...@@ -1,6 +1,7 @@
1#!/usr/bin/env python31#!/usr/bin/env python3
2"""Kill owned processes across local-cache/remote-file publication boundaries."""2"""Kill owned processes across local-cache/remote-file publication boundaries."""
3import argparse3import argparse
4import cache_images
4import hashlib5import hashlib
5import json6import json
6import os7import os
...@@ -186,7 +187,7 @@ def run(source, output):...@@ -186,7 +187,7 @@ def run(source, output):
186 try:187 try:
187 assert connection.execute('PRAGMA quick_check').fetchall() == [('ok',)]188 assert connection.execute('PRAGMA quick_check').fetchall() == [('ok',)]
188 assert connection.execute('PRAGMA foreign_key_check').fetchall() == []189 assert connection.execute('PRAGMA foreign_key_check').fetchall() == []
189 local_image = save_image(output, connection.execute('SELECT working FROM replica WHERE id=1').fetchone()[0])190 local_image = save_image(output, cache_images.working(connection))
190 finally:191 finally:
191 connection.close()192 connection.close()
192 result = {'case': index, 'phase': phase, 'during_confirmation': confirmation, 'before_status': before['status'], 'after_status': after['status'],193 result = {'case': index, 'phase': phase, 'during_confirmation': confirmation, 'before_status': before['status'], 'after_status': after['status'],
tools/test_offline_history.py+2-2
...@@ -141,7 +141,7 @@ class OfflineLedgerTests(unittest.TestCase):...@@ -141,7 +141,7 @@ class OfflineLedgerTests(unittest.TestCase):
141 connection = sqlite3.connect(output / 'rust' / f'{actor}.sqlite')141 connection = sqlite3.connect(output / 'rust' / f'{actor}.sqlite')
142 connection.executescript('CREATE TABLE receipts(edit_id INTEGER, revision TEXT); CREATE TABLE edits(id INTEGER); CREATE TABLE attempt(id INTEGER); CREATE TABLE conflicts(id INTEGER); CREATE TABLE replica(id INTEGER, base BLOB, working BLOB);')142 connection.executescript('CREATE TABLE receipts(edit_id INTEGER, revision TEXT); CREATE TABLE edits(id INTEGER); CREATE TABLE attempt(id INTEGER); CREATE TABLE conflicts(id INTEGER); CREATE TABLE replica(id INTEGER, base BLOB, working BLOB);')
143 connection.executemany('INSERT INTO receipts VALUES (?,?)', [(op+1, f'{actor}-{op}') for op in range(2)])143 connection.executemany('INSERT INTO receipts VALUES (?,?)', [(op+1, f'{actor}-{op}') for op in range(2)])
144 connection.execute('INSERT INTO replica VALUES (1, ?, ?)', (b'opaque image', b'opaque image'))144 connection.execute('INSERT INTO replica VALUES (1, ?, ?)', (b'opaque image', b''))
145 connection.commit()145 connection.commit()
146 connection.close()146 connection.close()
147 for i in range(4):147 for i in range(4):
...@@ -166,7 +166,7 @@ class OfflineLedgerTests(unittest.TestCase):...@@ -166,7 +166,7 @@ class OfflineLedgerTests(unittest.TestCase):
166 connection = sqlite3.connect(output / 'rust/w0.sqlite')166 connection = sqlite3.connect(output / 'rust/w0.sqlite')
167 for sql, undo in [("UPDATE receipts SET revision='wrong' WHERE edit_id=1", "UPDATE receipts SET revision='w0-0' WHERE edit_id=1"),167 for sql, undo in [("UPDATE receipts SET revision='wrong' WHERE edit_id=1", "UPDATE receipts SET revision='w0-0' WHERE edit_id=1"),
168 ('INSERT INTO attempt VALUES (1)', 'DELETE FROM attempt'),168 ('INSERT INTO attempt VALUES (1)', 'DELETE FROM attempt'),
169 ("UPDATE replica SET working=X'00'", "UPDATE replica SET working=base")]:169 ("UPDATE replica SET working=X'00'", "UPDATE replica SET working=X''")]:
170 connection.execute(sql)170 connection.execute(sql)
171 connection.commit()171 connection.commit()
172 with self.assertRaises(AssertionError): verify(output)172 with self.assertRaises(AssertionError): verify(output)
tools/verify_offline.py+1-1
...@@ -46,7 +46,7 @@ def verify(output, max_gap=120):...@@ -46,7 +46,7 @@ def verify(output, max_gap=120):
46 persisted = dict(connection.execute('SELECT edit_id, revision FROM receipts'))46 persisted = dict(connection.execute('SELECT edit_id, revision FROM receipts'))
47 assert persisted == {id: event['revision'] for id, event in receipts.items()}, 'SQLite receipts differ from observed acknowledgements'47 assert persisted == {id: event['revision'] for id, event in receipts.items()}, 'SQLite receipts differ from observed acknowledgements'
48 base, working = connection.execute('SELECT base, working FROM replica WHERE id=1').fetchone()48 base, working = connection.execute('SELECT base, working FROM replica WHERE id=1').fetchone()
49 assert base == working, 'A drained cache retained a divergent local branch'49 assert working == b'', 'A drained cache retained a divergent local branch'
50 caches[actor] = {'receipts': len(persisted), 'image_bytes': len(base), 'image_sha256': hashlib.sha256(base).hexdigest(), 'database_sha256': hashlib.sha256(path.read_bytes()).hexdigest()}50 caches[actor] = {'receipts': len(persisted), 'image_bytes': len(base), 'image_sha256': hashlib.sha256(base).hexdigest(), 'database_sha256': hashlib.sha256(path.read_bytes()).hexdigest()}
51 finally:51 finally:
52 connection.close()52 connection.close()