authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-10-01 17:35:32-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-10-01 19:43:20-07:00
loga37218b17ff30f35310862f62150db6f4a6dca04
tree92524f427ee40144bea0e7dfe76d81d93f7ab0f2
parent563a6671858bb6446fcc24546f424b3e379f9063
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

feat: the web build keeps its files in OPFS

The browser's notebooks, replicas and settings move from IndexedDB to the origin's private file system. A storage worker owns OPFS's synchronous handles, which only workers get, and writes the byte ranges each burst changed in place: notebook::fs now tracks what was written since the host last asked, so a commit writes its appended revision and the header rather than the whole section, and SQLite its pages. The app itself stays on the main thread, where the shared code's synchronous dialogs work. IndexedDB's files, and the first web build's sections, move over once. Assisted-by: claude-opus-5.5

5 files changed, 241 insertions(+), 113 deletions(-)

arc/platforms.md+5-3
......@@ -156,9 +156,11 @@ keyboard, the toolbar and the macOS menu bar all run commands from it.
156156- The page has one thread. The notebook's section thread, sync worker and background run as
157157 tasks on its event loop (`notebook::task`), and work the desktop gives a thread runs once
158158 the frame is done (`spawn`).
159- Files are `notebook::fs`'s: std's elsewhere, here held in memory and kept in IndexedDB,
160 with SQLite's VFS over the same files, so replicas and notebooks live side by side under
161 `/Notebooks` and `/Cache`. One tab at a time holds them.
159- Files are `notebook::fs`'s: std's elsewhere, here held in memory, with SQLite's VFS over
160 the same files, so replicas and notebooks live side by side under `/Notebooks` and
161 `/Cache`. A storage worker keeps them in the origin's private file system, writing the
162 byte ranges each burst changed through OPFS's synchronous handles, which only workers get.
163 One tab at a time holds them.
162164- Menus are the kit's own, as on Linux, with the PC's chords and ⌘ for Ctrl on a Mac; the
163165 browser keeps its own window and tab chords. Dialogs are the browser's; Open and Insert
164166 ask for files to copy in; printing downloads the PDF. Servers, recording and accessibility
crates/notebook/src/fs/web.rs+92-35
......@@ -1,7 +1,7 @@
11//! The browser's file system: std's calls over files held in memory, which the host loads
2//! before anything opens (`restore`) and writes out as they change (`changes`). SQLite reaches
3//! the same files through `sqlite`, its default VFS here, so a replica is a file like any other.
4//! One thread: no file is ever locked.
2//! before anything opens (`restore`) and writes out as they change, by the byte ranges that
3//! changed (`changes`). SQLite reaches the same files through `sqlite`, its default VFS here,
4//! so a replica is a file like any other. One thread: no file is ever locked.
55
66use onestore::{CommitError, CommitIo, CommitState, Stamp, Transaction};
77use std::{
......@@ -9,6 +9,7 @@ use std::{
99 collections::{BTreeMap, BTreeSet},
1010 ffi::OsString,
1111 io::{self, ErrorKind, Read, Seek, SeekFrom, Write},
12 ops::Range,
1213 path::{Component, Path, PathBuf},
1314 rc::Rc,
1415 time::{Duration, SystemTime, UNIX_EPOCH},
......@@ -23,6 +24,47 @@ struct Data {
2324 modified: f64,
2425 /// Where the file is, which a rename moves; none once it is removed.
2526 path: Option<PathBuf>,
27 /// The ranges written since the host last wrote the file out, or none for all of it.
28 unwritten: Option<Vec<Range<usize>>>,
29}
30
31/// More ranges than this write the whole file out instead.
32const RANGES: usize = 64;
33
34impl Data {
35 fn new(
36 bytes: Vec<u8>,
37 modified: f64,
38 path: PathBuf,
39 unwritten: Option<Vec<Range<usize>>>,
40 ) -> Shared {
41 Rc::new(RefCell::new(Self {
42 bytes,
43 modified,
44 path: Some(path),
45 unwritten,
46 }))
47 }
48
49 /// Notes `range` written now, merged into a written range it meets.
50 fn wrote(&mut self, range: Range<usize>) {
51 self.modified = now();
52 if let Some(ranges) = &mut self.unwritten {
53 match ranges
54 .iter_mut()
55 .find(|known| known.start <= range.end && range.start <= known.end)
56 {
57 Some(known) => *known = known.start.min(range.start)..known.end.max(range.end),
58 None => ranges.push(range),
59 }
60 if ranges.len() > RANGES {
61 self.unwritten = None;
62 }
63 }
64 if let Some(path) = self.path.clone() {
65 with(|files| files.changed.insert(path));
66 }
67 }
2668}
2769
2870type Shared = Rc<RefCell<Data>>;
......@@ -59,35 +101,58 @@ pub fn restore(path: impl AsRef<Path>, saved: Saved) {
59101 FILES.with_borrow_mut(|files| {
60102 let node = match saved {
61103 Saved::Directory => Node::Directory,
62 Saved::File(bytes, modified) => Node::File(Rc::new(RefCell::new(Data {
63 bytes,
64 modified,
65 path: Some(path.clone()),
66 }))),
104 Saved::File(bytes, modified) => {
105 Node::File(Data::new(bytes, modified, path.clone(), Some(Vec::new())))
106 }
67107 };
68108 files.nodes.insert(path, node);
69109 });
70110}
71111
112/// How a path changed, as the host writes it out.
113pub enum Change {
114 Removed,
115 Directory,
116 /// The file's length, and the ranges that changed with their bytes.
117 File {
118 length: u64,
119 ranges: Vec<(u64, Vec<u8>)>,
120 },
121}
122
72123/// Whether any path changed since `changes` was last called.
73124pub fn changed() -> bool {
74125 FILES.with_borrow(|files| !files.changed.is_empty())
75126}
76127
77/// The paths changed since the last call, each with what it holds now, if anything.
78pub fn changes() -> Vec<(PathBuf, Option<Saved>)> {
128/// The paths changed since the last call, each with how; a folder before what it holds.
129pub fn changes() -> Vec<(PathBuf, Change)> {
79130 FILES.with_borrow_mut(|files| {
80131 std::mem::take(&mut files.changed)
81132 .into_iter()
82133 .map(|path| {
83 let saved = files.nodes.get(&path).map(|node| match node {
84 Node::Directory => Saved::Directory,
85 Node::File(data) => {
86 let data = data.borrow();
87 Saved::File(data.bytes.clone(), data.modified)
134 let change = match files.nodes.get(&path) {
135 None => Change::Removed,
136 Some(Node::Directory) => Change::Directory,
137 Some(Node::File(data)) => {
138 let mut data = data.borrow_mut();
139 let length = data.bytes.len();
140 let ranges = data
141 .unwritten
142 .replace(Vec::new())
143 .unwrap_or_else(|| std::iter::once(0..length).collect());
144 Change::File {
145 length: length as u64,
146 ranges: ranges
147 .into_iter()
148 .map(|range| range.start.min(length)..range.end.min(length))
149 .filter(|range| !range.is_empty())
150 .map(|range| (range.start as u64, data.bytes[range].to_vec()))
151 .collect(),
152 }
88153 }
89 });
90 (path, saved)
154 };
155 (path, change)
91156 })
92157 .collect()
93158 })
......@@ -139,11 +204,7 @@ impl Files {
139204 if self.nodes.contains_key(&path) {
140205 return Err(ErrorKind::AlreadyExists.into());
141206 }
142 let data = Rc::new(RefCell::new(Data {
143 bytes: Vec::new(),
144 modified: now(),
145 path: Some(path.clone()),
146 }));
207 let data = Data::new(Vec::new(), now(), path.clone(), None);
147208 self.nodes
148209 .insert(path.clone(), Node::File(Rc::clone(&data)));
149210 self.changed.insert(path);
......@@ -163,14 +224,6 @@ fn with<T>(act: impl FnOnce(&mut Files) -> T) -> T {
163224 FILES.with_borrow_mut(act)
164225}
165226
166/// Marks `data` changed now, where it still has a path.
167fn touched(data: &mut Data) {
168 data.modified = now();
169 if let Some(path) = data.path.clone() {
170 with(|files| files.changed.insert(path));
171 }
172}
173
174227#[derive(Clone, Copy, Debug, PartialEq, Eq)]
175228pub struct FileType(bool);
176229
......@@ -295,7 +348,8 @@ pub fn write(path: impl AsRef<Path>, contents: impl AsRef<[u8]>) -> io::Result<(
295348 let data = with(|files| files.file(&path).or_else(|_| files.create(path.clone())))?;
296349 let mut data = data.borrow_mut();
297350 data.bytes = contents.as_ref().to_vec();
298 touched(&mut data);
351 let length = data.bytes.len();
352 data.wrote(0..length);
299353 Ok(())
300354}
301355
......@@ -425,7 +479,9 @@ pub fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
425479 };
426480 let target = to.join(each.strip_prefix(&from).unwrap_or(Path::new("")));
427481 if let Node::File(data) = &node {
428 data.borrow_mut().path = Some(target.clone());
482 let mut data = data.borrow_mut();
483 data.path = Some(target.clone());
484 data.unwritten = None;
429485 }
430486 files.changed.insert(each);
431487 files.changed.insert(target.clone());
......@@ -546,7 +602,7 @@ impl OpenOptions {
546602 if self.truncate && self.write {
547603 let mut data = data.borrow_mut();
548604 data.bytes.clear();
549 touched(&mut data);
605 data.wrote(0..0);
550606 }
551607 Ok(File {
552608 data,
......@@ -598,8 +654,9 @@ impl File {
598654
599655 pub fn set_len(&self, size: u64) -> io::Result<()> {
600656 let mut data = self.data.borrow_mut();
657 let old = data.bytes.len();
601658 data.bytes.resize(size as usize, 0);
602 touched(&mut data);
659 data.wrote(old.min(size as usize)..size as usize);
603660 Ok(())
604661 }
605662
......@@ -618,7 +675,7 @@ impl File {
618675 data.bytes.resize(end, 0);
619676 }
620677 data.bytes[offset as usize..end].copy_from_slice(input);
621 touched(&mut data);
678 data.wrote(offset as usize..end);
622679 }
623680}
624681
crates/notebook/src/fs/web/sqlite.rs+3-3
......@@ -2,7 +2,7 @@
22//! log and the notebook it caches are kept and written out alike.
33#![allow(unsafe_code)]
44
5use super::{FILES, Node, Shared, normal, touched, with};
5use super::{FILES, Node, Shared, normal, with};
66use rsqlite_vfs::{
77 OsCallback, SQLiteIoMethods, SQLiteVfs, SQLiteVfsFile, VfsError, VfsFile, VfsResult, VfsStore,
88 ffi::{SQLITE_CANTOPEN, SQLITE_IOERR, SQLITE_IOERR_DELETE, sqlite3_vfs},
......@@ -37,14 +37,14 @@ impl VfsFile for File {
3737 data.bytes.resize(end, 0);
3838 }
3939 data.bytes[offset..end].copy_from_slice(input);
40 touched(&mut data);
40 data.wrote(offset..end);
4141 Ok(())
4242 }
4343
4444 fn truncate(&mut self, size: usize) -> VfsResult<()> {
4545 let mut data = self.0.borrow_mut();
4646 data.bytes.truncate(size);
47 touched(&mut data);
47 data.wrote(size..size);
4848 Ok(())
4949 }
5050
crates/snowbound/src/web.rs+20-11
......@@ -27,7 +27,7 @@ use winit::{
2727#[wasm_bindgen(module = "/web/glue.js")]
2828extern "C" {
2929 fn attach(module: JsValue);
30 /// The files IndexedDB kept: `[path, bytes or null for a folder, modified]`.
30 /// The files the browser kept: `[path, bytes or null for a folder, modified]`.
3131 #[wasm_bindgen(js_name = loadFiles, catch)]
3232 async fn load_files() -> Result<js_sys::Array, JsValue>;
3333 #[wasm_bindgen(js_name = requestFrame)]
......@@ -58,8 +58,8 @@ extern "C" {
5858 fn tell(message: &str);
5959 #[wasm_bindgen(js_name = openLink)]
6060 fn open_link(url: &str);
61 /// Writes changed files to IndexedDB: `[path, bytes or null for a folder or undefined
62 /// for a removal, modified]` triples.
61 /// Writes changes out: `[path]` removed, `[path, null]` a folder, and `[path, length,
62 /// [[offset, bytes], ...]]` a file's new length and the ranges that changed.
6363 #[wasm_bindgen(js_name = storeFiles)]
6464 fn store_files(changes: js_sys::Array);
6565}
......@@ -710,19 +710,28 @@ fn restore(files: js_sys::Array) {
710710 }
711711}
712712
713/// Writes the files changed since the last time out to IndexedDB.
713/// Writes the files changed since the last time out to the browser's storage.
714714fn store() {
715 use notebook::fs::Change;
715716 STORE_DUE.set(None);
716717 let changes = js_sys::Array::new();
717 for (path, saved) in notebook::fs::changes() {
718 for (path, change) in notebook::fs::changes() {
718719 let path = JsValue::from_str(&path.to_string_lossy());
719 let entry = match saved {
720 None => js_sys::Array::of1(&path),
721 Some(notebook::fs::Saved::Directory) => js_sys::Array::of2(&path, &JsValue::NULL),
722 Some(notebook::fs::Saved::File(bytes, modified)) => js_sys::Array::of3(
720 let entry = match change {
721 Change::Removed => js_sys::Array::of1(&path),
722 Change::Directory => js_sys::Array::of2(&path, &JsValue::NULL),
723 Change::File { length, ranges } => js_sys::Array::of3(
723724 &path,
724 &js_sys::Uint8Array::from(bytes.as_slice()),
725 &modified.into(),
725 &(length as f64).into(),
726 &ranges
727 .into_iter()
728 .map(|(offset, bytes)| {
729 js_sys::Array::of2(
730 &(offset as f64).into(),
731 &js_sys::Uint8Array::from(bytes.as_slice()),
732 )
733 })
734 .collect::<js_sys::Array>(),
726735 ),
727736 };
728737 changes.push(&entry);
crates/snowbound/web/glue.js+121-61
......@@ -1,5 +1,6 @@
11// The browser's half of Snowbound (src/web.rs): the page's input, its dialogs and files, and
2// IndexedDB, which keeps the files `notebook::fs` holds between visits.
2// the files `notebook::fs` holds, which a storage worker keeps in the origin's private file
3// system (OPFS) between visits.
34
45let wasm;
56let canvas;
......@@ -7,75 +8,134 @@ let input;
78let picker;
89let framePending = false;
910let wakeTimer;
11let storage;
1012
11const DB = "snowbound";
12// `files` holds every path: `{path, bytes}`, `bytes` null for a folder. `sections` is the first
13// web build's store, whose sections move into a notebook once.
14const FILES = "files";
15const SECTIONS = "sections";
16const MIGRATED = "/Notebooks/Web Notebook";
13/** The storage worker: the only place OPFS hands out synchronous handles, which write a
14 * file's changed ranges in place. Runs as a worker of its own, from this source. */
15function storageWorker() {
16 // IndexedDB held the files before OPFS did: `files` by path, and before that the first web
17 // build's `sections`, which move into a notebook of their own.
18 const DB = "snowbound";
19 const MIGRATED = "/Notebooks/Web Notebook";
20 const handles = new Map();
21 let root;
22 let queue = Promise.resolve();
1723
18function database() {
19 return new Promise((resolve, reject) => {
20 const open = indexedDB.open(DB, 2);
21 open.onupgradeneeded = () => {
22 const db = open.result;
23 if (!db.objectStoreNames.contains(FILES)) db.createObjectStore(FILES, { keyPath: "path" });
24 if (!db.objectStoreNames.contains(SECTIONS))
25 db.createObjectStore(SECTIONS, { keyPath: "file" });
26 };
27 open.onsuccess = () => resolve(open.result);
28 open.onerror = () => reject(open.error);
29 });
24 const names = (path) => path.split("/").filter(Boolean);
25 const folder = async (parts) => {
26 let dir = root;
27 for (const name of parts) dir = await dir.getDirectoryHandle(name, { create: true });
28 return dir;
29 };
30 const handle = async (path) => {
31 if (!handles.has(path)) {
32 const parts = names(path);
33 const file = await (await folder(parts.slice(0, -1))).getFileHandle(parts.at(-1), { create: true });
34 handles.set(path, await file.createSyncAccessHandle());
35 }
36 return handles.get(path);
37 };
38 const close = (path) => {
39 for (const [held, open] of handles)
40 if (held === path || held.startsWith(`${path}/`)) {
41 open.close();
42 handles.delete(held);
43 }
44 };
45 const write = async (path, length, ranges) => {
46 const open = await handle(path);
47 for (const [offset, bytes] of ranges) open.write(bytes, { at: offset });
48 open.truncate(length);
49 open.flush();
50 };
51 const remove = async (path) => {
52 close(path);
53 const parts = names(path);
54 try {
55 await (await folder(parts.slice(0, -1))).removeEntry(parts.at(-1), { recursive: true });
56 } catch (error) {
57 if (error.name !== "NotFoundError") throw error;
58 }
59 };
60 const list = async (dir, path, out) => {
61 for await (const [name, entry] of dir.entries()) {
62 const at = `${path}/${name}`;
63 if (entry.kind === "directory") {
64 out.push([at, null, 0]);
65 await list(entry, at, out);
66 } else {
67 const file = await entry.getFile();
68 out.push([at, new Uint8Array(await file.arrayBuffer()), file.lastModified]);
69 }
70 }
71 return out;
72 };
73 const migrate = async () => {
74 const db = await new Promise((resolve, reject) => {
75 const open = indexedDB.open(DB);
76 open.onsuccess = () => resolve(open.result);
77 open.onerror = () => reject(open.error);
78 });
79 const stores = [...db.objectStoreNames];
80 const all = (store) =>
81 new Promise((resolve, reject) => {
82 const request = db.transaction(store).objectStore(store).getAll();
83 request.onsuccess = () => resolve(request.result);
84 request.onerror = () => reject(request.error);
85 });
86 const files = stores.includes("files") ? await all("files") : [];
87 const sections = stores.includes("sections") ? await all("sections") : [];
88 db.close();
89 const moved = files.length
90 ? files
91 : sections
92 .sort((a, b) => a.order - b.order)
93 .map(({ file, bytes }) => ({ path: `${MIGRATED}/${file}`, bytes }));
94 for (const { path, bytes } of moved)
95 if (bytes) await write(path, bytes.length, [[0, bytes]]);
96 else await folder(names(path));
97 await new Promise((resolve) => {
98 const removal = indexedDB.deleteDatabase(DB);
99 removal.onsuccess = removal.onerror = removal.onblocked = resolve;
100 });
101 };
102
103 onmessage = ({ data }) => {
104 queue = queue
105 .then(async () => {
106 if (data.kind === "load") {
107 root = await navigator.storage.getDirectory();
108 let files = await list(root, "", []);
109 if (!files.length) {
110 await migrate();
111 files = await list(root, "", []);
112 }
113 postMessage(files, files.flatMap(([, bytes]) => (bytes ? [bytes.buffer] : [])));
114 } else
115 for (const [path, length, ranges] of data.changes)
116 if (length === undefined) await remove(path);
117 else if (length === null) await folder(names(path));
118 else await write(path, length, ranges);
119 })
120 .catch((error) => console.error("Keeping files", error));
121 };
30122}
31123
32function done(transaction) {
124/** Every file kept, as `[path, bytes or null for a folder, modified]`. */
125export function loadFiles() {
126 const source = URL.createObjectURL(new Blob([`(${storageWorker})()`], { type: "text/javascript" }));
127 storage = new Worker(source);
128 storage.postMessage({ kind: "load" });
33129 return new Promise((resolve, reject) => {
34 transaction.oncomplete = resolve;
35 transaction.onerror = () => reject(transaction.error);
36 transaction.onabort = () => reject(transaction.error);
130 storage.onmessage = ({ data }) => resolve(data);
131 storage.onerror = (error) => reject(new Error(`The storage worker failed: ${error.message}`));
37132 });
38133}
39134
40/** Every file kept, as `[path, bytes or null, modified]`, the first web build's sections moved
41 * into a notebook of their own first. */
42export async function loadFiles() {
43 const db = await database();
44 const read = db.transaction([FILES, SECTIONS], "readonly");
45 const files = read.objectStore(FILES).getAll();
46 const sections = read.objectStore(SECTIONS).getAll();
47 await done(read);
48 let rows = files.result;
49 if (!rows.length && sections.result.length) {
50 const moved = [
51 { path: "/Notebooks", bytes: null },
52 { path: MIGRATED, bytes: null },
53 ...sections.result
54 .sort((a, b) => a.order - b.order)
55 .map(({ file, bytes }) => ({ path: `${MIGRATED}/${file}`, bytes, modified: Date.now() })),
56 ];
57 const write = db.transaction([FILES, SECTIONS], "readwrite");
58 for (const row of moved) write.objectStore(FILES).put(row);
59 write.objectStore(SECTIONS).clear();
60 await done(write);
61 rows = moved;
62 }
63 return rows.map(({ path, bytes, modified }) => [path, bytes, modified ?? 0]);
64}
65
66/** Writes `[path]` (removed), `[path, null]` (a folder) and `[path, bytes, modified]` entries. */
135/** Writes `[path]` (removed), `[path, null]` (a folder) and `[path, length, ranges]` entries. */
67136export function storeFiles(changes) {
68 database()
69 .then((db) => {
70 const transaction = db.transaction(FILES, "readwrite");
71 const store = transaction.objectStore(FILES);
72 for (const [path, bytes, modified] of changes) {
73 if (bytes === undefined) store.delete(path);
74 else store.put({ path, bytes, modified });
75 }
76 return done(transaction);
77 })
78 .catch((error) => console.error("Keeping files", error));
137 const buffers = changes.flatMap(([, , ranges]) => (ranges ?? []).map(([, bytes]) => bytes.buffer));
138 storage.postMessage({ kind: "store", changes }, buffers);
79139}
80140
81141export function requestFrame() {