From a37218b17ff30f35310862f62150db6f4a6dca04 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Thu, 1 Oct 2026 17:35:32 -0700 Subject: [PATCH] feat: the web build keeps its files in OPFS The browser's notebooks, replicas and settings move from IndexedDB to the origin's private file system. A storage worker owns OPFS's synchronous handles, which only workers get, and writes the byte ranges each burst changed in place: notebook::fs now tracks what was written since the host last asked, so a commit writes its appended revision and the header rather than the whole section, and SQLite its pages. The app itself stays on the main thread, where the shared code's synchronous dialogs work. IndexedDB's files, and the first web build's sections, move over once. Assisted-by: claude-opus-5.5 --- arc/platforms.md | 8 +- crates/notebook/src/fs/web.rs | 127 +++++++++++++------ crates/notebook/src/fs/web/sqlite.rs | 6 +- crates/snowbound/src/web.rs | 31 +++-- crates/snowbound/web/glue.js | 182 ++++++++++++++++++--------- 5 files changed, 241 insertions(+), 113 deletions(-) diff --git a/arc/platforms.md b/arc/platforms.md index 2753dcfc15e846a2ce29a3296833c281fdaecb5b..29378ae749ac538a4344dfe8cd04b4f83406f011 100644 --- a/arc/platforms.md +++ b/arc/platforms.md @@ -156,9 +156,11 @@ keyboard, the toolbar and the macOS menu bar all run commands from it. - The page has one thread. The notebook's section thread, sync worker and background run as tasks on its event loop (`notebook::task`), and work the desktop gives a thread runs once the frame is done (`spawn`). -- Files are `notebook::fs`'s: std's elsewhere, here held in memory and kept in IndexedDB, - with SQLite's VFS over the same files, so replicas and notebooks live side by side under - `/Notebooks` and `/Cache`. One tab at a time holds them. +- Files are `notebook::fs`'s: std's elsewhere, here held in memory, with SQLite's VFS over + the same files, so replicas and notebooks live side by side under `/Notebooks` and + `/Cache`. A storage worker keeps them in the origin's private file system, writing the + byte ranges each burst changed through OPFS's synchronous handles, which only workers get. + One tab at a time holds them. - Menus are the kit's own, as on Linux, with the PC's chords and ⌘ for Ctrl on a Mac; the browser keeps its own window and tab chords. Dialogs are the browser's; Open and Insert ask for files to copy in; printing downloads the PDF. Servers, recording and accessibility diff --git a/crates/notebook/src/fs/web.rs b/crates/notebook/src/fs/web.rs index c130e7e830c1f01ef810ca2a2280e6cbc4e7876c..2ee83f22727f963b5f4e854cf219e2b54b626f3b 100644 --- a/crates/notebook/src/fs/web.rs +++ b/crates/notebook/src/fs/web.rs @@ -1,7 +1,7 @@ //! The browser's file system: std's calls over files held in memory, which the host loads -//! before anything opens (`restore`) and writes out as they change (`changes`). SQLite reaches -//! the same files through `sqlite`, its default VFS here, so a replica is a file like any other. -//! One thread: no file is ever locked. +//! before anything opens (`restore`) and writes out as they change, by the byte ranges that +//! changed (`changes`). SQLite reaches the same files through `sqlite`, its default VFS here, +//! so a replica is a file like any other. One thread: no file is ever locked. use onestore::{CommitError, CommitIo, CommitState, Stamp, Transaction}; use std::{ @@ -9,6 +9,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, ffi::OsString, io::{self, ErrorKind, Read, Seek, SeekFrom, Write}, + ops::Range, path::{Component, Path, PathBuf}, rc::Rc, time::{Duration, SystemTime, UNIX_EPOCH}, @@ -23,6 +24,47 @@ struct Data { modified: f64, /// Where the file is, which a rename moves; none once it is removed. path: Option, + /// The ranges written since the host last wrote the file out, or none for all of it. + unwritten: Option>>, +} + +/// More ranges than this write the whole file out instead. +const RANGES: usize = 64; + +impl Data { + fn new( + bytes: Vec, + modified: f64, + path: PathBuf, + unwritten: Option>>, + ) -> Shared { + Rc::new(RefCell::new(Self { + bytes, + modified, + path: Some(path), + unwritten, + })) + } + + /// Notes `range` written now, merged into a written range it meets. + fn wrote(&mut self, range: Range) { + self.modified = now(); + if let Some(ranges) = &mut self.unwritten { + match ranges + .iter_mut() + .find(|known| known.start <= range.end && range.start <= known.end) + { + Some(known) => *known = known.start.min(range.start)..known.end.max(range.end), + None => ranges.push(range), + } + if ranges.len() > RANGES { + self.unwritten = None; + } + } + if let Some(path) = self.path.clone() { + with(|files| files.changed.insert(path)); + } + } } type Shared = Rc>; @@ -59,35 +101,58 @@ pub fn restore(path: impl AsRef, saved: Saved) { FILES.with_borrow_mut(|files| { let node = match saved { Saved::Directory => Node::Directory, - Saved::File(bytes, modified) => Node::File(Rc::new(RefCell::new(Data { - bytes, - modified, - path: Some(path.clone()), - }))), + Saved::File(bytes, modified) => { + Node::File(Data::new(bytes, modified, path.clone(), Some(Vec::new()))) + } }; files.nodes.insert(path, node); }); } +/// How a path changed, as the host writes it out. +pub enum Change { + Removed, + Directory, + /// The file's length, and the ranges that changed with their bytes. + File { + length: u64, + ranges: Vec<(u64, Vec)>, + }, +} + /// Whether any path changed since `changes` was last called. pub fn changed() -> bool { FILES.with_borrow(|files| !files.changed.is_empty()) } -/// The paths changed since the last call, each with what it holds now, if anything. -pub fn changes() -> Vec<(PathBuf, Option)> { +/// The paths changed since the last call, each with how; a folder before what it holds. +pub fn changes() -> Vec<(PathBuf, Change)> { FILES.with_borrow_mut(|files| { std::mem::take(&mut files.changed) .into_iter() .map(|path| { - let saved = files.nodes.get(&path).map(|node| match node { - Node::Directory => Saved::Directory, - Node::File(data) => { - let data = data.borrow(); - Saved::File(data.bytes.clone(), data.modified) + let change = match files.nodes.get(&path) { + None => Change::Removed, + Some(Node::Directory) => Change::Directory, + Some(Node::File(data)) => { + let mut data = data.borrow_mut(); + let length = data.bytes.len(); + let ranges = data + .unwritten + .replace(Vec::new()) + .unwrap_or_else(|| std::iter::once(0..length).collect()); + Change::File { + length: length as u64, + ranges: ranges + .into_iter() + .map(|range| range.start.min(length)..range.end.min(length)) + .filter(|range| !range.is_empty()) + .map(|range| (range.start as u64, data.bytes[range].to_vec())) + .collect(), + } } - }); - (path, saved) + }; + (path, change) }) .collect() }) @@ -139,11 +204,7 @@ impl Files { if self.nodes.contains_key(&path) { return Err(ErrorKind::AlreadyExists.into()); } - let data = Rc::new(RefCell::new(Data { - bytes: Vec::new(), - modified: now(), - path: Some(path.clone()), - })); + let data = Data::new(Vec::new(), now(), path.clone(), None); self.nodes .insert(path.clone(), Node::File(Rc::clone(&data))); self.changed.insert(path); @@ -163,14 +224,6 @@ fn with(act: impl FnOnce(&mut Files) -> T) -> T { FILES.with_borrow_mut(act) } -/// Marks `data` changed now, where it still has a path. -fn touched(data: &mut Data) { - data.modified = now(); - if let Some(path) = data.path.clone() { - with(|files| files.changed.insert(path)); - } -} - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct FileType(bool); @@ -295,7 +348,8 @@ pub fn write(path: impl AsRef, contents: impl AsRef<[u8]>) -> io::Result<( let data = with(|files| files.file(&path).or_else(|_| files.create(path.clone())))?; let mut data = data.borrow_mut(); data.bytes = contents.as_ref().to_vec(); - touched(&mut data); + let length = data.bytes.len(); + data.wrote(0..length); Ok(()) } @@ -425,7 +479,9 @@ pub fn rename(from: impl AsRef, to: impl AsRef) -> io::Result<()> { }; let target = to.join(each.strip_prefix(&from).unwrap_or(Path::new(""))); if let Node::File(data) = &node { - data.borrow_mut().path = Some(target.clone()); + let mut data = data.borrow_mut(); + data.path = Some(target.clone()); + data.unwritten = None; } files.changed.insert(each); files.changed.insert(target.clone()); @@ -546,7 +602,7 @@ impl OpenOptions { if self.truncate && self.write { let mut data = data.borrow_mut(); data.bytes.clear(); - touched(&mut data); + data.wrote(0..0); } Ok(File { data, @@ -598,8 +654,9 @@ impl File { pub fn set_len(&self, size: u64) -> io::Result<()> { let mut data = self.data.borrow_mut(); + let old = data.bytes.len(); data.bytes.resize(size as usize, 0); - touched(&mut data); + data.wrote(old.min(size as usize)..size as usize); Ok(()) } @@ -618,7 +675,7 @@ impl File { data.bytes.resize(end, 0); } data.bytes[offset as usize..end].copy_from_slice(input); - touched(&mut data); + data.wrote(offset as usize..end); } } diff --git a/crates/notebook/src/fs/web/sqlite.rs b/crates/notebook/src/fs/web/sqlite.rs index c8dae1c369229f2ccffc175e45d55f155fdee056..f9915a8b4192dfcba54f0ba6d03c45bcd92792eb 100644 --- a/crates/notebook/src/fs/web/sqlite.rs +++ b/crates/notebook/src/fs/web/sqlite.rs @@ -2,7 +2,7 @@ //! log and the notebook it caches are kept and written out alike. #![allow(unsafe_code)] -use super::{FILES, Node, Shared, normal, touched, with}; +use super::{FILES, Node, Shared, normal, with}; use rsqlite_vfs::{ OsCallback, SQLiteIoMethods, SQLiteVfs, SQLiteVfsFile, VfsError, VfsFile, VfsResult, VfsStore, ffi::{SQLITE_CANTOPEN, SQLITE_IOERR, SQLITE_IOERR_DELETE, sqlite3_vfs}, @@ -37,14 +37,14 @@ impl VfsFile for File { data.bytes.resize(end, 0); } data.bytes[offset..end].copy_from_slice(input); - touched(&mut data); + data.wrote(offset..end); Ok(()) } fn truncate(&mut self, size: usize) -> VfsResult<()> { let mut data = self.0.borrow_mut(); data.bytes.truncate(size); - touched(&mut data); + data.wrote(size..size); Ok(()) } diff --git a/crates/snowbound/src/web.rs b/crates/snowbound/src/web.rs index 88ad9b1f7286c72fe830a018cb71dcd2ed02fd05..bb658efbc4529ab8057ca76492930b307ebd948d 100644 --- a/crates/snowbound/src/web.rs +++ b/crates/snowbound/src/web.rs @@ -27,7 +27,7 @@ use winit::{ #[wasm_bindgen(module = "/web/glue.js")] extern "C" { fn attach(module: JsValue); - /// The files IndexedDB kept: `[path, bytes or null for a folder, modified]`. + /// The files the browser kept: `[path, bytes or null for a folder, modified]`. #[wasm_bindgen(js_name = loadFiles, catch)] async fn load_files() -> Result; #[wasm_bindgen(js_name = requestFrame)] @@ -58,8 +58,8 @@ extern "C" { fn tell(message: &str); #[wasm_bindgen(js_name = openLink)] fn open_link(url: &str); - /// Writes changed files to IndexedDB: `[path, bytes or null for a folder or undefined - /// for a removal, modified]` triples. + /// Writes changes out: `[path]` removed, `[path, null]` a folder, and `[path, length, + /// [[offset, bytes], ...]]` a file's new length and the ranges that changed. #[wasm_bindgen(js_name = storeFiles)] fn store_files(changes: js_sys::Array); } @@ -710,19 +710,28 @@ fn restore(files: js_sys::Array) { } } -/// Writes the files changed since the last time out to IndexedDB. +/// Writes the files changed since the last time out to the browser's storage. fn store() { + use notebook::fs::Change; STORE_DUE.set(None); let changes = js_sys::Array::new(); - for (path, saved) in notebook::fs::changes() { + for (path, change) in notebook::fs::changes() { let path = JsValue::from_str(&path.to_string_lossy()); - let entry = match saved { - None => js_sys::Array::of1(&path), - Some(notebook::fs::Saved::Directory) => js_sys::Array::of2(&path, &JsValue::NULL), - Some(notebook::fs::Saved::File(bytes, modified)) => js_sys::Array::of3( + let entry = match change { + Change::Removed => js_sys::Array::of1(&path), + Change::Directory => js_sys::Array::of2(&path, &JsValue::NULL), + Change::File { length, ranges } => js_sys::Array::of3( &path, - &js_sys::Uint8Array::from(bytes.as_slice()), - &modified.into(), + &(length as f64).into(), + &ranges + .into_iter() + .map(|(offset, bytes)| { + js_sys::Array::of2( + &(offset as f64).into(), + &js_sys::Uint8Array::from(bytes.as_slice()), + ) + }) + .collect::(), ), }; changes.push(&entry); diff --git a/crates/snowbound/web/glue.js b/crates/snowbound/web/glue.js index 39922340f7521d04e2887f6bf4e1062f345b30ab..509683f6cc8e8a320cd43db71682d95eed322101 100644 --- a/crates/snowbound/web/glue.js +++ b/crates/snowbound/web/glue.js @@ -1,5 +1,6 @@ // The browser's half of Snowbound (src/web.rs): the page's input, its dialogs and files, and -// IndexedDB, which keeps the files `notebook::fs` holds between visits. +// the files `notebook::fs` holds, which a storage worker keeps in the origin's private file +// system (OPFS) between visits. let wasm; let canvas; @@ -7,75 +8,134 @@ let input; let picker; let framePending = false; let wakeTimer; +let storage; -const DB = "snowbound"; -// `files` holds every path: `{path, bytes}`, `bytes` null for a folder. `sections` is the first -// web build's store, whose sections move into a notebook once. -const FILES = "files"; -const SECTIONS = "sections"; -const MIGRATED = "/Notebooks/Web Notebook"; +/** The storage worker: the only place OPFS hands out synchronous handles, which write a + * file's changed ranges in place. Runs as a worker of its own, from this source. */ +function storageWorker() { + // IndexedDB held the files before OPFS did: `files` by path, and before that the first web + // build's `sections`, which move into a notebook of their own. + const DB = "snowbound"; + const MIGRATED = "/Notebooks/Web Notebook"; + const handles = new Map(); + let root; + let queue = Promise.resolve(); -function database() { - return new Promise((resolve, reject) => { - const open = indexedDB.open(DB, 2); - open.onupgradeneeded = () => { - const db = open.result; - if (!db.objectStoreNames.contains(FILES)) db.createObjectStore(FILES, { keyPath: "path" }); - if (!db.objectStoreNames.contains(SECTIONS)) - db.createObjectStore(SECTIONS, { keyPath: "file" }); - }; - open.onsuccess = () => resolve(open.result); - open.onerror = () => reject(open.error); - }); + const names = (path) => path.split("/").filter(Boolean); + const folder = async (parts) => { + let dir = root; + for (const name of parts) dir = await dir.getDirectoryHandle(name, { create: true }); + return dir; + }; + const handle = async (path) => { + if (!handles.has(path)) { + const parts = names(path); + const file = await (await folder(parts.slice(0, -1))).getFileHandle(parts.at(-1), { create: true }); + handles.set(path, await file.createSyncAccessHandle()); + } + return handles.get(path); + }; + const close = (path) => { + for (const [held, open] of handles) + if (held === path || held.startsWith(`${path}/`)) { + open.close(); + handles.delete(held); + } + }; + const write = async (path, length, ranges) => { + const open = await handle(path); + for (const [offset, bytes] of ranges) open.write(bytes, { at: offset }); + open.truncate(length); + open.flush(); + }; + const remove = async (path) => { + close(path); + const parts = names(path); + try { + await (await folder(parts.slice(0, -1))).removeEntry(parts.at(-1), { recursive: true }); + } catch (error) { + if (error.name !== "NotFoundError") throw error; + } + }; + const list = async (dir, path, out) => { + for await (const [name, entry] of dir.entries()) { + const at = `${path}/${name}`; + if (entry.kind === "directory") { + out.push([at, null, 0]); + await list(entry, at, out); + } else { + const file = await entry.getFile(); + out.push([at, new Uint8Array(await file.arrayBuffer()), file.lastModified]); + } + } + return out; + }; + const migrate = async () => { + const db = await new Promise((resolve, reject) => { + const open = indexedDB.open(DB); + open.onsuccess = () => resolve(open.result); + open.onerror = () => reject(open.error); + }); + const stores = [...db.objectStoreNames]; + const all = (store) => + new Promise((resolve, reject) => { + const request = db.transaction(store).objectStore(store).getAll(); + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); + const files = stores.includes("files") ? await all("files") : []; + const sections = stores.includes("sections") ? await all("sections") : []; + db.close(); + const moved = files.length + ? files + : sections + .sort((a, b) => a.order - b.order) + .map(({ file, bytes }) => ({ path: `${MIGRATED}/${file}`, bytes })); + for (const { path, bytes } of moved) + if (bytes) await write(path, bytes.length, [[0, bytes]]); + else await folder(names(path)); + await new Promise((resolve) => { + const removal = indexedDB.deleteDatabase(DB); + removal.onsuccess = removal.onerror = removal.onblocked = resolve; + }); + }; + + onmessage = ({ data }) => { + queue = queue + .then(async () => { + if (data.kind === "load") { + root = await navigator.storage.getDirectory(); + let files = await list(root, "", []); + if (!files.length) { + await migrate(); + files = await list(root, "", []); + } + postMessage(files, files.flatMap(([, bytes]) => (bytes ? [bytes.buffer] : []))); + } else + for (const [path, length, ranges] of data.changes) + if (length === undefined) await remove(path); + else if (length === null) await folder(names(path)); + else await write(path, length, ranges); + }) + .catch((error) => console.error("Keeping files", error)); + }; } -function done(transaction) { +/** Every file kept, as `[path, bytes or null for a folder, modified]`. */ +export function loadFiles() { + const source = URL.createObjectURL(new Blob([`(${storageWorker})()`], { type: "text/javascript" })); + storage = new Worker(source); + storage.postMessage({ kind: "load" }); return new Promise((resolve, reject) => { - transaction.oncomplete = resolve; - transaction.onerror = () => reject(transaction.error); - transaction.onabort = () => reject(transaction.error); + storage.onmessage = ({ data }) => resolve(data); + storage.onerror = (error) => reject(new Error(`The storage worker failed: ${error.message}`)); }); } -/** Every file kept, as `[path, bytes or null, modified]`, the first web build's sections moved - * into a notebook of their own first. */ -export async function loadFiles() { - const db = await database(); - const read = db.transaction([FILES, SECTIONS], "readonly"); - const files = read.objectStore(FILES).getAll(); - const sections = read.objectStore(SECTIONS).getAll(); - await done(read); - let rows = files.result; - if (!rows.length && sections.result.length) { - const moved = [ - { path: "/Notebooks", bytes: null }, - { path: MIGRATED, bytes: null }, - ...sections.result - .sort((a, b) => a.order - b.order) - .map(({ file, bytes }) => ({ path: `${MIGRATED}/${file}`, bytes, modified: Date.now() })), - ]; - const write = db.transaction([FILES, SECTIONS], "readwrite"); - for (const row of moved) write.objectStore(FILES).put(row); - write.objectStore(SECTIONS).clear(); - await done(write); - rows = moved; - } - return rows.map(({ path, bytes, modified }) => [path, bytes, modified ?? 0]); -} - -/** Writes `[path]` (removed), `[path, null]` (a folder) and `[path, bytes, modified]` entries. */ +/** Writes `[path]` (removed), `[path, null]` (a folder) and `[path, length, ranges]` entries. */ export function storeFiles(changes) { - database() - .then((db) => { - const transaction = db.transaction(FILES, "readwrite"); - const store = transaction.objectStore(FILES); - for (const [path, bytes, modified] of changes) { - if (bytes === undefined) store.delete(path); - else store.put({ path, bytes, modified }); - } - return done(transaction); - }) - .catch((error) => console.error("Keeping files", error)); + const buffers = changes.flatMap(([, , ranges]) => (ranges ?? []).map(([, bytes]) => bytes.buffer)); + storage.postMessage({ kind: "store", changes }, buffers); } export function requestFrame() { -- 2.54.0