diff --git a/arc/platforms.md b/arc/platforms.md index 2753dcfc15e846a2ce29a3296833c281fdaecb5b..29378ae749ac538a4344dfe8cd04b4f83406f011 100644 --- a/arc/platforms.md +++ b/arc/platforms.md @@ -156,9 +156,11 @@ keyboard, the toolbar and the macOS menu bar all run commands from it. - The page has one thread. The notebook's section thread, sync worker and background run as tasks on its event loop (`notebook::task`), and work the desktop gives a thread runs once the frame is done (`spawn`). -- Files are `notebook::fs`'s: std's elsewhere, here held in memory and kept in IndexedDB, - with SQLite's VFS over the same files, so replicas and notebooks live side by side under - `/Notebooks` and `/Cache`. One tab at a time holds them. +- Files are `notebook::fs`'s: std's elsewhere, here held in memory, with SQLite's VFS over + the same files, so replicas and notebooks live side by side under `/Notebooks` and + `/Cache`. A storage worker keeps them in the origin's private file system, writing the + byte ranges each burst changed through OPFS's synchronous handles, which only workers get. + One tab at a time holds them. - Menus are the kit's own, as on Linux, with the PC's chords and ⌘ for Ctrl on a Mac; the browser keeps its own window and tab chords. Dialogs are the browser's; Open and Insert ask for files to copy in; printing downloads the PDF. Servers, recording and accessibility diff --git a/crates/notebook/src/fs/web.rs b/crates/notebook/src/fs/web.rs index c130e7e830c1f01ef810ca2a2280e6cbc4e7876c..2ee83f22727f963b5f4e854cf219e2b54b626f3b 100644 --- a/crates/notebook/src/fs/web.rs +++ b/crates/notebook/src/fs/web.rs @@ -1,7 +1,7 @@ //! The browser's file system: std's calls over files held in memory, which the host loads -//! before anything opens (`restore`) and writes out as they change (`changes`). SQLite reaches -//! the same files through `sqlite`, its default VFS here, so a replica is a file like any other. -//! One thread: no file is ever locked. +//! before anything opens (`restore`) and writes out as they change, by the byte ranges that +//! changed (`changes`). SQLite reaches the same files through `sqlite`, its default VFS here, +//! so a replica is a file like any other. One thread: no file is ever locked. use onestore::{CommitError, CommitIo, CommitState, Stamp, Transaction}; use std::{ @@ -9,6 +9,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, ffi::OsString, io::{self, ErrorKind, Read, Seek, SeekFrom, Write}, + ops::Range, path::{Component, Path, PathBuf}, rc::Rc, time::{Duration, SystemTime, UNIX_EPOCH}, @@ -23,6 +24,47 @@ struct Data { modified: f64, /// Where the file is, which a rename moves; none once it is removed. path: Option, + /// The ranges written since the host last wrote the file out, or none for all of it. + unwritten: Option>>, +} + +/// More ranges than this write the whole file out instead. +const RANGES: usize = 64; + +impl Data { + fn new( + bytes: Vec, + modified: f64, + path: PathBuf, + unwritten: Option>>, + ) -> Shared { + Rc::new(RefCell::new(Self { + bytes, + modified, + path: Some(path), + unwritten, + })) + } + + /// Notes `range` written now, merged into a written range it meets. + fn wrote(&mut self, range: Range) { + self.modified = now(); + if let Some(ranges) = &mut self.unwritten { + match ranges + .iter_mut() + .find(|known| known.start <= range.end && range.start <= known.end) + { + Some(known) => *known = known.start.min(range.start)..known.end.max(range.end), + None => ranges.push(range), + } + if ranges.len() > RANGES { + self.unwritten = None; + } + } + if let Some(path) = self.path.clone() { + with(|files| files.changed.insert(path)); + } + } } type Shared = Rc>; @@ -59,35 +101,58 @@ pub fn restore(path: impl AsRef, saved: Saved) { FILES.with_borrow_mut(|files| { let node = match saved { Saved::Directory => Node::Directory, - Saved::File(bytes, modified) => Node::File(Rc::new(RefCell::new(Data { - bytes, - modified, - path: Some(path.clone()), - }))), + Saved::File(bytes, modified) => { + Node::File(Data::new(bytes, modified, path.clone(), Some(Vec::new()))) + } }; files.nodes.insert(path, node); }); } +/// How a path changed, as the host writes it out. +pub enum Change { + Removed, + Directory, + /// The file's length, and the ranges that changed with their bytes. + File { + length: u64, + ranges: Vec<(u64, Vec)>, + }, +} + /// Whether any path changed since `changes` was last called. pub fn changed() -> bool { FILES.with_borrow(|files| !files.changed.is_empty()) } -/// The paths changed since the last call, each with what it holds now, if anything. -pub fn changes() -> Vec<(PathBuf, Option)> { +/// The paths changed since the last call, each with how; a folder before what it holds. +pub fn changes() -> Vec<(PathBuf, Change)> { FILES.with_borrow_mut(|files| { std::mem::take(&mut files.changed) .into_iter() .map(|path| { - let saved = files.nodes.get(&path).map(|node| match node { - Node::Directory => Saved::Directory, - Node::File(data) => { - let data = data.borrow(); - Saved::File(data.bytes.clone(), data.modified) + let change = match files.nodes.get(&path) { + None => Change::Removed, + Some(Node::Directory) => Change::Directory, + Some(Node::File(data)) => { + let mut data = data.borrow_mut(); + let length = data.bytes.len(); + let ranges = data + .unwritten + .replace(Vec::new()) + .unwrap_or_else(|| std::iter::once(0..length).collect()); + Change::File { + length: length as u64, + ranges: ranges + .into_iter() + .map(|range| range.start.min(length)..range.end.min(length)) + .filter(|range| !range.is_empty()) + .map(|range| (range.start as u64, data.bytes[range].to_vec())) + .collect(), + } } - }); - (path, saved) + }; + (path, change) }) .collect() }) @@ -139,11 +204,7 @@ impl Files { if self.nodes.contains_key(&path) { return Err(ErrorKind::AlreadyExists.into()); } - let data = Rc::new(RefCell::new(Data { - bytes: Vec::new(), - modified: now(), - path: Some(path.clone()), - })); + let data = Data::new(Vec::new(), now(), path.clone(), None); self.nodes .insert(path.clone(), Node::File(Rc::clone(&data))); self.changed.insert(path); @@ -163,14 +224,6 @@ fn with(act: impl FnOnce(&mut Files) -> T) -> T { FILES.with_borrow_mut(act) } -/// Marks `data` changed now, where it still has a path. -fn touched(data: &mut Data) { - data.modified = now(); - if let Some(path) = data.path.clone() { - with(|files| files.changed.insert(path)); - } -} - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct FileType(bool); @@ -295,7 +348,8 @@ pub fn write(path: impl AsRef, contents: impl AsRef<[u8]>) -> io::Result<( let data = with(|files| files.file(&path).or_else(|_| files.create(path.clone())))?; let mut data = data.borrow_mut(); data.bytes = contents.as_ref().to_vec(); - touched(&mut data); + let length = data.bytes.len(); + data.wrote(0..length); Ok(()) } @@ -425,7 +479,9 @@ pub fn rename(from: impl AsRef, to: impl AsRef) -> io::Result<()> { }; let target = to.join(each.strip_prefix(&from).unwrap_or(Path::new(""))); if let Node::File(data) = &node { - data.borrow_mut().path = Some(target.clone()); + let mut data = data.borrow_mut(); + data.path = Some(target.clone()); + data.unwritten = None; } files.changed.insert(each); files.changed.insert(target.clone()); @@ -546,7 +602,7 @@ impl OpenOptions { if self.truncate && self.write { let mut data = data.borrow_mut(); data.bytes.clear(); - touched(&mut data); + data.wrote(0..0); } Ok(File { data, @@ -598,8 +654,9 @@ impl File { pub fn set_len(&self, size: u64) -> io::Result<()> { let mut data = self.data.borrow_mut(); + let old = data.bytes.len(); data.bytes.resize(size as usize, 0); - touched(&mut data); + data.wrote(old.min(size as usize)..size as usize); Ok(()) } @@ -618,7 +675,7 @@ impl File { data.bytes.resize(end, 0); } data.bytes[offset as usize..end].copy_from_slice(input); - touched(&mut data); + data.wrote(offset as usize..end); } } diff --git a/crates/notebook/src/fs/web/sqlite.rs b/crates/notebook/src/fs/web/sqlite.rs index c8dae1c369229f2ccffc175e45d55f155fdee056..f9915a8b4192dfcba54f0ba6d03c45bcd92792eb 100644 --- a/crates/notebook/src/fs/web/sqlite.rs +++ b/crates/notebook/src/fs/web/sqlite.rs @@ -2,7 +2,7 @@ //! log and the notebook it caches are kept and written out alike. #![allow(unsafe_code)] -use super::{FILES, Node, Shared, normal, touched, with}; +use super::{FILES, Node, Shared, normal, with}; use rsqlite_vfs::{ OsCallback, SQLiteIoMethods, SQLiteVfs, SQLiteVfsFile, VfsError, VfsFile, VfsResult, VfsStore, ffi::{SQLITE_CANTOPEN, SQLITE_IOERR, SQLITE_IOERR_DELETE, sqlite3_vfs}, @@ -37,14 +37,14 @@ impl VfsFile for File { data.bytes.resize(end, 0); } data.bytes[offset..end].copy_from_slice(input); - touched(&mut data); + data.wrote(offset..end); Ok(()) } fn truncate(&mut self, size: usize) -> VfsResult<()> { let mut data = self.0.borrow_mut(); data.bytes.truncate(size); - touched(&mut data); + data.wrote(size..size); Ok(()) } diff --git a/crates/snowbound/src/web.rs b/crates/snowbound/src/web.rs index 88ad9b1f7286c72fe830a018cb71dcd2ed02fd05..bb658efbc4529ab8057ca76492930b307ebd948d 100644 --- a/crates/snowbound/src/web.rs +++ b/crates/snowbound/src/web.rs @@ -27,7 +27,7 @@ use winit::{ #[wasm_bindgen(module = "/web/glue.js")] extern "C" { fn attach(module: JsValue); - /// The files IndexedDB kept: `[path, bytes or null for a folder, modified]`. + /// The files the browser kept: `[path, bytes or null for a folder, modified]`. #[wasm_bindgen(js_name = loadFiles, catch)] async fn load_files() -> Result; #[wasm_bindgen(js_name = requestFrame)] @@ -58,8 +58,8 @@ extern "C" { fn tell(message: &str); #[wasm_bindgen(js_name = openLink)] fn open_link(url: &str); - /// Writes changed files to IndexedDB: `[path, bytes or null for a folder or undefined - /// for a removal, modified]` triples. + /// Writes changes out: `[path]` removed, `[path, null]` a folder, and `[path, length, + /// [[offset, bytes], ...]]` a file's new length and the ranges that changed. #[wasm_bindgen(js_name = storeFiles)] fn store_files(changes: js_sys::Array); } @@ -710,19 +710,28 @@ fn restore(files: js_sys::Array) { } } -/// Writes the files changed since the last time out to IndexedDB. +/// Writes the files changed since the last time out to the browser's storage. fn store() { + use notebook::fs::Change; STORE_DUE.set(None); let changes = js_sys::Array::new(); - for (path, saved) in notebook::fs::changes() { + for (path, change) in notebook::fs::changes() { let path = JsValue::from_str(&path.to_string_lossy()); - let entry = match saved { - None => js_sys::Array::of1(&path), - Some(notebook::fs::Saved::Directory) => js_sys::Array::of2(&path, &JsValue::NULL), - Some(notebook::fs::Saved::File(bytes, modified)) => js_sys::Array::of3( + let entry = match change { + Change::Removed => js_sys::Array::of1(&path), + Change::Directory => js_sys::Array::of2(&path, &JsValue::NULL), + Change::File { length, ranges } => js_sys::Array::of3( &path, - &js_sys::Uint8Array::from(bytes.as_slice()), - &modified.into(), + &(length as f64).into(), + &ranges + .into_iter() + .map(|(offset, bytes)| { + js_sys::Array::of2( + &(offset as f64).into(), + &js_sys::Uint8Array::from(bytes.as_slice()), + ) + }) + .collect::(), ), }; changes.push(&entry); diff --git a/crates/snowbound/web/glue.js b/crates/snowbound/web/glue.js index 39922340f7521d04e2887f6bf4e1062f345b30ab..509683f6cc8e8a320cd43db71682d95eed322101 100644 --- a/crates/snowbound/web/glue.js +++ b/crates/snowbound/web/glue.js @@ -1,5 +1,6 @@ // The browser's half of Snowbound (src/web.rs): the page's input, its dialogs and files, and -// IndexedDB, which keeps the files `notebook::fs` holds between visits. +// the files `notebook::fs` holds, which a storage worker keeps in the origin's private file +// system (OPFS) between visits. let wasm; let canvas; @@ -7,75 +8,134 @@ let input; let picker; let framePending = false; let wakeTimer; +let storage; -const DB = "snowbound"; -// `files` holds every path: `{path, bytes}`, `bytes` null for a folder. `sections` is the first -// web build's store, whose sections move into a notebook once. -const FILES = "files"; -const SECTIONS = "sections"; -const MIGRATED = "/Notebooks/Web Notebook"; +/** The storage worker: the only place OPFS hands out synchronous handles, which write a + * file's changed ranges in place. Runs as a worker of its own, from this source. */ +function storageWorker() { + // IndexedDB held the files before OPFS did: `files` by path, and before that the first web + // build's `sections`, which move into a notebook of their own. + const DB = "snowbound"; + const MIGRATED = "/Notebooks/Web Notebook"; + const handles = new Map(); + let root; + let queue = Promise.resolve(); -function database() { - return new Promise((resolve, reject) => { - const open = indexedDB.open(DB, 2); - open.onupgradeneeded = () => { - const db = open.result; - if (!db.objectStoreNames.contains(FILES)) db.createObjectStore(FILES, { keyPath: "path" }); - if (!db.objectStoreNames.contains(SECTIONS)) - db.createObjectStore(SECTIONS, { keyPath: "file" }); - }; - open.onsuccess = () => resolve(open.result); - open.onerror = () => reject(open.error); - }); + const names = (path) => path.split("/").filter(Boolean); + const folder = async (parts) => { + let dir = root; + for (const name of parts) dir = await dir.getDirectoryHandle(name, { create: true }); + return dir; + }; + const handle = async (path) => { + if (!handles.has(path)) { + const parts = names(path); + const file = await (await folder(parts.slice(0, -1))).getFileHandle(parts.at(-1), { create: true }); + handles.set(path, await file.createSyncAccessHandle()); + } + return handles.get(path); + }; + const close = (path) => { + for (const [held, open] of handles) + if (held === path || held.startsWith(`${path}/`)) { + open.close(); + handles.delete(held); + } + }; + const write = async (path, length, ranges) => { + const open = await handle(path); + for (const [offset, bytes] of ranges) open.write(bytes, { at: offset }); + open.truncate(length); + open.flush(); + }; + const remove = async (path) => { + close(path); + const parts = names(path); + try { + await (await folder(parts.slice(0, -1))).removeEntry(parts.at(-1), { recursive: true }); + } catch (error) { + if (error.name !== "NotFoundError") throw error; + } + }; + const list = async (dir, path, out) => { + for await (const [name, entry] of dir.entries()) { + const at = `${path}/${name}`; + if (entry.kind === "directory") { + out.push([at, null, 0]); + await list(entry, at, out); + } else { + const file = await entry.getFile(); + out.push([at, new Uint8Array(await file.arrayBuffer()), file.lastModified]); + } + } + return out; + }; + const migrate = async () => { + const db = await new Promise((resolve, reject) => { + const open = indexedDB.open(DB); + open.onsuccess = () => resolve(open.result); + open.onerror = () => reject(open.error); + }); + const stores = [...db.objectStoreNames]; + const all = (store) => + new Promise((resolve, reject) => { + const request = db.transaction(store).objectStore(store).getAll(); + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); + const files = stores.includes("files") ? await all("files") : []; + const sections = stores.includes("sections") ? await all("sections") : []; + db.close(); + const moved = files.length + ? files + : sections + .sort((a, b) => a.order - b.order) + .map(({ file, bytes }) => ({ path: `${MIGRATED}/${file}`, bytes })); + for (const { path, bytes } of moved) + if (bytes) await write(path, bytes.length, [[0, bytes]]); + else await folder(names(path)); + await new Promise((resolve) => { + const removal = indexedDB.deleteDatabase(DB); + removal.onsuccess = removal.onerror = removal.onblocked = resolve; + }); + }; + + onmessage = ({ data }) => { + queue = queue + .then(async () => { + if (data.kind === "load") { + root = await navigator.storage.getDirectory(); + let files = await list(root, "", []); + if (!files.length) { + await migrate(); + files = await list(root, "", []); + } + postMessage(files, files.flatMap(([, bytes]) => (bytes ? [bytes.buffer] : []))); + } else + for (const [path, length, ranges] of data.changes) + if (length === undefined) await remove(path); + else if (length === null) await folder(names(path)); + else await write(path, length, ranges); + }) + .catch((error) => console.error("Keeping files", error)); + }; } -function done(transaction) { +/** Every file kept, as `[path, bytes or null for a folder, modified]`. */ +export function loadFiles() { + const source = URL.createObjectURL(new Blob([`(${storageWorker})()`], { type: "text/javascript" })); + storage = new Worker(source); + storage.postMessage({ kind: "load" }); return new Promise((resolve, reject) => { - transaction.oncomplete = resolve; - transaction.onerror = () => reject(transaction.error); - transaction.onabort = () => reject(transaction.error); + storage.onmessage = ({ data }) => resolve(data); + storage.onerror = (error) => reject(new Error(`The storage worker failed: ${error.message}`)); }); } -/** Every file kept, as `[path, bytes or null, modified]`, the first web build's sections moved - * into a notebook of their own first. */ -export async function loadFiles() { - const db = await database(); - const read = db.transaction([FILES, SECTIONS], "readonly"); - const files = read.objectStore(FILES).getAll(); - const sections = read.objectStore(SECTIONS).getAll(); - await done(read); - let rows = files.result; - if (!rows.length && sections.result.length) { - const moved = [ - { path: "/Notebooks", bytes: null }, - { path: MIGRATED, bytes: null }, - ...sections.result - .sort((a, b) => a.order - b.order) - .map(({ file, bytes }) => ({ path: `${MIGRATED}/${file}`, bytes, modified: Date.now() })), - ]; - const write = db.transaction([FILES, SECTIONS], "readwrite"); - for (const row of moved) write.objectStore(FILES).put(row); - write.objectStore(SECTIONS).clear(); - await done(write); - rows = moved; - } - return rows.map(({ path, bytes, modified }) => [path, bytes, modified ?? 0]); -} - -/** Writes `[path]` (removed), `[path, null]` (a folder) and `[path, bytes, modified]` entries. */ +/** Writes `[path]` (removed), `[path, null]` (a folder) and `[path, length, ranges]` entries. */ export function storeFiles(changes) { - database() - .then((db) => { - const transaction = db.transaction(FILES, "readwrite"); - const store = transaction.objectStore(FILES); - for (const [path, bytes, modified] of changes) { - if (bytes === undefined) store.delete(path); - else store.put({ path, bytes, modified }); - } - return done(transaction); - }) - .catch((error) => console.error("Keeping files", error)); + const buffers = changes.flatMap(([, , ranges]) => (ranges ?? []).map(([, bytes]) => bytes.buffer)); + storage.postMessage({ kind: "store", changes }, buffers); } export function requestFrame() {