authorgravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-10-05 22:50:33-07:00
committergravatar for git@paperclover.netclover caruso <git@paperclover.net> 2026-10-06 00:27:38-07:00
log01883927d36e29e1f069a2d3cdd3f50a0b6195cd
treeb26dc9028e138f105641d8a86f70dc6b7cde3bb2
parent0f94ab4eccec40bad42aa31b3adc6dbf9cb50d7a
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

fix: retry browser storage flush failures

Preserve fatal sync errors while retrying recoverable OPFS flush rejection. Add actual-glue queue and browser recovery regressions. Fixes #97 Assisted-by: gpt-6.1-sol Assisted-by: gpt-6

6 files changed, 173 insertions(+), 5 deletions(-)

crates/notebook/src/fs/web.rs+2-1
...@@ -876,7 +876,8 @@ pub async fn durable() -> io::Result<()> {...@@ -876,7 +876,8 @@ pub async fn durable() -> io::Result<()> {
876 fn flush_storage() -> Result<js_sys::Promise, JsValue>;876 fn flush_storage() -> Result<js_sys::Promise, JsValue>;
877 }877 }
878 let failure = |error: JsValue| {878 let failure = |error: JsValue| {
879 io::Error::other(879 io::Error::new(
880 ErrorKind::WouldBlock,
880 error881 error
881 .as_string()882 .as_string()
882 .unwrap_or_else(|| "Browser storage failed".into()),883 .unwrap_or_else(|| "Browser storage failed".into()),
crates/notebook/src/sync.rs+4-2
...@@ -232,8 +232,10 @@ impl Replica {...@@ -232,8 +232,10 @@ impl Replica {
232 /// The same guarded synchronization step, awaiting non-blocking remote operations.232 /// The same guarded synchronization step, awaiting non-blocking remote operations.
233 pub async fn sync_once_async(&self, remote: &mut impl Remote) -> Result<Synced> {233 pub async fn sync_once_async(&self, remote: &mut impl Remote) -> Result<Synced> {
234 let result = self.sync_once_inner(remote).await;234 let result = self.sync_once_inner(remote).await;
235 crate::fs::durable().await?;235 let durable = crate::fs::durable().await;
236 result236 let synced = result?;
237 durable?;
238 Ok(synced)
237 }239 }
238240
239 /// Ownership uses `try_lock`; the cache mutex is released before every await.241 /// Ownership uses `try_lock`; the cache mutex is released before every await.
tools/TESTING.md+1-1
...@@ -20,7 +20,7 @@ another, and the exit status is the result....@@ -20,7 +20,7 @@ another, and the exit status is the result.
20| `windows-aarch64`, `linux-*` | Clippy `-D warnings` on what ships (libraries and binaries but `mobile`), then the `snowbound` build; Linux through `platform/linux/cargo.sh`, which links with zig against glibc 2.17 |20| `windows-aarch64`, `linux-*` | Clippy `-D warnings` on what ships (libraries and binaries but `mobile`), then the `snowbound` build; Linux through `platform/linux/cargo.sh`, which links with zig against glibc 2.17 |
21| `ios` | `xcodebuild` of the simulator app, unsigned |21| `ios` | `xcodebuild` of the simulator app, unsigned |
22| `web` | Clippy `-D warnings` on `snowbound` for `wasm32-unknown-unknown`, SQLite built by nixpkgs' clang; `release_web.py` links, optimizes and deploys the static folder |22| `web` | Clippy `-D warnings` on `snowbound` for `wasm32-unknown-unknown`, SQLite built by nixpkgs' clang; `release_web.py` links, optimizes and deploys the static folder |
23| `web-js` | Node boundary tests for Live Share browser sockets |23| `web-js` | Node boundary tests for Live Share browser sockets and storage |
24| `macos-10.6` | `platform/snow-leopard/cargo.sh` build of `snowbound`; skipped, saying why, without the SDK or nightly `rust-src` |24| `macos-10.6` | `platform/snow-leopard/cargo.sh` build of `snowbound`; skipped, saying why, without the SDK or nightly `rust-src` |
2525
26```sh26```sh
tools/ci.py+1-1
...@@ -80,7 +80,7 @@ def lanes():...@@ -80,7 +80,7 @@ def lanes():
80 missing=None if shutil.which('zig') else 'needs zig'))80 missing=None if shutil.which('zig') else 'needs zig'))
81 result.append(dict(81 result.append(dict(
82 name='web-js', minutes=5, packages=['notebook'], paths=('tools/web/',),82 name='web-js', minutes=5, packages=['notebook'], paths=('tools/web/',),
83 commands=[['node', '--test', 'tools/web/test_live.mjs']],83 commands=[['node', '--test', 'tools/web/test_*.mjs']],
84 missing=None if shutil.which('node') else 'needs node'))84 missing=None if shutil.which('node') else 'needs node'))
85 targets = subprocess.run(['rustup', 'target', 'list', '--installed'], capture_output=True, text=True).stdout85 targets = subprocess.run(['rustup', 'target', 'list', '--installed'], capture_output=True, text=True).stdout
86 wasm = web_environment()86 wasm = web_environment()
tools/web/storage_recovery.mjs created+92
...@@ -0,0 +1,92 @@
1// Generates ux-testing/drive.mjs steps for a local build and a disposable .one fixture.
2import { readFile, writeFile } from 'node:fs/promises';
3
4const [url, fixture, output, phase = '0'] = process.argv.slice(2);
5if (!url || !fixture || !output) throw new Error('Usage: node storage_recovery.mjs APP_URL FIXTURE.one STEPS.json [0|1]');
6if (!['0', '1'].includes(phase)) throw new Error('Storage phase must be 0 or 1');
7const bytes = (await readFile(fixture)).toString('base64');
8
9async function ready() {
10 const end = Date.now() + 40000;
11 while ((!globalThis.snowboundFlushStorage || document.querySelector('#shell')) && Date.now() < end)
12 await new Promise(resolve => setTimeout(resolve, 100));
13 if (!globalThis.snowboundFlushStorage || document.querySelector('#shell')) throw new Error('App did not start');
14 const script = [...document.scripts].find(script => script.textContent.includes('import init, * as snowbound'));
15 const path = script?.textContent.match(/import init, \* as snowbound from ["']([^"']+)["']/)?.[1];
16 if (!path) throw new Error('App module import was not found');
17 const app = await import(new URL(path, location.href).href);
18 app.accessibility(true);
19 globalThis.storageProbe = {
20 app,
21 original: globalThis.snowboundFlushStorage,
22 text() {
23 const root = document.querySelector('#a11y');
24 return [root?.textContent, ...Array.from(root?.querySelectorAll('[aria-label]') ?? [], node => node.getAttribute('aria-label'))].join(' ');
25 },
26 async bytes() {
27 const root = await navigator.storage.getDirectory();
28 async function find(dir) {
29 for await (const [name, entry] of dir.entries()) {
30 if (entry.kind === 'directory') {
31 const found = await find(entry);
32 if (found) return found;
33 } else if (name === 'Storage Regression.one') {
34 return new Uint8Array(await (await entry.getFile()).arrayBuffer());
35 }
36 }
37 }
38 return find(root);
39 },
40 async disk() {
41 const bytes = await this.bytes();
42 if (!bytes) return null;
43 return { transactions: new DataView(bytes.buffer).getUint32(96, true), length: bytes.length };
44 },
45 async until(check, message) {
46 const end = Date.now() + 20000;
47 while (Date.now() < end) {
48 if (await check()) return;
49 await new Promise(resolve => setTimeout(resolve, 100));
50 }
51 throw new Error(message);
52 },
53 };
54}
55
56function arm(successfulBeforeFailure) {
57 const probe = globalThis.storageProbe;
58 probe.rejected = [];
59 probe.succeeded = 0;
60 probe.unavailable = true;
61 globalThis.snowboundFlushStorage = async () => {
62 if (probe.unavailable && probe.succeeded >= successfulBeforeFailure) {
63 probe.rejected.push(Date.now());
64 throw new Error('Injected recoverable browser storage rejection');
65 }
66 await probe.original();
67 probe.succeeded++;
68 };
69}
70
71const expression = (fn, ...args) => `(${fn})(${args.map(arg => JSON.stringify(arg)).join(',')})`;
72const steps = [
73 { goto: url },
74 { eval: expression(ready) },
75 { eval: `(async()=>{const p=storageProbe;p.app.files('open',[['Storage Regression.one',Uint8Array.from(atob(${JSON.stringify(bytes)}),c=>c.charCodeAt(0))]]);await p.until(async()=>await p.disk()&&p.text().includes('Storage Regression'),'Fixture did not open and reach OPFS');await p.original();})()` },
76];
77
78for (const [before, marker] of [[0, ' storage recovered before publication'], [1, ' storage recovered after acknowledgment']].filter(([before]) => before === Number(phase))) {
79 steps.push(
80 { eval: '(async()=>{storageProbe.base=await storageProbe.disk();})()' },
81 { eval: expression(arm, before) },
82 { click: [100, 126] },
83 { key: ['End', 'End', 35, 6] },
84 { type: marker },
85 { eval: `(async()=>{const p=storageProbe;await p.until(()=>p.rejected.length>=3,'Automatic sync did not retry the rejected flush');if(p.rejected.length>8||p.rejected.at(-1)-p.rejected[0]<1000)throw Error('Automatic sync retry loop is unbounded');const disk=await p.disk();if(disk.transactions!==p.base.transactions+${before})throw Error('Unexpected publication count during storage failure');return {rejected:p.rejected.length,disk};})()`, print: true },
86 { eval: `(async()=>{const p=storageProbe;const succeeded=p.succeeded;p.unavailable=false;await p.until(async()=>p.succeeded>succeeded&&(await p.disk()).transactions===p.base.transactions+1,'Same worker did not recover and publish exactly once');await p.original();if(!p.text().includes(${JSON.stringify(marker)}))throw Error('Recovered text is absent from the accessibility tree');return {rejected:p.rejected.length,succeeded:p.succeeded,disk:await p.disk()};})()`, print: true },
87 { goto: url },
88 { eval: expression(ready) },
89 { eval: `(async()=>{const p=storageProbe;const bytes=await p.bytes();if(!bytes)throw Error('Stored fixture is absent');p.app.files('open',[['Readback.one',bytes]]);await p.until(()=>p.text().includes(${JSON.stringify(marker)}),'Recovered edit did not survive cold opening from OPFS');return p.disk();})()`, print: true },
90 );
91}
92await writeFile(output, `${JSON.stringify(steps, null, 2)}\n`);
tools/web/test_storage.mjs created+73
...@@ -0,0 +1,73 @@
1import assert from 'node:assert/strict';
2import { readFile } from 'node:fs/promises';
3import test from 'node:test';
4import vm from 'node:vm';
5
6const source = await readFile(new URL('../../crates/snowbound/web/glue.js', import.meta.url), 'utf8');
7
8test('an OPFS flush rejection retains ordered writes until storage recovers', async () => {
9 const files = new Map([['existing', new Uint8Array([0])]]);
10 const flushed = [];
11 const replies = [];
12 let unavailable = false;
13 const root = {
14 async *entries() {
15 for (const [name, bytes] of files) {
16 yield [name, { kind: 'file', getFile: async () => ({ arrayBuffer: async () => bytes.buffer }) }];
17 }
18 },
19 async getFileHandle(path) {
20 let bytes = files.get(path)?.slice() ?? new Uint8Array();
21 return {
22 async createSyncAccessHandle() {
23 return {
24 write(part, { at }) {
25 if (at + part.length > bytes.length) {
26 const extended = new Uint8Array(at + part.length);
27 extended.set(bytes);
28 bytes = extended;
29 }
30 bytes.set(part, at);
31 return part.length;
32 },
33 truncate(length) { bytes = bytes.slice(0, length); },
34 flush() {
35 if (unavailable) throw new Error('Injected OPFS outage');
36 files.set(path, bytes.slice());
37 flushed.push([path, [...bytes]]);
38 },
39 };
40 },
41 };
42 },
43 };
44 const context = vm.createContext({
45 navigator: { storage: { getDirectory: async () => root } },
46 postMessage: (message) => replies.push(message),
47 console: { error() {} },
48 });
49 vm.runInContext(source.replace(/\bexport /g, '').replaceAll('import.meta.url', JSON.stringify(import.meta.url)), context);
50 vm.runInContext('storageWorker()', context);
51 const send = async (data) => {
52 context.onmessage({ data });
53 const expected = replies.length + 1;
54 if (data.kind === 'store') context.onmessage({ data: { kind: 'settle' } });
55 for (let turns = 0; replies.length < expected; turns++) {
56 assert.ok(turns < 100, 'storage worker did not answer');
57 await Promise.resolve();
58 }
59 return replies.at(-1);
60 };
61 await send({ kind: 'load' });
62 unavailable = true;
63 const first = await send({ kind: 'store', changes: [['replica-wal', 2, [[0, new Uint8Array([1, 2])]]]] });
64 assert.match(first.error, /Injected OPFS outage/);
65 await send({ kind: 'store', changes: [['replica-wal', 2, [[1, new Uint8Array([3])]]], ['replica', 1, [[0, new Uint8Array([4])]]]] });
66 assert.equal(files.has('replica'), false);
67 assert.equal(flushed.length, 0);
68 unavailable = false;
69 assert.equal((await send({ kind: 'settle' })).error, null);
70 assert.deepEqual(flushed, [['replica-wal', [1, 2]], ['replica-wal', [1, 3]], ['replica', [4]]]);
71 assert.equal((await send({ kind: 'settle' })).error, null);
72 assert.equal(flushed.length, 3);
73});