| author |  clover caruso <git@paperclover.net> 2026-10-05 22:50:33-07:00 |
| committer | clover caruso <git@paperclover.net> 2026-10-06 00:27:38-07:00 |
| log | 01883927d36e29e1f069a2d3cdd3f50a0b6195cd |
| tree | b26dc9028e138f105641d8a86f70dc6b7cde3bb2 |
| parent | 0f94ab4eccec40bad42aa31b3adc6dbf9cb50d7a |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
fix: retry browser storage flush failures
Preserve fatal sync errors while retrying recoverable OPFS flush rejection.
Add actual-glue queue and browser recovery regressions.
Fixes #97
Assisted-by: gpt-6.1-sol
Assisted-by: gpt-66 files changed, 173 insertions(+), 5 deletions(-)
crates/notebook/src/fs/web.rs+2-1
| ... | ... | @@ -876,7 +876,8 @@ pub async fn durable() -> io::Result<()> { |
| 876 | 876 | fn flush_storage() -> Result<js_sys::Promise, JsValue>; |
| 877 | 877 | } |
| 878 | 878 | let failure = |error: JsValue| { |
| 879 | io::Error::other( | |
| 879 | io::Error::new( | |
| 880 | ErrorKind::WouldBlock, | |
| 880 | 881 | error |
| 881 | 882 | .as_string() |
| 882 | 883 | .unwrap_or_else(|| "Browser storage failed".into()), |
crates/notebook/src/sync.rs+4-2
| ... | ... | @@ -232,8 +232,10 @@ impl Replica { |
| 232 | 232 | /// The same guarded synchronization step, awaiting non-blocking remote operations. |
| 233 | 233 | pub async fn sync_once_async(&self, remote: &mut impl Remote) -> Result<Synced> { |
| 234 | 234 | let result = self.sync_once_inner(remote).await; |
| 235 | crate::fs::durable().await?; | |
| 236 | result | |
| 235 | let durable = crate::fs::durable().await; | |
| 236 | let synced = result?; | |
| 237 | durable?; | |
| 238 | Ok(synced) | |
| 237 | 239 | } |
| 238 | 240 | |
| 239 | 241 | /// Ownership uses `try_lock`; the cache mutex is released before every await. |
tools/TESTING.md+1-1
| ... | ... | @@ -20,7 +20,7 @@ another, and the exit status is the result. |
| 20 | 20 | | `windows-aarch64`, `linux-*` | Clippy `-D warnings` on what ships (libraries and binaries but `mobile`), then the `snowbound` build; Linux through `platform/linux/cargo.sh`, which links with zig against glibc 2.17 | |
| 21 | 21 | | `ios` | `xcodebuild` of the simulator app, unsigned | |
| 22 | 22 | | `web` | Clippy `-D warnings` on `snowbound` for `wasm32-unknown-unknown`, SQLite built by nixpkgs' clang; `release_web.py` links, optimizes and deploys the static folder | |
| 23 | | `web-js` | Node boundary tests for Live Share browser sockets | | |
| 23 | | `web-js` | Node boundary tests for Live Share browser sockets and storage | | |
| 24 | 24 | | `macos-10.6` | `platform/snow-leopard/cargo.sh` build of `snowbound`; skipped, saying why, without the SDK or nightly `rust-src` | |
| 25 | 25 | |
| 26 | 26 | ```sh |
tools/ci.py+1-1
| ... | ... | @@ -80,7 +80,7 @@ def lanes(): |
| 80 | 80 | missing=None if shutil.which('zig') else 'needs zig')) |
| 81 | 81 | result.append(dict( |
| 82 | 82 | name='web-js', minutes=5, packages=['notebook'], paths=('tools/web/',), |
| 83 | commands=[['node', '--test', 'tools/web/test_live.mjs']], | |
| 83 | commands=[['node', '--test', 'tools/web/test_*.mjs']], | |
| 84 | 84 | missing=None if shutil.which('node') else 'needs node')) |
| 85 | 85 | targets = subprocess.run(['rustup', 'target', 'list', '--installed'], capture_output=True, text=True).stdout |
| 86 | 86 | wasm = web_environment() |
tools/web/storage_recovery.mjs created+92
| ... | ... | @@ -0,0 +1,92 @@ |
| 1 | // Generates ux-testing/drive.mjs steps for a local build and a disposable .one fixture. | |
| 2 | import { readFile, writeFile } from 'node:fs/promises'; | |
| 3 | ||
| 4 | const [url, fixture, output, phase = '0'] = process.argv.slice(2); | |
| 5 | if (!url || !fixture || !output) throw new Error('Usage: node storage_recovery.mjs APP_URL FIXTURE.one STEPS.json [0|1]'); | |
| 6 | if (!['0', '1'].includes(phase)) throw new Error('Storage phase must be 0 or 1'); | |
| 7 | const bytes = (await readFile(fixture)).toString('base64'); | |
| 8 | ||
| 9 | async function ready() { | |
| 10 | const end = Date.now() + 40000; | |
| 11 | while ((!globalThis.snowboundFlushStorage || document.querySelector('#shell')) && Date.now() < end) | |
| 12 | await new Promise(resolve => setTimeout(resolve, 100)); | |
| 13 | if (!globalThis.snowboundFlushStorage || document.querySelector('#shell')) throw new Error('App did not start'); | |
| 14 | const script = [...document.scripts].find(script => script.textContent.includes('import init, * as snowbound')); | |
| 15 | const path = script?.textContent.match(/import init, \* as snowbound from ["']([^"']+)["']/)?.[1]; | |
| 16 | if (!path) throw new Error('App module import was not found'); | |
| 17 | const app = await import(new URL(path, location.href).href); | |
| 18 | app.accessibility(true); | |
| 19 | globalThis.storageProbe = { | |
| 20 | app, | |
| 21 | original: globalThis.snowboundFlushStorage, | |
| 22 | text() { | |
| 23 | const root = document.querySelector('#a11y'); | |
| 24 | return [root?.textContent, ...Array.from(root?.querySelectorAll('[aria-label]') ?? [], node => node.getAttribute('aria-label'))].join(' '); | |
| 25 | }, | |
| 26 | async bytes() { | |
| 27 | const root = await navigator.storage.getDirectory(); | |
| 28 | async function find(dir) { | |
| 29 | for await (const [name, entry] of dir.entries()) { | |
| 30 | if (entry.kind === 'directory') { | |
| 31 | const found = await find(entry); | |
| 32 | if (found) return found; | |
| 33 | } else if (name === 'Storage Regression.one') { | |
| 34 | return new Uint8Array(await (await entry.getFile()).arrayBuffer()); | |
| 35 | } | |
| 36 | } | |
| 37 | } | |
| 38 | return find(root); | |
| 39 | }, | |
| 40 | async disk() { | |
| 41 | const bytes = await this.bytes(); | |
| 42 | if (!bytes) return null; | |
| 43 | return { transactions: new DataView(bytes.buffer).getUint32(96, true), length: bytes.length }; | |
| 44 | }, | |
| 45 | async until(check, message) { | |
| 46 | const end = Date.now() + 20000; | |
| 47 | while (Date.now() < end) { | |
| 48 | if (await check()) return; | |
| 49 | await new Promise(resolve => setTimeout(resolve, 100)); | |
| 50 | } | |
| 51 | throw new Error(message); | |
| 52 | }, | |
| 53 | }; | |
| 54 | } | |
| 55 | ||
| 56 | function arm(successfulBeforeFailure) { | |
| 57 | const probe = globalThis.storageProbe; | |
| 58 | probe.rejected = []; | |
| 59 | probe.succeeded = 0; | |
| 60 | probe.unavailable = true; | |
| 61 | globalThis.snowboundFlushStorage = async () => { | |
| 62 | if (probe.unavailable && probe.succeeded >= successfulBeforeFailure) { | |
| 63 | probe.rejected.push(Date.now()); | |
| 64 | throw new Error('Injected recoverable browser storage rejection'); | |
| 65 | } | |
| 66 | await probe.original(); | |
| 67 | probe.succeeded++; | |
| 68 | }; | |
| 69 | } | |
| 70 | ||
| 71 | const expression = (fn, ...args) => `(${fn})(${args.map(arg => JSON.stringify(arg)).join(',')})`; | |
| 72 | const steps = [ | |
| 73 | { goto: url }, | |
| 74 | { eval: expression(ready) }, | |
| 75 | { eval: `(async()=>{const p=storageProbe;p.app.files('open',[['Storage Regression.one',Uint8Array.from(atob(${JSON.stringify(bytes)}),c=>c.charCodeAt(0))]]);await p.until(async()=>await p.disk()&&p.text().includes('Storage Regression'),'Fixture did not open and reach OPFS');await p.original();})()` }, | |
| 76 | ]; | |
| 77 | ||
| 78 | for (const [before, marker] of [[0, ' storage recovered before publication'], [1, ' storage recovered after acknowledgment']].filter(([before]) => before === Number(phase))) { | |
| 79 | steps.push( | |
| 80 | { eval: '(async()=>{storageProbe.base=await storageProbe.disk();})()' }, | |
| 81 | { eval: expression(arm, before) }, | |
| 82 | { click: [100, 126] }, | |
| 83 | { key: ['End', 'End', 35, 6] }, | |
| 84 | { type: marker }, | |
| 85 | { eval: `(async()=>{const p=storageProbe;await p.until(()=>p.rejected.length>=3,'Automatic sync did not retry the rejected flush');if(p.rejected.length>8||p.rejected.at(-1)-p.rejected[0]<1000)throw Error('Automatic sync retry loop is unbounded');const disk=await p.disk();if(disk.transactions!==p.base.transactions+${before})throw Error('Unexpected publication count during storage failure');return {rejected:p.rejected.length,disk};})()`, print: true }, | |
| 86 | { eval: `(async()=>{const p=storageProbe;const succeeded=p.succeeded;p.unavailable=false;await p.until(async()=>p.succeeded>succeeded&&(await p.disk()).transactions===p.base.transactions+1,'Same worker did not recover and publish exactly once');await p.original();if(!p.text().includes(${JSON.stringify(marker)}))throw Error('Recovered text is absent from the accessibility tree');return {rejected:p.rejected.length,succeeded:p.succeeded,disk:await p.disk()};})()`, print: true }, | |
| 87 | { goto: url }, | |
| 88 | { eval: expression(ready) }, | |
| 89 | { eval: `(async()=>{const p=storageProbe;const bytes=await p.bytes();if(!bytes)throw Error('Stored fixture is absent');p.app.files('open',[['Readback.one',bytes]]);await p.until(()=>p.text().includes(${JSON.stringify(marker)}),'Recovered edit did not survive cold opening from OPFS');return p.disk();})()`, print: true }, | |
| 90 | ); | |
| 91 | } | |
| 92 | await writeFile(output, `${JSON.stringify(steps, null, 2)}\n`); |
tools/web/test_storage.mjs created+73
| ... | ... | @@ -0,0 +1,73 @@ |
| 1 | import assert from 'node:assert/strict'; | |
| 2 | import { readFile } from 'node:fs/promises'; | |
| 3 | import test from 'node:test'; | |
| 4 | import vm from 'node:vm'; | |
| 5 | ||
| 6 | const source = await readFile(new URL('../../crates/snowbound/web/glue.js', import.meta.url), 'utf8'); | |
| 7 | ||
| 8 | test('an OPFS flush rejection retains ordered writes until storage recovers', async () => { | |
| 9 | const files = new Map([['existing', new Uint8Array([0])]]); | |
| 10 | const flushed = []; | |
| 11 | const replies = []; | |
| 12 | let unavailable = false; | |
| 13 | const root = { | |
| 14 | async *entries() { | |
| 15 | for (const [name, bytes] of files) { | |
| 16 | yield [name, { kind: 'file', getFile: async () => ({ arrayBuffer: async () => bytes.buffer }) }]; | |
| 17 | } | |
| 18 | }, | |
| 19 | async getFileHandle(path) { | |
| 20 | let bytes = files.get(path)?.slice() ?? new Uint8Array(); | |
| 21 | return { | |
| 22 | async createSyncAccessHandle() { | |
| 23 | return { | |
| 24 | write(part, { at }) { | |
| 25 | if (at + part.length > bytes.length) { | |
| 26 | const extended = new Uint8Array(at + part.length); | |
| 27 | extended.set(bytes); | |
| 28 | bytes = extended; | |
| 29 | } | |
| 30 | bytes.set(part, at); | |
| 31 | return part.length; | |
| 32 | }, | |
| 33 | truncate(length) { bytes = bytes.slice(0, length); }, | |
| 34 | flush() { | |
| 35 | if (unavailable) throw new Error('Injected OPFS outage'); | |
| 36 | files.set(path, bytes.slice()); | |
| 37 | flushed.push([path, [...bytes]]); | |
| 38 | }, | |
| 39 | }; | |
| 40 | }, | |
| 41 | }; | |
| 42 | }, | |
| 43 | }; | |
| 44 | const context = vm.createContext({ | |
| 45 | navigator: { storage: { getDirectory: async () => root } }, | |
| 46 | postMessage: (message) => replies.push(message), | |
| 47 | console: { error() {} }, | |
| 48 | }); | |
| 49 | vm.runInContext(source.replace(/\bexport /g, '').replaceAll('import.meta.url', JSON.stringify(import.meta.url)), context); | |
| 50 | vm.runInContext('storageWorker()', context); | |
| 51 | const send = async (data) => { | |
| 52 | context.onmessage({ data }); | |
| 53 | const expected = replies.length + 1; | |
| 54 | if (data.kind === 'store') context.onmessage({ data: { kind: 'settle' } }); | |
| 55 | for (let turns = 0; replies.length < expected; turns++) { | |
| 56 | assert.ok(turns < 100, 'storage worker did not answer'); | |
| 57 | await Promise.resolve(); | |
| 58 | } | |
| 59 | return replies.at(-1); | |
| 60 | }; | |
| 61 | await send({ kind: 'load' }); | |
| 62 | unavailable = true; | |
| 63 | const first = await send({ kind: 'store', changes: [['replica-wal', 2, [[0, new Uint8Array([1, 2])]]]] }); | |
| 64 | assert.match(first.error, /Injected OPFS outage/); | |
| 65 | await send({ kind: 'store', changes: [['replica-wal', 2, [[1, new Uint8Array([3])]]], ['replica', 1, [[0, new Uint8Array([4])]]]] }); | |
| 66 | assert.equal(files.has('replica'), false); | |
| 67 | assert.equal(flushed.length, 0); | |
| 68 | unavailable = false; | |
| 69 | assert.equal((await send({ kind: 'settle' })).error, null); | |
| 70 | assert.deepEqual(flushed, [['replica-wal', [1, 2]], ['replica-wal', [1, 3]], ['replica', [4]]]); | |
| 71 | assert.equal((await send({ kind: 'settle' })).error, null); | |
| 72 | assert.equal(flushed.length, 3); | |
| 73 | }); |