diff --git a/crates/notebook/src/fs/web.rs b/crates/notebook/src/fs/web.rs index 11f0e22bc7c515eca9b433f1ebc59421d7e49571..71094d0dfce208ccb15715862adf055f77afbaec 100644 --- a/crates/notebook/src/fs/web.rs +++ b/crates/notebook/src/fs/web.rs @@ -876,7 +876,8 @@ pub async fn durable() -> io::Result<()> { fn flush_storage() -> Result; } let failure = |error: JsValue| { - io::Error::other( + io::Error::new( + ErrorKind::WouldBlock, error .as_string() .unwrap_or_else(|| "Browser storage failed".into()), diff --git a/crates/notebook/src/sync.rs b/crates/notebook/src/sync.rs index 49f2260d3d9d15e96fe85c1c77788cd028755e9a..111154225d140534d106a027c834d240225a1670 100644 --- a/crates/notebook/src/sync.rs +++ b/crates/notebook/src/sync.rs @@ -232,8 +232,10 @@ impl Replica { /// The same guarded synchronization step, awaiting non-blocking remote operations. pub async fn sync_once_async(&self, remote: &mut impl Remote) -> Result { let result = self.sync_once_inner(remote).await; - crate::fs::durable().await?; - result + let durable = crate::fs::durable().await; + let synced = result?; + durable?; + Ok(synced) } /// Ownership uses `try_lock`; the cache mutex is released before every await. diff --git a/tools/TESTING.md b/tools/TESTING.md index ede71e01e1ca73c17b5be5b0d5372dad95479142..cb38492137dbdaa1f63e8ff94af7d0c8fbadbe44 100644 --- a/tools/TESTING.md +++ b/tools/TESTING.md @@ -20,7 +20,7 @@ another, and the exit status is the result. | `windows-aarch64`, `linux-*` | Clippy `-D warnings` on what ships (libraries and binaries but `mobile`), then the `snowbound` build; Linux through `platform/linux/cargo.sh`, which links with zig against glibc 2.17 | | `ios` | `xcodebuild` of the simulator app, unsigned | | `web` | Clippy `-D warnings` on `snowbound` for `wasm32-unknown-unknown`, SQLite built by nixpkgs' clang; `release_web.py` links, optimizes and deploys the static folder | -| `web-js` | Node boundary tests for Live Share browser sockets | +| `web-js` | Node boundary tests for Live Share browser sockets and storage | | `macos-10.6` | `platform/snow-leopard/cargo.sh` build of `snowbound`; skipped, saying why, without the SDK or nightly `rust-src` | ```sh diff --git a/tools/ci.py b/tools/ci.py index 7cc1135851170accbd8fcf3a9fdf2ba2a5c55219..d140ba8a75823b15a7a5ce990b8594f411e1d220 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -80,7 +80,7 @@ def lanes(): missing=None if shutil.which('zig') else 'needs zig')) result.append(dict( name='web-js', minutes=5, packages=['notebook'], paths=('tools/web/',), - commands=[['node', '--test', 'tools/web/test_live.mjs']], + commands=[['node', '--test', 'tools/web/test_*.mjs']], missing=None if shutil.which('node') else 'needs node')) targets = subprocess.run(['rustup', 'target', 'list', '--installed'], capture_output=True, text=True).stdout wasm = web_environment() diff --git a/tools/web/storage_recovery.mjs b/tools/web/storage_recovery.mjs new file mode 100644 index 0000000000000000000000000000000000000000..b2cb1f3038374eaf068ece2be93e9bf94e7ce682 --- /dev/null +++ b/tools/web/storage_recovery.mjs @@ -0,0 +1,92 @@ +// Generates ux-testing/drive.mjs steps for a local build and a disposable .one fixture. +import { readFile, writeFile } from 'node:fs/promises'; + +const [url, fixture, output, phase = '0'] = process.argv.slice(2); +if (!url || !fixture || !output) throw new Error('Usage: node storage_recovery.mjs APP_URL FIXTURE.one STEPS.json [0|1]'); +if (!['0', '1'].includes(phase)) throw new Error('Storage phase must be 0 or 1'); +const bytes = (await readFile(fixture)).toString('base64'); + +async function ready() { + const end = Date.now() + 40000; + while ((!globalThis.snowboundFlushStorage || document.querySelector('#shell')) && Date.now() < end) + await new Promise(resolve => setTimeout(resolve, 100)); + if (!globalThis.snowboundFlushStorage || document.querySelector('#shell')) throw new Error('App did not start'); + const script = [...document.scripts].find(script => script.textContent.includes('import init, * as snowbound')); + const path = script?.textContent.match(/import init, \* as snowbound from ["']([^"']+)["']/)?.[1]; + if (!path) throw new Error('App module import was not found'); + const app = await import(new URL(path, location.href).href); + app.accessibility(true); + globalThis.storageProbe = { + app, + original: globalThis.snowboundFlushStorage, + text() { + const root = document.querySelector('#a11y'); + return [root?.textContent, ...Array.from(root?.querySelectorAll('[aria-label]') ?? [], node => node.getAttribute('aria-label'))].join(' '); + }, + async bytes() { + const root = await navigator.storage.getDirectory(); + async function find(dir) { + for await (const [name, entry] of dir.entries()) { + if (entry.kind === 'directory') { + const found = await find(entry); + if (found) return found; + } else if (name === 'Storage Regression.one') { + return new Uint8Array(await (await entry.getFile()).arrayBuffer()); + } + } + } + return find(root); + }, + async disk() { + const bytes = await this.bytes(); + if (!bytes) return null; + return { transactions: new DataView(bytes.buffer).getUint32(96, true), length: bytes.length }; + }, + async until(check, message) { + const end = Date.now() + 20000; + while (Date.now() < end) { + if (await check()) return; + await new Promise(resolve => setTimeout(resolve, 100)); + } + throw new Error(message); + }, + }; +} + +function arm(successfulBeforeFailure) { + const probe = globalThis.storageProbe; + probe.rejected = []; + probe.succeeded = 0; + probe.unavailable = true; + globalThis.snowboundFlushStorage = async () => { + if (probe.unavailable && probe.succeeded >= successfulBeforeFailure) { + probe.rejected.push(Date.now()); + throw new Error('Injected recoverable browser storage rejection'); + } + await probe.original(); + probe.succeeded++; + }; +} + +const expression = (fn, ...args) => `(${fn})(${args.map(arg => JSON.stringify(arg)).join(',')})`; +const steps = [ + { goto: url }, + { eval: expression(ready) }, + { eval: `(async()=>{const p=storageProbe;p.app.files('open',[['Storage Regression.one',Uint8Array.from(atob(${JSON.stringify(bytes)}),c=>c.charCodeAt(0))]]);await p.until(async()=>await p.disk()&&p.text().includes('Storage Regression'),'Fixture did not open and reach OPFS');await p.original();})()` }, +]; + +for (const [before, marker] of [[0, ' storage recovered before publication'], [1, ' storage recovered after acknowledgment']].filter(([before]) => before === Number(phase))) { + steps.push( + { eval: '(async()=>{storageProbe.base=await storageProbe.disk();})()' }, + { eval: expression(arm, before) }, + { click: [100, 126] }, + { key: ['End', 'End', 35, 6] }, + { type: marker }, + { eval: `(async()=>{const p=storageProbe;await p.until(()=>p.rejected.length>=3,'Automatic sync did not retry the rejected flush');if(p.rejected.length>8||p.rejected.at(-1)-p.rejected[0]<1000)throw Error('Automatic sync retry loop is unbounded');const disk=await p.disk();if(disk.transactions!==p.base.transactions+${before})throw Error('Unexpected publication count during storage failure');return {rejected:p.rejected.length,disk};})()`, print: true }, + { eval: `(async()=>{const p=storageProbe;const succeeded=p.succeeded;p.unavailable=false;await p.until(async()=>p.succeeded>succeeded&&(await p.disk()).transactions===p.base.transactions+1,'Same worker did not recover and publish exactly once');await p.original();if(!p.text().includes(${JSON.stringify(marker)}))throw Error('Recovered text is absent from the accessibility tree');return {rejected:p.rejected.length,succeeded:p.succeeded,disk:await p.disk()};})()`, print: true }, + { goto: url }, + { eval: expression(ready) }, + { eval: `(async()=>{const p=storageProbe;const bytes=await p.bytes();if(!bytes)throw Error('Stored fixture is absent');p.app.files('open',[['Readback.one',bytes]]);await p.until(()=>p.text().includes(${JSON.stringify(marker)}),'Recovered edit did not survive cold opening from OPFS');return p.disk();})()`, print: true }, + ); +} +await writeFile(output, `${JSON.stringify(steps, null, 2)}\n`); diff --git a/tools/web/test_storage.mjs b/tools/web/test_storage.mjs new file mode 100644 index 0000000000000000000000000000000000000000..839d5207cb9a88506230b3168829a784d1b6583e --- /dev/null +++ b/tools/web/test_storage.mjs @@ -0,0 +1,73 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import test from 'node:test'; +import vm from 'node:vm'; + +const source = await readFile(new URL('../../crates/snowbound/web/glue.js', import.meta.url), 'utf8'); + +test('an OPFS flush rejection retains ordered writes until storage recovers', async () => { + const files = new Map([['existing', new Uint8Array([0])]]); + const flushed = []; + const replies = []; + let unavailable = false; + const root = { + async *entries() { + for (const [name, bytes] of files) { + yield [name, { kind: 'file', getFile: async () => ({ arrayBuffer: async () => bytes.buffer }) }]; + } + }, + async getFileHandle(path) { + let bytes = files.get(path)?.slice() ?? new Uint8Array(); + return { + async createSyncAccessHandle() { + return { + write(part, { at }) { + if (at + part.length > bytes.length) { + const extended = new Uint8Array(at + part.length); + extended.set(bytes); + bytes = extended; + } + bytes.set(part, at); + return part.length; + }, + truncate(length) { bytes = bytes.slice(0, length); }, + flush() { + if (unavailable) throw new Error('Injected OPFS outage'); + files.set(path, bytes.slice()); + flushed.push([path, [...bytes]]); + }, + }; + }, + }; + }, + }; + const context = vm.createContext({ + navigator: { storage: { getDirectory: async () => root } }, + postMessage: (message) => replies.push(message), + console: { error() {} }, + }); + vm.runInContext(source.replace(/\bexport /g, '').replaceAll('import.meta.url', JSON.stringify(import.meta.url)), context); + vm.runInContext('storageWorker()', context); + const send = async (data) => { + context.onmessage({ data }); + const expected = replies.length + 1; + if (data.kind === 'store') context.onmessage({ data: { kind: 'settle' } }); + for (let turns = 0; replies.length < expected; turns++) { + assert.ok(turns < 100, 'storage worker did not answer'); + await Promise.resolve(); + } + return replies.at(-1); + }; + await send({ kind: 'load' }); + unavailable = true; + const first = await send({ kind: 'store', changes: [['replica-wal', 2, [[0, new Uint8Array([1, 2])]]]] }); + assert.match(first.error, /Injected OPFS outage/); + await send({ kind: 'store', changes: [['replica-wal', 2, [[1, new Uint8Array([3])]]], ['replica', 1, [[0, new Uint8Array([4])]]]] }); + assert.equal(files.has('replica'), false); + assert.equal(flushed.length, 0); + unavailable = false; + assert.equal((await send({ kind: 'settle' })).error, null); + assert.deepEqual(flushed, [['replica-wal', [1, 2]], ['replica-wal', [1, 3]], ['replica', [4]]]); + assert.equal((await send({ kind: 'settle' })).error, null); + assert.equal(flushed.length, 3); +});