From 6586db88bc31255e09f30e9ff799c62057736d5d Mon Sep 17 00:00:00 2001 From: clover caruso Date: Mon, 16 Nov 2026 23:48:22 -0800 Subject: [PATCH] Batch fragmented VM uploads and finish failed-transfer cleanup Coalesce HTTP fragments into bounded 64 KiB guest writes. Attempt cleanup before returning a known upload failure, retaining background cleanup for disconnects and uncertain failures. Verified real Linux and Windows binary round trips and destination preservation after short uploads. The OAuth integration covers fragmented, empty, oversized and incomplete bodies; the old binary made 201 writes for 200000 bytes, the new binary makes four. 61 Rust tests pass. Assisted-by: gpt-6.1-sol --- dashboard/src/computer.rs | 30 ++++++++++++++-------- tools/computer-client-test.py | 48 +++++++++++++++++++++++++++++++---- 2 files changed, 63 insertions(+), 15 deletions(-) diff --git a/dashboard/src/computer.rs b/dashboard/src/computer.rs index 4675da3010a437b4c0e58d5911b56daf4bbcee41..74837c77511d5accb6a6b88d9cf6558bad1224b7 100644 --- a/dashboard/src/computer.rs +++ b/dashboard/src/computer.rs @@ -315,22 +315,32 @@ async fn transfer( let result: Result = tokio::time::timeout_at(transfer.expires.into(), async { let mut stream=request.into_body().into_data_stream(); let mut offset=0_u64; - while let Some(chunk)=stream.next().await { - let chunk=chunk.map_err(|_|Error::new(400,"The upload ended early. Request a new URL and retry."))?; - for bytes in chunk.chunks(65536) { - if offset+bytes.len() as u64>transfer.size { return Err(Error::new(413,"The upload exceeds its declared size.")); } - transfer_access(&computer,&transfer).await?; - dispatch(&transfer.vm,json!({"action":"file","operation":"write","path":temporary,"offset":offset,"data":base64::engine::general_purpose::STANDARD.encode(bytes)})).await?; - offset+=bytes.len() as u64; + let mut chunk=bytes::Bytes::new(); + let mut buffer=Vec::with_capacity(65536); + loop { + buffer.clear(); + while buffer.len()<65536 { + if chunk.is_empty() { + let Some(next)=stream.next().await else { break }; + chunk=next.map_err(|_|Error::new(400,"The upload ended early. Request a new URL and retry."))?; + } + let length=(65536-buffer.len()).min(chunk.len()); + buffer.extend_from_slice(&chunk.split_to(length)); } + if offset+buffer.len() as u64>transfer.size { return Err(Error::new(413,"The upload exceeds its declared size.")); } + if buffer.is_empty() && offset>0 { break; } + transfer_access(&computer,&transfer).await?; + dispatch(&transfer.vm,json!({"action":"file","operation":"write","path":temporary,"offset":offset,"data":base64::engine::general_purpose::STANDARD.encode(&buffer)})).await?; + offset+=buffer.len() as u64; + if buffer.len()<65536 { break; } } if offset!=transfer.size { return Err(Error::new(400,"The upload ended early. Request a new URL and retry.")); } - if offset==0 { dispatch(&transfer.vm,json!({"action":"file","operation":"write","path":temporary,"offset":0,"data":""})).await?; } transfer_access(&computer,&transfer).await?; dispatch(&transfer.vm,json!({"action":"file","operation":"commit","path":temporary,"destination":transfer.path,"overwrite":transfer.overwrite})).await?; Ok(json!({"path":transfer.path,"bytes":offset})) - }).await.map_err(|_|Error::new(410,"This transfer expired. Request a new URL."))?; - if result.is_ok() { + }).await.unwrap_or_else(|_|Err(Error::new(410,"This transfer expired. Request a new URL."))); + if result.is_ok() || matches!(tokio::time::timeout(Duration::from_secs(15), + dispatch(&transfer.vm,json!({"action":"file","operation":"remove","path":temporary}))).await, Ok(Ok(_))) { cleanup.path = None; } Ok(axum::Json(result?).into_response()) diff --git a/tools/computer-client-test.py b/tools/computer-client-test.py index dc33c2887dde4c6268319db7ed6b785579519a6e..0958859bfa22a6129566772e7b2286adcaf1ca5c 100644 --- a/tools/computer-client-test.py +++ b/tools/computer-client-test.py @@ -30,9 +30,13 @@ class Host(socketserver.StreamRequestHandler): owner = None files = {} power_actions = [] + writes = [] def handle(self): - request = json.loads(self.rfile.readline()) + line = self.rfile.readline() + if not line: + return + request = json.loads(line) try: reply = {"value": self.answer(request["operation"], request.get("payload") or {})} except KeyError as missing: @@ -63,13 +67,16 @@ class Host(socketserver.StreamRequestHandler): if operation == "stat": return {"directory": True} if path.endswith("/") else {"directory": False, "size": len(files[path])} if operation == "write": - files[path] = files.get(path, b"")[:request["offset"]] + base64.b64decode(request["data"]) + data = base64.b64decode(request["data"]) + self.writes.append((path, len(data))) + files[path] = files.get(path, b"")[:request["offset"]] + data elif operation == "commit": files[request["destination"]] = files.pop(path) elif operation == "read": data = files[path][request["offset"]:request["offset"] + request["length"]] return {"data": base64.b64encode(data).decode()} elif operation == "remove": + time.sleep(.05) files.pop(path, None) return {} @@ -84,8 +91,8 @@ class Client(unittest.TestCase): with socket.socket() as available: available.bind(("127.0.0.1", 0)) port = available.getsockname()[1] - host = socketserver.ThreadingUnixStreamServer(str(data / "host.sock"), Host) - threading.Thread(target=host.serve_forever, daemon=True).start() + cls.host = socketserver.ThreadingUnixStreamServer(str(data / "host.sock"), Host) + threading.Thread(target=cls.host.serve_forever, daemon=True).start() environment = {**os.environ, "SNOWGLOBE_DOMAIN": "paperclover.net", "SNOWGLOBE_DATA_DIR": str(data), "SNOWGLOBE_PUBLIC_ORIGIN": "https://" + HOST, "SNOWGLOBE_AUTH_RP_ID": "auth.paperclover.net", "SNOWGLOBE_FILE_ORIGIN": "https://file.paperclover.net", "SNOWGLOBE_AUTH_REQUIRED": "1", @@ -96,7 +103,8 @@ class Client(unittest.TestCase): with sqlite3.connect(data / "accounts.sqlite") as db: Host.owner = db.execute("SELECT id FROM users WHERE username='preview'").fetchone()[0] cls.log = data / "server.log" - cls.server = subprocess.Popen([BINARY], env=environment, stdout=cls.log.open("wb"), stderr=subprocess.STDOUT) + with cls.log.open("wb") as log: + cls.server = subprocess.Popen([BINARY], env=environment, stdout=log, stderr=subprocess.STDOUT) deadline = time.monotonic() + 30 while True: try: @@ -130,6 +138,8 @@ class Client(unittest.TestCase): def tearDownClass(cls): cls.server.terminate() cls.server.wait(timeout=10) + cls.host.shutdown() + cls.host.server_close() cls.temporary.cleanup() @classmethod @@ -141,6 +151,7 @@ class Client(unittest.TestCase): connection.request(method, path, json.dumps(body) if body is not None else None, headers) response = connection.getresponse() content = response.read() + connection.close() for cookie in response.headers.get_all("set-cookie") or []: key, value = cookie.split(";", 1)[0].split("=", 1) cls.cookies[key] = value @@ -181,6 +192,33 @@ class Client(unittest.TestCase): vm.download("test-vm", "/tmp/payload.bin", source.with_suffix(".back")) self.assertEqual(source.with_suffix(".back").read_bytes(), source.read_bytes()) + data = source.read_bytes() + for declared, content, expected in [(len(data), data, 200), (0, b"", 200), + (len(data) - 1, data, 413), (len(data) + 1, data, 400)]: + destination = f"/tmp/fragmented-{declared}" + if expected != 200: + Host.files[destination] = b"retained" + ticket = vm.call("computer_upload", vm="test-vm", path=destination, size=declared, overwrite=True) + def fragments(): + for offset in range(0, len(content), 997): + yield content[offset:offset + 997] + time.sleep(.001) + connection = http.client.HTTPConnection("127.0.0.1", self.port, timeout=30) + connection.request("PUT", urlsplit(ticket["url"]).path, fragments(), self.headers, encode_chunked=True) + response = connection.getresponse() + body = response.read() + connection.close() + self.assertEqual(response.status, expected, body) + if expected == 200: + self.assertEqual(Host.files[destination], content) + sizes = [size for path, size in Host.writes if path.startswith(destination + ".snowglobe-")] + self.assertEqual(len(sizes), max(1, (len(content) + 65535) // 65536)) + self.assertEqual(sum(sizes), len(content)) + else: + self.assertEqual(Host.files[destination], b"retained") + self.assertFalse(any(path.startswith(destination + ".snowglobe-") for path in Host.files), + "upload returned before removing its temporary file") + self.assertIn("png", vm.call("computer_shot", vm="test-vm")) state = json.loads(computer.STATE.read_text()) -- 2.54.1