authoravatarclover caruso<git@paperclover.net>2026-11-16 23:48:22-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-22 00:44:02-08:00
log6586db88bc31255e09f30e9ff799c62057736d5d
tree47b0b05eda086079d6e5e8be02eca6e34f9bbd8c
parentffeeff93bd50abd5d2198b8d2890329265d550a1
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Batch fragmented VM uploads and finish failed-transfer cleanup

Coalesce HTTP fragments into bounded 64 KiB guest writes. Attempt cleanup before returning a known upload failure, retaining background cleanup for disconnects and uncertain failures. Verified real Linux and Windows binary round trips and destination preservation after short uploads. The OAuth integration covers fragmented, empty, oversized and incomplete bodies; the old binary made 201 writes for 200000 bytes, the new binary makes four. 61 Rust tests pass. Assisted-by: gpt-6.1-sol

2 files changed, 63 insertions(+), 15 deletions(-)

dashboard/src/computer.rs+20-10
......@@ -315,22 +315,32 @@ async fn transfer(
315315 let result: Result<Value> = tokio::time::timeout_at(transfer.expires.into(), async {
316316 let mut stream=request.into_body().into_data_stream();
317317 let mut offset=0_u64;
318 while let Some(chunk)=stream.next().await {
319 let chunk=chunk.map_err(|_|Error::new(400,"The upload ended early. Request a new URL and retry."))?;
320 for bytes in chunk.chunks(65536) {
321 if offset+bytes.len() as u64>transfer.size { return Err(Error::new(413,"The upload exceeds its declared size.")); }
322 transfer_access(&computer,&transfer).await?;
323 dispatch(&transfer.vm,json!({"action":"file","operation":"write","path":temporary,"offset":offset,"data":base64::engine::general_purpose::STANDARD.encode(bytes)})).await?;
324 offset+=bytes.len() as u64;
318 let mut chunk=bytes::Bytes::new();
319 let mut buffer=Vec::with_capacity(65536);
320 loop {
321 buffer.clear();
322 while buffer.len()<65536 {
323 if chunk.is_empty() {
324 let Some(next)=stream.next().await else { break };
325 chunk=next.map_err(|_|Error::new(400,"The upload ended early. Request a new URL and retry."))?;
326 }
327 let length=(65536-buffer.len()).min(chunk.len());
328 buffer.extend_from_slice(&chunk.split_to(length));
325329 }
330 if offset+buffer.len() as u64>transfer.size { return Err(Error::new(413,"The upload exceeds its declared size.")); }
331 if buffer.is_empty() && offset>0 { break; }
332 transfer_access(&computer,&transfer).await?;
333 dispatch(&transfer.vm,json!({"action":"file","operation":"write","path":temporary,"offset":offset,"data":base64::engine::general_purpose::STANDARD.encode(&buffer)})).await?;
334 offset+=buffer.len() as u64;
335 if buffer.len()<65536 { break; }
326336 }
327337 if offset!=transfer.size { return Err(Error::new(400,"The upload ended early. Request a new URL and retry.")); }
328 if offset==0 { dispatch(&transfer.vm,json!({"action":"file","operation":"write","path":temporary,"offset":0,"data":""})).await?; }
329338 transfer_access(&computer,&transfer).await?;
330339 dispatch(&transfer.vm,json!({"action":"file","operation":"commit","path":temporary,"destination":transfer.path,"overwrite":transfer.overwrite})).await?;
331340 Ok(json!({"path":transfer.path,"bytes":offset}))
332 }).await.map_err(|_|Error::new(410,"This transfer expired. Request a new URL."))?;
333 if result.is_ok() {
341 }).await.unwrap_or_else(|_|Err(Error::new(410,"This transfer expired. Request a new URL.")));
342 if result.is_ok() || matches!(tokio::time::timeout(Duration::from_secs(15),
343 dispatch(&transfer.vm,json!({"action":"file","operation":"remove","path":temporary}))).await, Ok(Ok(_))) {
334344 cleanup.path = None;
335345 }
336346 Ok(axum::Json(result?).into_response())
tools/computer-client-test.py+43-5
......@@ -30,9 +30,13 @@ class Host(socketserver.StreamRequestHandler):
3030 owner = None
3131 files = {}
3232 power_actions = []
33 writes = []
3334
3435 def handle(self):
35 request = json.loads(self.rfile.readline())
36 line = self.rfile.readline()
37 if not line:
38 return
39 request = json.loads(line)
3640 try:
3741 reply = {"value": self.answer(request["operation"], request.get("payload") or {})}
3842 except KeyError as missing:
......@@ -63,13 +67,16 @@ class Host(socketserver.StreamRequestHandler):
6367 if operation == "stat":
6468 return {"directory": True} if path.endswith("/") else {"directory": False, "size": len(files[path])}
6569 if operation == "write":
66 files[path] = files.get(path, b"")[:request["offset"]] + base64.b64decode(request["data"])
70 data = base64.b64decode(request["data"])
71 self.writes.append((path, len(data)))
72 files[path] = files.get(path, b"")[:request["offset"]] + data
6773 elif operation == "commit":
6874 files[request["destination"]] = files.pop(path)
6975 elif operation == "read":
7076 data = files[path][request["offset"]:request["offset"] + request["length"]]
7177 return {"data": base64.b64encode(data).decode()}
7278 elif operation == "remove":
79 time.sleep(.05)
7380 files.pop(path, None)
7481 return {}
7582
......@@ -84,8 +91,8 @@ class Client(unittest.TestCase):
8491 with socket.socket() as available:
8592 available.bind(("127.0.0.1", 0))
8693 port = available.getsockname()[1]
87 host = socketserver.ThreadingUnixStreamServer(str(data / "host.sock"), Host)
88 threading.Thread(target=host.serve_forever, daemon=True).start()
94 cls.host = socketserver.ThreadingUnixStreamServer(str(data / "host.sock"), Host)
95 threading.Thread(target=cls.host.serve_forever, daemon=True).start()
8996 environment = {**os.environ, "SNOWGLOBE_DOMAIN": "paperclover.net", "SNOWGLOBE_DATA_DIR": str(data),
9097 "SNOWGLOBE_PUBLIC_ORIGIN": "https://" + HOST, "SNOWGLOBE_AUTH_RP_ID": "auth.paperclover.net",
9198 "SNOWGLOBE_FILE_ORIGIN": "https://file.paperclover.net", "SNOWGLOBE_AUTH_REQUIRED": "1",
......@@ -96,7 +103,8 @@ class Client(unittest.TestCase):
96103 with sqlite3.connect(data / "accounts.sqlite") as db:
97104 Host.owner = db.execute("SELECT id FROM users WHERE username='preview'").fetchone()[0]
98105 cls.log = data / "server.log"
99 cls.server = subprocess.Popen([BINARY], env=environment, stdout=cls.log.open("wb"), stderr=subprocess.STDOUT)
106 with cls.log.open("wb") as log:
107 cls.server = subprocess.Popen([BINARY], env=environment, stdout=log, stderr=subprocess.STDOUT)
100108 deadline = time.monotonic() + 30
101109 while True:
102110 try:
......@@ -130,6 +138,8 @@ class Client(unittest.TestCase):
130138 def tearDownClass(cls):
131139 cls.server.terminate()
132140 cls.server.wait(timeout=10)
141 cls.host.shutdown()
142 cls.host.server_close()
133143 cls.temporary.cleanup()
134144
135145 @classmethod
......@@ -141,6 +151,7 @@ class Client(unittest.TestCase):
141151 connection.request(method, path, json.dumps(body) if body is not None else None, headers)
142152 response = connection.getresponse()
143153 content = response.read()
154 connection.close()
144155 for cookie in response.headers.get_all("set-cookie") or []:
145156 key, value = cookie.split(";", 1)[0].split("=", 1)
146157 cls.cookies[key] = value
......@@ -181,6 +192,33 @@ class Client(unittest.TestCase):
181192 vm.download("test-vm", "/tmp/payload.bin", source.with_suffix(".back"))
182193 self.assertEqual(source.with_suffix(".back").read_bytes(), source.read_bytes())
183194
195 data = source.read_bytes()
196 for declared, content, expected in [(len(data), data, 200), (0, b"", 200),
197 (len(data) - 1, data, 413), (len(data) + 1, data, 400)]:
198 destination = f"/tmp/fragmented-{declared}"
199 if expected != 200:
200 Host.files[destination] = b"retained"
201 ticket = vm.call("computer_upload", vm="test-vm", path=destination, size=declared, overwrite=True)
202 def fragments():
203 for offset in range(0, len(content), 997):
204 yield content[offset:offset + 997]
205 time.sleep(.001)
206 connection = http.client.HTTPConnection("127.0.0.1", self.port, timeout=30)
207 connection.request("PUT", urlsplit(ticket["url"]).path, fragments(), self.headers, encode_chunked=True)
208 response = connection.getresponse()
209 body = response.read()
210 connection.close()
211 self.assertEqual(response.status, expected, body)
212 if expected == 200:
213 self.assertEqual(Host.files[destination], content)
214 sizes = [size for path, size in Host.writes if path.startswith(destination + ".snowglobe-")]
215 self.assertEqual(len(sizes), max(1, (len(content) + 65535) // 65536))
216 self.assertEqual(sum(sizes), len(content))
217 else:
218 self.assertEqual(Host.files[destination], b"retained")
219 self.assertFalse(any(path.startswith(destination + ".snowglobe-") for path in Host.files),
220 "upload returned before removing its temporary file")
221
184222 self.assertIn("png", vm.call("computer_shot", vm="test-vm"))
185223
186224 state = json.loads(computer.STATE.read_text())