authoravatarclover caruso<git@paperclover.net>2026-11-16 23:48:22-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-22 00:44:02-08:00
log6586db88bc31255e09f30e9ff799c62057736d5d
tree47b0b05eda086079d6e5e8be02eca6e34f9bbd8c
parentffeeff93bd50abd5d2198b8d2890329265d550a1
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Batch fragmented VM uploads and finish failed-transfer cleanup

Coalesce HTTP fragments into bounded 64 KiB guest writes. Attempt cleanup before returning a known upload failure, retaining background cleanup for disconnects and uncertain failures. Verified real Linux and Windows binary round trips and destination preservation after short uploads. The OAuth integration covers fragmented, empty, oversized and incomplete bodies; the old binary made 201 writes for 200000 bytes, the new binary makes four. 61 Rust tests pass. Assisted-by: gpt-6.1-sol

2 files changed, 63 insertions(+), 15 deletions(-)

dashboard/src/computer.rs+20-10
...@@ -315,22 +315,32 @@ async fn transfer(...@@ -315,22 +315,32 @@ async fn transfer(
315 let result: Result<Value> = tokio::time::timeout_at(transfer.expires.into(), async {315 let result: Result<Value> = tokio::time::timeout_at(transfer.expires.into(), async {
316 let mut stream=request.into_body().into_data_stream();316 let mut stream=request.into_body().into_data_stream();
317 let mut offset=0_u64;317 let mut offset=0_u64;
318 while let Some(chunk)=stream.next().await {318 let mut chunk=bytes::Bytes::new();
319 let chunk=chunk.map_err(|_|Error::new(400,"The upload ended early. Request a new URL and retry."))?;319 let mut buffer=Vec::with_capacity(65536);
320 for bytes in chunk.chunks(65536) {320 loop {
321 if offset+bytes.len() as u64>transfer.size { return Err(Error::new(413,"The upload exceeds its declared size.")); }321 buffer.clear();
322 transfer_access(&computer,&transfer).await?;322 while buffer.len()<65536 {
323 dispatch(&transfer.vm,json!({"action":"file","operation":"write","path":temporary,"offset":offset,"data":base64::engine::general_purpose::STANDARD.encode(bytes)})).await?;323 if chunk.is_empty() {
324 offset+=bytes.len() as u64;324 let Some(next)=stream.next().await else { break };
325 chunk=next.map_err(|_|Error::new(400,"The upload ended early. Request a new URL and retry."))?;
326 }
327 let length=(65536-buffer.len()).min(chunk.len());
328 buffer.extend_from_slice(&chunk.split_to(length));
325 }329 }
330 if offset+buffer.len() as u64>transfer.size { return Err(Error::new(413,"The upload exceeds its declared size.")); }
331 if buffer.is_empty() && offset>0 { break; }
332 transfer_access(&computer,&transfer).await?;
333 dispatch(&transfer.vm,json!({"action":"file","operation":"write","path":temporary,"offset":offset,"data":base64::engine::general_purpose::STANDARD.encode(&buffer)})).await?;
334 offset+=buffer.len() as u64;
335 if buffer.len()<65536 { break; }
326 }336 }
327 if offset!=transfer.size { return Err(Error::new(400,"The upload ended early. Request a new URL and retry.")); }337 if offset!=transfer.size { return Err(Error::new(400,"The upload ended early. Request a new URL and retry.")); }
328 if offset==0 { dispatch(&transfer.vm,json!({"action":"file","operation":"write","path":temporary,"offset":0,"data":""})).await?; }
329 transfer_access(&computer,&transfer).await?;338 transfer_access(&computer,&transfer).await?;
330 dispatch(&transfer.vm,json!({"action":"file","operation":"commit","path":temporary,"destination":transfer.path,"overwrite":transfer.overwrite})).await?;339 dispatch(&transfer.vm,json!({"action":"file","operation":"commit","path":temporary,"destination":transfer.path,"overwrite":transfer.overwrite})).await?;
331 Ok(json!({"path":transfer.path,"bytes":offset}))340 Ok(json!({"path":transfer.path,"bytes":offset}))
332 }).await.map_err(|_|Error::new(410,"This transfer expired. Request a new URL."))?;341 }).await.unwrap_or_else(|_|Err(Error::new(410,"This transfer expired. Request a new URL.")));
333 if result.is_ok() {342 if result.is_ok() || matches!(tokio::time::timeout(Duration::from_secs(15),
343 dispatch(&transfer.vm,json!({"action":"file","operation":"remove","path":temporary}))).await, Ok(Ok(_))) {
334 cleanup.path = None;344 cleanup.path = None;
335 }345 }
336 Ok(axum::Json(result?).into_response())346 Ok(axum::Json(result?).into_response())
tools/computer-client-test.py+43-5
...@@ -30,9 +30,13 @@ class Host(socketserver.StreamRequestHandler):...@@ -30,9 +30,13 @@ class Host(socketserver.StreamRequestHandler):
30 owner = None30 owner = None
31 files = {}31 files = {}
32 power_actions = []32 power_actions = []
33 writes = []
3334
34 def handle(self):35 def handle(self):
35 request = json.loads(self.rfile.readline())36 line = self.rfile.readline()
37 if not line:
38 return
39 request = json.loads(line)
36 try:40 try:
37 reply = {"value": self.answer(request["operation"], request.get("payload") or {})}41 reply = {"value": self.answer(request["operation"], request.get("payload") or {})}
38 except KeyError as missing:42 except KeyError as missing:
...@@ -63,13 +67,16 @@ class Host(socketserver.StreamRequestHandler):...@@ -63,13 +67,16 @@ class Host(socketserver.StreamRequestHandler):
63 if operation == "stat":67 if operation == "stat":
64 return {"directory": True} if path.endswith("/") else {"directory": False, "size": len(files[path])}68 return {"directory": True} if path.endswith("/") else {"directory": False, "size": len(files[path])}
65 if operation == "write":69 if operation == "write":
66 files[path] = files.get(path, b"")[:request["offset"]] + base64.b64decode(request["data"])70 data = base64.b64decode(request["data"])
71 self.writes.append((path, len(data)))
72 files[path] = files.get(path, b"")[:request["offset"]] + data
67 elif operation == "commit":73 elif operation == "commit":
68 files[request["destination"]] = files.pop(path)74 files[request["destination"]] = files.pop(path)
69 elif operation == "read":75 elif operation == "read":
70 data = files[path][request["offset"]:request["offset"] + request["length"]]76 data = files[path][request["offset"]:request["offset"] + request["length"]]
71 return {"data": base64.b64encode(data).decode()}77 return {"data": base64.b64encode(data).decode()}
72 elif operation == "remove":78 elif operation == "remove":
79 time.sleep(.05)
73 files.pop(path, None)80 files.pop(path, None)
74 return {}81 return {}
7582
...@@ -84,8 +91,8 @@ class Client(unittest.TestCase):...@@ -84,8 +91,8 @@ class Client(unittest.TestCase):
84 with socket.socket() as available:91 with socket.socket() as available:
85 available.bind(("127.0.0.1", 0))92 available.bind(("127.0.0.1", 0))
86 port = available.getsockname()[1]93 port = available.getsockname()[1]
87 host = socketserver.ThreadingUnixStreamServer(str(data / "host.sock"), Host)94 cls.host = socketserver.ThreadingUnixStreamServer(str(data / "host.sock"), Host)
88 threading.Thread(target=host.serve_forever, daemon=True).start()95 threading.Thread(target=cls.host.serve_forever, daemon=True).start()
89 environment = {**os.environ, "SNOWGLOBE_DOMAIN": "paperclover.net", "SNOWGLOBE_DATA_DIR": str(data),96 environment = {**os.environ, "SNOWGLOBE_DOMAIN": "paperclover.net", "SNOWGLOBE_DATA_DIR": str(data),
90 "SNOWGLOBE_PUBLIC_ORIGIN": "https://" + HOST, "SNOWGLOBE_AUTH_RP_ID": "auth.paperclover.net",97 "SNOWGLOBE_PUBLIC_ORIGIN": "https://" + HOST, "SNOWGLOBE_AUTH_RP_ID": "auth.paperclover.net",
91 "SNOWGLOBE_FILE_ORIGIN": "https://file.paperclover.net", "SNOWGLOBE_AUTH_REQUIRED": "1",98 "SNOWGLOBE_FILE_ORIGIN": "https://file.paperclover.net", "SNOWGLOBE_AUTH_REQUIRED": "1",
...@@ -96,7 +103,8 @@ class Client(unittest.TestCase):...@@ -96,7 +103,8 @@ class Client(unittest.TestCase):
96 with sqlite3.connect(data / "accounts.sqlite") as db:103 with sqlite3.connect(data / "accounts.sqlite") as db:
97 Host.owner = db.execute("SELECT id FROM users WHERE username='preview'").fetchone()[0]104 Host.owner = db.execute("SELECT id FROM users WHERE username='preview'").fetchone()[0]
98 cls.log = data / "server.log"105 cls.log = data / "server.log"
99 cls.server = subprocess.Popen([BINARY], env=environment, stdout=cls.log.open("wb"), stderr=subprocess.STDOUT)106 with cls.log.open("wb") as log:
107 cls.server = subprocess.Popen([BINARY], env=environment, stdout=log, stderr=subprocess.STDOUT)
100 deadline = time.monotonic() + 30108 deadline = time.monotonic() + 30
101 while True:109 while True:
102 try:110 try:
...@@ -130,6 +138,8 @@ class Client(unittest.TestCase):...@@ -130,6 +138,8 @@ class Client(unittest.TestCase):
130 def tearDownClass(cls):138 def tearDownClass(cls):
131 cls.server.terminate()139 cls.server.terminate()
132 cls.server.wait(timeout=10)140 cls.server.wait(timeout=10)
141 cls.host.shutdown()
142 cls.host.server_close()
133 cls.temporary.cleanup()143 cls.temporary.cleanup()
134144
135 @classmethod145 @classmethod
...@@ -141,6 +151,7 @@ class Client(unittest.TestCase):...@@ -141,6 +151,7 @@ class Client(unittest.TestCase):
141 connection.request(method, path, json.dumps(body) if body is not None else None, headers)151 connection.request(method, path, json.dumps(body) if body is not None else None, headers)
142 response = connection.getresponse()152 response = connection.getresponse()
143 content = response.read()153 content = response.read()
154 connection.close()
144 for cookie in response.headers.get_all("set-cookie") or []:155 for cookie in response.headers.get_all("set-cookie") or []:
145 key, value = cookie.split(";", 1)[0].split("=", 1)156 key, value = cookie.split(";", 1)[0].split("=", 1)
146 cls.cookies[key] = value157 cls.cookies[key] = value
...@@ -181,6 +192,33 @@ class Client(unittest.TestCase):...@@ -181,6 +192,33 @@ class Client(unittest.TestCase):
181 vm.download("test-vm", "/tmp/payload.bin", source.with_suffix(".back"))192 vm.download("test-vm", "/tmp/payload.bin", source.with_suffix(".back"))
182 self.assertEqual(source.with_suffix(".back").read_bytes(), source.read_bytes())193 self.assertEqual(source.with_suffix(".back").read_bytes(), source.read_bytes())
183194
195 data = source.read_bytes()
196 for declared, content, expected in [(len(data), data, 200), (0, b"", 200),
197 (len(data) - 1, data, 413), (len(data) + 1, data, 400)]:
198 destination = f"/tmp/fragmented-{declared}"
199 if expected != 200:
200 Host.files[destination] = b"retained"
201 ticket = vm.call("computer_upload", vm="test-vm", path=destination, size=declared, overwrite=True)
202 def fragments():
203 for offset in range(0, len(content), 997):
204 yield content[offset:offset + 997]
205 time.sleep(.001)
206 connection = http.client.HTTPConnection("127.0.0.1", self.port, timeout=30)
207 connection.request("PUT", urlsplit(ticket["url"]).path, fragments(), self.headers, encode_chunked=True)
208 response = connection.getresponse()
209 body = response.read()
210 connection.close()
211 self.assertEqual(response.status, expected, body)
212 if expected == 200:
213 self.assertEqual(Host.files[destination], content)
214 sizes = [size for path, size in Host.writes if path.startswith(destination + ".snowglobe-")]
215 self.assertEqual(len(sizes), max(1, (len(content) + 65535) // 65536))
216 self.assertEqual(sum(sizes), len(content))
217 else:
218 self.assertEqual(Host.files[destination], b"retained")
219 self.assertFalse(any(path.startswith(destination + ".snowglobe-") for path in Host.files),
220 "upload returned before removing its temporary file")
221
184 self.assertIn("png", vm.call("computer_shot", vm="test-vm"))222 self.assertIn("png", vm.call("computer_shot", vm="test-vm"))
185223
186 state = json.loads(computer.STATE.read_text())224 state = json.loads(computer.STATE.read_text())