diff --git a/dashboard/src/computer.rs b/dashboard/src/computer.rs index 4675da3010a437b4c0e58d5911b56daf4bbcee41..74837c77511d5accb6a6b88d9cf6558bad1224b7 100644 --- a/dashboard/src/computer.rs +++ b/dashboard/src/computer.rs @@ -315,22 +315,32 @@ async fn transfer( let result: Result = tokio::time::timeout_at(transfer.expires.into(), async { let mut stream=request.into_body().into_data_stream(); let mut offset=0_u64; - while let Some(chunk)=stream.next().await { - let chunk=chunk.map_err(|_|Error::new(400,"The upload ended early. Request a new URL and retry."))?; - for bytes in chunk.chunks(65536) { - if offset+bytes.len() as u64>transfer.size { return Err(Error::new(413,"The upload exceeds its declared size.")); } - transfer_access(&computer,&transfer).await?; - dispatch(&transfer.vm,json!({"action":"file","operation":"write","path":temporary,"offset":offset,"data":base64::engine::general_purpose::STANDARD.encode(bytes)})).await?; - offset+=bytes.len() as u64; + let mut chunk=bytes::Bytes::new(); + let mut buffer=Vec::with_capacity(65536); + loop { + buffer.clear(); + while buffer.len()<65536 { + if chunk.is_empty() { + let Some(next)=stream.next().await else { break }; + chunk=next.map_err(|_|Error::new(400,"The upload ended early. Request a new URL and retry."))?; + } + let length=(65536-buffer.len()).min(chunk.len()); + buffer.extend_from_slice(&chunk.split_to(length)); } + if offset+buffer.len() as u64>transfer.size { return Err(Error::new(413,"The upload exceeds its declared size.")); } + if buffer.is_empty() && offset>0 { break; } + transfer_access(&computer,&transfer).await?; + dispatch(&transfer.vm,json!({"action":"file","operation":"write","path":temporary,"offset":offset,"data":base64::engine::general_purpose::STANDARD.encode(&buffer)})).await?; + offset+=buffer.len() as u64; + if buffer.len()<65536 { break; } } if offset!=transfer.size { return Err(Error::new(400,"The upload ended early. Request a new URL and retry.")); } - if offset==0 { dispatch(&transfer.vm,json!({"action":"file","operation":"write","path":temporary,"offset":0,"data":""})).await?; } transfer_access(&computer,&transfer).await?; dispatch(&transfer.vm,json!({"action":"file","operation":"commit","path":temporary,"destination":transfer.path,"overwrite":transfer.overwrite})).await?; Ok(json!({"path":transfer.path,"bytes":offset})) - }).await.map_err(|_|Error::new(410,"This transfer expired. Request a new URL."))?; - if result.is_ok() { + }).await.unwrap_or_else(|_|Err(Error::new(410,"This transfer expired. Request a new URL."))); + if result.is_ok() || matches!(tokio::time::timeout(Duration::from_secs(15), + dispatch(&transfer.vm,json!({"action":"file","operation":"remove","path":temporary}))).await, Ok(Ok(_))) { cleanup.path = None; } Ok(axum::Json(result?).into_response()) diff --git a/tools/computer-client-test.py b/tools/computer-client-test.py index dc33c2887dde4c6268319db7ed6b785579519a6e..0958859bfa22a6129566772e7b2286adcaf1ca5c 100644 --- a/tools/computer-client-test.py +++ b/tools/computer-client-test.py @@ -30,9 +30,13 @@ class Host(socketserver.StreamRequestHandler): owner = None files = {} power_actions = [] + writes = [] def handle(self): - request = json.loads(self.rfile.readline()) + line = self.rfile.readline() + if not line: + return + request = json.loads(line) try: reply = {"value": self.answer(request["operation"], request.get("payload") or {})} except KeyError as missing: @@ -63,13 +67,16 @@ class Host(socketserver.StreamRequestHandler): if operation == "stat": return {"directory": True} if path.endswith("/") else {"directory": False, "size": len(files[path])} if operation == "write": - files[path] = files.get(path, b"")[:request["offset"]] + base64.b64decode(request["data"]) + data = base64.b64decode(request["data"]) + self.writes.append((path, len(data))) + files[path] = files.get(path, b"")[:request["offset"]] + data elif operation == "commit": files[request["destination"]] = files.pop(path) elif operation == "read": data = files[path][request["offset"]:request["offset"] + request["length"]] return {"data": base64.b64encode(data).decode()} elif operation == "remove": + time.sleep(.05) files.pop(path, None) return {} @@ -84,8 +91,8 @@ class Client(unittest.TestCase): with socket.socket() as available: available.bind(("127.0.0.1", 0)) port = available.getsockname()[1] - host = socketserver.ThreadingUnixStreamServer(str(data / "host.sock"), Host) - threading.Thread(target=host.serve_forever, daemon=True).start() + cls.host = socketserver.ThreadingUnixStreamServer(str(data / "host.sock"), Host) + threading.Thread(target=cls.host.serve_forever, daemon=True).start() environment = {**os.environ, "SNOWGLOBE_DOMAIN": "paperclover.net", "SNOWGLOBE_DATA_DIR": str(data), "SNOWGLOBE_PUBLIC_ORIGIN": "https://" + HOST, "SNOWGLOBE_AUTH_RP_ID": "auth.paperclover.net", "SNOWGLOBE_FILE_ORIGIN": "https://file.paperclover.net", "SNOWGLOBE_AUTH_REQUIRED": "1", @@ -96,7 +103,8 @@ class Client(unittest.TestCase): with sqlite3.connect(data / "accounts.sqlite") as db: Host.owner = db.execute("SELECT id FROM users WHERE username='preview'").fetchone()[0] cls.log = data / "server.log" - cls.server = subprocess.Popen([BINARY], env=environment, stdout=cls.log.open("wb"), stderr=subprocess.STDOUT) + with cls.log.open("wb") as log: + cls.server = subprocess.Popen([BINARY], env=environment, stdout=log, stderr=subprocess.STDOUT) deadline = time.monotonic() + 30 while True: try: @@ -130,6 +138,8 @@ class Client(unittest.TestCase): def tearDownClass(cls): cls.server.terminate() cls.server.wait(timeout=10) + cls.host.shutdown() + cls.host.server_close() cls.temporary.cleanup() @classmethod @@ -141,6 +151,7 @@ class Client(unittest.TestCase): connection.request(method, path, json.dumps(body) if body is not None else None, headers) response = connection.getresponse() content = response.read() + connection.close() for cookie in response.headers.get_all("set-cookie") or []: key, value = cookie.split(";", 1)[0].split("=", 1) cls.cookies[key] = value @@ -181,6 +192,33 @@ class Client(unittest.TestCase): vm.download("test-vm", "/tmp/payload.bin", source.with_suffix(".back")) self.assertEqual(source.with_suffix(".back").read_bytes(), source.read_bytes()) + data = source.read_bytes() + for declared, content, expected in [(len(data), data, 200), (0, b"", 200), + (len(data) - 1, data, 413), (len(data) + 1, data, 400)]: + destination = f"/tmp/fragmented-{declared}" + if expected != 200: + Host.files[destination] = b"retained" + ticket = vm.call("computer_upload", vm="test-vm", path=destination, size=declared, overwrite=True) + def fragments(): + for offset in range(0, len(content), 997): + yield content[offset:offset + 997] + time.sleep(.001) + connection = http.client.HTTPConnection("127.0.0.1", self.port, timeout=30) + connection.request("PUT", urlsplit(ticket["url"]).path, fragments(), self.headers, encode_chunked=True) + response = connection.getresponse() + body = response.read() + connection.close() + self.assertEqual(response.status, expected, body) + if expected == 200: + self.assertEqual(Host.files[destination], content) + sizes = [size for path, size in Host.writes if path.startswith(destination + ".snowglobe-")] + self.assertEqual(len(sizes), max(1, (len(content) + 65535) // 65536)) + self.assertEqual(sum(sizes), len(content)) + else: + self.assertEqual(Host.files[destination], b"retained") + self.assertFalse(any(path.startswith(destination + ".snowglobe-") for path in Host.files), + "upload returned before removing its temporary file") + self.assertIn("png", vm.call("computer_shot", vm="test-vm")) state = json.loads(computer.STATE.read_text())