| author | |
| committer | |
| log | dc25dbf0f9528632f2d593b93197f9390e0567ac |
| tree | f35cf0235baba556e6a6b2d0de7b37b29909e429 |
| parent | 1c09d8d7fbaeda2067216015d56eee511114f461 |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Use bounded guest-advertised batches after a compatible 64 KiB start, retaining small QEMU agent writes. Keep the one-use link expiry separate from the active transfer budget so large files can finish on slower disks. Preserve per-batch authorization and atomic destination replacement.
Assisted-by: gpt-6.1-sol7 files changed, 75 insertions(+), 19 deletions(-)
dashboard/src/computer-tools.json+1-1| ... | @@ -487,7 +487,7 @@ | ... | @@ -487,7 +487,7 @@ |
| 487 | }, | 487 | }, |
| 488 | { | 488 | { |
| 489 | "name": "computer_upload", | 489 | "name": "computer_upload", |
| 490 | "description": "Create a one-use HTTP PUT URL for uploading a local file into the VM without putting bytes in model context. Send the raw file body to the URL within 10 minutes. At most 64 MiB; destination replacement requires overwrite=true. The upload commits only after all declared bytes arrive.", | 490 | "description": "Create a one-use HTTP PUT URL for uploading a local file into the VM without putting bytes in model context. Start the raw HTTP PUT within 10 minutes. At most 64 MiB; destination replacement requires overwrite=true. The upload commits only after all declared bytes arrive.", |
| 491 | "inputSchema": { | 491 | "inputSchema": { |
| 492 | "type": "object", | 492 | "type": "object", |
| 493 | "properties": { | 493 | "properties": { |
dashboard/src/computer.rs+17-6| ... | @@ -262,13 +262,19 @@ async fn transfer_access(computer: &Computer, transfer: &Transfer) -> Result<()> | ... | @@ -262,13 +262,19 @@ async fn transfer_access(computer: &Computer, transfer: &Transfer) -> Result<()> |
| 262 | permitted(&grant, &transfer.vm, "computer:control", &owner) | 262 | permitted(&grant, &transfer.vm, "computer:control", &owner) |
| 263 | } | 263 | } |
| 264 | 264 | ||
| 265 | fn transfer_chunk(reply: &Value) -> usize { | ||
| 266 | reply["chunk_bytes"].as_u64() | ||
| 267 | .filter(|size| (65536..=host::LIMITS.file_chunk as u64).contains(size)) | ||
| 268 | .unwrap_or(65536) as usize | ||
| 269 | } | ||
| 270 | |||
| 265 | async fn transfer( | 271 | async fn transfer( |
| 266 | State(computer): State<Computer>, | 272 | State(computer): State<Computer>, |
| 267 | axum::extract::Path(id): axum::extract::Path<String>, | 273 | axum::extract::Path(id): axum::extract::Path<String>, |
| 268 | request: Request, | 274 | request: Request, |
| 269 | ) -> Result<Response> { | 275 | ) -> Result<Response> { |
| 270 | use base64::Engine; | 276 | use base64::Engine; |
| 271 | let transfer = computer | 277 | let mut transfer = computer |
| 272 | .transfers | 278 | .transfers |
| 273 | .lock() | 279 | .lock() |
| 274 | .unwrap() | 280 | .unwrap() |
| ... | @@ -302,6 +308,7 @@ async fn transfer( | ... | @@ -302,6 +308,7 @@ async fn transfer( |
| 302 | "Use the HTTP method returned with this URL.", | 308 | "Use the HTTP method returned with this URL.", |
| 303 | )); | 309 | )); |
| 304 | } | 310 | } |
| 311 | transfer.expires = std::time::Instant::now() + Duration::from_secs(1800); | ||
| 305 | if transfer.upload { | 312 | if transfer.upload { |
| 306 | let temporary = format!( | 313 | let temporary = format!( |
| 307 | "{}.snowglobe-{}", | 314 | "{}.snowglobe-{}", |
| ... | @@ -316,23 +323,25 @@ async fn transfer( | ... | @@ -316,23 +323,25 @@ async fn transfer( |
| 316 | let mut stream=request.into_body().into_data_stream(); | 323 | let mut stream=request.into_body().into_data_stream(); |
| 317 | let mut offset=0_u64; | 324 | let mut offset=0_u64; |
| 318 | let mut chunk=bytes::Bytes::new(); | 325 | let mut chunk=bytes::Bytes::new(); |
| 326 | let mut chunk_size=65536; | ||
| 319 | let mut buffer=Vec::with_capacity(65536); | 327 | let mut buffer=Vec::with_capacity(65536); |
| 320 | loop { | 328 | loop { |
| 321 | buffer.clear(); | 329 | buffer.clear(); |
| 322 | while buffer.len()<65536 { | 330 | while buffer.len()<chunk_size { |
| 323 | if chunk.is_empty() { | 331 | if chunk.is_empty() { |
| 324 | let Some(next)=stream.next().await else { break }; | 332 | let Some(next)=stream.next().await else { break }; |
| 325 | chunk=next.map_err(|_|Error::new(400,"The upload ended early. Request a new URL and retry."))?; | 333 | chunk=next.map_err(|_|Error::new(400,"The upload ended early. Request a new URL and retry."))?; |
| 326 | } | 334 | } |
| 327 | let length=(65536-buffer.len()).min(chunk.len()); | 335 | let length=(chunk_size-buffer.len()).min(chunk.len()); |
| 328 | buffer.extend_from_slice(&chunk.split_to(length)); | 336 | buffer.extend_from_slice(&chunk.split_to(length)); |
| 329 | } | 337 | } |
| 330 | if offset+buffer.len() as u64>transfer.size { return Err(Error::new(413,"The upload exceeds its declared size.")); } | 338 | if offset+buffer.len() as u64>transfer.size { return Err(Error::new(413,"The upload exceeds its declared size.")); } |
| 331 | if buffer.is_empty() && offset>0 { break; } | 339 | if buffer.is_empty() && offset>0 { break; } |
| 332 | transfer_access(&computer,&transfer).await?; | 340 | transfer_access(&computer,&transfer).await?; |
| 333 | dispatch(&transfer.vm,json!({"action":"file","operation":"write","path":temporary,"offset":offset,"data":base64::engine::general_purpose::STANDARD.encode(&buffer)})).await?; | 341 | let reply=dispatch(&transfer.vm,json!({"action":"file","operation":"write","path":temporary,"offset":offset,"data":base64::engine::general_purpose::STANDARD.encode(&buffer)})).await?; |
| 334 | offset+=buffer.len() as u64; | 342 | offset+=buffer.len() as u64; |
| 335 | if buffer.len()<65536 { break; } | 343 | if buffer.len()<chunk_size { break; } |
| 344 | chunk_size=transfer_chunk(&reply); | ||
| 336 | } | 345 | } |
| 337 | if offset!=transfer.size { return Err(Error::new(400,"The upload ended early. Request a new URL and retry.")); } | 346 | if offset!=transfer.size { return Err(Error::new(400,"The upload ended early. Request a new URL and retry.")); } |
| 338 | transfer_access(&computer,&transfer).await?; | 347 | transfer_access(&computer,&transfer).await?; |
| ... | @@ -351,10 +360,11 @@ async fn transfer( | ... | @@ -351,10 +360,11 @@ async fn transfer( |
| 351 | let _permit = permit; | 360 | let _permit = permit; |
| 352 | let result: Result<()> = tokio::time::timeout_at(transfer.expires.into(), async { | 361 | let result: Result<()> = tokio::time::timeout_at(transfer.expires.into(), async { |
| 353 | let mut offset = 0; | 362 | let mut offset = 0; |
| 363 | let mut chunk_size = 65536; | ||
| 354 | while offset < transfer.size { | 364 | while offset < transfer.size { |
| 355 | if sender.is_closed() { return Ok(()); } | 365 | if sender.is_closed() { return Ok(()); } |
| 356 | transfer_access(&computer, &transfer).await?; | 366 | transfer_access(&computer, &transfer).await?; |
| 357 | let length = (transfer.size - offset).min(65536); | 367 | let length = (transfer.size - offset).min(chunk_size as u64); |
| 358 | let reply = dispatch(&transfer.vm, json!({"action":"file","operation":"read","path":transfer.path,"offset":offset,"length":length})).await?; | 368 | let reply = dispatch(&transfer.vm, json!({"action":"file","operation":"read","path":transfer.path,"offset":offset,"length":length})).await?; |
| 359 | let bytes = base64::engine::general_purpose::STANDARD.decode(string(&reply["data"])) | 369 | let bytes = base64::engine::general_purpose::STANDARD.decode(string(&reply["data"])) |
| 360 | .map_err(|_| Error::new(502, "The guest returned an incomplete file."))?; | 370 | .map_err(|_| Error::new(502, "The guest returned an incomplete file."))?; |
| ... | @@ -363,6 +373,7 @@ async fn transfer( | ... | @@ -363,6 +373,7 @@ async fn transfer( |
| 363 | } | 373 | } |
| 364 | if sender.send(Ok(Bytes::from(bytes))).await.is_err() { return Ok(()); } | 374 | if sender.send(Ok(Bytes::from(bytes))).await.is_err() { return Ok(()); } |
| 365 | offset += length; | 375 | offset += length; |
| 376 | chunk_size = transfer_chunk(&reply); | ||
| 366 | } | 377 | } |
| 367 | Ok(()) | 378 | Ok(()) |
| 368 | }).await.unwrap_or_else(|_| Err(Error::new(410, "This transfer expired. Request a new URL."))); | 379 | }).await.unwrap_or_else(|_| Err(Error::new(410, "This transfer expired. Request a new URL."))); |
dashboard/src/host-protocol.json+1-1| ... | @@ -1 +1 @@ | ... | @@ -1 +1 @@ |
| 1 | {"request":2097152,"response":16777216,"memory":536870912} | 1 | {"request":2097152,"response":16777216,"memory":536870912,"file_chunk":1048576} |
dashboard/src/host.rs+2-2| ... | @@ -2,8 +2,8 @@ use crate::*; | ... | @@ -2,8 +2,8 @@ use crate::*; |
| 2 | use tokio::io::{AsyncReadExt, AsyncWriteExt}; | 2 | use tokio::io::{AsyncReadExt, AsyncWriteExt}; |
| 3 | 3 | ||
| 4 | #[derive(serde::Deserialize)] | 4 | #[derive(serde::Deserialize)] |
| 5 | struct Limits { request: usize, response: usize } | 5 | pub(crate) struct Limits { request: usize, response: usize, pub(crate) file_chunk: usize } |
| 6 | static LIMITS: std::sync::LazyLock<Limits> = std::sync::LazyLock::new(|| serde_json::from_str(include_str!("host-protocol.json")).expect("host protocol limits")); | 6 | pub(crate) static LIMITS: std::sync::LazyLock<Limits> = std::sync::LazyLock::new(|| serde_json::from_str(include_str!("host-protocol.json")).expect("host protocol limits")); |
| 7 | 7 | ||
| 8 | static SLOTS: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(4); | 8 | static SLOTS: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(4); |
| 9 | 9 |
tools/computer-client-test.py+30-2| ... | @@ -32,6 +32,8 @@ class Host(socketserver.StreamRequestHandler): | ... | @@ -32,6 +32,8 @@ class Host(socketserver.StreamRequestHandler): |
| 32 | files = {} | 32 | files = {} |
| 33 | power_actions = [] | 33 | power_actions = [] |
| 34 | writes = [] | 34 | writes = [] |
| 35 | reads = [] | ||
| 36 | chunk_bytes = None | ||
| 35 | started = threading.Semaphore(0) | 37 | started = threading.Semaphore(0) |
| 36 | release = threading.Event() | 38 | release = threading.Event() |
| 37 | 39 | ||
| ... | @@ -44,6 +46,12 @@ class Host(socketserver.StreamRequestHandler): | ... | @@ -44,6 +46,12 @@ class Host(socketserver.StreamRequestHandler): |
| 44 | reply = {"value": self.answer(request["operation"], request.get("payload") or {})} | 46 | reply = {"value": self.answer(request["operation"], request.get("payload") or {})} |
| 45 | except KeyError as missing: | 47 | except KeyError as missing: |
| 46 | reply = {"error": f"missing {missing}", "status": 404} | 48 | reply = {"error": f"missing {missing}", "status": 404} |
| 49 | except BlockingIOError as error: | ||
| 50 | reply = {"error": str(error), "status": 409} | ||
| 51 | except TimeoutError as error: | ||
| 52 | reply = {"error": str(error), "status": 504} | ||
| 53 | except ValueError as error: | ||
| 54 | reply = {"error": str(error), "status": 400} | ||
| 47 | body = json.dumps(reply).encode() | 55 | body = json.dumps(reply).encode() |
| 48 | try: | 56 | try: |
| 49 | self.wfile.write(struct.pack(">I", len(body)) + body) | 57 | self.wfile.write(struct.pack(">I", len(body)) + body) |
| ... | @@ -83,12 +91,16 @@ class Host(socketserver.StreamRequestHandler): | ... | @@ -83,12 +91,16 @@ class Host(socketserver.StreamRequestHandler): |
| 83 | elif operation == "commit": | 91 | elif operation == "commit": |
| 84 | files[request["destination"]] = files.pop(path) | 92 | files[request["destination"]] = files.pop(path) |
| 85 | elif operation == "read": | 93 | elif operation == "read": |
| 94 | self.reads.append((path, request["length"])) | ||
| 86 | data = files[path][request["offset"]:request["offset"] + request["length"]] | 95 | data = files[path][request["offset"]:request["offset"] + request["length"]] |
| 87 | return {"data": base64.b64encode(data).decode()} | 96 | reply = {"data": base64.b64encode(data).decode()} |
| 97 | if self.chunk_bytes is not None: | ||
| 98 | reply["chunk_bytes"] = self.chunk_bytes | ||
| 99 | return reply | ||
| 88 | elif operation == "remove": | 100 | elif operation == "remove": |
| 89 | time.sleep(.05) | 101 | time.sleep(.05) |
| 90 | files.pop(path, None) | 102 | files.pop(path, None) |
| 91 | return {} | 103 | return {"chunk_bytes": self.chunk_bytes} if operation == "write" and self.chunk_bytes is not None else {} |
| 92 | 104 | ||
| 93 | 105 | ||
| 94 | class Client(unittest.TestCase): | 106 | class Client(unittest.TestCase): |
| ... | @@ -245,6 +257,22 @@ class Client(unittest.TestCase): | ... | @@ -245,6 +257,22 @@ class Client(unittest.TestCase): |
| 245 | self.assertFalse(any(path.startswith(destination + ".snowglobe-") for path in Host.files), | 257 | self.assertFalse(any(path.startswith(destination + ".snowglobe-") for path in Host.files), |
| 246 | "upload returned before removing its temporary file") | 258 | "upload returned before removing its temporary file") |
| 247 | 259 | ||
| 260 | for index, cap in enumerate([2**20, 0, -1, True, "1048576", 2**20 + 1, 1.5, 2**100]): | ||
| 261 | Host.chunk_bytes = cap | ||
| 262 | content = os.urandom(2**21 + 17 if index == 0 else 200_000) | ||
| 263 | source.write_bytes(content) | ||
| 264 | destination = f"/tmp/negotiated-{index}" | ||
| 265 | Host.writes.clear() | ||
| 266 | Host.reads.clear() | ||
| 267 | vm.upload("test-vm", source, destination) | ||
| 268 | back = source.with_suffix(".negotiated") | ||
| 269 | vm.download("test-vm", destination, back) | ||
| 270 | self.assertEqual(back.read_bytes(), content) | ||
| 271 | expected = [65536, 2**20, len(content) - 65536 - 2**20] if index == 0 else [65536, 65536, 65536, 3392] | ||
| 272 | self.assertEqual([size for _, size in Host.writes], expected) | ||
| 273 | self.assertEqual([size for _, size in Host.reads], expected) | ||
| 274 | Host.chunk_bytes = None | ||
| 275 | |||
| 248 | self.assertIn("png", vm.call("computer_shot", vm="test-vm")) | 276 | self.assertIn("png", vm.call("computer_shot", vm="test-vm")) |
| 249 | 277 | ||
| 250 | state = json.loads(computer.STATE.read_text()) | 278 | state = json.loads(computer.STATE.read_text()) |
tools/computer-test.py+18-2| ... | @@ -31,8 +31,8 @@ class Protocol(unittest.TestCase): | ... | @@ -31,8 +31,8 @@ class Protocol(unittest.TestCase): |
| 31 | computer.validate({'action':'input','actions':[action],'screenshot':False}) | 31 | computer.validate({'action':'input','actions':[action],'screenshot':False}) |
| 32 | 32 | ||
| 33 | def test_rejects_unbounded_files_and_ui_targets(self): | 33 | def test_rejects_unbounded_files_and_ui_targets(self): |
| 34 | for request in [{'action':'file','operation':'read','path':'/tmp/x','length':65537}, | 34 | for request in [{'action':'file','operation':'read','path':'/tmp/x','length':computer.FILE_CHUNK+1}, |
| 35 | {'action':'file','operation':'write','path':'/tmp/x','data':'a'*90001}, | 35 | {'action':'file','operation':'write','path':'/tmp/x','data':base64.b64encode(b'x'*(computer.FILE_CHUNK+1)).decode()}, |
| 36 | {'action':'file','operation':'read','path':'/tmp/x','offset':True}, | 36 | {'action':'file','operation':'read','path':'/tmp/x','offset':True}, |
| 37 | {'action':'ui','pid':True}, | 37 | {'action':'ui','pid':True}, |
| 38 | {'action':'ui_action','target':{'name':'x'*5000},'operation':'press'}]: | 38 | {'action':'ui_action','target':{'name':'x'*5000},'operation':'press'}]: |
| ... | @@ -84,6 +84,22 @@ class GuestCancellation(unittest.TestCase): | ... | @@ -84,6 +84,22 @@ class GuestCancellation(unittest.TestCase): |
| 84 | patcher.start() | 84 | patcher.start() |
| 85 | self.addCleanup(patcher.stop) | 85 | self.addCleanup(patcher.stop) |
| 86 | 86 | ||
| 87 | def test_large_windows_transfer_keeps_agent_writes_within_verified_size(self): | ||
| 88 | data = b'\0binary\xff' * (2**17 + 1) | ||
| 89 | written = [] | ||
| 90 | def qga(name, operation, **arguments): | ||
| 91 | if operation == 'guest-file-open': | ||
| 92 | return 1 | ||
| 93 | if operation == 'guest-file-write': | ||
| 94 | chunk = base64.b64decode(arguments['buf-b64']) | ||
| 95 | self.assertLessEqual(len(chunk), 65536) | ||
| 96 | written.append(chunk) | ||
| 97 | return {'count': len(chunk)} | ||
| 98 | self.assertEqual(operation, 'guest-file-close') | ||
| 99 | with patch.object(computer, 'qga', qga): | ||
| 100 | computer.write_file('fixture', 'owned-file', data) | ||
| 101 | self.assertEqual(b''.join(written), data) | ||
| 102 | |||
| 87 | def test_windows_powershell_reads_redirected_output_without_console_capture(self): | 103 | def test_windows_powershell_reads_redirected_output_without_console_capture(self): |
| 88 | with patch.object(computer, 'write_file'), patch.object(computer, 'guest_exec', return_value=b''), patch.object(computer, 'read_file', return_value=b'file output'): | 104 | with patch.object(computer, 'write_file'), patch.object(computer, 'guest_exec', return_value=b''), patch.object(computer, 'read_file', return_value=b'file output'): |
| 89 | self.assertEqual(computer.powershell('fixture', "Write-Output 'file output'"), b'file output') | 105 | self.assertEqual(computer.powershell('fixture', "Write-Output 'file output'"), b'file output') |
tools/vm-computer.py+6-5| ... | @@ -22,6 +22,7 @@ rfb = import_module('vm-rfb') | ... | @@ -22,6 +22,7 @@ rfb = import_module('vm-rfb') |
| 22 | SCREEN_SLOT = threading.BoundedSemaphore(1) | 22 | SCREEN_SLOT = threading.BoundedSemaphore(1) |
| 23 | DEADLINE = ContextVar('computer_deadline', default=float('inf')) | 23 | DEADLINE = ContextVar('computer_deadline', default=float('inf')) |
| 24 | OPERATION = ContextVar('computer_operation', default=None) | 24 | OPERATION = ContextVar('computer_operation', default=None) |
| 25 | FILE_CHUNK = json.loads((Path(__file__).resolve().parent.parent / 'dashboard/src/host-protocol.json').read_text())['file_chunk'] | ||
| 25 | 26 | ||
| 26 | 27 | ||
| 27 | def remaining(limit): | 28 | def remaining(limit): |
| ... | @@ -314,10 +315,10 @@ def validate(request): | ... | @@ -314,10 +315,10 @@ def validate(request): |
| 314 | raise ValueError('Choose a transfer offset within 64 MiB.') | 315 | raise ValueError('Choose a transfer offset within 64 MiB.') |
| 315 | if operation=='read': | 316 | if operation=='read': |
| 316 | length=request.get('length',65536) | 317 | length=request.get('length',65536) |
| 317 | if type(length) is not int or not 1<=length<=65536: raise ValueError('Read at most 64 KiB at a time.') | 318 | if type(length) is not int or not 1<=length<=FILE_CHUNK: raise ValueError('Read at most 1 MiB at a time.') |
| 318 | else: | 319 | else: |
| 319 | if not isinstance(request.get('data'),str) or len(request['data'])>90000 or len(base64.b64decode(request['data'],validate=True))>65536: | 320 | if not isinstance(request.get('data'),str) or len(request['data'])>((FILE_CHUNK+2)//3)*4 or len(base64.b64decode(request['data'],validate=True))>FILE_CHUNK: |
| 320 | raise ValueError('Write at most 64 KiB at a time.') | 321 | raise ValueError('Write at most 1 MiB at a time.') |
| 321 | if operation=='commit' and (not isinstance(request.get('destination'),str) or not 1<=len(request['destination'])<=4096 or '\x00' in request['destination'] or type(request.get('overwrite',False)) is not bool): | 322 | if operation=='commit' and (not isinstance(request.get('destination'),str) or not 1<=len(request['destination'])<=4096 or '\x00' in request['destination'] or type(request.get('overwrite',False)) is not bool): |
| 322 | raise ValueError('Choose an absolute guest destination and whether to replace it.') | 323 | raise ValueError('Choose an absolute guest destination and whether to replace it.') |
| 323 | if action == 'process' and (not isinstance(request.get('id'),str) or len(request['id'])>128 or request.get('operation') not in ('status','stop')): | 324 | if action == 'process' and (not isinstance(request.get('id'),str) or len(request['id'])>128 or request.get('operation') not in ('status','stop')): |
| ... | @@ -617,10 +618,10 @@ def handle(name, request, *, active_snapshot=None): | ... | @@ -617,10 +618,10 @@ def handle(name, request, *, active_snapshot=None): |
| 617 | if len(path)<3 or path[1:3] not in (':\\',':/') or not path[0].isalpha(): raise ValueError('Use an absolute guest file path.') | 618 | if len(path)<3 or path[1:3] not in (':\\',':/') or not path[0].isalpha(): raise ValueError('Use an absolute guest file path.') |
| 618 | if request['operation']=='read': | 619 | if request['operation']=='read': |
| 619 | data=read_file(name,path,request.get('offset',0),request.get('length',65536)) | 620 | data=read_file(name,path,request.get('offset',0),request.get('length',65536)) |
| 620 | return {'data':base64.b64encode(data).decode(),'bytes':len(data)} | 621 | return {'data':base64.b64encode(data).decode(),'bytes':len(data),'chunk_bytes':FILE_CHUNK} |
| 621 | data=base64.b64decode(request['data'],validate=True) | 622 | data=base64.b64decode(request['data'],validate=True) |
| 622 | write_file(name,path,data,offset=request.get('offset',0)) | 623 | write_file(name,path,data,offset=request.get('offset',0)) |
| 623 | return {'bytes':len(data)} | 624 | return {'bytes':len(data),'chunk_bytes':FILE_CHUNK} |
| 624 | result = worker(name,platform,request) | 625 | result = worker(name,platform,request) |
| 625 | if request.get('screenshot'): | 626 | if request.get('screenshot'): |
| 626 | try: result.update(asyncio.run(desktop(name))) | 627 | try: result.update(asyncio.run(desktop(name))) |