From dc25dbf0f9528632f2d593b93197f9390e0567ac Mon Sep 17 00:00:00 2001 From: clover caruso Date: Tue, 17 Nov 2026 02:40:16 -0800 Subject: [PATCH] Negotiate larger Windows file transfer batches Use bounded guest-advertised batches after a compatible 64 KiB start, retaining small QEMU agent writes. Keep the one-use link expiry separate from the active transfer budget so large files can finish on slower disks. Preserve per-batch authorization and atomic destination replacement. Assisted-by: gpt-6.1-sol --- dashboard/src/computer-tools.json | 2 +- dashboard/src/computer.rs | 23 ++++++++++++++++------ dashboard/src/host-protocol.json | 2 +- dashboard/src/host.rs | 4 ++-- tools/computer-client-test.py | 32 +++++++++++++++++++++++++++++-- tools/computer-test.py | 20 +++++++++++++++++-- tools/vm-computer.py | 11 ++++++----- 7 files changed, 75 insertions(+), 19 deletions(-) diff --git a/dashboard/src/computer-tools.json b/dashboard/src/computer-tools.json index b40ae925ce85b28ff94c44c01b084fc81efa55aa..94dbcb9775bb1ec35d5db93fb25e5f76d6045564 100644 --- a/dashboard/src/computer-tools.json +++ b/dashboard/src/computer-tools.json @@ -487,7 +487,7 @@ }, { "name": "computer_upload", - "description": "Create a one-use HTTP PUT URL for uploading a local file into the VM without putting bytes in model context. Send the raw file body to the URL within 10 minutes. At most 64 MiB; destination replacement requires overwrite=true. The upload commits only after all declared bytes arrive.", + "description": "Create a one-use HTTP PUT URL for uploading a local file into the VM without putting bytes in model context. Start the raw HTTP PUT within 10 minutes. At most 64 MiB; destination replacement requires overwrite=true. The upload commits only after all declared bytes arrive.", "inputSchema": { "type": "object", "properties": { diff --git a/dashboard/src/computer.rs b/dashboard/src/computer.rs index 74837c77511d5accb6a6b88d9cf6558bad1224b7..d6271596764f2c0adced1d327ccf0e4e16091bb6 100644 --- a/dashboard/src/computer.rs +++ b/dashboard/src/computer.rs @@ -262,13 +262,19 @@ async fn transfer_access(computer: &Computer, transfer: &Transfer) -> Result<()> permitted(&grant, &transfer.vm, "computer:control", &owner) } +fn transfer_chunk(reply: &Value) -> usize { + reply["chunk_bytes"].as_u64() + .filter(|size| (65536..=host::LIMITS.file_chunk as u64).contains(size)) + .unwrap_or(65536) as usize +} + async fn transfer( State(computer): State, axum::extract::Path(id): axum::extract::Path, request: Request, ) -> Result { use base64::Engine; - let transfer = computer + let mut transfer = computer .transfers .lock() .unwrap() @@ -302,6 +308,7 @@ async fn transfer( "Use the HTTP method returned with this URL.", )); } + transfer.expires = std::time::Instant::now() + Duration::from_secs(1800); if transfer.upload { let temporary = format!( "{}.snowglobe-{}", @@ -316,23 +323,25 @@ async fn transfer( let mut stream=request.into_body().into_data_stream(); let mut offset=0_u64; let mut chunk=bytes::Bytes::new(); + let mut chunk_size=65536; let mut buffer=Vec::with_capacity(65536); loop { buffer.clear(); - while buffer.len()<65536 { + while buffer.len()transfer.size { return Err(Error::new(413,"The upload exceeds its declared size.")); } if buffer.is_empty() && offset>0 { break; } transfer_access(&computer,&transfer).await?; - dispatch(&transfer.vm,json!({"action":"file","operation":"write","path":temporary,"offset":offset,"data":base64::engine::general_purpose::STANDARD.encode(&buffer)})).await?; + let reply=dispatch(&transfer.vm,json!({"action":"file","operation":"write","path":temporary,"offset":offset,"data":base64::engine::general_purpose::STANDARD.encode(&buffer)})).await?; offset+=buffer.len() as u64; - if buffer.len()<65536 { break; } + if buffer.len() = tokio::time::timeout_at(transfer.expires.into(), async { let mut offset = 0; + let mut chunk_size = 65536; while offset < transfer.size { if sender.is_closed() { return Ok(()); } transfer_access(&computer, &transfer).await?; - let length = (transfer.size - offset).min(65536); + let length = (transfer.size - offset).min(chunk_size as u64); let reply = dispatch(&transfer.vm, json!({"action":"file","operation":"read","path":transfer.path,"offset":offset,"length":length})).await?; let bytes = base64::engine::general_purpose::STANDARD.decode(string(&reply["data"])) .map_err(|_| Error::new(502, "The guest returned an incomplete file."))?; @@ -363,6 +373,7 @@ async fn transfer( } if sender.send(Ok(Bytes::from(bytes))).await.is_err() { return Ok(()); } offset += length; + chunk_size = transfer_chunk(&reply); } Ok(()) }).await.unwrap_or_else(|_| Err(Error::new(410, "This transfer expired. Request a new URL."))); diff --git a/dashboard/src/host-protocol.json b/dashboard/src/host-protocol.json index 676610e003afc31b7dcf4f41e8d7b14387bdb3a0..3139c769a15dd6966d4f3c6113e266cad972f16f 100644 --- a/dashboard/src/host-protocol.json +++ b/dashboard/src/host-protocol.json @@ -1 +1 @@ -{"request":2097152,"response":16777216,"memory":536870912} +{"request":2097152,"response":16777216,"memory":536870912,"file_chunk":1048576} diff --git a/dashboard/src/host.rs b/dashboard/src/host.rs index 6be684de5cfd03c4987760f724aaf93148eab6c8..4bfde5dcf6658f0776530367487198012c2b0894 100644 --- a/dashboard/src/host.rs +++ b/dashboard/src/host.rs @@ -2,8 +2,8 @@ use crate::*; use tokio::io::{AsyncReadExt, AsyncWriteExt}; #[derive(serde::Deserialize)] -struct Limits { request: usize, response: usize } -static LIMITS: std::sync::LazyLock = std::sync::LazyLock::new(|| serde_json::from_str(include_str!("host-protocol.json")).expect("host protocol limits")); +pub(crate) struct Limits { request: usize, response: usize, pub(crate) file_chunk: usize } +pub(crate) static LIMITS: std::sync::LazyLock = std::sync::LazyLock::new(|| serde_json::from_str(include_str!("host-protocol.json")).expect("host protocol limits")); static SLOTS: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(4); diff --git a/tools/computer-client-test.py b/tools/computer-client-test.py index c22be0b72c9d79928541bc239837f7b1b00e7fa5..ff56faaa8c923fbe7d85f45eeb86d3e2cd2b7c4f 100644 --- a/tools/computer-client-test.py +++ b/tools/computer-client-test.py @@ -32,6 +32,8 @@ class Host(socketserver.StreamRequestHandler): files = {} power_actions = [] writes = [] + reads = [] + chunk_bytes = None started = threading.Semaphore(0) release = threading.Event() @@ -44,6 +46,12 @@ class Host(socketserver.StreamRequestHandler): reply = {"value": self.answer(request["operation"], request.get("payload") or {})} except KeyError as missing: reply = {"error": f"missing {missing}", "status": 404} + except BlockingIOError as error: + reply = {"error": str(error), "status": 409} + except TimeoutError as error: + reply = {"error": str(error), "status": 504} + except ValueError as error: + reply = {"error": str(error), "status": 400} body = json.dumps(reply).encode() try: self.wfile.write(struct.pack(">I", len(body)) + body) @@ -83,12 +91,16 @@ class Host(socketserver.StreamRequestHandler): elif operation == "commit": files[request["destination"]] = files.pop(path) elif operation == "read": + self.reads.append((path, request["length"])) data = files[path][request["offset"]:request["offset"] + request["length"]] - return {"data": base64.b64encode(data).decode()} + reply = {"data": base64.b64encode(data).decode()} + if self.chunk_bytes is not None: + reply["chunk_bytes"] = self.chunk_bytes + return reply elif operation == "remove": time.sleep(.05) files.pop(path, None) - return {} + return {"chunk_bytes": self.chunk_bytes} if operation == "write" and self.chunk_bytes is not None else {} class Client(unittest.TestCase): @@ -245,6 +257,22 @@ class Client(unittest.TestCase): self.assertFalse(any(path.startswith(destination + ".snowglobe-") for path in Host.files), "upload returned before removing its temporary file") + for index, cap in enumerate([2**20, 0, -1, True, "1048576", 2**20 + 1, 1.5, 2**100]): + Host.chunk_bytes = cap + content = os.urandom(2**21 + 17 if index == 0 else 200_000) + source.write_bytes(content) + destination = f"/tmp/negotiated-{index}" + Host.writes.clear() + Host.reads.clear() + vm.upload("test-vm", source, destination) + back = source.with_suffix(".negotiated") + vm.download("test-vm", destination, back) + self.assertEqual(back.read_bytes(), content) + expected = [65536, 2**20, len(content) - 65536 - 2**20] if index == 0 else [65536, 65536, 65536, 3392] + self.assertEqual([size for _, size in Host.writes], expected) + self.assertEqual([size for _, size in Host.reads], expected) + Host.chunk_bytes = None + self.assertIn("png", vm.call("computer_shot", vm="test-vm")) state = json.loads(computer.STATE.read_text()) diff --git a/tools/computer-test.py b/tools/computer-test.py index 10e6f4e55574d32c2053d9c2d59eae8c789c3c3f..25de1190cdffdf2c9fcb382aeeb6169667275ee7 100644 --- a/tools/computer-test.py +++ b/tools/computer-test.py @@ -31,8 +31,8 @@ class Protocol(unittest.TestCase): computer.validate({'action':'input','actions':[action],'screenshot':False}) def test_rejects_unbounded_files_and_ui_targets(self): - for request in [{'action':'file','operation':'read','path':'/tmp/x','length':65537}, - {'action':'file','operation':'write','path':'/tmp/x','data':'a'*90001}, + for request in [{'action':'file','operation':'read','path':'/tmp/x','length':computer.FILE_CHUNK+1}, + {'action':'file','operation':'write','path':'/tmp/x','data':base64.b64encode(b'x'*(computer.FILE_CHUNK+1)).decode()}, {'action':'file','operation':'read','path':'/tmp/x','offset':True}, {'action':'ui','pid':True}, {'action':'ui_action','target':{'name':'x'*5000},'operation':'press'}]: @@ -84,6 +84,22 @@ class GuestCancellation(unittest.TestCase): patcher.start() self.addCleanup(patcher.stop) + def test_large_windows_transfer_keeps_agent_writes_within_verified_size(self): + data = b'\0binary\xff' * (2**17 + 1) + written = [] + def qga(name, operation, **arguments): + if operation == 'guest-file-open': + return 1 + if operation == 'guest-file-write': + chunk = base64.b64decode(arguments['buf-b64']) + self.assertLessEqual(len(chunk), 65536) + written.append(chunk) + return {'count': len(chunk)} + self.assertEqual(operation, 'guest-file-close') + with patch.object(computer, 'qga', qga): + computer.write_file('fixture', 'owned-file', data) + self.assertEqual(b''.join(written), data) + def test_windows_powershell_reads_redirected_output_without_console_capture(self): with patch.object(computer, 'write_file'), patch.object(computer, 'guest_exec', return_value=b''), patch.object(computer, 'read_file', return_value=b'file output'): self.assertEqual(computer.powershell('fixture', "Write-Output 'file output'"), b'file output') diff --git a/tools/vm-computer.py b/tools/vm-computer.py index 75cc5ed01fd055d0e98bfe191796e2bd77b9873b..1766ac4d3f338bbaf6739f3ef08e31cf88b13a8d 100644 --- a/tools/vm-computer.py +++ b/tools/vm-computer.py @@ -22,6 +22,7 @@ rfb = import_module('vm-rfb') SCREEN_SLOT = threading.BoundedSemaphore(1) DEADLINE = ContextVar('computer_deadline', default=float('inf')) OPERATION = ContextVar('computer_operation', default=None) +FILE_CHUNK = json.loads((Path(__file__).resolve().parent.parent / 'dashboard/src/host-protocol.json').read_text())['file_chunk'] def remaining(limit): @@ -314,10 +315,10 @@ def validate(request): raise ValueError('Choose a transfer offset within 64 MiB.') if operation=='read': length=request.get('length',65536) - if type(length) is not int or not 1<=length<=65536: raise ValueError('Read at most 64 KiB at a time.') + if type(length) is not int or not 1<=length<=FILE_CHUNK: raise ValueError('Read at most 1 MiB at a time.') else: - if not isinstance(request.get('data'),str) or len(request['data'])>90000 or len(base64.b64decode(request['data'],validate=True))>65536: - raise ValueError('Write at most 64 KiB at a time.') + if not isinstance(request.get('data'),str) or len(request['data'])>((FILE_CHUNK+2)//3)*4 or len(base64.b64decode(request['data'],validate=True))>FILE_CHUNK: + raise ValueError('Write at most 1 MiB at a time.') if operation=='commit' and (not isinstance(request.get('destination'),str) or not 1<=len(request['destination'])<=4096 or '\x00' in request['destination'] or type(request.get('overwrite',False)) is not bool): raise ValueError('Choose an absolute guest destination and whether to replace it.') if action == 'process' and (not isinstance(request.get('id'),str) or len(request['id'])>128 or request.get('operation') not in ('status','stop')): @@ -617,10 +618,10 @@ def handle(name, request, *, active_snapshot=None): if len(path)<3 or path[1:3] not in (':\\',':/') or not path[0].isalpha(): raise ValueError('Use an absolute guest file path.') if request['operation']=='read': data=read_file(name,path,request.get('offset',0),request.get('length',65536)) - return {'data':base64.b64encode(data).decode(),'bytes':len(data)} + return {'data':base64.b64encode(data).decode(),'bytes':len(data),'chunk_bytes':FILE_CHUNK} data=base64.b64decode(request['data'],validate=True) write_file(name,path,data,offset=request.get('offset',0)) - return {'bytes':len(data)} + return {'bytes':len(data),'chunk_bytes':FILE_CHUNK} result = worker(name,platform,request) if request.get('screenshot'): try: result.update(asyncio.run(desktop(name))) -- 2.54.1