authoravatarclover caruso<git@paperclover.net>2026-11-17 02:40:16-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-22 00:44:02-08:00
logdc25dbf0f9528632f2d593b93197f9390e0567ac
treef35cf0235baba556e6a6b2d0de7b37b29909e429
parent1c09d8d7fbaeda2067216015d56eee511114f461
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Negotiate larger Windows file transfer batches

Use bounded guest-advertised batches after a compatible 64 KiB start, retaining small QEMU agent writes. Keep the one-use link expiry separate from the active transfer budget so large files can finish on slower disks. Preserve per-batch authorization and atomic destination replacement. Assisted-by: gpt-6.1-sol

7 files changed, 75 insertions(+), 19 deletions(-)

dashboard/src/computer-tools.json+1-1
...@@ -487,7 +487,7 @@...@@ -487,7 +487,7 @@
487 },487 },
488 {488 {
489 "name": "computer_upload",489 "name": "computer_upload",
490 "description": "Create a one-use HTTP PUT URL for uploading a local file into the VM without putting bytes in model context. Send the raw file body to the URL within 10 minutes. At most 64 MiB; destination replacement requires overwrite=true. The upload commits only after all declared bytes arrive.",490 "description": "Create a one-use HTTP PUT URL for uploading a local file into the VM without putting bytes in model context. Start the raw HTTP PUT within 10 minutes. At most 64 MiB; destination replacement requires overwrite=true. The upload commits only after all declared bytes arrive.",
491 "inputSchema": {491 "inputSchema": {
492 "type": "object",492 "type": "object",
493 "properties": {493 "properties": {
dashboard/src/computer.rs+17-6
...@@ -262,13 +262,19 @@ async fn transfer_access(computer: &Computer, transfer: &Transfer) -> Result<()>...@@ -262,13 +262,19 @@ async fn transfer_access(computer: &Computer, transfer: &Transfer) -> Result<()>
262 permitted(&grant, &transfer.vm, "computer:control", &owner)262 permitted(&grant, &transfer.vm, "computer:control", &owner)
263}263}
264264
265fn transfer_chunk(reply: &Value) -> usize {
266 reply["chunk_bytes"].as_u64()
267 .filter(|size| (65536..=host::LIMITS.file_chunk as u64).contains(size))
268 .unwrap_or(65536) as usize
269}
270
265async fn transfer(271async fn transfer(
266 State(computer): State<Computer>,272 State(computer): State<Computer>,
267 axum::extract::Path(id): axum::extract::Path<String>,273 axum::extract::Path(id): axum::extract::Path<String>,
268 request: Request,274 request: Request,
269) -> Result<Response> {275) -> Result<Response> {
270 use base64::Engine;276 use base64::Engine;
271 let transfer = computer277 let mut transfer = computer
272 .transfers278 .transfers
273 .lock()279 .lock()
274 .unwrap()280 .unwrap()
...@@ -302,6 +308,7 @@ async fn transfer(...@@ -302,6 +308,7 @@ async fn transfer(
302 "Use the HTTP method returned with this URL.",308 "Use the HTTP method returned with this URL.",
303 ));309 ));
304 }310 }
311 transfer.expires = std::time::Instant::now() + Duration::from_secs(1800);
305 if transfer.upload {312 if transfer.upload {
306 let temporary = format!(313 let temporary = format!(
307 "{}.snowglobe-{}",314 "{}.snowglobe-{}",
...@@ -316,23 +323,25 @@ async fn transfer(...@@ -316,23 +323,25 @@ async fn transfer(
316 let mut stream=request.into_body().into_data_stream();323 let mut stream=request.into_body().into_data_stream();
317 let mut offset=0_u64;324 let mut offset=0_u64;
318 let mut chunk=bytes::Bytes::new();325 let mut chunk=bytes::Bytes::new();
326 let mut chunk_size=65536;
319 let mut buffer=Vec::with_capacity(65536);327 let mut buffer=Vec::with_capacity(65536);
320 loop {328 loop {
321 buffer.clear();329 buffer.clear();
322 while buffer.len()<65536 {330 while buffer.len()<chunk_size {
323 if chunk.is_empty() {331 if chunk.is_empty() {
324 let Some(next)=stream.next().await else { break };332 let Some(next)=stream.next().await else { break };
325 chunk=next.map_err(|_|Error::new(400,"The upload ended early. Request a new URL and retry."))?;333 chunk=next.map_err(|_|Error::new(400,"The upload ended early. Request a new URL and retry."))?;
326 }334 }
327 let length=(65536-buffer.len()).min(chunk.len());335 let length=(chunk_size-buffer.len()).min(chunk.len());
328 buffer.extend_from_slice(&chunk.split_to(length));336 buffer.extend_from_slice(&chunk.split_to(length));
329 }337 }
330 if offset+buffer.len() as u64>transfer.size { return Err(Error::new(413,"The upload exceeds its declared size.")); }338 if offset+buffer.len() as u64>transfer.size { return Err(Error::new(413,"The upload exceeds its declared size.")); }
331 if buffer.is_empty() && offset>0 { break; }339 if buffer.is_empty() && offset>0 { break; }
332 transfer_access(&computer,&transfer).await?;340 transfer_access(&computer,&transfer).await?;
333 dispatch(&transfer.vm,json!({"action":"file","operation":"write","path":temporary,"offset":offset,"data":base64::engine::general_purpose::STANDARD.encode(&buffer)})).await?;341 let reply=dispatch(&transfer.vm,json!({"action":"file","operation":"write","path":temporary,"offset":offset,"data":base64::engine::general_purpose::STANDARD.encode(&buffer)})).await?;
334 offset+=buffer.len() as u64;342 offset+=buffer.len() as u64;
335 if buffer.len()<65536 { break; }343 if buffer.len()<chunk_size { break; }
344 chunk_size=transfer_chunk(&reply);
336 }345 }
337 if offset!=transfer.size { return Err(Error::new(400,"The upload ended early. Request a new URL and retry.")); }346 if offset!=transfer.size { return Err(Error::new(400,"The upload ended early. Request a new URL and retry.")); }
338 transfer_access(&computer,&transfer).await?;347 transfer_access(&computer,&transfer).await?;
...@@ -351,10 +360,11 @@ async fn transfer(...@@ -351,10 +360,11 @@ async fn transfer(
351 let _permit = permit;360 let _permit = permit;
352 let result: Result<()> = tokio::time::timeout_at(transfer.expires.into(), async {361 let result: Result<()> = tokio::time::timeout_at(transfer.expires.into(), async {
353 let mut offset = 0;362 let mut offset = 0;
363 let mut chunk_size = 65536;
354 while offset < transfer.size {364 while offset < transfer.size {
355 if sender.is_closed() { return Ok(()); }365 if sender.is_closed() { return Ok(()); }
356 transfer_access(&computer, &transfer).await?;366 transfer_access(&computer, &transfer).await?;
357 let length = (transfer.size - offset).min(65536);367 let length = (transfer.size - offset).min(chunk_size as u64);
358 let reply = dispatch(&transfer.vm, json!({"action":"file","operation":"read","path":transfer.path,"offset":offset,"length":length})).await?;368 let reply = dispatch(&transfer.vm, json!({"action":"file","operation":"read","path":transfer.path,"offset":offset,"length":length})).await?;
359 let bytes = base64::engine::general_purpose::STANDARD.decode(string(&reply["data"]))369 let bytes = base64::engine::general_purpose::STANDARD.decode(string(&reply["data"]))
360 .map_err(|_| Error::new(502, "The guest returned an incomplete file."))?;370 .map_err(|_| Error::new(502, "The guest returned an incomplete file."))?;
...@@ -363,6 +373,7 @@ async fn transfer(...@@ -363,6 +373,7 @@ async fn transfer(
363 }373 }
364 if sender.send(Ok(Bytes::from(bytes))).await.is_err() { return Ok(()); }374 if sender.send(Ok(Bytes::from(bytes))).await.is_err() { return Ok(()); }
365 offset += length;375 offset += length;
376 chunk_size = transfer_chunk(&reply);
366 }377 }
367 Ok(())378 Ok(())
368 }).await.unwrap_or_else(|_| Err(Error::new(410, "This transfer expired. Request a new URL.")));379 }).await.unwrap_or_else(|_| Err(Error::new(410, "This transfer expired. Request a new URL.")));
dashboard/src/host-protocol.json+1-1
...@@ -1 +1 @@...@@ -1 +1 @@
1{"request":2097152,"response":16777216,"memory":536870912}1{"request":2097152,"response":16777216,"memory":536870912,"file_chunk":1048576}
dashboard/src/host.rs+2-2
...@@ -2,8 +2,8 @@ use crate::*;...@@ -2,8 +2,8 @@ use crate::*;
2use tokio::io::{AsyncReadExt, AsyncWriteExt};2use tokio::io::{AsyncReadExt, AsyncWriteExt};
33
4#[derive(serde::Deserialize)]4#[derive(serde::Deserialize)]
5struct Limits { request: usize, response: usize }5pub(crate) struct Limits { request: usize, response: usize, pub(crate) file_chunk: usize }
6static LIMITS: std::sync::LazyLock<Limits> = std::sync::LazyLock::new(|| serde_json::from_str(include_str!("host-protocol.json")).expect("host protocol limits"));6pub(crate) static LIMITS: std::sync::LazyLock<Limits> = std::sync::LazyLock::new(|| serde_json::from_str(include_str!("host-protocol.json")).expect("host protocol limits"));
77
8static SLOTS: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(4);8static SLOTS: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(4);
99
tools/computer-client-test.py+30-2
...@@ -32,6 +32,8 @@ class Host(socketserver.StreamRequestHandler):...@@ -32,6 +32,8 @@ class Host(socketserver.StreamRequestHandler):
32 files = {}32 files = {}
33 power_actions = []33 power_actions = []
34 writes = []34 writes = []
35 reads = []
36 chunk_bytes = None
35 started = threading.Semaphore(0)37 started = threading.Semaphore(0)
36 release = threading.Event()38 release = threading.Event()
3739
...@@ -44,6 +46,12 @@ class Host(socketserver.StreamRequestHandler):...@@ -44,6 +46,12 @@ class Host(socketserver.StreamRequestHandler):
44 reply = {"value": self.answer(request["operation"], request.get("payload") or {})}46 reply = {"value": self.answer(request["operation"], request.get("payload") or {})}
45 except KeyError as missing:47 except KeyError as missing:
46 reply = {"error": f"missing {missing}", "status": 404}48 reply = {"error": f"missing {missing}", "status": 404}
49 except BlockingIOError as error:
50 reply = {"error": str(error), "status": 409}
51 except TimeoutError as error:
52 reply = {"error": str(error), "status": 504}
53 except ValueError as error:
54 reply = {"error": str(error), "status": 400}
47 body = json.dumps(reply).encode()55 body = json.dumps(reply).encode()
48 try:56 try:
49 self.wfile.write(struct.pack(">I", len(body)) + body)57 self.wfile.write(struct.pack(">I", len(body)) + body)
...@@ -83,12 +91,16 @@ class Host(socketserver.StreamRequestHandler):...@@ -83,12 +91,16 @@ class Host(socketserver.StreamRequestHandler):
83 elif operation == "commit":91 elif operation == "commit":
84 files[request["destination"]] = files.pop(path)92 files[request["destination"]] = files.pop(path)
85 elif operation == "read":93 elif operation == "read":
94 self.reads.append((path, request["length"]))
86 data = files[path][request["offset"]:request["offset"] + request["length"]]95 data = files[path][request["offset"]:request["offset"] + request["length"]]
87 return {"data": base64.b64encode(data).decode()}96 reply = {"data": base64.b64encode(data).decode()}
97 if self.chunk_bytes is not None:
98 reply["chunk_bytes"] = self.chunk_bytes
99 return reply
88 elif operation == "remove":100 elif operation == "remove":
89 time.sleep(.05)101 time.sleep(.05)
90 files.pop(path, None)102 files.pop(path, None)
91 return {}103 return {"chunk_bytes": self.chunk_bytes} if operation == "write" and self.chunk_bytes is not None else {}
92104
93105
94class Client(unittest.TestCase):106class Client(unittest.TestCase):
...@@ -245,6 +257,22 @@ class Client(unittest.TestCase):...@@ -245,6 +257,22 @@ class Client(unittest.TestCase):
245 self.assertFalse(any(path.startswith(destination + ".snowglobe-") for path in Host.files),257 self.assertFalse(any(path.startswith(destination + ".snowglobe-") for path in Host.files),
246 "upload returned before removing its temporary file")258 "upload returned before removing its temporary file")
247259
260 for index, cap in enumerate([2**20, 0, -1, True, "1048576", 2**20 + 1, 1.5, 2**100]):
261 Host.chunk_bytes = cap
262 content = os.urandom(2**21 + 17 if index == 0 else 200_000)
263 source.write_bytes(content)
264 destination = f"/tmp/negotiated-{index}"
265 Host.writes.clear()
266 Host.reads.clear()
267 vm.upload("test-vm", source, destination)
268 back = source.with_suffix(".negotiated")
269 vm.download("test-vm", destination, back)
270 self.assertEqual(back.read_bytes(), content)
271 expected = [65536, 2**20, len(content) - 65536 - 2**20] if index == 0 else [65536, 65536, 65536, 3392]
272 self.assertEqual([size for _, size in Host.writes], expected)
273 self.assertEqual([size for _, size in Host.reads], expected)
274 Host.chunk_bytes = None
275
248 self.assertIn("png", vm.call("computer_shot", vm="test-vm"))276 self.assertIn("png", vm.call("computer_shot", vm="test-vm"))
249277
250 state = json.loads(computer.STATE.read_text())278 state = json.loads(computer.STATE.read_text())
tools/computer-test.py+18-2
...@@ -31,8 +31,8 @@ class Protocol(unittest.TestCase):...@@ -31,8 +31,8 @@ class Protocol(unittest.TestCase):
31 computer.validate({'action':'input','actions':[action],'screenshot':False})31 computer.validate({'action':'input','actions':[action],'screenshot':False})
3232
33 def test_rejects_unbounded_files_and_ui_targets(self):33 def test_rejects_unbounded_files_and_ui_targets(self):
34 for request in [{'action':'file','operation':'read','path':'/tmp/x','length':65537},34 for request in [{'action':'file','operation':'read','path':'/tmp/x','length':computer.FILE_CHUNK+1},
35 {'action':'file','operation':'write','path':'/tmp/x','data':'a'*90001},35 {'action':'file','operation':'write','path':'/tmp/x','data':base64.b64encode(b'x'*(computer.FILE_CHUNK+1)).decode()},
36 {'action':'file','operation':'read','path':'/tmp/x','offset':True},36 {'action':'file','operation':'read','path':'/tmp/x','offset':True},
37 {'action':'ui','pid':True},37 {'action':'ui','pid':True},
38 {'action':'ui_action','target':{'name':'x'*5000},'operation':'press'}]:38 {'action':'ui_action','target':{'name':'x'*5000},'operation':'press'}]:
...@@ -84,6 +84,22 @@ class GuestCancellation(unittest.TestCase):...@@ -84,6 +84,22 @@ class GuestCancellation(unittest.TestCase):
84 patcher.start()84 patcher.start()
85 self.addCleanup(patcher.stop)85 self.addCleanup(patcher.stop)
8686
87 def test_large_windows_transfer_keeps_agent_writes_within_verified_size(self):
88 data = b'\0binary\xff' * (2**17 + 1)
89 written = []
90 def qga(name, operation, **arguments):
91 if operation == 'guest-file-open':
92 return 1
93 if operation == 'guest-file-write':
94 chunk = base64.b64decode(arguments['buf-b64'])
95 self.assertLessEqual(len(chunk), 65536)
96 written.append(chunk)
97 return {'count': len(chunk)}
98 self.assertEqual(operation, 'guest-file-close')
99 with patch.object(computer, 'qga', qga):
100 computer.write_file('fixture', 'owned-file', data)
101 self.assertEqual(b''.join(written), data)
102
87 def test_windows_powershell_reads_redirected_output_without_console_capture(self):103 def test_windows_powershell_reads_redirected_output_without_console_capture(self):
88 with patch.object(computer, 'write_file'), patch.object(computer, 'guest_exec', return_value=b''), patch.object(computer, 'read_file', return_value=b'file output'):104 with patch.object(computer, 'write_file'), patch.object(computer, 'guest_exec', return_value=b''), patch.object(computer, 'read_file', return_value=b'file output'):
89 self.assertEqual(computer.powershell('fixture', "Write-Output 'file output'"), b'file output')105 self.assertEqual(computer.powershell('fixture', "Write-Output 'file output'"), b'file output')
tools/vm-computer.py+6-5
...@@ -22,6 +22,7 @@ rfb = import_module('vm-rfb')...@@ -22,6 +22,7 @@ rfb = import_module('vm-rfb')
22SCREEN_SLOT = threading.BoundedSemaphore(1)22SCREEN_SLOT = threading.BoundedSemaphore(1)
23DEADLINE = ContextVar('computer_deadline', default=float('inf'))23DEADLINE = ContextVar('computer_deadline', default=float('inf'))
24OPERATION = ContextVar('computer_operation', default=None)24OPERATION = ContextVar('computer_operation', default=None)
25FILE_CHUNK = json.loads((Path(__file__).resolve().parent.parent / 'dashboard/src/host-protocol.json').read_text())['file_chunk']
2526
2627
27def remaining(limit):28def remaining(limit):
...@@ -314,10 +315,10 @@ def validate(request):...@@ -314,10 +315,10 @@ def validate(request):
314 raise ValueError('Choose a transfer offset within 64 MiB.')315 raise ValueError('Choose a transfer offset within 64 MiB.')
315 if operation=='read':316 if operation=='read':
316 length=request.get('length',65536)317 length=request.get('length',65536)
317 if type(length) is not int or not 1<=length<=65536: raise ValueError('Read at most 64 KiB at a time.')318 if type(length) is not int or not 1<=length<=FILE_CHUNK: raise ValueError('Read at most 1 MiB at a time.')
318 else:319 else:
319 if not isinstance(request.get('data'),str) or len(request['data'])>90000 or len(base64.b64decode(request['data'],validate=True))>65536:320 if not isinstance(request.get('data'),str) or len(request['data'])>((FILE_CHUNK+2)//3)*4 or len(base64.b64decode(request['data'],validate=True))>FILE_CHUNK:
320 raise ValueError('Write at most 64 KiB at a time.')321 raise ValueError('Write at most 1 MiB at a time.')
321 if operation=='commit' and (not isinstance(request.get('destination'),str) or not 1<=len(request['destination'])<=4096 or '\x00' in request['destination'] or type(request.get('overwrite',False)) is not bool):322 if operation=='commit' and (not isinstance(request.get('destination'),str) or not 1<=len(request['destination'])<=4096 or '\x00' in request['destination'] or type(request.get('overwrite',False)) is not bool):
322 raise ValueError('Choose an absolute guest destination and whether to replace it.')323 raise ValueError('Choose an absolute guest destination and whether to replace it.')
323 if action == 'process' and (not isinstance(request.get('id'),str) or len(request['id'])>128 or request.get('operation') not in ('status','stop')):324 if action == 'process' and (not isinstance(request.get('id'),str) or len(request['id'])>128 or request.get('operation') not in ('status','stop')):
...@@ -617,10 +618,10 @@ def handle(name, request, *, active_snapshot=None):...@@ -617,10 +618,10 @@ def handle(name, request, *, active_snapshot=None):
617 if len(path)<3 or path[1:3] not in (':\\',':/') or not path[0].isalpha(): raise ValueError('Use an absolute guest file path.')618 if len(path)<3 or path[1:3] not in (':\\',':/') or not path[0].isalpha(): raise ValueError('Use an absolute guest file path.')
618 if request['operation']=='read':619 if request['operation']=='read':
619 data=read_file(name,path,request.get('offset',0),request.get('length',65536))620 data=read_file(name,path,request.get('offset',0),request.get('length',65536))
620 return {'data':base64.b64encode(data).decode(),'bytes':len(data)}621 return {'data':base64.b64encode(data).decode(),'bytes':len(data),'chunk_bytes':FILE_CHUNK}
621 data=base64.b64decode(request['data'],validate=True)622 data=base64.b64decode(request['data'],validate=True)
622 write_file(name,path,data,offset=request.get('offset',0))623 write_file(name,path,data,offset=request.get('offset',0))
623 return {'bytes':len(data)}624 return {'bytes':len(data),'chunk_bytes':FILE_CHUNK}
624 result = worker(name,platform,request)625 result = worker(name,platform,request)
625 if request.get('screenshot'):626 if request.get('screenshot'):
626 try: result.update(asyncio.run(desktop(name)))627 try: result.update(asyncio.run(desktop(name)))