authoravatarclover caruso<git@paperclover.net>2026-11-16 19:18:24-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-22 00:44:02-08:00
log7f67e70dc86c47a0645b001e4b3ae556376ea05e
tree8f25e0aca747bb32589b0bd7712df6104f440a1f
parent711e7f082e04608ad1e120898526ee9c9870483f
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Expose cancellable VM operation handles during execution

Keep active operation inspection independent of guest execution, reject stale handles, and retain uncertain cleanup state until the guest acknowledges it. Assisted-by: gpt-6.1-sol

5 files changed, 240 insertions(+), 25 deletions(-)

dashboard/src/computer-tools.json+2-2
......@@ -14,7 +14,7 @@
1414 },
1515 {
1616 "name": "computer_status",
17 "description": "Report the selected VM’s available runtimes, native UI support, permissions and desktop capabilities.",
17 "description": "Report VM power, command readiness, runtimes and desktop capabilities. Status and screenshots remain available during commands on Linux-hosted VMs. Use active_operation.id with computer_process to inspect or cancel an active operation. Available runtimes do not prove a particular command succeeded.",
1818 "inputSchema": {
1919 "type": "object",
2020 "properties": {
......@@ -423,7 +423,7 @@
423423 },
424424 {
425425 "name": "computer_process",
426 "description": "Inspect or stop a process launched by computer_spawn. Use its returned id; handles reject PID reuse.",
426 "description": "Inspect or stop a process from computer_spawn or an active operation from computer_status. Active operation handles remain available while execution is busy. Stop requests can require cleanup; inspect until running is false. A null running value means cleanup remains unconfirmed. Spawned process handles reject PID reuse.",
427427 "inputSchema": {
428428 "type": "object",
429429 "properties": {
tools/computer-test.py+125-1
......@@ -6,7 +6,10 @@ import asyncio
66import time
77import json
88import tempfile
9import threading
910import unittest
11from concurrent.futures import ThreadPoolExecutor
12from contextlib import contextmanager
1013from pathlib import Path
1114from unittest.mock import patch, AsyncMock, Mock
1215
......@@ -160,7 +163,7 @@ class GuestCancellation(unittest.TestCase):
160163 with patch.object(computer, 'finish_operation', side_effect=AssertionError('Status must not wait for cleanup')):
161164 status = computer.handle('fixture', {'action': 'status'})
162165 self.assertEqual(status['readiness'], 'recovery_required')
163 self.assertEqual(status['active_operation'], operation)
166 self.assertEqual(status['active_operation']['id'], 'operation:' + identity.removeprefix('snowglobe-computer-request-'))
164167 with patch.object(computer, 'guest_exec', side_effect=TimeoutError()), patch.object(computer, 'powershell', side_effect=TimeoutError()), self.assertRaises(BlockingIOError):
165168 computer.handle('fixture', {'action': 'exec', 'runtime': 'default', 'script': 'true', 'timeout_ms': 1000})
166169 work.assert_not_called()
......@@ -194,6 +197,127 @@ class GuestCancellation(unittest.TestCase):
194197 self.assertEqual(status['capabilities'], [])
195198 work.assert_not_called()
196199
200 def test_active_operation_can_be_inspected_and_cancelled_during_execution(self):
201 started = threading.Event()
202 def virsh(action, *args, **kwargs):
203 return 'running' if action == 'domstate' else '<domain/>'
204 def work(*args):
205 started.set()
206 while True:
207 computer.remaining(1)
208 time.sleep(.005)
209 with patch.object(computer.vms, 'virsh', virsh), patch.object(computer, 'worker', work), ThreadPoolExecutor(max_workers=1) as pool:
210 call = pool.submit(computer.handle, 'fixture', {'action': 'exec', 'script': 'long operation', 'runtime': 'default', 'timeout_ms': 1000})
211 self.assertTrue(started.wait(2))
212 status = computer.handle('fixture', {'action': 'status'})
213 identity = status['active_operation']['id']
214 inspect = {'action': 'process', 'id': identity, 'operation': 'status'}
215 self.assertTrue(computer.handle('fixture', inspect)['running'])
216 stopped = computer.handle('fixture', {**inspect, 'operation': 'stop'})
217 self.assertTrue(stopped['stopping'])
218 with self.assertRaises(InterruptedError):
219 call.result(timeout=2)
220 self.assertFalse(computer.handle('fixture', inspect)['running'])
221
222 def test_stale_operation_handle_cannot_cancel_a_later_call(self):
223 with computer.vms.computer_lock('fixture', 'exec') as previous:
224 identity = previous['id']
225 with computer.vms.computer_lock('fixture', 'exec') as active:
226 token = computer.OPERATION.set(('fixture', active['id']))
227 try:
228 result = computer.handle('fixture', {'action': 'process', 'id': identity, 'operation': 'stop'})
229 self.assertFalse(result['running'])
230 self.assertGreater(computer.remaining(1), 0)
231 finally:
232 computer.OPERATION.reset(token)
233
234 def test_cancelled_operation_cannot_start_another_guest_command(self):
235 with computer.vms.computer_lock('fixture', 'exec') as active:
236 token = computer.OPERATION.set(('fixture', active['id']))
237 try:
238 computer.handle('fixture', {'action': 'process', 'id': active['id'], 'operation': 'stop'})
239 with patch.object(computer.subprocess, 'run') as run, self.assertRaises(InterruptedError):
240 computer.qga('fixture', 'guest-ping')
241 run.assert_not_called()
242 finally:
243 computer.OPERATION.reset(token)
244
245 def test_pending_operation_remains_uncertain_until_cleanup_is_acknowledged(self):
246 operation = {'platform': 'windows', 'id': 'a' * 32}
247 computer.record_operation('fixture', operation)
248 request = {'action': 'process', 'id': 'operation:' + operation['id'], 'operation': 'status'}
249 status = computer.handle('fixture', request)
250 self.assertIsNone(status['running'])
251 self.assertTrue(status['recovery_required'])
252 with patch.object(computer, 'powershell', side_effect=TimeoutError()), self.assertRaises(BlockingIOError):
253 computer.handle('fixture', {**request, 'operation': 'stop'})
254 self.assertTrue(computer.handle('fixture', request)['recovery_required'])
255 with patch.object(computer, 'powershell', return_value=b''):
256 result = computer.handle('fixture', {**request, 'operation': 'stop'})
257 self.assertFalse(result['running'])
258 self.assertTrue(result['cancelled'])
259
260 def test_cleanup_does_not_remove_a_newer_pending_operation(self):
261 previous = {'platform': 'windows', 'id': 'a' * 32}
262 current = {'platform': 'windows', 'id': 'b' * 32}
263 computer.record_operation('fixture', previous)
264 lock = computer.vms.computer_lock
265 @contextmanager
266 def interleaved(name, action, identity=None):
267 computer.record_operation(name, current)
268 with lock(name, action, identity) as active:
269 yield active
270 with patch.object(computer.vms, 'computer_lock', interleaved), patch.object(computer, 'powershell') as stop:
271 result = computer.handle('fixture', {'action': 'process', 'id': 'operation:' + previous['id'], 'operation': 'stop'})
272 self.assertFalse(result['running'])
273 stop.assert_not_called()
274 self.assertEqual(json.loads((computer.vms.DISKS / 'fixture/computer-operation.json').read_text()), current)
275
276 def test_cleanup_keeps_transport_available_after_cancellation(self):
277 with computer.vms.computer_lock('fixture', 'exec') as active:
278 token = computer.OPERATION.set(('fixture', active['id']))
279 try:
280 computer.handle('fixture', {'action': 'process', 'id': active['id'], 'operation': 'stop'})
281 operation = {'platform': 'windows', 'id': active['id'].removeprefix('operation:')}
282 computer.record_operation('fixture', operation)
283 def cleanup(*args, **kwargs):
284 self.assertIsNone(computer.OPERATION.get())
285 self.assertGreater(computer.remaining(10), 0)
286 return b''
287 with patch.object(computer, 'powershell', cleanup):
288 computer.finish_operation('fixture', operation)
289 with self.assertRaises(InterruptedError):
290 computer.remaining(1)
291 finally:
292 computer.OPERATION.reset(token)
293
294 def test_guest_files_close_after_cancellation_and_deadline_expiry(self):
295 for write in (False, True):
296 operation = computer.OPERATION.set(('fixture', 'operation:' + 'a' * 32))
297 deadline = computer.DEADLINE.set(time.monotonic() + 10)
298 calls = []
299 def qga(name, action, **arguments):
300 calls.append(action)
301 if action == 'guest-file-open':
302 return 17
303 if action == 'guest-file-close':
304 self.assertEqual(arguments['handle'], 17)
305 self.assertIsNone(computer.OPERATION.get())
306 self.assertGreater(computer.remaining(10), 0)
307 return {}
308 computer.DEADLINE.set(time.monotonic() - 1)
309 raise InterruptedError('Fixture cancellation')
310 try:
311 with self.subTest(write=write), patch.object(computer, 'qga', qga), self.assertRaises(InterruptedError):
312 if write:
313 computer.write_file('fixture', 'owned-file', b'payload')
314 else:
315 computer.read_file('fixture', 'owned-file')
316 self.assertEqual(calls[-1], 'guest-file-close')
317 finally:
318 computer.DEADLINE.reset(deadline)
319 computer.OPERATION.reset(operation)
320
197321 def test_spawn_returns_its_handle_when_cleanup_must_wait_for_reconnection(self):
198322 result = {'id': '123:456', 'running': True}
199323 def execute(name, script, **kwargs):
tools/dashboard-host.py+4-2
......@@ -207,8 +207,10 @@ class Host:
207207 if action == "computer":
208208 try:
209209 return importlib.import_module("vm-computer").handle(payload["name"], payload["request"])
210 except BlockingIOError:
211 raise dashboard_runs.Error(409, "Another computer operation is running. Wait for it to finish.") from None
210 except (BlockingIOError, InterruptedError) as error:
211 raise dashboard_runs.Error(409, str(error) or "Another computer operation is running. Read computer_status before retrying.") from None
212 except (TimeoutError, subprocess.TimeoutExpired):
213 raise dashboard_runs.Error(504, "The computer operation timed out. Read computer_status before retrying.") from None
212214 except ValueError as error:
213215 raise Rejected(str(error)) from error
214216 if action == "clipboard":
tools/vm-computer.py+73-14
......@@ -21,12 +21,19 @@ import vms
2121rfb = import_module('vm-rfb')
2222SCREEN_SLOT = threading.BoundedSemaphore(1)
2323DEADLINE = ContextVar('computer_deadline', default=float('inf'))
24OPERATION = ContextVar('computer_operation', default=None)
2425
2526
2627def remaining(limit):
28 operation = OPERATION.get()
29 if operation is not None:
30 name, identity = operation
31 with vms.computer_control(name) as control:
32 if control.read() == identity:
33 raise InterruptedError('The computer operation was cancelled. Inspect its result before retrying.')
2734 seconds = min(limit, DEADLINE.get() - time.monotonic())
2835 if seconds <= 0:
29 raise TimeoutError('The input batch timed out. Check the desktop before continuing.')
36 raise TimeoutError('The computer operation timed out. Inspect its status before retrying.')
3037 return seconds
3138
3239
......@@ -75,6 +82,17 @@ def guest_exec(name, path, args, data=None, timeout=20):
7582 raise ValueError('The guest operation is still running. Check its status before retrying.')
7683
7784
85def close_file(name, handle):
86 """Guest file handles outlive the request unless explicitly closed."""
87 deadline = DEADLINE.set(time.monotonic() + 15)
88 operation = OPERATION.set(None)
89 try:
90 qga(name, 'guest-file-close', handle=handle)
91 finally:
92 OPERATION.reset(operation)
93 DEADLINE.reset(deadline)
94
95
7896def read_file(name, path, offset=0, length=65536):
7997 handle = qga(name, 'guest-file-open', path=path, mode='rb')
8098 try:
......@@ -82,7 +100,7 @@ def read_file(name, path, offset=0, length=65536):
82100 value = qga(name, 'guest-file-read', handle=handle, count=length)
83101 return base64.b64decode(value['buf-b64'], validate=True)
84102 finally:
85 qga(name, 'guest-file-close', handle=handle)
103 close_file(name, handle)
86104
87105
88106def write_file(name, path, data, offset=0):
......@@ -98,7 +116,7 @@ def write_file(name, path, data, offset=0):
98116 if result['count'] != len(chunk):
99117 raise ValueError('The guest disk could not accept the file. Check free space.')
100118 finally:
101 qga(name, 'guest-file-close', handle=handle)
119 close_file(name, handle)
102120
103121
104122def powershell(name, script, timeout=60):
......@@ -147,6 +165,7 @@ async def capture(name, actions, platform, screenshot=True, native_text=True):
147165 await asyncio.wait_for(ready.wait(), 10)
148166 if actions:
149167 for action in actions:
168 remaining(180)
150169 kind = action['type']
151170 if kind in ('move', 'click', 'drag', 'scroll'):
152171 x, y = action['x'], action['y']
......@@ -346,6 +365,7 @@ def record_operation(name, operation):
346365
347366def finish_operation(name, operation):
348367 token = DEADLINE.set(time.monotonic() + 60)
368 active = OPERATION.set(None)
349369 try:
350370 if operation['platform'] == 'linux':
351371 guest_exec(name, '/etc/qemu-ga/fsfreeze-hook.d/snowglobe-computer', [],
......@@ -370,6 +390,7 @@ def finish_operation(name, operation):
370390 except Exception as error:
371391 raise BlockingIOError('The previous operation has not stopped. Check its screenshot, then retry when the guest agent reconnects.') from error
372392 finally:
393 OPERATION.reset(active)
373394 DEADLINE.reset(token)
374395
375396
......@@ -388,7 +409,8 @@ def worker(name, platform, request):
388409 return reply
389410 if platform == 'linux':
390411 source = Path(__file__).with_name('mac-computer.py').read_text()
391 unit = 'snowglobe-computer-request-' + uuid.uuid4().hex
412 identity = OPERATION.get()
413 unit = 'snowglobe-computer-request-' + (identity[1].removeprefix('operation:') if identity else uuid.uuid4().hex)
392414 guest_exec(name, '/etc/qemu-ga/fsfreeze-hook.d/snowglobe-computer', [],
393415 json.dumps({'_prepare': unit}).encode())
394416 operation = {'platform': platform, 'id': unit}
......@@ -408,7 +430,8 @@ def worker(name, platform, request):
408430 raise ValueError(result['error'])
409431 return result
410432 if platform == 'windows':
411 run = uuid.uuid4().hex
433 identity = OPERATION.get()
434 run = identity[1].removeprefix('operation:') if identity else uuid.uuid4().hex
412435 directory = 'C:\\ProgramData\\SnowglobeComputer\\' + run
413436 source = Path(__file__).resolve().parent.parent / 'guest/computer-windows.cs'
414437 content = source.read_bytes()
......@@ -495,14 +518,52 @@ $registered.Run($null) | Out-Null
495518def handle(name, request):
496519 validate(request)
497520 vms.ensure_name(name)
521 if request['action'] == 'process' and request['id'].startswith('operation:'):
522 if vms.mac_vms.CONFIG and name == vms.mac_vms.CONFIG['name']:
523 raise ValueError('This VM does not provide active operation handles.')
524 identity = request['id']
525 with vms.computer_control(name) as control:
526 active = vms.computer_activity(name)
527 if active is not None:
528 if active.get('id') != identity:
529 return {'running': False}
530 if request['operation'] == 'stop':
531 control.seek(0)
532 control.truncate()
533 control.write(identity)
534 control.flush()
535 else:
536 active['stopping'] = control.read() == identity
537 return {'running': True, **active, 'stopping': request['operation'] == 'stop' or active.get('stopping', False)}
538 try:
539 with os.fdopen(vms.open_regular(vms.DISKS / name / 'computer-operation.json')) as incoming:
540 unfinished = json.load(incoming)
541 except FileNotFoundError:
542 return {'running': False}
543 if 'operation:' + unfinished['id'].removeprefix('snowglobe-computer-request-') != identity:
544 return {'running': False}
545 if request['operation'] == 'status':
546 return {'running': None, 'recovery_required': True, 'id': identity}
547 with vms.computer_lock(name, 'cancel', identity):
548 try:
549 with os.fdopen(vms.open_regular(vms.DISKS / name / 'computer-operation.json')) as incoming:
550 unfinished = json.load(incoming)
551 except FileNotFoundError:
552 return {'running': False}
553 if 'operation:' + unfinished['id'].removeprefix('snowglobe-computer-request-') != identity:
554 return {'running': False}
555 finish_operation(name, unfinished)
556 return {'running': False, 'cancelled': True, 'id': identity}
498557 if vms.mac_vms.CONFIG and name == vms.mac_vms.CONFIG['name']:
499558 return vms.mac_vms.handle('computer', {'name':name,'request':request})
500559 token = DEADLINE.set(time.monotonic() + 210)
560 operation_token = None
501561 try:
502562 with contextlib.ExitStack() as scope:
503563 busy = False
504564 try:
505 scope.enter_context(vms.computer_lock(name, request['action']))
565 active = scope.enter_context(vms.computer_lock(name, request['action']))
566 operation_token = OPERATION.set((name, active['id']))
506567 except BlockingIOError:
507568 if request['action'] not in ('status', 'shot'):
508569 raise BlockingIOError('Another computer operation is running. Read computer_status before retrying.') from None
......@@ -523,13 +584,8 @@ def handle(name, request):
523584 if request['action'] == 'shot':
524585 return asyncio.run(desktop(name))
525586 if busy:
526 with os.fdopen(vms.open_regular(vms.DISKS / name / 'computer.lock')) as incoming:
527 try:
528 operation = json.load(incoming)
529 except ValueError:
530 operation = {'action': 'computer'}
531 operation['age_seconds'] = round(max(0, time.time() - os.fstat(incoming.fileno()).st_mtime), 1)
532 return {**status, 'readiness': 'busy', 'capabilities': ['shot'], 'active_operation': operation}
587 active = vms.computer_activity(name)
588 return {**status, 'readiness': 'busy', 'capabilities': ['shot'], 'active_operation': active}
533589 try:
534590 with os.fdopen(vms.open_regular(vms.DISKS / name / 'computer-operation.json')) as incoming:
535591 unfinished = json.load(incoming)
......@@ -537,7 +593,8 @@ def handle(name, request):
537593 pass
538594 else:
539595 if request['action'] == 'status':
540 return {**status, 'readiness': 'recovery_required', 'capabilities': ['shot'], 'active_operation': unfinished,
596 active = {**unfinished, 'id': 'operation:' + unfinished['id'].removeprefix('snowglobe-computer-request-')}
597 return {**status, 'readiness': 'recovery_required', 'capabilities': ['shot'], 'active_operation': active,
541598 'worker_error': 'The previous operation needs cleanup. Inspect its screenshot or restart the VM before retrying.'}
542599 finish_operation(name, unfinished)
543600 native_text = True
......@@ -573,4 +630,6 @@ def handle(name, request):
573630 except (ValueError,OSError,TimeoutError): result['capture_error']='Screen capture failed. Retry computer_shot.'
574631 return result
575632 finally:
633 if operation_token is not None:
634 OPERATION.reset(operation_token)
576635 DEADLINE.reset(token)
tools/vms.py+36-6
......@@ -208,7 +208,7 @@ def validate(action, payload):
208208
209209
210210@contextlib.contextmanager
211def computer_lock(name, action="computer"):
211def computer_lock(name, action="computer", identity=None):
212212 ensure_name(name)
213213 directory = DISKS / name
214214 if directory.is_symlink():
......@@ -216,11 +216,41 @@ def computer_lock(name, action="computer"):
216216 descriptor = os.open(directory / "computer.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
217217 with os.fdopen(descriptor, "w") as lock:
218218 fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
219 lock.seek(0)
220 lock.truncate()
221 json.dump({"action": action}, lock)
222 lock.flush()
223 yield
219 operation = {"id": identity or "operation:" + secrets.token_hex(16), "action": action}
220 with computer_control(name):
221 lock.truncate(0)
222 json.dump(operation, lock)
223 lock.flush()
224 yield operation
225
226
227@contextlib.contextmanager
228def computer_control(name):
229 ensure_name(name)
230 directory = DISKS / name
231 if directory.is_symlink():
232 raise ValueError("The VM disk directory is a symbolic link.")
233 descriptor = os.open(directory / "computer-control", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
234 with os.fdopen(descriptor, "r+") as control:
235 fcntl.flock(control, fcntl.LOCK_EX)
236 yield control
237
238
239def computer_activity(name):
240 try:
241 with os.fdopen(open_regular(DISKS / name / "computer.lock")) as incoming:
242 try:
243 fcntl.flock(incoming, fcntl.LOCK_SH | fcntl.LOCK_NB)
244 except BlockingIOError:
245 try:
246 operation = json.load(incoming)
247 except ValueError:
248 return {"action": "computer"}
249 operation["age_seconds"] = round(max(0, time.time() - os.fstat(incoming.fileno()).st_mtime), 1)
250 return operation
251 except FileNotFoundError:
252 pass
253 return None
224254
225255
226256def command(*args, pass_fds=(), timeout=None):