diff --git a/dashboard/src/computer-tools.json b/dashboard/src/computer-tools.json index 7fb2d32bcfc145d79b5aae1517fb3adb72394cfe..51b545620f44cebdeb1d98a904fa3bb01f94c457 100644 --- a/dashboard/src/computer-tools.json +++ b/dashboard/src/computer-tools.json @@ -14,7 +14,7 @@ }, { "name": "computer_status", - "description": "Report the selected VM’s available runtimes, native UI support, permissions and desktop capabilities.", + "description": "Report VM power, command readiness, runtimes and desktop capabilities. Status and screenshots remain available during commands on Linux-hosted VMs. Use active_operation.id with computer_process to inspect or cancel an active operation. Available runtimes do not prove a particular command succeeded.", "inputSchema": { "type": "object", "properties": { @@ -423,7 +423,7 @@ }, { "name": "computer_process", - "description": "Inspect or stop a process launched by computer_spawn. Use its returned id; handles reject PID reuse.", + "description": "Inspect or stop a process from computer_spawn or an active operation from computer_status. Active operation handles remain available while execution is busy. Stop requests can require cleanup; inspect until running is false. A null running value means cleanup remains unconfirmed. Spawned process handles reject PID reuse.", "inputSchema": { "type": "object", "properties": { diff --git a/tools/computer-test.py b/tools/computer-test.py index 36a9354a9147a584faaf2c8601b4855f0752b2b0..5214abb95ca2b7b2b302c3268459a6bab8762e5d 100644 --- a/tools/computer-test.py +++ b/tools/computer-test.py @@ -6,7 +6,10 @@ import asyncio import time import json import tempfile +import threading import unittest +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager from pathlib import Path from unittest.mock import patch, AsyncMock, Mock @@ -160,7 +163,7 @@ class GuestCancellation(unittest.TestCase): with patch.object(computer, 'finish_operation', side_effect=AssertionError('Status must not wait for cleanup')): status = computer.handle('fixture', {'action': 'status'}) self.assertEqual(status['readiness'], 'recovery_required') - self.assertEqual(status['active_operation'], operation) + self.assertEqual(status['active_operation']['id'], 'operation:' + identity.removeprefix('snowglobe-computer-request-')) with patch.object(computer, 'guest_exec', side_effect=TimeoutError()), patch.object(computer, 'powershell', side_effect=TimeoutError()), self.assertRaises(BlockingIOError): computer.handle('fixture', {'action': 'exec', 'runtime': 'default', 'script': 'true', 'timeout_ms': 1000}) work.assert_not_called() @@ -194,6 +197,127 @@ class GuestCancellation(unittest.TestCase): self.assertEqual(status['capabilities'], []) work.assert_not_called() + def test_active_operation_can_be_inspected_and_cancelled_during_execution(self): + started = threading.Event() + def virsh(action, *args, **kwargs): + return 'running' if action == 'domstate' else '' + def work(*args): + started.set() + while True: + computer.remaining(1) + time.sleep(.005) + with patch.object(computer.vms, 'virsh', virsh), patch.object(computer, 'worker', work), ThreadPoolExecutor(max_workers=1) as pool: + call = pool.submit(computer.handle, 'fixture', {'action': 'exec', 'script': 'long operation', 'runtime': 'default', 'timeout_ms': 1000}) + self.assertTrue(started.wait(2)) + status = computer.handle('fixture', {'action': 'status'}) + identity = status['active_operation']['id'] + inspect = {'action': 'process', 'id': identity, 'operation': 'status'} + self.assertTrue(computer.handle('fixture', inspect)['running']) + stopped = computer.handle('fixture', {**inspect, 'operation': 'stop'}) + self.assertTrue(stopped['stopping']) + with self.assertRaises(InterruptedError): + call.result(timeout=2) + self.assertFalse(computer.handle('fixture', inspect)['running']) + + def test_stale_operation_handle_cannot_cancel_a_later_call(self): + with computer.vms.computer_lock('fixture', 'exec') as previous: + identity = previous['id'] + with computer.vms.computer_lock('fixture', 'exec') as active: + token = computer.OPERATION.set(('fixture', active['id'])) + try: + result = computer.handle('fixture', {'action': 'process', 'id': identity, 'operation': 'stop'}) + self.assertFalse(result['running']) + self.assertGreater(computer.remaining(1), 0) + finally: + computer.OPERATION.reset(token) + + def test_cancelled_operation_cannot_start_another_guest_command(self): + with computer.vms.computer_lock('fixture', 'exec') as active: + token = computer.OPERATION.set(('fixture', active['id'])) + try: + computer.handle('fixture', {'action': 'process', 'id': active['id'], 'operation': 'stop'}) + with patch.object(computer.subprocess, 'run') as run, self.assertRaises(InterruptedError): + computer.qga('fixture', 'guest-ping') + run.assert_not_called() + finally: + computer.OPERATION.reset(token) + + def test_pending_operation_remains_uncertain_until_cleanup_is_acknowledged(self): + operation = {'platform': 'windows', 'id': 'a' * 32} + computer.record_operation('fixture', operation) + request = {'action': 'process', 'id': 'operation:' + operation['id'], 'operation': 'status'} + status = computer.handle('fixture', request) + self.assertIsNone(status['running']) + self.assertTrue(status['recovery_required']) + with patch.object(computer, 'powershell', side_effect=TimeoutError()), self.assertRaises(BlockingIOError): + computer.handle('fixture', {**request, 'operation': 'stop'}) + self.assertTrue(computer.handle('fixture', request)['recovery_required']) + with patch.object(computer, 'powershell', return_value=b''): + result = computer.handle('fixture', {**request, 'operation': 'stop'}) + self.assertFalse(result['running']) + self.assertTrue(result['cancelled']) + + def test_cleanup_does_not_remove_a_newer_pending_operation(self): + previous = {'platform': 'windows', 'id': 'a' * 32} + current = {'platform': 'windows', 'id': 'b' * 32} + computer.record_operation('fixture', previous) + lock = computer.vms.computer_lock + @contextmanager + def interleaved(name, action, identity=None): + computer.record_operation(name, current) + with lock(name, action, identity) as active: + yield active + with patch.object(computer.vms, 'computer_lock', interleaved), patch.object(computer, 'powershell') as stop: + result = computer.handle('fixture', {'action': 'process', 'id': 'operation:' + previous['id'], 'operation': 'stop'}) + self.assertFalse(result['running']) + stop.assert_not_called() + self.assertEqual(json.loads((computer.vms.DISKS / 'fixture/computer-operation.json').read_text()), current) + + def test_cleanup_keeps_transport_available_after_cancellation(self): + with computer.vms.computer_lock('fixture', 'exec') as active: + token = computer.OPERATION.set(('fixture', active['id'])) + try: + computer.handle('fixture', {'action': 'process', 'id': active['id'], 'operation': 'stop'}) + operation = {'platform': 'windows', 'id': active['id'].removeprefix('operation:')} + computer.record_operation('fixture', operation) + def cleanup(*args, **kwargs): + self.assertIsNone(computer.OPERATION.get()) + self.assertGreater(computer.remaining(10), 0) + return b'' + with patch.object(computer, 'powershell', cleanup): + computer.finish_operation('fixture', operation) + with self.assertRaises(InterruptedError): + computer.remaining(1) + finally: + computer.OPERATION.reset(token) + + def test_guest_files_close_after_cancellation_and_deadline_expiry(self): + for write in (False, True): + operation = computer.OPERATION.set(('fixture', 'operation:' + 'a' * 32)) + deadline = computer.DEADLINE.set(time.monotonic() + 10) + calls = [] + def qga(name, action, **arguments): + calls.append(action) + if action == 'guest-file-open': + return 17 + if action == 'guest-file-close': + self.assertEqual(arguments['handle'], 17) + self.assertIsNone(computer.OPERATION.get()) + self.assertGreater(computer.remaining(10), 0) + return {} + computer.DEADLINE.set(time.monotonic() - 1) + raise InterruptedError('Fixture cancellation') + try: + with self.subTest(write=write), patch.object(computer, 'qga', qga), self.assertRaises(InterruptedError): + if write: + computer.write_file('fixture', 'owned-file', b'payload') + else: + computer.read_file('fixture', 'owned-file') + self.assertEqual(calls[-1], 'guest-file-close') + finally: + computer.DEADLINE.reset(deadline) + computer.OPERATION.reset(operation) + def test_spawn_returns_its_handle_when_cleanup_must_wait_for_reconnection(self): result = {'id': '123:456', 'running': True} def execute(name, script, **kwargs): diff --git a/tools/dashboard-host.py b/tools/dashboard-host.py index 4c47f424292dde11e7c3ed8054176d0699469405..a9e55c18a9baf3b89b045c2579e0a8bb094b9b2c 100644 --- a/tools/dashboard-host.py +++ b/tools/dashboard-host.py @@ -207,8 +207,10 @@ class Host: if action == "computer": try: return importlib.import_module("vm-computer").handle(payload["name"], payload["request"]) - except BlockingIOError: - raise dashboard_runs.Error(409, "Another computer operation is running. Wait for it to finish.") from None + except (BlockingIOError, InterruptedError) as error: + raise dashboard_runs.Error(409, str(error) or "Another computer operation is running. Read computer_status before retrying.") from None + except (TimeoutError, subprocess.TimeoutExpired): + raise dashboard_runs.Error(504, "The computer operation timed out. Read computer_status before retrying.") from None except ValueError as error: raise Rejected(str(error)) from error if action == "clipboard": diff --git a/tools/vm-computer.py b/tools/vm-computer.py index 3ba16566fc58d4456d108c64af77ed627ce732ed..42ea10a20c583023b9c87f5d2c26e505a35347b8 100644 --- a/tools/vm-computer.py +++ b/tools/vm-computer.py @@ -21,12 +21,19 @@ import vms rfb = import_module('vm-rfb') SCREEN_SLOT = threading.BoundedSemaphore(1) DEADLINE = ContextVar('computer_deadline', default=float('inf')) +OPERATION = ContextVar('computer_operation', default=None) def remaining(limit): + operation = OPERATION.get() + if operation is not None: + name, identity = operation + with vms.computer_control(name) as control: + if control.read() == identity: + raise InterruptedError('The computer operation was cancelled. Inspect its result before retrying.') seconds = min(limit, DEADLINE.get() - time.monotonic()) if seconds <= 0: - raise TimeoutError('The input batch timed out. Check the desktop before continuing.') + raise TimeoutError('The computer operation timed out. Inspect its status before retrying.') return seconds @@ -75,6 +82,17 @@ def guest_exec(name, path, args, data=None, timeout=20): raise ValueError('The guest operation is still running. Check its status before retrying.') +def close_file(name, handle): + """Guest file handles outlive the request unless explicitly closed.""" + deadline = DEADLINE.set(time.monotonic() + 15) + operation = OPERATION.set(None) + try: + qga(name, 'guest-file-close', handle=handle) + finally: + OPERATION.reset(operation) + DEADLINE.reset(deadline) + + def read_file(name, path, offset=0, length=65536): handle = qga(name, 'guest-file-open', path=path, mode='rb') try: @@ -82,7 +100,7 @@ def read_file(name, path, offset=0, length=65536): value = qga(name, 'guest-file-read', handle=handle, count=length) return base64.b64decode(value['buf-b64'], validate=True) finally: - qga(name, 'guest-file-close', handle=handle) + close_file(name, handle) def write_file(name, path, data, offset=0): @@ -98,7 +116,7 @@ def write_file(name, path, data, offset=0): if result['count'] != len(chunk): raise ValueError('The guest disk could not accept the file. Check free space.') finally: - qga(name, 'guest-file-close', handle=handle) + close_file(name, handle) def powershell(name, script, timeout=60): @@ -147,6 +165,7 @@ async def capture(name, actions, platform, screenshot=True, native_text=True): await asyncio.wait_for(ready.wait(), 10) if actions: for action in actions: + remaining(180) kind = action['type'] if kind in ('move', 'click', 'drag', 'scroll'): x, y = action['x'], action['y'] @@ -346,6 +365,7 @@ def record_operation(name, operation): def finish_operation(name, operation): token = DEADLINE.set(time.monotonic() + 60) + active = OPERATION.set(None) try: if operation['platform'] == 'linux': guest_exec(name, '/etc/qemu-ga/fsfreeze-hook.d/snowglobe-computer', [], @@ -370,6 +390,7 @@ def finish_operation(name, operation): except Exception as error: raise BlockingIOError('The previous operation has not stopped. Check its screenshot, then retry when the guest agent reconnects.') from error finally: + OPERATION.reset(active) DEADLINE.reset(token) @@ -388,7 +409,8 @@ def worker(name, platform, request): return reply if platform == 'linux': source = Path(__file__).with_name('mac-computer.py').read_text() - unit = 'snowglobe-computer-request-' + uuid.uuid4().hex + identity = OPERATION.get() + unit = 'snowglobe-computer-request-' + (identity[1].removeprefix('operation:') if identity else uuid.uuid4().hex) guest_exec(name, '/etc/qemu-ga/fsfreeze-hook.d/snowglobe-computer', [], json.dumps({'_prepare': unit}).encode()) operation = {'platform': platform, 'id': unit} @@ -408,7 +430,8 @@ def worker(name, platform, request): raise ValueError(result['error']) return result if platform == 'windows': - run = uuid.uuid4().hex + identity = OPERATION.get() + run = identity[1].removeprefix('operation:') if identity else uuid.uuid4().hex directory = 'C:\\ProgramData\\SnowglobeComputer\\' + run source = Path(__file__).resolve().parent.parent / 'guest/computer-windows.cs' content = source.read_bytes() @@ -495,14 +518,52 @@ $registered.Run($null) | Out-Null def handle(name, request): validate(request) vms.ensure_name(name) + if request['action'] == 'process' and request['id'].startswith('operation:'): + if vms.mac_vms.CONFIG and name == vms.mac_vms.CONFIG['name']: + raise ValueError('This VM does not provide active operation handles.') + identity = request['id'] + with vms.computer_control(name) as control: + active = vms.computer_activity(name) + if active is not None: + if active.get('id') != identity: + return {'running': False} + if request['operation'] == 'stop': + control.seek(0) + control.truncate() + control.write(identity) + control.flush() + else: + active['stopping'] = control.read() == identity + return {'running': True, **active, 'stopping': request['operation'] == 'stop' or active.get('stopping', False)} + try: + with os.fdopen(vms.open_regular(vms.DISKS / name / 'computer-operation.json')) as incoming: + unfinished = json.load(incoming) + except FileNotFoundError: + return {'running': False} + if 'operation:' + unfinished['id'].removeprefix('snowglobe-computer-request-') != identity: + return {'running': False} + if request['operation'] == 'status': + return {'running': None, 'recovery_required': True, 'id': identity} + with vms.computer_lock(name, 'cancel', identity): + try: + with os.fdopen(vms.open_regular(vms.DISKS / name / 'computer-operation.json')) as incoming: + unfinished = json.load(incoming) + except FileNotFoundError: + return {'running': False} + if 'operation:' + unfinished['id'].removeprefix('snowglobe-computer-request-') != identity: + return {'running': False} + finish_operation(name, unfinished) + return {'running': False, 'cancelled': True, 'id': identity} if vms.mac_vms.CONFIG and name == vms.mac_vms.CONFIG['name']: return vms.mac_vms.handle('computer', {'name':name,'request':request}) token = DEADLINE.set(time.monotonic() + 210) + operation_token = None try: with contextlib.ExitStack() as scope: busy = False try: - scope.enter_context(vms.computer_lock(name, request['action'])) + active = scope.enter_context(vms.computer_lock(name, request['action'])) + operation_token = OPERATION.set((name, active['id'])) except BlockingIOError: if request['action'] not in ('status', 'shot'): raise BlockingIOError('Another computer operation is running. Read computer_status before retrying.') from None @@ -523,13 +584,8 @@ def handle(name, request): if request['action'] == 'shot': return asyncio.run(desktop(name)) if busy: - with os.fdopen(vms.open_regular(vms.DISKS / name / 'computer.lock')) as incoming: - try: - operation = json.load(incoming) - except ValueError: - operation = {'action': 'computer'} - operation['age_seconds'] = round(max(0, time.time() - os.fstat(incoming.fileno()).st_mtime), 1) - return {**status, 'readiness': 'busy', 'capabilities': ['shot'], 'active_operation': operation} + active = vms.computer_activity(name) + return {**status, 'readiness': 'busy', 'capabilities': ['shot'], 'active_operation': active} try: with os.fdopen(vms.open_regular(vms.DISKS / name / 'computer-operation.json')) as incoming: unfinished = json.load(incoming) @@ -537,7 +593,8 @@ def handle(name, request): pass else: if request['action'] == 'status': - return {**status, 'readiness': 'recovery_required', 'capabilities': ['shot'], 'active_operation': unfinished, + active = {**unfinished, 'id': 'operation:' + unfinished['id'].removeprefix('snowglobe-computer-request-')} + return {**status, 'readiness': 'recovery_required', 'capabilities': ['shot'], 'active_operation': active, 'worker_error': 'The previous operation needs cleanup. Inspect its screenshot or restart the VM before retrying.'} finish_operation(name, unfinished) native_text = True @@ -573,4 +630,6 @@ def handle(name, request): except (ValueError,OSError,TimeoutError): result['capture_error']='Screen capture failed. Retry computer_shot.' return result finally: + if operation_token is not None: + OPERATION.reset(operation_token) DEADLINE.reset(token) diff --git a/tools/vms.py b/tools/vms.py index 803e028ebb2621e5c18809a7dc48307a4cfa1315..dfeb44ff9754455ac84e5f012d0a1a9e8a04b5db 100644 --- a/tools/vms.py +++ b/tools/vms.py @@ -208,7 +208,7 @@ def validate(action, payload): @contextlib.contextmanager -def computer_lock(name, action="computer"): +def computer_lock(name, action="computer", identity=None): ensure_name(name) directory = DISKS / name if directory.is_symlink(): @@ -216,11 +216,41 @@ def computer_lock(name, action="computer"): descriptor = os.open(directory / "computer.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) with os.fdopen(descriptor, "w") as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) - lock.seek(0) - lock.truncate() - json.dump({"action": action}, lock) - lock.flush() - yield + operation = {"id": identity or "operation:" + secrets.token_hex(16), "action": action} + with computer_control(name): + lock.truncate(0) + json.dump(operation, lock) + lock.flush() + yield operation + + +@contextlib.contextmanager +def computer_control(name): + ensure_name(name) + directory = DISKS / name + if directory.is_symlink(): + raise ValueError("The VM disk directory is a symbolic link.") + descriptor = os.open(directory / "computer-control", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + with os.fdopen(descriptor, "r+") as control: + fcntl.flock(control, fcntl.LOCK_EX) + yield control + + +def computer_activity(name): + try: + with os.fdopen(open_regular(DISKS / name / "computer.lock")) as incoming: + try: + fcntl.flock(incoming, fcntl.LOCK_SH | fcntl.LOCK_NB) + except BlockingIOError: + try: + operation = json.load(incoming) + except ValueError: + return {"action": "computer"} + operation["age_seconds"] = round(max(0, time.time() - os.fstat(incoming.fileno()).st_mtime), 1) + return operation + except FileNotFoundError: + pass + return None def command(*args, pass_fds=(), timeout=None):