| author | |
| committer | |
| log | 7f67e70dc86c47a0645b001e4b3ae556376ea05e |
| tree | 8f25e0aca747bb32589b0bd7712df6104f440a1f |
| parent | 711e7f082e04608ad1e120898526ee9c9870483f |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Keep active operation inspection independent of guest execution, reject stale
handles, and retain uncertain cleanup state until the guest acknowledges it.
Assisted-by: gpt-6.1-sol5 files changed, 240 insertions(+), 25 deletions(-)
dashboard/src/computer-tools.json+2-2| ... | @@ -14,7 +14,7 @@ | ... | @@ -14,7 +14,7 @@ |
| 14 | }, | 14 | }, |
| 15 | { | 15 | { |
| 16 | "name": "computer_status", | 16 | "name": "computer_status", |
| 17 | "description": "Report the selected VM’s available runtimes, native UI support, permissions and desktop capabilities.", | 17 | "description": "Report VM power, command readiness, runtimes and desktop capabilities. Status and screenshots remain available during commands on Linux-hosted VMs. Use active_operation.id with computer_process to inspect or cancel an active operation. Available runtimes do not prove a particular command succeeded.", |
| 18 | "inputSchema": { | 18 | "inputSchema": { |
| 19 | "type": "object", | 19 | "type": "object", |
| 20 | "properties": { | 20 | "properties": { |
| ... | @@ -423,7 +423,7 @@ | ... | @@ -423,7 +423,7 @@ |
| 423 | }, | 423 | }, |
| 424 | { | 424 | { |
| 425 | "name": "computer_process", | 425 | "name": "computer_process", |
| 426 | "description": "Inspect or stop a process launched by computer_spawn. Use its returned id; handles reject PID reuse.", | 426 | "description": "Inspect or stop a process from computer_spawn or an active operation from computer_status. Active operation handles remain available while execution is busy. Stop requests can require cleanup; inspect until running is false. A null running value means cleanup remains unconfirmed. Spawned process handles reject PID reuse.", |
| 427 | "inputSchema": { | 427 | "inputSchema": { |
| 428 | "type": "object", | 428 | "type": "object", |
| 429 | "properties": { | 429 | "properties": { |
tools/computer-test.py+125-1| ... | @@ -6,7 +6,10 @@ import asyncio | ... | @@ -6,7 +6,10 @@ import asyncio |
| 6 | import time | 6 | import time |
| 7 | import json | 7 | import json |
| 8 | import tempfile | 8 | import tempfile |
| 9 | import threading | ||
| 9 | import unittest | 10 | import unittest |
| 11 | from concurrent.futures import ThreadPoolExecutor | ||
| 12 | from contextlib import contextmanager | ||
| 10 | from pathlib import Path | 13 | from pathlib import Path |
| 11 | from unittest.mock import patch, AsyncMock, Mock | 14 | from unittest.mock import patch, AsyncMock, Mock |
| 12 | 15 | ||
| ... | @@ -160,7 +163,7 @@ class GuestCancellation(unittest.TestCase): | ... | @@ -160,7 +163,7 @@ class GuestCancellation(unittest.TestCase): |
| 160 | with patch.object(computer, 'finish_operation', side_effect=AssertionError('Status must not wait for cleanup')): | 163 | with patch.object(computer, 'finish_operation', side_effect=AssertionError('Status must not wait for cleanup')): |
| 161 | status = computer.handle('fixture', {'action': 'status'}) | 164 | status = computer.handle('fixture', {'action': 'status'}) |
| 162 | self.assertEqual(status['readiness'], 'recovery_required') | 165 | self.assertEqual(status['readiness'], 'recovery_required') |
| 163 | self.assertEqual(status['active_operation'], operation) | 166 | self.assertEqual(status['active_operation']['id'], 'operation:' + identity.removeprefix('snowglobe-computer-request-')) |
| 164 | with patch.object(computer, 'guest_exec', side_effect=TimeoutError()), patch.object(computer, 'powershell', side_effect=TimeoutError()), self.assertRaises(BlockingIOError): | 167 | with patch.object(computer, 'guest_exec', side_effect=TimeoutError()), patch.object(computer, 'powershell', side_effect=TimeoutError()), self.assertRaises(BlockingIOError): |
| 165 | computer.handle('fixture', {'action': 'exec', 'runtime': 'default', 'script': 'true', 'timeout_ms': 1000}) | 168 | computer.handle('fixture', {'action': 'exec', 'runtime': 'default', 'script': 'true', 'timeout_ms': 1000}) |
| 166 | work.assert_not_called() | 169 | work.assert_not_called() |
| ... | @@ -194,6 +197,127 @@ class GuestCancellation(unittest.TestCase): | ... | @@ -194,6 +197,127 @@ class GuestCancellation(unittest.TestCase): |
| 194 | self.assertEqual(status['capabilities'], []) | 197 | self.assertEqual(status['capabilities'], []) |
| 195 | work.assert_not_called() | 198 | work.assert_not_called() |
| 196 | 199 | ||
| 200 | def test_active_operation_can_be_inspected_and_cancelled_during_execution(self): | ||
| 201 | started = threading.Event() | ||
| 202 | def virsh(action, *args, **kwargs): | ||
| 203 | return 'running' if action == 'domstate' else '<domain/>' | ||
| 204 | def work(*args): | ||
| 205 | started.set() | ||
| 206 | while True: | ||
| 207 | computer.remaining(1) | ||
| 208 | time.sleep(.005) | ||
| 209 | with patch.object(computer.vms, 'virsh', virsh), patch.object(computer, 'worker', work), ThreadPoolExecutor(max_workers=1) as pool: | ||
| 210 | call = pool.submit(computer.handle, 'fixture', {'action': 'exec', 'script': 'long operation', 'runtime': 'default', 'timeout_ms': 1000}) | ||
| 211 | self.assertTrue(started.wait(2)) | ||
| 212 | status = computer.handle('fixture', {'action': 'status'}) | ||
| 213 | identity = status['active_operation']['id'] | ||
| 214 | inspect = {'action': 'process', 'id': identity, 'operation': 'status'} | ||
| 215 | self.assertTrue(computer.handle('fixture', inspect)['running']) | ||
| 216 | stopped = computer.handle('fixture', {**inspect, 'operation': 'stop'}) | ||
| 217 | self.assertTrue(stopped['stopping']) | ||
| 218 | with self.assertRaises(InterruptedError): | ||
| 219 | call.result(timeout=2) | ||
| 220 | self.assertFalse(computer.handle('fixture', inspect)['running']) | ||
| 221 | |||
| 222 | def test_stale_operation_handle_cannot_cancel_a_later_call(self): | ||
| 223 | with computer.vms.computer_lock('fixture', 'exec') as previous: | ||
| 224 | identity = previous['id'] | ||
| 225 | with computer.vms.computer_lock('fixture', 'exec') as active: | ||
| 226 | token = computer.OPERATION.set(('fixture', active['id'])) | ||
| 227 | try: | ||
| 228 | result = computer.handle('fixture', {'action': 'process', 'id': identity, 'operation': 'stop'}) | ||
| 229 | self.assertFalse(result['running']) | ||
| 230 | self.assertGreater(computer.remaining(1), 0) | ||
| 231 | finally: | ||
| 232 | computer.OPERATION.reset(token) | ||
| 233 | |||
| 234 | def test_cancelled_operation_cannot_start_another_guest_command(self): | ||
| 235 | with computer.vms.computer_lock('fixture', 'exec') as active: | ||
| 236 | token = computer.OPERATION.set(('fixture', active['id'])) | ||
| 237 | try: | ||
| 238 | computer.handle('fixture', {'action': 'process', 'id': active['id'], 'operation': 'stop'}) | ||
| 239 | with patch.object(computer.subprocess, 'run') as run, self.assertRaises(InterruptedError): | ||
| 240 | computer.qga('fixture', 'guest-ping') | ||
| 241 | run.assert_not_called() | ||
| 242 | finally: | ||
| 243 | computer.OPERATION.reset(token) | ||
| 244 | |||
| 245 | def test_pending_operation_remains_uncertain_until_cleanup_is_acknowledged(self): | ||
| 246 | operation = {'platform': 'windows', 'id': 'a' * 32} | ||
| 247 | computer.record_operation('fixture', operation) | ||
| 248 | request = {'action': 'process', 'id': 'operation:' + operation['id'], 'operation': 'status'} | ||
| 249 | status = computer.handle('fixture', request) | ||
| 250 | self.assertIsNone(status['running']) | ||
| 251 | self.assertTrue(status['recovery_required']) | ||
| 252 | with patch.object(computer, 'powershell', side_effect=TimeoutError()), self.assertRaises(BlockingIOError): | ||
| 253 | computer.handle('fixture', {**request, 'operation': 'stop'}) | ||
| 254 | self.assertTrue(computer.handle('fixture', request)['recovery_required']) | ||
| 255 | with patch.object(computer, 'powershell', return_value=b''): | ||
| 256 | result = computer.handle('fixture', {**request, 'operation': 'stop'}) | ||
| 257 | self.assertFalse(result['running']) | ||
| 258 | self.assertTrue(result['cancelled']) | ||
| 259 | |||
| 260 | def test_cleanup_does_not_remove_a_newer_pending_operation(self): | ||
| 261 | previous = {'platform': 'windows', 'id': 'a' * 32} | ||
| 262 | current = {'platform': 'windows', 'id': 'b' * 32} | ||
| 263 | computer.record_operation('fixture', previous) | ||
| 264 | lock = computer.vms.computer_lock | ||
| 265 | @contextmanager | ||
| 266 | def interleaved(name, action, identity=None): | ||
| 267 | computer.record_operation(name, current) | ||
| 268 | with lock(name, action, identity) as active: | ||
| 269 | yield active | ||
| 270 | with patch.object(computer.vms, 'computer_lock', interleaved), patch.object(computer, 'powershell') as stop: | ||
| 271 | result = computer.handle('fixture', {'action': 'process', 'id': 'operation:' + previous['id'], 'operation': 'stop'}) | ||
| 272 | self.assertFalse(result['running']) | ||
| 273 | stop.assert_not_called() | ||
| 274 | self.assertEqual(json.loads((computer.vms.DISKS / 'fixture/computer-operation.json').read_text()), current) | ||
| 275 | |||
| 276 | def test_cleanup_keeps_transport_available_after_cancellation(self): | ||
| 277 | with computer.vms.computer_lock('fixture', 'exec') as active: | ||
| 278 | token = computer.OPERATION.set(('fixture', active['id'])) | ||
| 279 | try: | ||
| 280 | computer.handle('fixture', {'action': 'process', 'id': active['id'], 'operation': 'stop'}) | ||
| 281 | operation = {'platform': 'windows', 'id': active['id'].removeprefix('operation:')} | ||
| 282 | computer.record_operation('fixture', operation) | ||
| 283 | def cleanup(*args, **kwargs): | ||
| 284 | self.assertIsNone(computer.OPERATION.get()) | ||
| 285 | self.assertGreater(computer.remaining(10), 0) | ||
| 286 | return b'' | ||
| 287 | with patch.object(computer, 'powershell', cleanup): | ||
| 288 | computer.finish_operation('fixture', operation) | ||
| 289 | with self.assertRaises(InterruptedError): | ||
| 290 | computer.remaining(1) | ||
| 291 | finally: | ||
| 292 | computer.OPERATION.reset(token) | ||
| 293 | |||
| 294 | def test_guest_files_close_after_cancellation_and_deadline_expiry(self): | ||
| 295 | for write in (False, True): | ||
| 296 | operation = computer.OPERATION.set(('fixture', 'operation:' + 'a' * 32)) | ||
| 297 | deadline = computer.DEADLINE.set(time.monotonic() + 10) | ||
| 298 | calls = [] | ||
| 299 | def qga(name, action, **arguments): | ||
| 300 | calls.append(action) | ||
| 301 | if action == 'guest-file-open': | ||
| 302 | return 17 | ||
| 303 | if action == 'guest-file-close': | ||
| 304 | self.assertEqual(arguments['handle'], 17) | ||
| 305 | self.assertIsNone(computer.OPERATION.get()) | ||
| 306 | self.assertGreater(computer.remaining(10), 0) | ||
| 307 | return {} | ||
| 308 | computer.DEADLINE.set(time.monotonic() - 1) | ||
| 309 | raise InterruptedError('Fixture cancellation') | ||
| 310 | try: | ||
| 311 | with self.subTest(write=write), patch.object(computer, 'qga', qga), self.assertRaises(InterruptedError): | ||
| 312 | if write: | ||
| 313 | computer.write_file('fixture', 'owned-file', b'payload') | ||
| 314 | else: | ||
| 315 | computer.read_file('fixture', 'owned-file') | ||
| 316 | self.assertEqual(calls[-1], 'guest-file-close') | ||
| 317 | finally: | ||
| 318 | computer.DEADLINE.reset(deadline) | ||
| 319 | computer.OPERATION.reset(operation) | ||
| 320 | |||
| 197 | def test_spawn_returns_its_handle_when_cleanup_must_wait_for_reconnection(self): | 321 | def test_spawn_returns_its_handle_when_cleanup_must_wait_for_reconnection(self): |
| 198 | result = {'id': '123:456', 'running': True} | 322 | result = {'id': '123:456', 'running': True} |
| 199 | def execute(name, script, **kwargs): | 323 | def execute(name, script, **kwargs): |
tools/dashboard-host.py+4-2| ... | @@ -207,8 +207,10 @@ class Host: | ... | @@ -207,8 +207,10 @@ class Host: |
| 207 | if action == "computer": | 207 | if action == "computer": |
| 208 | try: | 208 | try: |
| 209 | return importlib.import_module("vm-computer").handle(payload["name"], payload["request"]) | 209 | return importlib.import_module("vm-computer").handle(payload["name"], payload["request"]) |
| 210 | except BlockingIOError: | 210 | except (BlockingIOError, InterruptedError) as error: |
| 211 | raise dashboard_runs.Error(409, "Another computer operation is running. Wait for it to finish.") from None | 211 | raise dashboard_runs.Error(409, str(error) or "Another computer operation is running. Read computer_status before retrying.") from None |
| 212 | except (TimeoutError, subprocess.TimeoutExpired): | ||
| 213 | raise dashboard_runs.Error(504, "The computer operation timed out. Read computer_status before retrying.") from None | ||
| 212 | except ValueError as error: | 214 | except ValueError as error: |
| 213 | raise Rejected(str(error)) from error | 215 | raise Rejected(str(error)) from error |
| 214 | if action == "clipboard": | 216 | if action == "clipboard": |
tools/vm-computer.py+73-14| ... | @@ -21,12 +21,19 @@ import vms | ... | @@ -21,12 +21,19 @@ import vms |
| 21 | rfb = import_module('vm-rfb') | 21 | rfb = import_module('vm-rfb') |
| 22 | SCREEN_SLOT = threading.BoundedSemaphore(1) | 22 | SCREEN_SLOT = threading.BoundedSemaphore(1) |
| 23 | DEADLINE = ContextVar('computer_deadline', default=float('inf')) | 23 | DEADLINE = ContextVar('computer_deadline', default=float('inf')) |
| 24 | OPERATION = ContextVar('computer_operation', default=None) | ||
| 24 | 25 | ||
| 25 | 26 | ||
| 26 | def remaining(limit): | 27 | def remaining(limit): |
| 28 | operation = OPERATION.get() | ||
| 29 | if operation is not None: | ||
| 30 | name, identity = operation | ||
| 31 | with vms.computer_control(name) as control: | ||
| 32 | if control.read() == identity: | ||
| 33 | raise InterruptedError('The computer operation was cancelled. Inspect its result before retrying.') | ||
| 27 | seconds = min(limit, DEADLINE.get() - time.monotonic()) | 34 | seconds = min(limit, DEADLINE.get() - time.monotonic()) |
| 28 | if seconds <= 0: | 35 | if seconds <= 0: |
| 29 | raise TimeoutError('The input batch timed out. Check the desktop before continuing.') | 36 | raise TimeoutError('The computer operation timed out. Inspect its status before retrying.') |
| 30 | return seconds | 37 | return seconds |
| 31 | 38 | ||
| 32 | 39 | ||
| ... | @@ -75,6 +82,17 @@ def guest_exec(name, path, args, data=None, timeout=20): | ... | @@ -75,6 +82,17 @@ def guest_exec(name, path, args, data=None, timeout=20): |
| 75 | raise ValueError('The guest operation is still running. Check its status before retrying.') | 82 | raise ValueError('The guest operation is still running. Check its status before retrying.') |
| 76 | 83 | ||
| 77 | 84 | ||
| 85 | def close_file(name, handle): | ||
| 86 | """Guest file handles outlive the request unless explicitly closed.""" | ||
| 87 | deadline = DEADLINE.set(time.monotonic() + 15) | ||
| 88 | operation = OPERATION.set(None) | ||
| 89 | try: | ||
| 90 | qga(name, 'guest-file-close', handle=handle) | ||
| 91 | finally: | ||
| 92 | OPERATION.reset(operation) | ||
| 93 | DEADLINE.reset(deadline) | ||
| 94 | |||
| 95 | |||
| 78 | def read_file(name, path, offset=0, length=65536): | 96 | def read_file(name, path, offset=0, length=65536): |
| 79 | handle = qga(name, 'guest-file-open', path=path, mode='rb') | 97 | handle = qga(name, 'guest-file-open', path=path, mode='rb') |
| 80 | try: | 98 | try: |
| ... | @@ -82,7 +100,7 @@ def read_file(name, path, offset=0, length=65536): | ... | @@ -82,7 +100,7 @@ def read_file(name, path, offset=0, length=65536): |
| 82 | value = qga(name, 'guest-file-read', handle=handle, count=length) | 100 | value = qga(name, 'guest-file-read', handle=handle, count=length) |
| 83 | return base64.b64decode(value['buf-b64'], validate=True) | 101 | return base64.b64decode(value['buf-b64'], validate=True) |
| 84 | finally: | 102 | finally: |
| 85 | qga(name, 'guest-file-close', handle=handle) | 103 | close_file(name, handle) |
| 86 | 104 | ||
| 87 | 105 | ||
| 88 | def write_file(name, path, data, offset=0): | 106 | def write_file(name, path, data, offset=0): |
| ... | @@ -98,7 +116,7 @@ def write_file(name, path, data, offset=0): | ... | @@ -98,7 +116,7 @@ def write_file(name, path, data, offset=0): |
| 98 | if result['count'] != len(chunk): | 116 | if result['count'] != len(chunk): |
| 99 | raise ValueError('The guest disk could not accept the file. Check free space.') | 117 | raise ValueError('The guest disk could not accept the file. Check free space.') |
| 100 | finally: | 118 | finally: |
| 101 | qga(name, 'guest-file-close', handle=handle) | 119 | close_file(name, handle) |
| 102 | 120 | ||
| 103 | 121 | ||
| 104 | def powershell(name, script, timeout=60): | 122 | def powershell(name, script, timeout=60): |
| ... | @@ -147,6 +165,7 @@ async def capture(name, actions, platform, screenshot=True, native_text=True): | ... | @@ -147,6 +165,7 @@ async def capture(name, actions, platform, screenshot=True, native_text=True): |
| 147 | await asyncio.wait_for(ready.wait(), 10) | 165 | await asyncio.wait_for(ready.wait(), 10) |
| 148 | if actions: | 166 | if actions: |
| 149 | for action in actions: | 167 | for action in actions: |
| 168 | remaining(180) | ||
| 150 | kind = action['type'] | 169 | kind = action['type'] |
| 151 | if kind in ('move', 'click', 'drag', 'scroll'): | 170 | if kind in ('move', 'click', 'drag', 'scroll'): |
| 152 | x, y = action['x'], action['y'] | 171 | x, y = action['x'], action['y'] |
| ... | @@ -346,6 +365,7 @@ def record_operation(name, operation): | ... | @@ -346,6 +365,7 @@ def record_operation(name, operation): |
| 346 | 365 | ||
| 347 | def finish_operation(name, operation): | 366 | def finish_operation(name, operation): |
| 348 | token = DEADLINE.set(time.monotonic() + 60) | 367 | token = DEADLINE.set(time.monotonic() + 60) |
| 368 | active = OPERATION.set(None) | ||
| 349 | try: | 369 | try: |
| 350 | if operation['platform'] == 'linux': | 370 | if operation['platform'] == 'linux': |
| 351 | guest_exec(name, '/etc/qemu-ga/fsfreeze-hook.d/snowglobe-computer', [], | 371 | guest_exec(name, '/etc/qemu-ga/fsfreeze-hook.d/snowglobe-computer', [], |
| ... | @@ -370,6 +390,7 @@ def finish_operation(name, operation): | ... | @@ -370,6 +390,7 @@ def finish_operation(name, operation): |
| 370 | except Exception as error: | 390 | except Exception as error: |
| 371 | raise BlockingIOError('The previous operation has not stopped. Check its screenshot, then retry when the guest agent reconnects.') from error | 391 | raise BlockingIOError('The previous operation has not stopped. Check its screenshot, then retry when the guest agent reconnects.') from error |
| 372 | finally: | 392 | finally: |
| 393 | OPERATION.reset(active) | ||
| 373 | DEADLINE.reset(token) | 394 | DEADLINE.reset(token) |
| 374 | 395 | ||
| 375 | 396 | ||
| ... | @@ -388,7 +409,8 @@ def worker(name, platform, request): | ... | @@ -388,7 +409,8 @@ def worker(name, platform, request): |
| 388 | return reply | 409 | return reply |
| 389 | if platform == 'linux': | 410 | if platform == 'linux': |
| 390 | source = Path(__file__).with_name('mac-computer.py').read_text() | 411 | source = Path(__file__).with_name('mac-computer.py').read_text() |
| 391 | unit = 'snowglobe-computer-request-' + uuid.uuid4().hex | 412 | identity = OPERATION.get() |
| 413 | unit = 'snowglobe-computer-request-' + (identity[1].removeprefix('operation:') if identity else uuid.uuid4().hex) | ||
| 392 | guest_exec(name, '/etc/qemu-ga/fsfreeze-hook.d/snowglobe-computer', [], | 414 | guest_exec(name, '/etc/qemu-ga/fsfreeze-hook.d/snowglobe-computer', [], |
| 393 | json.dumps({'_prepare': unit}).encode()) | 415 | json.dumps({'_prepare': unit}).encode()) |
| 394 | operation = {'platform': platform, 'id': unit} | 416 | operation = {'platform': platform, 'id': unit} |
| ... | @@ -408,7 +430,8 @@ def worker(name, platform, request): | ... | @@ -408,7 +430,8 @@ def worker(name, platform, request): |
| 408 | raise ValueError(result['error']) | 430 | raise ValueError(result['error']) |
| 409 | return result | 431 | return result |
| 410 | if platform == 'windows': | 432 | if platform == 'windows': |
| 411 | run = uuid.uuid4().hex | 433 | identity = OPERATION.get() |
| 434 | run = identity[1].removeprefix('operation:') if identity else uuid.uuid4().hex | ||
| 412 | directory = 'C:\\ProgramData\\SnowglobeComputer\\' + run | 435 | directory = 'C:\\ProgramData\\SnowglobeComputer\\' + run |
| 413 | source = Path(__file__).resolve().parent.parent / 'guest/computer-windows.cs' | 436 | source = Path(__file__).resolve().parent.parent / 'guest/computer-windows.cs' |
| 414 | content = source.read_bytes() | 437 | content = source.read_bytes() |
| ... | @@ -495,14 +518,52 @@ $registered.Run($null) | Out-Null | ... | @@ -495,14 +518,52 @@ $registered.Run($null) | Out-Null |
| 495 | def handle(name, request): | 518 | def handle(name, request): |
| 496 | validate(request) | 519 | validate(request) |
| 497 | vms.ensure_name(name) | 520 | vms.ensure_name(name) |
| 521 | if request['action'] == 'process' and request['id'].startswith('operation:'): | ||
| 522 | if vms.mac_vms.CONFIG and name == vms.mac_vms.CONFIG['name']: | ||
| 523 | raise ValueError('This VM does not provide active operation handles.') | ||
| 524 | identity = request['id'] | ||
| 525 | with vms.computer_control(name) as control: | ||
| 526 | active = vms.computer_activity(name) | ||
| 527 | if active is not None: | ||
| 528 | if active.get('id') != identity: | ||
| 529 | return {'running': False} | ||
| 530 | if request['operation'] == 'stop': | ||
| 531 | control.seek(0) | ||
| 532 | control.truncate() | ||
| 533 | control.write(identity) | ||
| 534 | control.flush() | ||
| 535 | else: | ||
| 536 | active['stopping'] = control.read() == identity | ||
| 537 | return {'running': True, **active, 'stopping': request['operation'] == 'stop' or active.get('stopping', False)} | ||
| 538 | try: | ||
| 539 | with os.fdopen(vms.open_regular(vms.DISKS / name / 'computer-operation.json')) as incoming: | ||
| 540 | unfinished = json.load(incoming) | ||
| 541 | except FileNotFoundError: | ||
| 542 | return {'running': False} | ||
| 543 | if 'operation:' + unfinished['id'].removeprefix('snowglobe-computer-request-') != identity: | ||
| 544 | return {'running': False} | ||
| 545 | if request['operation'] == 'status': | ||
| 546 | return {'running': None, 'recovery_required': True, 'id': identity} | ||
| 547 | with vms.computer_lock(name, 'cancel', identity): | ||
| 548 | try: | ||
| 549 | with os.fdopen(vms.open_regular(vms.DISKS / name / 'computer-operation.json')) as incoming: | ||
| 550 | unfinished = json.load(incoming) | ||
| 551 | except FileNotFoundError: | ||
| 552 | return {'running': False} | ||
| 553 | if 'operation:' + unfinished['id'].removeprefix('snowglobe-computer-request-') != identity: | ||
| 554 | return {'running': False} | ||
| 555 | finish_operation(name, unfinished) | ||
| 556 | return {'running': False, 'cancelled': True, 'id': identity} | ||
| 498 | if vms.mac_vms.CONFIG and name == vms.mac_vms.CONFIG['name']: | 557 | if vms.mac_vms.CONFIG and name == vms.mac_vms.CONFIG['name']: |
| 499 | return vms.mac_vms.handle('computer', {'name':name,'request':request}) | 558 | return vms.mac_vms.handle('computer', {'name':name,'request':request}) |
| 500 | token = DEADLINE.set(time.monotonic() + 210) | 559 | token = DEADLINE.set(time.monotonic() + 210) |
| 560 | operation_token = None | ||
| 501 | try: | 561 | try: |
| 502 | with contextlib.ExitStack() as scope: | 562 | with contextlib.ExitStack() as scope: |
| 503 | busy = False | 563 | busy = False |
| 504 | try: | 564 | try: |
| 505 | scope.enter_context(vms.computer_lock(name, request['action'])) | 565 | active = scope.enter_context(vms.computer_lock(name, request['action'])) |
| 566 | operation_token = OPERATION.set((name, active['id'])) | ||
| 506 | except BlockingIOError: | 567 | except BlockingIOError: |
| 507 | if request['action'] not in ('status', 'shot'): | 568 | if request['action'] not in ('status', 'shot'): |
| 508 | raise BlockingIOError('Another computer operation is running. Read computer_status before retrying.') from None | 569 | raise BlockingIOError('Another computer operation is running. Read computer_status before retrying.') from None |
| ... | @@ -523,13 +584,8 @@ def handle(name, request): | ... | @@ -523,13 +584,8 @@ def handle(name, request): |
| 523 | if request['action'] == 'shot': | 584 | if request['action'] == 'shot': |
| 524 | return asyncio.run(desktop(name)) | 585 | return asyncio.run(desktop(name)) |
| 525 | if busy: | 586 | if busy: |
| 526 | with os.fdopen(vms.open_regular(vms.DISKS / name / 'computer.lock')) as incoming: | 587 | active = vms.computer_activity(name) |
| 527 | try: | 588 | return {**status, 'readiness': 'busy', 'capabilities': ['shot'], 'active_operation': active} |
| 528 | operation = json.load(incoming) | ||
| 529 | except ValueError: | ||
| 530 | operation = {'action': 'computer'} | ||
| 531 | operation['age_seconds'] = round(max(0, time.time() - os.fstat(incoming.fileno()).st_mtime), 1) | ||
| 532 | return {**status, 'readiness': 'busy', 'capabilities': ['shot'], 'active_operation': operation} | ||
| 533 | try: | 589 | try: |
| 534 | with os.fdopen(vms.open_regular(vms.DISKS / name / 'computer-operation.json')) as incoming: | 590 | with os.fdopen(vms.open_regular(vms.DISKS / name / 'computer-operation.json')) as incoming: |
| 535 | unfinished = json.load(incoming) | 591 | unfinished = json.load(incoming) |
| ... | @@ -537,7 +593,8 @@ def handle(name, request): | ... | @@ -537,7 +593,8 @@ def handle(name, request): |
| 537 | pass | 593 | pass |
| 538 | else: | 594 | else: |
| 539 | if request['action'] == 'status': | 595 | if request['action'] == 'status': |
| 540 | return {**status, 'readiness': 'recovery_required', 'capabilities': ['shot'], 'active_operation': unfinished, | 596 | active = {**unfinished, 'id': 'operation:' + unfinished['id'].removeprefix('snowglobe-computer-request-')} |
| 597 | return {**status, 'readiness': 'recovery_required', 'capabilities': ['shot'], 'active_operation': active, | ||
| 541 | 'worker_error': 'The previous operation needs cleanup. Inspect its screenshot or restart the VM before retrying.'} | 598 | 'worker_error': 'The previous operation needs cleanup. Inspect its screenshot or restart the VM before retrying.'} |
| 542 | finish_operation(name, unfinished) | 599 | finish_operation(name, unfinished) |
| 543 | native_text = True | 600 | native_text = True |
| ... | @@ -573,4 +630,6 @@ def handle(name, request): | ... | @@ -573,4 +630,6 @@ def handle(name, request): |
| 573 | except (ValueError,OSError,TimeoutError): result['capture_error']='Screen capture failed. Retry computer_shot.' | 630 | except (ValueError,OSError,TimeoutError): result['capture_error']='Screen capture failed. Retry computer_shot.' |
| 574 | return result | 631 | return result |
| 575 | finally: | 632 | finally: |
| 633 | if operation_token is not None: | ||
| 634 | OPERATION.reset(operation_token) | ||
| 576 | DEADLINE.reset(token) | 635 | DEADLINE.reset(token) |
tools/vms.py+36-6| ... | @@ -208,7 +208,7 @@ def validate(action, payload): | ... | @@ -208,7 +208,7 @@ def validate(action, payload): |
| 208 | 208 | ||
| 209 | 209 | ||
| 210 | @contextlib.contextmanager | 210 | @contextlib.contextmanager |
| 211 | def computer_lock(name, action="computer"): | 211 | def computer_lock(name, action="computer", identity=None): |
| 212 | ensure_name(name) | 212 | ensure_name(name) |
| 213 | directory = DISKS / name | 213 | directory = DISKS / name |
| 214 | if directory.is_symlink(): | 214 | if directory.is_symlink(): |
| ... | @@ -216,11 +216,41 @@ def computer_lock(name, action="computer"): | ... | @@ -216,11 +216,41 @@ def computer_lock(name, action="computer"): |
| 216 | descriptor = os.open(directory / "computer.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) | 216 | descriptor = os.open(directory / "computer.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) |
| 217 | with os.fdopen(descriptor, "w") as lock: | 217 | with os.fdopen(descriptor, "w") as lock: |
| 218 | fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) | 218 | fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) |
| 219 | lock.seek(0) | 219 | operation = {"id": identity or "operation:" + secrets.token_hex(16), "action": action} |
| 220 | lock.truncate() | 220 | with computer_control(name): |
| 221 | json.dump({"action": action}, lock) | 221 | lock.truncate(0) |
| 222 | lock.flush() | 222 | json.dump(operation, lock) |
| 223 | yield | 223 | lock.flush() |
| 224 | yield operation | ||
| 225 | |||
| 226 | |||
| 227 | @contextlib.contextmanager | ||
| 228 | def computer_control(name): | ||
| 229 | ensure_name(name) | ||
| 230 | directory = DISKS / name | ||
| 231 | if directory.is_symlink(): | ||
| 232 | raise ValueError("The VM disk directory is a symbolic link.") | ||
| 233 | descriptor = os.open(directory / "computer-control", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) | ||
| 234 | with os.fdopen(descriptor, "r+") as control: | ||
| 235 | fcntl.flock(control, fcntl.LOCK_EX) | ||
| 236 | yield control | ||
| 237 | |||
| 238 | |||
| 239 | def computer_activity(name): | ||
| 240 | try: | ||
| 241 | with os.fdopen(open_regular(DISKS / name / "computer.lock")) as incoming: | ||
| 242 | try: | ||
| 243 | fcntl.flock(incoming, fcntl.LOCK_SH | fcntl.LOCK_NB) | ||
| 244 | except BlockingIOError: | ||
| 245 | try: | ||
| 246 | operation = json.load(incoming) | ||
| 247 | except ValueError: | ||
| 248 | return {"action": "computer"} | ||
| 249 | operation["age_seconds"] = round(max(0, time.time() - os.fstat(incoming.fileno()).st_mtime), 1) | ||
| 250 | return operation | ||
| 251 | except FileNotFoundError: | ||
| 252 | pass | ||
| 253 | return None | ||
| 224 | 254 | ||
| 225 | 255 | ||
| 226 | def command(*args, pass_fds=(), timeout=None): | 256 | def command(*args, pass_fds=(), timeout=None): |