1#!/usr/bin/env python3
2"""Computer protocol and guest file boundary tests using disposable files."""
3from importlib import import_module
4import base64
5import asyncio
6import time
7import json
8import io
9import tempfile
10import threading
11import unittest
12from concurrent.futures import ThreadPoolExecutor
13from contextlib import contextmanager
14from pathlib import Path
15from unittest.mock import patch, AsyncMock, Mock
16
17computer=import_module('vm-computer')
18worker=import_module('mac-computer')
19leases=import_module('vm-ai')
20
21
22class Protocol(unittest.TestCase):
23 def test_drag_defaults_count_toward_the_batch_budget(self):
24 drag = {'type': 'drag', 'x': 0, 'y': 0, 'to_x': 1, 'to_y': 1}
25 computer.validate({'action': 'input', 'actions': [drag] * 37 + [{'type': 'wait', 'duration_ms': 200}]})
26 with self.assertRaisesRegex(ValueError, '15 seconds'):
27 computer.validate({'action': 'input', 'actions': [drag] * 38})
28
29 def test_action_limit_errors_identify_the_action_field_and_limit(self):
30 for action, field, limit in [({'type': 'wait', 'duration_ms': 5001}, 'duration_ms', 5000),
31 ({'type': 'click', 'x': 0, 'y': 0, 'count': 4}, 'count', 3),
32 ({'type': 'scroll', 'x': 0, 'y': 0, 'direction': 'up', 'amount': 101}, 'amount', 100)]:
33 with self.subTest(field=field), self.assertRaisesRegex(ValueError, f'Action 2: {field}.*1.*{limit}'):
34 computer.validate({'action': 'input', 'actions': [{'type': 'text', 'text': 'not executed'}, action]})
35
36 def test_rejects_malformed_actions_before_input(self):
37 for action in [None,[],{'type':[]},{'type':'click','x':True,'y':0},
38 {'type':'drag','x':0,'y':0,'to_x':0,'to_y':-1},
39 {'type':'key','keys':['Invalid']},{'type':'key','keys':'Enter'},
40 {'type':'scroll','x':0,'y':0,'direction':'forward'},
41 {'type':'wait','duration_ms':5001},{'type':'text','text':'x'*16385},
42 {'type':'click','x':0,'y':0,'command':'id'}]:
43 with self.subTest(action=action),self.assertRaises(ValueError):
44 computer.validate({'action':'input','actions':[action],'screenshot':False})
45
46 def test_rejects_unbounded_files_and_ui_targets(self):
47 for request in [{'action':'file','operation':'read','path':'/tmp/x','length':computer.FILE_CHUNK+1},
48 {'action':'file','operation':'write','path':'/tmp/x','data':base64.b64encode(b'x'*(computer.FILE_CHUNK+1)).decode()},
49 {'action':'file','operation':'read','path':'/tmp/x','offset':True},
50 {'action':'ui','pid':True},
51 {'action':'ui_action','target':{'name':'x'*5000},'operation':'press'}]:
52 with self.subTest(request=request),self.assertRaises(ValueError):computer.validate(request)
53
54 def test_file_upload_is_exclusive_and_commit_does_not_replace_by_default(self):
55 with tempfile.TemporaryDirectory() as directory:
56 temporary=str(Path(directory)/'upload');destination=str(Path(directory)/'Unicode 🦊')
57 data=b'\0binary\xff\n'*8192
58 def call(operation,**values):return worker.execute({'action':'file','operation':operation,'path':temporary,**values})
59 call('write',data=base64.b64encode(data[:65536]).decode(),offset=0)
60 with self.assertRaises(FileExistsError):call('write',data='',offset=0)
61 with self.assertRaises(ValueError):call('write',data='',offset=42)
62 call('write',data=base64.b64encode(data[65536:]).decode(),offset=65536)
63 self.assertEqual(Path(temporary).read_bytes(),data)
64 Path(destination).write_bytes(b'existing')
65 with self.assertRaises(FileExistsError):call('commit',destination=destination,overwrite=False)
66 self.assertEqual(Path(destination).read_bytes(),b'existing')
67 call('commit',destination=destination,overwrite=True)
68 self.assertEqual(Path(destination).read_bytes(),data)
69 self.assertFalse(Path(temporary).exists())
70
71 def test_upload_refuses_symlink_and_directory(self):
72 with tempfile.TemporaryDirectory() as directory:
73 target=Path(directory)/'target';target.write_bytes(b'original')
74 link=Path(directory)/'link';link.symlink_to(target)
75 for path in [link,Path(directory)]:
76 with self.assertRaises(OSError):worker.execute({'action':'file','operation':'write','path':str(path),'data':'','offset':0})
77 self.assertEqual(target.read_bytes(),b'original')
78
79 def test_stale_process_handle_cannot_stop_reused_pid(self):
80 with patch.object(worker,'MAC',True),patch.object(worker,'process_identity',return_value='new-start'),patch.object(worker.os,'killpg') as kill:
81 self.assertEqual(worker.execute({'action':'process','id':'100:old-start','operation':'stop'}),{'running':False})
82 kill.assert_not_called()
83
84
85class GuestCancellation(unittest.TestCase):
86 def setUp(self):
87 directory = tempfile.TemporaryDirectory()
88 self.addCleanup(directory.cleanup)
89 root = Path(directory.name)
90 (root / 'fixture').mkdir()
91 artifact = root / 'worker.exe'
92 artifact.write_bytes(b'compiled worker fixture')
93 environment = patch.dict(computer.os.environ, {'SNOWGLOBE_COMPUTER_WINDOWS_WORKER': str(artifact)})
94 environment.start()
95 self.addCleanup(environment.stop)
96 patcher = patch.object(computer.vms, 'DISKS', root)
97 patcher.start()
98 self.addCleanup(patcher.stop)
99
100 def test_large_windows_transfer_keeps_agent_writes_within_verified_size(self):
101 data = b'\0binary\xff' * (2**17 + 1)
102 written = []
103 def qga(name, operation, **arguments):
104 if operation == 'guest-file-open':
105 return 1
106 if operation == 'guest-file-write':
107 chunk = base64.b64decode(arguments['buf-b64'])
108 self.assertLessEqual(len(chunk), 65536)
109 written.append(chunk)
110 return {'count': len(chunk)}
111 self.assertEqual(operation, 'guest-file-close')
112 with patch.object(computer, 'qga', qga):
113 computer.write_file('fixture', 'owned-file', data)
114 self.assertEqual(b''.join(written), data)
115
116 def test_agent_rejections_report_causes_without_echoing_guest_diagnostics(self):
117 private = 'private-guest-diagnostic'
118 cases = [
119 ('guest-exec', 'GenericError', 'Failed to execute child process (Invalid argument)', 'program could not start'),
120 ('guest-file-open', 'GenericError', 'The system cannot find the file specified.', 'file was not found'),
121 ('guest-file-open', 'GenericError', 'No such file or directory', 'file was not found'),
122 ('guest-file-open', 'GenericError', 'Permission denied', 'guest denied access'),
123 ('guest-file-write', 'GenericError', 'No space left on device', 'disk is full'),
124 ('guest-info', 'CommandNotFound', 'Unknown command', 'does not support'),
125 ('guest-info', 'GenericError', 'Unknown failure', 'rejected the operation'),
126 ]
127 for operation, kind, diagnostic, expected in cases:
128 for structured in [False, True]:
129 reply = Mock(stdout=json.dumps({'error': {'class': kind, 'desc': private + ': ' + diagnostic}}) if structured else '',
130 stderr='' if structured else 'error: guest agent command failed: ' + private + ': ' + kind + ': ' + diagnostic)
131 with self.subTest(operation=operation, diagnostic=diagnostic, structured=structured), \
132 patch.object(computer.subprocess, 'run', return_value=reply), \
133 self.assertRaisesRegex(ValueError, expected) as failure:
134 computer.qga('fixture', operation)
135 self.assertNotIn(private, str(failure.exception))
136
137 def test_agent_timeouts_are_uncertain_and_do_not_repeat_the_command(self):
138 replies = [Mock(stdout='', stderr="error: guest agent command timed out: guest agent didn't respond to command within '10' seconds"),
139 Mock(stdout='', stderr='error: Timed out during operation: guest agent command'),
140 computer.subprocess.TimeoutExpired(['virsh'], 15)]
141 for reply in replies:
142 with self.subTest(reply=type(reply)), patch.object(computer.subprocess, 'run') as run:
143 if isinstance(reply, Exception):
144 run.side_effect = reply
145 else:
146 run.return_value = reply
147 with self.assertRaisesRegex(TimeoutError, 'Inspect the operation status'):
148 computer.qga('fixture', 'guest-exec', path='owned-program')
149 run.assert_called_once()
150
151 def test_agent_success_is_not_overridden_by_stderr(self):
152 reply = Mock(stdout='{"return":{"pid":123}}', stderr='guest agent command failed: stale warning')
153 with patch.object(computer.subprocess, 'run', return_value=reply):
154 self.assertEqual(computer.qga('fixture', 'guest-exec'), {'pid': 123})
155
156 def test_windows_powershell_reads_redirected_output_without_console_capture(self):
157 with patch.object(computer, 'write_file'), patch.object(computer, 'guest_exec', return_value=b''), patch.object(computer, 'read_file', return_value=b'file output'):
158 self.assertEqual(computer.powershell('fixture', "Write-Output 'file output'"), b'file output')
159
160 def test_windows_powershell_rejects_failed_execution_before_reading_output(self):
161 with patch.object(computer, 'write_file'), patch.object(computer, 'guest_exec', side_effect=ValueError('Failed')), patch.object(computer, 'read_file') as read:
162 with self.assertRaises(ValueError):
163 computer.powershell('fixture', 'exit 7')
164 read.assert_not_called()
165
166 def test_reused_guest_pid_waits_for_current_process(self):
167 replies = [{'pid': 2596}, {'exited': True, 'exitcode': 1}, {'exited': False},
168 {'exited': True, 'exitcode': 0, 'out-data': base64.b64encode(b'current').decode()},
169 ProcessLookupError()]
170 with patch.object(computer, 'qga', side_effect=replies), patch.object(computer.time, 'sleep'):
171 self.assertEqual(computer.guest_exec('fixture', 'cmd.exe', []), b'current')
172
173 def test_fast_reused_guest_pid_returns_last_completed_record(self):
174 replies = [{'pid': 2596}, {'exited': True, 'exitcode': 0, 'out-data': 'b2xk'},
175 {'exited': True, 'exitcode': 0, 'out-data': 'bmV3'}, ProcessLookupError()]
176 with patch.object(computer, 'qga', side_effect=replies):
177 self.assertEqual(computer.guest_exec('fixture', 'cmd.exe', []), b'new')
178
179 def test_guest_completion_requires_explicit_end_of_records(self):
180 for error in [ValueError('Disconnected'), TimeoutError()]:
181 replies = [{'pid': 2596}, {'exited': True, 'exitcode': 0}, error]
182 with self.subTest(error=type(error)), patch.object(computer, 'qga', side_effect=replies):
183 with self.assertRaises(type(error)):
184 computer.guest_exec('fixture', 'cmd.exe', [])
185
186 def test_guest_pid_absence_is_distinct_from_transport_failure(self):
187 for message in ['PID 2596 does not exist', 'PID ld does not exist', 'PID lld does not exist', "Invalid parameter 'pid'", 'Guest agent is not responding']:
188 response = Mock(stdout='', stderr='error: ' + message + '\nerror: Failed to create virsh cache: Read-only file system\n')
189 with self.subTest(message=message), patch.object(computer.subprocess, 'run', return_value=response):
190 with self.assertRaises(ValueError if message == 'Guest agent is not responding' else ProcessLookupError):
191 computer.qga('fixture', 'guest-exec-status', pid=2596)
192
193 def test_windows_stops_guest_task_after_its_operation_budget_expires(self):
194 running = []
195 def execute(name, executable, arguments, **kwargs):
196 if arguments[0] == '--launch':
197 running.append(True)
198 computer.DEADLINE.set(time.monotonic() - 1)
199 return b''
200 if arguments[0] == '--cleanup':
201 self.assertGreater(computer.remaining(20), 0)
202 running.clear()
203 return b''
204 token = computer.DEADLINE.set(time.monotonic() + 30)
205 try:
206 with patch.object(computer, 'guest_exec', execute), patch.object(computer, 'write_file'), patch.object(computer, 'read_file', return_value=b'M'):
207 with self.assertRaises(TimeoutError):
208 computer.worker('fixture', 'windows', {'action': 'input', 'actions': [{'type': 'text', 'text': 'hello'}]})
209 self.assertEqual(running, [])
210 finally:
211 computer.DEADLINE.reset(token)
212
213 def test_cancelled_cold_install_cleans_staging_without_launching_a_request(self):
214 calls = []
215 with computer.vms.computer_lock('fixture', 'status') as active:
216 operation = computer.OPERATION.set(('fixture', active['id']))
217 deadline = computer.DEADLINE.set(time.monotonic() + 30)
218 def execute(name, executable, arguments, **kwargs):
219 computer.remaining(1)
220 calls.append(arguments)
221 if arguments[0] == '--install':
222 with computer.vms.computer_control(name) as control:
223 computer.vms.computer_cancel(control, active['id'])
224 computer.DEADLINE.set(time.monotonic() - 1)
225 return b''
226 try:
227 with patch.object(computer, 'read_file', side_effect=ValueError('Missing cache')), \
228 patch.object(computer, 'write_file'), patch.object(computer, 'guest_exec', execute), \
229 self.assertRaises(InterruptedError):
230 computer.worker('fixture', 'windows', {'action': 'status'})
231 self.assertTrue(any('rmdir' in arguments for arguments in calls))
232 self.assertFalse(any(arguments[0] == '--launch' for arguments in calls))
233 self.assertFalse((computer.vms.DISKS / 'fixture/computer-operation.json').exists())
234 finally:
235 computer.OPERATION.reset(operation)
236 computer.DEADLINE.reset(deadline)
237
238 def test_console_remains_available_without_guest_agent(self):
239 for platform in ['windows', 'linux']:
240 def virsh(action, *args, **kwargs):
241 return 'running' if action == 'domstate' else '<domain><metadata><vm xmlns="https://paperclover.net/snowglobe"><platform>'+platform+'</platform></vm></metadata></domain>'
242 with self.subTest(platform=platform), patch.object(computer.vms, 'virsh', virsh), patch.object(computer, 'qga', side_effect=ValueError('Disconnected')), patch.object(computer, 'worker') as work:
243 status = computer.handle('fixture', {'action': 'status'})
244 self.assertEqual(status['readiness'], 'agent_unavailable')
245 self.assertEqual(status['capabilities'], ['shot', 'input'])
246 self.assertEqual(status['text_encoding'], 'ascii')
247 actions = [{'type': 'text', 'text': 'recovery console'}]
248 with patch.object(computer, 'desktop', new_callable=AsyncMock, return_value={'completed': 1}) as desktop:
249 self.assertEqual(computer.handle('fixture', {'action': 'input', 'actions': actions, 'screenshot': False}), {'completed': 1})
250 desktop.assert_awaited_once_with('fixture', actions, platform, False, False)
251 work.assert_not_called()
252 self.assertFalse((computer.vms.DISKS / 'fixture' / 'computer-operation.json').exists())
253
254 def test_status_distinguishes_disabled_commands_from_a_starting_worker(self):
255 def virsh(action, *args, **kwargs):
256 return 'running' if action == 'domstate' else '<domain/>'
257 with patch.object(computer.vms, 'virsh', virsh), patch.object(computer, 'worker') as work:
258 with patch.object(computer, 'qga', return_value={'supported_commands':[{'name':'guest-exec','enabled':False}]}):
259 status = computer.handle('fixture', {'action':'status'})
260 self.assertEqual(status['readiness'], 'agent_unsupported')
261 work.assert_not_called()
262 with patch.object(computer, 'qga', return_value={'supported_commands':[{'name':'guest-exec','enabled':True}]}):
263 work.side_effect = ValueError('Sign in to the VM desktop before using this tool.')
264 status = computer.handle('fixture', {'action':'status'})
265 self.assertEqual(status['readiness'], 'worker_starting')
266 self.assertIn('Sign in', status['worker_error'])
267 work.side_effect = TimeoutError('The guest agent timed out.')
268 status = computer.handle('fixture', {'action':'status'})
269 self.assertEqual(status['readiness'], 'worker_starting')
270 self.assertIn('timed out', status['worker_error'])
271 work.side_effect = None
272 work.return_value = {'runtimes':['bash']}
273 status = computer.handle('fixture', {'action':'status'})
274 self.assertEqual(status['readiness'], 'ready')
275 self.assertEqual(status['runtimes'], ['bash'])
276
277 def test_legacy_mac_status_keeps_its_private_transport(self):
278 def virsh(action, *args, **kwargs):
279 return 'running' if action == 'domstate' else '<domain><metadata><vm xmlns="https://paperclover.net/snowglobe"><platform>macos</platform></vm></metadata></domain>'
280 with patch.object(computer.vms, 'virsh', virsh), patch.object(computer, 'qga', side_effect=AssertionError('This Mac has no QGA channel')), patch.object(computer, 'worker', return_value={'runtimes':['bash']}) as work:
281 self.assertEqual(computer.handle('fixture', {'action':'status'})['readiness'], 'ready')
282 work.assert_called_once_with('fixture', 'macos', {'action':'status'})
283
284 def test_windows_status_waits_for_personalization_before_using_template_session(self):
285 root = '<domain><metadata><vm xmlns="https://paperclover.net/snowglobe"><platform>windows</platform><setup>windows-v4</setup></vm></metadata></domain>'
286 with patch.object(computer.vms, 'virsh', side_effect=lambda action, *a, **k: 'running' if action == 'domstate' else root), \
287 patch.object(computer, 'qga', return_value={'supported_commands':[{'name':'guest-exec','enabled':True}]}), \
288 patch.object(computer.vms, 'windows_setup') as setup, patch.object(computer, 'worker') as work:
289 for phase in ['starting', 'personalizing', 'signingIn', 'failed']:
290 setup.return_value = phase
291 status = computer.handle('fixture', {'action':'status'})
292 self.assertEqual(status['readiness'], 'setup_failed' if phase == 'failed' else 'personalizing')
293 self.assertEqual(status['setup'], phase)
294 self.assertEqual(status['capabilities'], ['shot', 'input'])
295 work.assert_not_called()
296 setup.return_value = 'ready'
297 work.return_value = {'execution_context':{'user':'fixture'}}
298 status = computer.handle('fixture', {'action':'status'})
299 self.assertEqual(status['readiness'], 'ready')
300 self.assertEqual(status['execution_context']['user'], 'fixture')
301
302 def test_failed_cleanup_keeps_status_available_and_blocks_later_control(self):
303 for platform, identity in [('linux', 'snowglobe-computer-request-' + 'a' * 32), ('windows', 'a' * 32)]:
304 operation = {'platform': platform, 'id': identity}
305 computer.record_operation('fixture', operation)
306 with patch.object(computer, 'guest_exec', side_effect=TimeoutError()), patch.object(computer, 'powershell', side_effect=TimeoutError()), self.assertRaises(BlockingIOError):
307 computer.finish_operation('fixture', operation)
308 pending = computer.vms.DISKS / 'fixture' / 'computer-operation.json'
309 self.assertEqual(json.loads(pending.read_text()), operation)
310 def virsh(action, *args, **kwargs):
311 return 'running' if action == 'domstate' else '<domain/>'
312 with patch.object(computer.vms, 'virsh', virsh), patch.object(computer, 'worker') as work:
313 with patch.object(computer, 'finish_operation', side_effect=AssertionError('Status must not wait for cleanup')):
314 status = computer.handle('fixture', {'action': 'status'})
315 self.assertEqual(status['readiness'], 'recovery_required')
316 self.assertEqual(status['active_operation']['id'], 'operation:' + identity.removeprefix('snowglobe-computer-request-'))
317 with patch.object(computer, 'guest_exec', side_effect=TimeoutError()), patch.object(computer, 'powershell', side_effect=TimeoutError()), self.assertRaises(BlockingIOError):
318 computer.handle('fixture', {'action': 'exec', 'runtime': 'default', 'script': 'true', 'timeout_ms': 1000})
319 work.assert_not_called()
320 with patch.object(computer, 'guest_exec', return_value=b'{}'), patch.object(computer, 'powershell', return_value=b'{}'):
321 computer.handle('fixture', {'action': 'exec', 'runtime': 'default', 'script': 'true', 'timeout_ms': 1000})
322 work.assert_called_once()
323 self.assertFalse(pending.exists())
324
325 def test_busy_execution_keeps_status_and_screenshots_available(self):
326 def virsh(action, *args, **kwargs):
327 return 'running' if action == 'domstate' else '<domain/>'
328 with computer.vms.computer_lock('fixture', 'exec'), patch.object(computer.vms, 'virsh', virsh), patch.object(computer, 'worker') as work:
329 status = computer.handle('fixture', {'action': 'status'})
330 self.assertEqual(status['readiness'], 'busy')
331 self.assertEqual(status['active_operation']['action'], 'exec')
332 self.assertGreaterEqual(status['active_operation']['age_seconds'], 0)
333 with patch.object(computer, 'desktop', new_callable=AsyncMock, return_value={'png': 'fixture'}) as desktop:
334 self.assertEqual(computer.handle('fixture', {'action': 'shot'}), {'png': 'fixture'})
335 desktop.assert_awaited_once_with('fixture')
336 with self.assertRaisesRegex(BlockingIOError, 'computer_status'):
337 computer.handle('fixture', {'action': 'exec', 'script': 'true', 'runtime': 'default', 'timeout_ms': 1000})
338 work.assert_not_called()
339
340 def test_stopped_vm_reports_status_without_starting_a_guest_worker(self):
341 def virsh(action, *args, **kwargs):
342 return 'shut off' if action == 'domstate' else '<domain/>'
343 with patch.object(computer.vms, 'virsh', virsh), patch.object(computer, 'worker') as work:
344 status = computer.handle('fixture', {'action': 'status'})
345 self.assertEqual(status['power_state'], 'shut off')
346 self.assertEqual(status['readiness'], 'stopped')
347 self.assertEqual(status['capabilities'], [])
348 work.assert_not_called()
349
350 def test_control_snapshot_cannot_turn_into_slow_guest_setup_or_cleanup(self):
351 with computer.vms.computer_lock('fixture', 'exec'):
352 active = computer.vms.computer_activity('fixture')
353 operation = {'platform': 'windows', 'id': active['id'].removeprefix('operation:')}
354 computer.record_operation('fixture', operation)
355 def virsh(action, *args, **kwargs):
356 return 'running' if action == 'domstate' else '<domain/>'
357 with patch.object(computer.vms, 'virsh', virsh), patch.object(computer, 'worker') as worker, \
358 patch.object(computer, 'finish_operation') as cleanup:
359 status = computer.handle('fixture', {'action': 'status'}, active_snapshot=active)
360 self.assertEqual(status['readiness'], 'busy')
361 self.assertEqual(status['active_operation'], active)
362 stopped = computer.handle('fixture', {'action': 'process', 'id': active['id'], 'operation': 'stop'}, active_snapshot=active)
363 self.assertIsNone(stopped['running'])
364 self.assertTrue(stopped['recovery_required'])
365 worker.assert_not_called()
366 cleanup.assert_not_called()
367
368 def test_active_operation_can_be_inspected_and_cancelled_during_execution(self):
369 started = threading.Event()
370 def virsh(action, *args, **kwargs):
371 return 'running' if action == 'domstate' else '<domain/>'
372 def work(*args):
373 started.set()
374 while True:
375 computer.remaining(1)
376 time.sleep(.005)
377 with patch.object(computer.vms, 'virsh', virsh), patch.object(computer, 'worker', work), ThreadPoolExecutor(max_workers=1) as pool:
378 call = pool.submit(computer.handle, 'fixture', {'action': 'exec', 'script': 'long operation', 'runtime': 'default', 'timeout_ms': 1000})
379 self.assertTrue(started.wait(2))
380 status = computer.handle('fixture', {'action': 'status'})
381 identity = status['active_operation']['id']
382 inspect = {'action': 'process', 'id': identity, 'operation': 'status'}
383 self.assertTrue(computer.handle('fixture', inspect)['running'])
384 stopped = computer.handle('fixture', {**inspect, 'operation': 'stop'})
385 self.assertTrue(stopped['stopping'])
386 with self.assertRaises(InterruptedError):
387 call.result(timeout=2)
388 self.assertFalse(computer.handle('fixture', inspect)['running'])
389
390 def test_stale_operation_handle_cannot_cancel_a_later_call(self):
391 with computer.vms.computer_lock('fixture', 'exec') as previous:
392 identity = previous['id']
393 with computer.vms.computer_lock('fixture', 'exec') as active:
394 token = computer.OPERATION.set(('fixture', active['id']))
395 try:
396 result = computer.handle('fixture', {'action': 'process', 'id': identity, 'operation': 'stop'})
397 self.assertFalse(result['running'])
398 self.assertGreater(computer.remaining(1), 0)
399 finally:
400 computer.OPERATION.reset(token)
401
402 def test_cancelled_operation_cannot_start_another_guest_command(self):
403 with computer.vms.computer_lock('fixture', 'exec') as active:
404 token = computer.OPERATION.set(('fixture', active['id']))
405 try:
406 computer.handle('fixture', {'action': 'process', 'id': active['id'], 'operation': 'stop'})
407 with patch.object(computer.subprocess, 'run') as run, self.assertRaises(InterruptedError):
408 computer.qga('fixture', 'guest-ping')
409 run.assert_not_called()
410 finally:
411 computer.OPERATION.reset(token)
412
413 def test_pending_operation_remains_uncertain_until_cleanup_is_acknowledged(self):
414 operation = {'platform': 'windows', 'id': 'a' * 32}
415 computer.record_operation('fixture', operation)
416 request = {'action': 'process', 'id': 'operation:' + operation['id'], 'operation': 'status'}
417 status = computer.handle('fixture', request)
418 self.assertIsNone(status['running'])
419 self.assertTrue(status['recovery_required'])
420 with patch.object(computer, 'powershell', side_effect=TimeoutError()), self.assertRaises(BlockingIOError):
421 computer.handle('fixture', {**request, 'operation': 'stop'})
422 self.assertTrue(computer.handle('fixture', request)['recovery_required'])
423 with patch.object(computer, 'powershell', return_value=b''):
424 result = computer.handle('fixture', {**request, 'operation': 'stop'})
425 self.assertFalse(result['running'])
426 self.assertTrue(result['cancelled'])
427
428 def test_cleanup_does_not_remove_a_newer_pending_operation(self):
429 previous = {'platform': 'windows', 'id': 'a' * 32}
430 current = {'platform': 'windows', 'id': 'b' * 32}
431 computer.record_operation('fixture', previous)
432 lock = computer.vms.computer_lock
433 @contextmanager
434 def interleaved(name, action, identity=None):
435 computer.record_operation(name, current)
436 with lock(name, action, identity) as active:
437 yield active
438 with patch.object(computer.vms, 'computer_lock', interleaved), patch.object(computer, 'powershell') as stop:
439 result = computer.handle('fixture', {'action': 'process', 'id': 'operation:' + previous['id'], 'operation': 'stop'})
440 self.assertFalse(result['running'])
441 stop.assert_not_called()
442 self.assertEqual(json.loads((computer.vms.DISKS / 'fixture/computer-operation.json').read_text()), current)
443
444 def test_cleanup_keeps_transport_available_after_cancellation(self):
445 with computer.vms.computer_lock('fixture', 'exec') as active:
446 token = computer.OPERATION.set(('fixture', active['id']))
447 try:
448 computer.handle('fixture', {'action': 'process', 'id': active['id'], 'operation': 'stop'})
449 operation = {'platform': 'windows', 'id': active['id'].removeprefix('operation:')}
450 computer.record_operation('fixture', operation)
451 def cleanup(*args, **kwargs):
452 self.assertIsNone(computer.OPERATION.get())
453 self.assertGreater(computer.remaining(10), 0)
454 return b''
455 with patch.object(computer, 'powershell', cleanup):
456 computer.finish_operation('fixture', operation)
457 with self.assertRaises(InterruptedError):
458 computer.remaining(1)
459 finally:
460 computer.OPERATION.reset(token)
461
462 def test_guest_files_close_after_cancellation_and_deadline_expiry(self):
463 for write in (False, True):
464 operation = computer.OPERATION.set(('fixture', 'operation:' + 'a' * 32))
465 deadline = computer.DEADLINE.set(time.monotonic() + 10)
466 calls = []
467 def qga(name, action, **arguments):
468 calls.append(action)
469 if action == 'guest-file-open':
470 return 17
471 if action == 'guest-file-close':
472 self.assertEqual(arguments['handle'], 17)
473 self.assertIsNone(computer.OPERATION.get())
474 self.assertGreater(computer.remaining(10), 0)
475 return {}
476 computer.DEADLINE.set(time.monotonic() - 1)
477 raise InterruptedError('Fixture cancellation')
478 try:
479 with self.subTest(write=write), patch.object(computer, 'qga', qga), self.assertRaises(InterruptedError):
480 if write:
481 computer.write_file('fixture', 'owned-file', b'payload')
482 else:
483 computer.read_file('fixture', 'owned-file')
484 self.assertEqual(calls[-1], 'guest-file-close')
485 finally:
486 computer.DEADLINE.reset(deadline)
487 computer.OPERATION.reset(operation)
488
489 def test_spawn_returns_its_handle_when_cleanup_must_wait_for_reconnection(self):
490 result = {'id': '123:456', 'running': True}
491 def execute(name, executable, arguments, **kwargs):
492 if arguments[0] == '--release':
493 raise TimeoutError()
494 return b''
495 with patch.object(computer, 'guest_exec', execute), patch.object(computer, 'write_file'), patch.object(computer, 'read_file', return_value=json.dumps(result).encode()):
496 self.assertEqual(computer.worker('fixture', 'windows', {'action': 'spawn'}), result)
497 pending = computer.vms.DISKS / 'fixture' / 'computer-operation.json'
498 operation = json.loads(pending.read_text())
499 self.assertTrue(operation['keep_spawn'])
500 with patch.object(computer, 'guest_exec', return_value=b'') as execute:
501 computer.finish_operation('fixture', operation)
502 self.assertEqual(execute.call_args.args[2][0], '--release')
503 self.assertFalse(pending.exists())
504
505 def test_spawn_is_stopped_if_its_success_cannot_be_persisted(self):
506 record = computer.record_operation
507 def persist(name, operation):
508 if operation.get('keep_spawn'):
509 raise OSError('Disk full')
510 record(name, operation)
511 with patch.object(computer, 'record_operation', persist), patch.object(computer, 'powershell', return_value=b'[true,true]'), patch.object(computer, 'guest_exec', return_value=b'') as execute, patch.object(computer, 'write_file'), patch.object(computer, 'read_file', return_value=b'{"id":"123:456"}'):
512 with self.assertRaises(OSError):
513 computer.worker('fixture', 'windows', {'action': 'spawn'})
514 self.assertEqual(execute.call_args.args[2][0], '--cleanup')
515 self.assertFalse((computer.vms.DISKS / 'fixture' / 'computer-operation.json').exists())
516
517 def test_cleanup_uses_the_worker_that_started_the_operation(self):
518 operation = {'platform': 'windows', 'id': 'a' * 32,
519 'worker': 'C:\\ProgramData\\SnowglobeComputer\\worker-previous.exe'}
520 computer.record_operation('fixture', operation)
521 with patch.object(computer, 'guest_exec', return_value=b'') as execute, patch.object(computer, 'powershell') as legacy:
522 computer.finish_operation('fixture', operation)
523 self.assertEqual(execute.call_args.args[1], operation['worker'])
524 self.assertEqual(execute.call_args.args[2][0], '--cleanup')
525 legacy.assert_not_called()
526
527 def test_warm_windows_commands_do_not_start_powershell_for_coordination(self):
528 def read(name, path, **kwargs):
529 return b'{"exit":0,"stdout":"ready"}' if path.endswith('response.json') else b'M'
530 with patch.object(computer, 'read_file', read), patch.object(computer, 'write_file'), patch.object(computer, 'guest_exec', return_value=b'') as execute, patch.object(computer, 'powershell', side_effect=AssertionError('Warm coordination must stay native')):
531 result = computer.worker('fixture', 'windows', {'action': 'exec', 'runtime': 'cmd', 'script': 'echo ready'})
532 self.assertEqual(result['stdout'], 'ready')
533 self.assertEqual([call.args[2][0] for call in execute.call_args_list], ['--prepare', '--launch', '--cleanup'])
534
535 def test_failed_native_cleanup_recovers_through_the_pinned_worker(self):
536 operation = {'platform': 'windows', 'id': 'a' * 32,
537 'worker': 'C:\\ProgramData\\SnowglobeComputer\\worker-previous.exe'}
538 computer.record_operation('fixture', operation)
539 with patch.object(computer, 'guest_exec', side_effect=ValueError('Worker exited 1')), patch.object(computer, 'powershell', return_value=b'') as fallback:
540 computer.finish_operation('fixture', operation)
541 script = fallback.call_args.args[1]
542 self.assertIn("LoadFrom('" + operation['worker'] + "')", script)
543 self.assertIn('[ComputerProcess]::Stop', script)
544 self.assertFalse((computer.vms.DISKS / 'fixture' / 'computer-operation.json').exists())
545
546 def test_failed_native_and_fallback_cleanup_preserves_recovery(self):
547 operation = {'platform': 'windows', 'id': 'a' * 32, 'worker': 'previous.exe'}
548 computer.record_operation('fixture', operation)
549 with patch.object(computer, 'guest_exec', side_effect=ValueError('Worker exited 1')), patch.object(computer, 'powershell', side_effect=ValueError('Disconnected')):
550 with self.assertRaises(BlockingIOError):
551 computer.finish_operation('fixture', operation)
552 self.assertEqual(json.loads((computer.vms.DISKS / 'fixture' / 'computer-operation.json').read_text()), operation)
553
554 def test_spawn_cleanup_fallback_keeps_the_process_running(self):
555 operation = {'platform': 'windows', 'id': 'a' * 32, 'worker': 'previous.exe', 'keep_spawn': True}
556 computer.record_operation('fixture', operation)
557 with patch.object(computer, 'guest_exec', side_effect=ValueError('Worker exited 1')), patch.object(computer, 'powershell', return_value=b'') as fallback:
558 computer.finish_operation('fixture', operation)
559 self.assertNotIn('[ComputerProcess]::Stop', fallback.call_args.args[1])
560 self.assertNotIn('$t.Stop(0)', fallback.call_args.args[1])
561
562 def test_linux_prepare_does_not_claim_a_request_before_systemd_is_accessible(self):
563 source = (Path(__file__).resolve().parent.parent / 'guest/computer-linux.py').read_text()
564 request = io.StringIO(json.dumps({'_prepare':'snowglobe-computer-request-' + 'a' * 32}))
565 denied = computer.subprocess.CalledProcessError(1, 'systemctl', stderr=b'Access denied')
566 with patch.object(computer.sys, 'stdin', request), patch.object(computer.subprocess, 'run', side_effect=denied), \
567 patch.object(computer.os, 'open') as claim:
568 with self.assertRaises(computer.subprocess.CalledProcessError):
569 exec(compile(source, '<guest computer>', 'exec'), {'__name__':'__main__'})
570 claim.assert_not_called()
571
572 def test_linux_stops_the_exact_request_unit_with_a_fresh_cleanup_budget(self):
573 units = []
574 def execute(name, executable, arguments, data=None, **kwargs):
575 request = json.loads(data)
576 if '_prepare' in request:
577 return b'{"prepared":true}'
578 if '_unit' in request:
579 units.append(request['_unit'])
580 computer.DEADLINE.set(time.monotonic() - 1)
581 raise TimeoutError()
582 self.assertEqual(request, {'_stop': units.pop()})
583 self.assertGreater(computer.remaining(20), 0)
584 return b'{"stopped":true}'
585 token = computer.DEADLINE.set(time.monotonic() + 30)
586 try:
587 with patch.object(computer, 'guest_exec', execute), self.assertRaises(TimeoutError):
588 computer.worker('fixture', 'linux', {'action': 'status'})
589 self.assertEqual(units, [])
590 finally:
591 computer.DEADLINE.reset(token)
592
593
594class DesktopBounds(unittest.IsolatedAsyncioTestCase):
595 async def asyncSetUp(self):
596 self.writer = Mock(drain=AsyncMock(), wait_closed=AsyncMock())
597 self.screen = Mock(width=3840, height=2160, start=AsyncMock())
598 async def read(ready, clear):
599 ready()
600 await asyncio.Future()
601 self.screen.read = read
602 self.patches = [patch.object(computer.vms, 'console_target', return_value={'port': 5900}),
603 patch.object(computer.asyncio, 'open_connection', AsyncMock(return_value=(None, self.writer))),
604 patch.object(computer.rfb, 'Desktop', return_value=self.screen)]
605 for item in self.patches:
606 item.start()
607 self.addCleanup(item.stop)
608
609 async def test_unsupported_fallback_text_cannot_partially_execute_a_batch(self):
610 for text in ['雪', '\x1b']:
611 actions = [{'type': 'click', 'x': 1, 'y': 1}, {'type': 'text', 'text': text}]
612 with self.subTest(text=text), self.assertRaisesRegex(ValueError, 'ASCII text only'):
613 await computer.desktop('fixture', actions, 'linux', False, False)
614 self.screen.input.assert_not_called()
615
616 async def test_native_text_still_accepts_unicode(self):
617 action = {'type': 'text', 'text': '雪🦊'}
618 with patch.object(computer, 'worker') as worker:
619 result = await computer.desktop('fixture', [action], 'windows', False)
620 self.assertEqual(result, {'completed': 1})
621 worker.assert_called_once_with('fixture', 'windows', {'action': 'input', 'actions': [action], 'screenshot': False})
622
623 async def test_input_without_screenshot_never_encodes_or_requests_another_frame(self):
624 with patch.object(computer.rfb, 'png', side_effect=AssertionError('No capture requested')):
625 result = await computer.desktop('fixture', [{'type': 'click', 'x': 1, 'y': 1}], 'linux', False)
626 self.assertEqual(result, {'completed': 1})
627 self.screen.request.assert_not_called()
628
629 async def test_cancelled_batch_keeps_capture_slot_until_text_worker_returns(self):
630 finished = []
631 def type_text(*args):
632 self.assertLess(computer.remaining(999), 181)
633 time.sleep(.06)
634 finished.append(True)
635 wait_for = asyncio.wait_for
636 with patch.object(computer, 'worker', type_text), patch.object(computer.asyncio, 'wait_for', lambda task, seconds: wait_for(task, .02 if seconds > 170 else seconds)):
637 with self.assertRaises(asyncio.TimeoutError):
638 await computer.desktop('fixture', [{'type': 'text', 'text': 'one'}] * 100, 'linux', False)
639 self.assertEqual(finished, [True])
640 self.assertTrue(computer.SCREEN_SLOT.acquire(blocking=False))
641 computer.SCREEN_SLOT.release()
642
643 async def test_expired_batch_never_starts_another_guest_command(self):
644 token = computer.DEADLINE.set(time.monotonic() - 1)
645 try:
646 with patch.object(computer.subprocess, 'run') as run, self.assertRaises(TimeoutError):
647 computer.qga('fixture', 'guest-ping')
648 run.assert_not_called()
649 finally:
650 computer.DEADLINE.reset(token)
651
652
653class TestVmExpiry(unittest.TestCase):
654 def test_creation_catalog_excludes_unprepared_windows_and_oversized_disks(self):
655 prepared = {'id': 'ready', 'platform': 'windows', 'setup': 'windows-v4', 'capacity': 40 * 2**30, 'os': 'Windows', 'arch': 'x86_64'}
656 linux = {**prepared, 'id': 'linux', 'platform': 'linux'}
657 broken = {**prepared, 'id': 'unprepared', 'setup': None}
658 oversized = {**linux, 'id': 'oversized', 'capacity': 81 * 2**30}
659 with patch.object(leases.vms, 'library', return_value={'presets': [prepared, linux, broken, oversized]}), patch.object(leases.vms, 'create') as create:
660 catalog = leases.manage({'action': 'presets', 'owner': 'fixture'})
661 self.assertEqual([p['id'] for p in catalog['presets']], ['ready', 'linux'])
662 for preset in ['unprepared', 'oversized']:
663 with self.assertRaisesRegex(ValueError, 'prepared preset'):
664 leases.create({'action': 'create', 'owner': 'fixture', 'preset': preset, 'name': 'test', 'vcpus': 2, 'memory': 4})
665 create.assert_not_called()
666
667 def test_creation_checks_ssd_capacity_instead_of_metadata_storage(self):
668 with tempfile.TemporaryDirectory() as directory:
669 root = Path(directory)
670 ssd = root / 'ssd'
671 ssd.mkdir()
672 with patch.object(leases.vms, 'DISKS', root), patch.object(leases.vms, 'AI_DISKS', ssd), \
673 patch.object(Path, 'is_mount', return_value=True), patch.object(leases.vms, 'virsh', return_value=''), \
674 patch.object(leases, 'presets', return_value=[{'id':'fixture','capacity':64*2**30}]), \
675 patch.object(leases.shutil, 'disk_usage', return_value=Mock(free=2**30)) as usage, \
676 patch.object(leases.vms, 'create') as create:
677 with self.assertRaisesRegex(ValueError, 'free space'):
678 leases.create({'action':'create','owner':'fixture','preset':'fixture','name':'fixture','vcpus':2,'memory':4})
679 usage.assert_called_once_with(ssd)
680 create.assert_not_called()
681
682 def test_expiry_preserves_personal_vms_and_skips_active_calls(self):
683 with tempfile.TemporaryDirectory() as directory, patch.object(leases.vms, 'DISKS', Path(directory)), patch.object(leases.time, 'time', return_value=10000):
684 for name, expiry in [('personal', 1), ('ai-prefix-only', None), ('ai-expired', 9999), ('ai-live', 10001), ('ai-busy', 9999)]:
685 root=Path(directory)/name;root.mkdir()
686 if expiry is not None:(root/'ai.json').write_text(json.dumps({'expiresAt':expiry}))
687 with leases.vms.computer_lock('ai-busy'), patch.object(leases.vms, 'remove') as remove:
688 leases.reap()
689 remove.assert_called_once_with({'name':'ai-expired','disks':True},missing_ok=True)
690
691 def test_activity_renews_only_test_vms_and_cannot_resurrect_expired_vms(self):
692 with tempfile.TemporaryDirectory() as directory, patch.object(leases.vms, 'DISKS', Path(directory)), patch.object(leases.time, 'time', return_value=10000):
693 root=Path(directory)/'ai-fixture';root.mkdir()
694 path=root/'ai.json';path.write_text(json.dumps({'expiresAt':10001}))
695 self.assertEqual(leases.lease('ai-fixture'),{'expiresAt':10001})
696 self.assertEqual(leases.lease('ai-fixture',3600),{'expiresAt':13600})
697 self.assertEqual(leases.lease('ai-fixture',100),{'expiresAt':13600})
698 self.assertIsNone(leases.lease('personal',3600))
699 path.write_text(json.dumps({'expiresAt':9999}))
700 with self.assertRaisesRegex(ValueError,'expired'):leases.lease('ai-fixture',86400)
701
702 def test_foreign_owner_and_personal_vm_cannot_be_deleted_or_extended(self):
703 with tempfile.TemporaryDirectory() as directory, patch.object(leases.vms, 'DISKS', Path(directory)), patch.object(leases.vms, 'remove') as remove:
704 for name in ['ai-fixture','personal']:(Path(directory)/name).mkdir()
705 (Path(directory)/'ai-fixture/ai.json').write_text(json.dumps({'expiresAt':20000}))
706 with patch.object(leases.vms,'virsh',return_value='<domain><metadata xmlns:s="https://paperclover.net/snowglobe"><s:vm><s:owner>owner</s:owner></s:vm></metadata></domain>'):
707 for name, owner in [('ai-fixture','other'),('personal','owner')]:
708 for action in ['delete','extend']:
709 with self.assertRaises(ValueError):leases.manage({'action':action,'name':name,'owner':owner,'seconds':3600})
710 remove.assert_not_called()
711
712
713if __name__=='__main__':unittest.main()