diff --git a/dashboard/src/computer-tools.json b/dashboard/src/computer-tools.json index 94dbcb9775bb1ec35d5db93fb25e5f76d6045564..88fd9ef9dfcd54d774144355c1bda228aa419b39 100644 --- a/dashboard/src/computer-tools.json +++ b/dashboard/src/computer-tools.json @@ -64,6 +64,7 @@ }, "actions": { "type": "array", + "description": "Waits and drags, including default drag durations, may total at most 15 seconds.", "items": { "oneOf": [ { diff --git a/tools/computer-test.py b/tools/computer-test.py index fa812b14d74971cfec2b100306584dea1bd683e8..8feb0a283c6e6e4628d2fefe1deb839b33464df9 100644 --- a/tools/computer-test.py +++ b/tools/computer-test.py @@ -20,6 +20,19 @@ leases=import_module('vm-ai') class Protocol(unittest.TestCase): + def test_drag_defaults_count_toward_the_batch_budget(self): + drag = {'type': 'drag', 'x': 0, 'y': 0, 'to_x': 1, 'to_y': 1} + computer.validate({'action': 'input', 'actions': [drag] * 37 + [{'type': 'wait', 'duration_ms': 200}]}) + with self.assertRaisesRegex(ValueError, '15 seconds'): + computer.validate({'action': 'input', 'actions': [drag] * 38}) + + def test_action_limit_errors_identify_the_action_field_and_limit(self): + for action, field, limit in [({'type': 'wait', 'duration_ms': 5001}, 'duration_ms', 5000), + ({'type': 'click', 'x': 0, 'y': 0, 'count': 4}, 'count', 3), + ({'type': 'scroll', 'x': 0, 'y': 0, 'direction': 'up', 'amount': 101}, 'amount', 100)]: + with self.subTest(field=field), self.assertRaisesRegex(ValueError, f'Action 2: {field}.*1.*{limit}'): + computer.validate({'action': 'input', 'actions': [{'type': 'text', 'text': 'not executed'}, action]}) + def test_rejects_malformed_actions_before_input(self): for action in [None,[],{'type':[]},{'type':'click','x':True,'y':0}, {'type':'drag','x':0,'y':0,'to_x':0,'to_y':-1}, diff --git a/tools/vm-computer.py b/tools/vm-computer.py index 9de713d0f7ede8047088f2f96e9e46ca601720ed..afcf4e3ff4c01925370b8eba9dac745f2d6d11e8 100644 --- a/tools/vm-computer.py +++ b/tools/vm-computer.py @@ -23,6 +23,7 @@ SCREEN_SLOT = threading.BoundedSemaphore(1) DEADLINE = ContextVar('computer_deadline', default=float('inf')) OPERATION = ContextVar('computer_operation', default=None) FILE_CHUNK = json.loads((Path(__file__).resolve().parent.parent / 'dashboard/src/host-protocol.json').read_text())['file_chunk'] +DRAG_DURATION_MS = 400 def remaining(limit): @@ -213,7 +214,7 @@ async def capture(name, actions, platform, screenshot=True, native_text=True): for step in range(1, 21): screen.input({'type': 'pointer', 'x': round(x+(end_x-x)*step/20), 'y': round(y+(end_y-y)*step/20), 'buttons': button}) - await asyncio.sleep(action.get('duration_ms', 400)/20000) + await asyncio.sleep(action.get('duration_ms', DRAG_DURATION_MS)/20000) screen.input({'type': 'release'}) elif kind == 'scroll': wheel = {'up': 8, 'down': 16, 'left': 32, 'right': 64}[action['direction']] @@ -349,7 +350,7 @@ def validate(request): actions = request.get('actions') if not isinstance(actions,list) or not 1 <= len(actions) <= 100: raise ValueError('Send between 1 and 100 desktop actions.') - for item in actions: + for index, item in enumerate(actions, 1): if not isinstance(item,dict): raise ValueError('Send a desktop action object.') kind = item.get('type') @@ -365,7 +366,7 @@ def validate(request): raise ValueError('Choose the left, right, or middle mouse button.') for key,maximum in [('count',3),('amount',100),('duration_ms',5000)]: if key in item and (type(item[key]) is not int or not 1 <= item[key] <= maximum): - raise ValueError('The desktop action exceeds its limit.') + raise ValueError(f'Action {index}: {key} must be an integer from 1 to {maximum}.') if kind == 'scroll' and item['direction'] not in ('up','down','left','right'): raise ValueError('Choose a scroll direction.') if kind == 'key': @@ -374,8 +375,8 @@ def validate(request): for key in item['keys']: keycode(key) if kind == 'text' and (not isinstance(item['text'],str) or len(item['text'].encode()) > 16384): raise ValueError('Type at most 16 KiB of text at a time.') - if sum(a.get('duration_ms',0) for a in actions) > 15000: - raise ValueError('Keep total waits and drags under 15 seconds.') + if sum(a.get('duration_ms', DRAG_DURATION_MS if a['type'] == 'drag' else 0) for a in actions) > 15000: + raise ValueError('Waits and drags must total at most 15 seconds.') def record_operation(name, operation):