From a6e4327e47cc55ebb24fa311e6f9408bc61ec773 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Tue, 17 Nov 2026 05:15:31 -0800 Subject: [PATCH] Count default drag durations in desktop action budgets Apply the same drag duration to execution and batch admission. Report the action number, field and limit for invalid numeric arguments, and expose the combined wait and drag budget in the MCP schema. Assisted-by: gpt-6.1-sol --- dashboard/src/computer-tools.json | 1 + tools/computer-test.py | 13 +++++++++++++ tools/vm-computer.py | 11 ++++++----- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/dashboard/src/computer-tools.json b/dashboard/src/computer-tools.json index 94dbcb9775bb1ec35d5db93fb25e5f76d6045564..88fd9ef9dfcd54d774144355c1bda228aa419b39 100644 --- a/dashboard/src/computer-tools.json +++ b/dashboard/src/computer-tools.json @@ -64,6 +64,7 @@ }, "actions": { "type": "array", + "description": "Waits and drags, including default drag durations, may total at most 15 seconds.", "items": { "oneOf": [ { diff --git a/tools/computer-test.py b/tools/computer-test.py index fa812b14d74971cfec2b100306584dea1bd683e8..8feb0a283c6e6e4628d2fefe1deb839b33464df9 100644 --- a/tools/computer-test.py +++ b/tools/computer-test.py @@ -20,6 +20,19 @@ leases=import_module('vm-ai') class Protocol(unittest.TestCase): + def test_drag_defaults_count_toward_the_batch_budget(self): + drag = {'type': 'drag', 'x': 0, 'y': 0, 'to_x': 1, 'to_y': 1} + computer.validate({'action': 'input', 'actions': [drag] * 37 + [{'type': 'wait', 'duration_ms': 200}]}) + with self.assertRaisesRegex(ValueError, '15 seconds'): + computer.validate({'action': 'input', 'actions': [drag] * 38}) + + def test_action_limit_errors_identify_the_action_field_and_limit(self): + for action, field, limit in [({'type': 'wait', 'duration_ms': 5001}, 'duration_ms', 5000), + ({'type': 'click', 'x': 0, 'y': 0, 'count': 4}, 'count', 3), + ({'type': 'scroll', 'x': 0, 'y': 0, 'direction': 'up', 'amount': 101}, 'amount', 100)]: + with self.subTest(field=field), self.assertRaisesRegex(ValueError, f'Action 2: {field}.*1.*{limit}'): + computer.validate({'action': 'input', 'actions': [{'type': 'text', 'text': 'not executed'}, action]}) + def test_rejects_malformed_actions_before_input(self): for action in [None,[],{'type':[]},{'type':'click','x':True,'y':0}, {'type':'drag','x':0,'y':0,'to_x':0,'to_y':-1}, diff --git a/tools/vm-computer.py b/tools/vm-computer.py index 9de713d0f7ede8047088f2f96e9e46ca601720ed..afcf4e3ff4c01925370b8eba9dac745f2d6d11e8 100644 --- a/tools/vm-computer.py +++ b/tools/vm-computer.py @@ -23,6 +23,7 @@ SCREEN_SLOT = threading.BoundedSemaphore(1) DEADLINE = ContextVar('computer_deadline', default=float('inf')) OPERATION = ContextVar('computer_operation', default=None) FILE_CHUNK = json.loads((Path(__file__).resolve().parent.parent / 'dashboard/src/host-protocol.json').read_text())['file_chunk'] +DRAG_DURATION_MS = 400 def remaining(limit): @@ -213,7 +214,7 @@ async def capture(name, actions, platform, screenshot=True, native_text=True): for step in range(1, 21): screen.input({'type': 'pointer', 'x': round(x+(end_x-x)*step/20), 'y': round(y+(end_y-y)*step/20), 'buttons': button}) - await asyncio.sleep(action.get('duration_ms', 400)/20000) + await asyncio.sleep(action.get('duration_ms', DRAG_DURATION_MS)/20000) screen.input({'type': 'release'}) elif kind == 'scroll': wheel = {'up': 8, 'down': 16, 'left': 32, 'right': 64}[action['direction']] @@ -349,7 +350,7 @@ def validate(request): actions = request.get('actions') if not isinstance(actions,list) or not 1 <= len(actions) <= 100: raise ValueError('Send between 1 and 100 desktop actions.') - for item in actions: + for index, item in enumerate(actions, 1): if not isinstance(item,dict): raise ValueError('Send a desktop action object.') kind = item.get('type') @@ -365,7 +366,7 @@ def validate(request): raise ValueError('Choose the left, right, or middle mouse button.') for key,maximum in [('count',3),('amount',100),('duration_ms',5000)]: if key in item and (type(item[key]) is not int or not 1 <= item[key] <= maximum): - raise ValueError('The desktop action exceeds its limit.') + raise ValueError(f'Action {index}: {key} must be an integer from 1 to {maximum}.') if kind == 'scroll' and item['direction'] not in ('up','down','left','right'): raise ValueError('Choose a scroll direction.') if kind == 'key': @@ -374,8 +375,8 @@ def validate(request): for key in item['keys']: keycode(key) if kind == 'text' and (not isinstance(item['text'],str) or len(item['text'].encode()) > 16384): raise ValueError('Type at most 16 KiB of text at a time.') - if sum(a.get('duration_ms',0) for a in actions) > 15000: - raise ValueError('Keep total waits and drags under 15 seconds.') + if sum(a.get('duration_ms', DRAG_DURATION_MS if a['type'] == 'drag' else 0) for a in actions) > 15000: + raise ValueError('Waits and drags must total at most 15 seconds.') def record_operation(name, operation): -- 2.54.1