authoravatarclover caruso<git@paperclover.net>2026-11-17 05:15:31-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-22 00:44:02-08:00
loga6e4327e47cc55ebb24fa311e6f9408bc61ec773
tree154f69212602473e848faed3c5232181f0e36000
parent6f8ec57f945851d1989b77d1e949a5daa7588fdc
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Count default drag durations in desktop action budgets

Apply the same drag duration to execution and batch admission. Report the action number, field and limit for invalid numeric arguments, and expose the combined wait and drag budget in the MCP schema. Assisted-by: gpt-6.1-sol

3 files changed, 20 insertions(+), 5 deletions(-)

dashboard/src/computer-tools.json+1
...@@ -64,6 +64,7 @@...@@ -64,6 +64,7 @@
64 },64 },
65 "actions": {65 "actions": {
66 "type": "array",66 "type": "array",
67 "description": "Waits and drags, including default drag durations, may total at most 15 seconds.",
67 "items": {68 "items": {
68 "oneOf": [69 "oneOf": [
69 {70 {
tools/computer-test.py+13
...@@ -20,6 +20,19 @@ leases=import_module('vm-ai')...@@ -20,6 +20,19 @@ leases=import_module('vm-ai')
2020
2121
22class Protocol(unittest.TestCase):22class Protocol(unittest.TestCase):
23 def test_drag_defaults_count_toward_the_batch_budget(self):
24 drag = {'type': 'drag', 'x': 0, 'y': 0, 'to_x': 1, 'to_y': 1}
25 computer.validate({'action': 'input', 'actions': [drag] * 37 + [{'type': 'wait', 'duration_ms': 200}]})
26 with self.assertRaisesRegex(ValueError, '15 seconds'):
27 computer.validate({'action': 'input', 'actions': [drag] * 38})
28
29 def test_action_limit_errors_identify_the_action_field_and_limit(self):
30 for action, field, limit in [({'type': 'wait', 'duration_ms': 5001}, 'duration_ms', 5000),
31 ({'type': 'click', 'x': 0, 'y': 0, 'count': 4}, 'count', 3),
32 ({'type': 'scroll', 'x': 0, 'y': 0, 'direction': 'up', 'amount': 101}, 'amount', 100)]:
33 with self.subTest(field=field), self.assertRaisesRegex(ValueError, f'Action 2: {field}.*1.*{limit}'):
34 computer.validate({'action': 'input', 'actions': [{'type': 'text', 'text': 'not executed'}, action]})
35
23 def test_rejects_malformed_actions_before_input(self):36 def test_rejects_malformed_actions_before_input(self):
24 for action in [None,[],{'type':[]},{'type':'click','x':True,'y':0},37 for action in [None,[],{'type':[]},{'type':'click','x':True,'y':0},
25 {'type':'drag','x':0,'y':0,'to_x':0,'to_y':-1},38 {'type':'drag','x':0,'y':0,'to_x':0,'to_y':-1},
tools/vm-computer.py+6-5
...@@ -23,6 +23,7 @@ SCREEN_SLOT = threading.BoundedSemaphore(1)...@@ -23,6 +23,7 @@ SCREEN_SLOT = threading.BoundedSemaphore(1)
23DEADLINE = ContextVar('computer_deadline', default=float('inf'))23DEADLINE = ContextVar('computer_deadline', default=float('inf'))
24OPERATION = ContextVar('computer_operation', default=None)24OPERATION = ContextVar('computer_operation', default=None)
25FILE_CHUNK = json.loads((Path(__file__).resolve().parent.parent / 'dashboard/src/host-protocol.json').read_text())['file_chunk']25FILE_CHUNK = json.loads((Path(__file__).resolve().parent.parent / 'dashboard/src/host-protocol.json').read_text())['file_chunk']
26DRAG_DURATION_MS = 400
2627
2728
28def remaining(limit):29def remaining(limit):
...@@ -213,7 +214,7 @@ async def capture(name, actions, platform, screenshot=True, native_text=True):...@@ -213,7 +214,7 @@ async def capture(name, actions, platform, screenshot=True, native_text=True):
213 for step in range(1, 21):214 for step in range(1, 21):
214 screen.input({'type': 'pointer', 'x': round(x+(end_x-x)*step/20),215 screen.input({'type': 'pointer', 'x': round(x+(end_x-x)*step/20),
215 'y': round(y+(end_y-y)*step/20), 'buttons': button})216 'y': round(y+(end_y-y)*step/20), 'buttons': button})
216 await asyncio.sleep(action.get('duration_ms', 400)/20000)217 await asyncio.sleep(action.get('duration_ms', DRAG_DURATION_MS)/20000)
217 screen.input({'type': 'release'})218 screen.input({'type': 'release'})
218 elif kind == 'scroll':219 elif kind == 'scroll':
219 wheel = {'up': 8, 'down': 16, 'left': 32, 'right': 64}[action['direction']]220 wheel = {'up': 8, 'down': 16, 'left': 32, 'right': 64}[action['direction']]
...@@ -349,7 +350,7 @@ def validate(request):...@@ -349,7 +350,7 @@ def validate(request):
349 actions = request.get('actions')350 actions = request.get('actions')
350 if not isinstance(actions,list) or not 1 <= len(actions) <= 100:351 if not isinstance(actions,list) or not 1 <= len(actions) <= 100:
351 raise ValueError('Send between 1 and 100 desktop actions.')352 raise ValueError('Send between 1 and 100 desktop actions.')
352 for item in actions:353 for index, item in enumerate(actions, 1):
353 if not isinstance(item,dict):354 if not isinstance(item,dict):
354 raise ValueError('Send a desktop action object.')355 raise ValueError('Send a desktop action object.')
355 kind = item.get('type')356 kind = item.get('type')
...@@ -365,7 +366,7 @@ def validate(request):...@@ -365,7 +366,7 @@ def validate(request):
365 raise ValueError('Choose the left, right, or middle mouse button.')366 raise ValueError('Choose the left, right, or middle mouse button.')
366 for key,maximum in [('count',3),('amount',100),('duration_ms',5000)]:367 for key,maximum in [('count',3),('amount',100),('duration_ms',5000)]:
367 if key in item and (type(item[key]) is not int or not 1 <= item[key] <= maximum):368 if key in item and (type(item[key]) is not int or not 1 <= item[key] <= maximum):
368 raise ValueError('The desktop action exceeds its limit.')369 raise ValueError(f'Action {index}: {key} must be an integer from 1 to {maximum}.')
369 if kind == 'scroll' and item['direction'] not in ('up','down','left','right'):370 if kind == 'scroll' and item['direction'] not in ('up','down','left','right'):
370 raise ValueError('Choose a scroll direction.')371 raise ValueError('Choose a scroll direction.')
371 if kind == 'key':372 if kind == 'key':
...@@ -374,8 +375,8 @@ def validate(request):...@@ -374,8 +375,8 @@ def validate(request):
374 for key in item['keys']: keycode(key)375 for key in item['keys']: keycode(key)
375 if kind == 'text' and (not isinstance(item['text'],str) or len(item['text'].encode()) > 16384):376 if kind == 'text' and (not isinstance(item['text'],str) or len(item['text'].encode()) > 16384):
376 raise ValueError('Type at most 16 KiB of text at a time.')377 raise ValueError('Type at most 16 KiB of text at a time.')
377 if sum(a.get('duration_ms',0) for a in actions) > 15000:378 if sum(a.get('duration_ms', DRAG_DURATION_MS if a['type'] == 'drag' else 0) for a in actions) > 15000:
378 raise ValueError('Keep total waits and drags under 15 seconds.')379 raise ValueError('Waits and drags must total at most 15 seconds.')
379380
380381
381def record_operation(name, operation):382def record_operation(name, operation):