authoravatarclover caruso<git@paperclover.net>2026-11-17 05:15:31-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-22 00:44:02-08:00
loga6e4327e47cc55ebb24fa311e6f9408bc61ec773
tree154f69212602473e848faed3c5232181f0e36000
parent6f8ec57f945851d1989b77d1e949a5daa7588fdc
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Count default drag durations in desktop action budgets

Apply the same drag duration to execution and batch admission. Report the action number, field and limit for invalid numeric arguments, and expose the combined wait and drag budget in the MCP schema. Assisted-by: gpt-6.1-sol

3 files changed, 20 insertions(+), 5 deletions(-)

dashboard/src/computer-tools.json+1
......@@ -64,6 +64,7 @@
6464 },
6565 "actions": {
6666 "type": "array",
67 "description": "Waits and drags, including default drag durations, may total at most 15 seconds.",
6768 "items": {
6869 "oneOf": [
6970 {
tools/computer-test.py+13
......@@ -20,6 +20,19 @@ leases=import_module('vm-ai')
2020
2121
2222class Protocol(unittest.TestCase):
23 def test_drag_defaults_count_toward_the_batch_budget(self):
24 drag = {'type': 'drag', 'x': 0, 'y': 0, 'to_x': 1, 'to_y': 1}
25 computer.validate({'action': 'input', 'actions': [drag] * 37 + [{'type': 'wait', 'duration_ms': 200}]})
26 with self.assertRaisesRegex(ValueError, '15 seconds'):
27 computer.validate({'action': 'input', 'actions': [drag] * 38})
28
29 def test_action_limit_errors_identify_the_action_field_and_limit(self):
30 for action, field, limit in [({'type': 'wait', 'duration_ms': 5001}, 'duration_ms', 5000),
31 ({'type': 'click', 'x': 0, 'y': 0, 'count': 4}, 'count', 3),
32 ({'type': 'scroll', 'x': 0, 'y': 0, 'direction': 'up', 'amount': 101}, 'amount', 100)]:
33 with self.subTest(field=field), self.assertRaisesRegex(ValueError, f'Action 2: {field}.*1.*{limit}'):
34 computer.validate({'action': 'input', 'actions': [{'type': 'text', 'text': 'not executed'}, action]})
35
2336 def test_rejects_malformed_actions_before_input(self):
2437 for action in [None,[],{'type':[]},{'type':'click','x':True,'y':0},
2538 {'type':'drag','x':0,'y':0,'to_x':0,'to_y':-1},
tools/vm-computer.py+6-5
......@@ -23,6 +23,7 @@ SCREEN_SLOT = threading.BoundedSemaphore(1)
2323DEADLINE = ContextVar('computer_deadline', default=float('inf'))
2424OPERATION = ContextVar('computer_operation', default=None)
2525FILE_CHUNK = json.loads((Path(__file__).resolve().parent.parent / 'dashboard/src/host-protocol.json').read_text())['file_chunk']
26DRAG_DURATION_MS = 400
2627
2728
2829def remaining(limit):
......@@ -213,7 +214,7 @@ async def capture(name, actions, platform, screenshot=True, native_text=True):
213214 for step in range(1, 21):
214215 screen.input({'type': 'pointer', 'x': round(x+(end_x-x)*step/20),
215216 'y': round(y+(end_y-y)*step/20), 'buttons': button})
216 await asyncio.sleep(action.get('duration_ms', 400)/20000)
217 await asyncio.sleep(action.get('duration_ms', DRAG_DURATION_MS)/20000)
217218 screen.input({'type': 'release'})
218219 elif kind == 'scroll':
219220 wheel = {'up': 8, 'down': 16, 'left': 32, 'right': 64}[action['direction']]
......@@ -349,7 +350,7 @@ def validate(request):
349350 actions = request.get('actions')
350351 if not isinstance(actions,list) or not 1 <= len(actions) <= 100:
351352 raise ValueError('Send between 1 and 100 desktop actions.')
352 for item in actions:
353 for index, item in enumerate(actions, 1):
353354 if not isinstance(item,dict):
354355 raise ValueError('Send a desktop action object.')
355356 kind = item.get('type')
......@@ -365,7 +366,7 @@ def validate(request):
365366 raise ValueError('Choose the left, right, or middle mouse button.')
366367 for key,maximum in [('count',3),('amount',100),('duration_ms',5000)]:
367368 if key in item and (type(item[key]) is not int or not 1 <= item[key] <= maximum):
368 raise ValueError('The desktop action exceeds its limit.')
369 raise ValueError(f'Action {index}: {key} must be an integer from 1 to {maximum}.')
369370 if kind == 'scroll' and item['direction'] not in ('up','down','left','right'):
370371 raise ValueError('Choose a scroll direction.')
371372 if kind == 'key':
......@@ -374,8 +375,8 @@ def validate(request):
374375 for key in item['keys']: keycode(key)
375376 if kind == 'text' and (not isinstance(item['text'],str) or len(item['text'].encode()) > 16384):
376377 raise ValueError('Type at most 16 KiB of text at a time.')
377 if sum(a.get('duration_ms',0) for a in actions) > 15000:
378 raise ValueError('Keep total waits and drags under 15 seconds.')
378 if sum(a.get('duration_ms', DRAG_DURATION_MS if a['type'] == 'drag' else 0) for a in actions) > 15000:
379 raise ValueError('Waits and drags must total at most 15 seconds.')
379380
380381
381382def record_operation(name, operation):