authoravatarclover caruso<git@paperclover.net>2026-11-17 00:10:51-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-22 00:44:02-08:00
log2a5ae6ab62a20d20e9077bde2718caf8c50fce89
tree288529eadda1f7f3656124a0d75ac3d0286b4b62
parent6586db88bc31255e09f30e9ff799c62057736d5d
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Reserve broker capacity for VM inspection and recovery

Keep twelve ordinary host workers and reserve four connection workers for VM metadata, busy status, active-operation inspection/cancellation and forced power controls. Preserve admission snapshots so a finishing command cannot turn a reserved request into slow guest setup or cleanup. Match the listener backlog to the sixteen-connection bound and preserve preview VM isolation. The Rust client permits recovery calls while its four ordinary slots are occupied. Real Linux Unix-peer load testing, preview access checks, 43 computer protocol tests, 28 VM boundary tests, 61 Rust tests and the complete OAuth client integration pass. Assisted-by: gpt-6.1-sol

8 files changed, 220 insertions(+), 36 deletions(-)

dashboard/src/host.rs+15-8
......@@ -16,14 +16,21 @@ pub async fn sample(app: Arc<App>) -> Result<Arc<Document>> {
1616}
1717
1818pub async fn call(request: Value) -> Result<Value> {
19 let _slot = tokio::time::timeout(Duration::from_secs(70), SLOTS.acquire())
20 .await
21 .map_err(|_| {
22 Error::new(
23 504,
24 "The host operation is taking too long. Check its logs, then retry.",
25 )
26 })??;
19 let computer = &request["payload"]["request"];
20 let control = matches!(request["operation"].as_str(), Some("vm.owner" | "vm.computers"))
21 || (request["operation"] == "vm.act"
22 && matches!(request["payload"]["action"].as_str(), Some("reset" | "destroy")))
23 || (request["operation"] == "vm.computer"
24 && (matches!(computer["action"].as_str(), Some("status" | "shot"))
25 || (computer["action"] == "process"
26 && computer["id"].as_str().is_some_and(|id| id.starts_with("operation:")))));
27 let _slot = if control {
28 None
29 } else {
30 Some(tokio::time::timeout(Duration::from_secs(70), SLOTS.acquire())
31 .await
32 .map_err(|_| Error::new(504, "The host operation is taking too long. Check its logs, then retry."))??)
33 };
2734 open(request).await.map(|(value, _)| value)
2835}
2936
tools/computer-client-test.py+26
......@@ -15,6 +15,7 @@ import tempfile
1515import threading
1616import time
1717import unittest
18from concurrent.futures import ThreadPoolExecutor
1819from pathlib import Path
1920from urllib.parse import urlsplit
2021from urllib.request import Request
......@@ -31,6 +32,8 @@ class Host(socketserver.StreamRequestHandler):
3132 files = {}
3233 power_actions = []
3334 writes = []
35 started = threading.Semaphore(0)
36 release = threading.Event()
3437
3538 def handle(self):
3639 line = self.rfile.readline()
......@@ -60,7 +63,14 @@ class Host(socketserver.StreamRequestHandler):
6063 raise KeyError(operation)
6164 request, files = payload["request"], self.files
6265 if request["action"] == "exec":
66 if request["script"].startswith("held-"):
67 self.started.release()
68 self.release.wait(10)
6369 return {"exit": 0, "stdout": "ran " + request["script"], "png": PNG}
70 if request["action"] == "status":
71 return {"readiness": "busy", "active_operation": {"id": "operation:" + "a" * 32}}
72 if request["action"] == "process":
73 return {"running": True, "stopping": request["operation"] == "stop"}
6474 if request["action"] == "shot":
6575 return {"png": PNG}
6676 operation, path = request["operation"], request["path"]
......@@ -183,6 +193,22 @@ class Client(unittest.TestCase):
183193 self.assertEqual(vm.call("computer_vm", vm="test-vm", operation=operation),
184194 {"id":"test-vm", "requested":operation})
185195 self.assertEqual(Host.power_actions, ["destroy", "reset"])
196 with ThreadPoolExecutor(max_workers=4) as pool:
197 held = [pool.submit(vm.call, "computer_exec", vm="test-vm", script=f"held-{index}") for index in range(4)]
198 try:
199 for _ in held:
200 self.assertTrue(Host.started.acquire(timeout=3))
201 started = time.monotonic()
202 status = vm.call("computer_status", vm="test-vm")
203 self.assertEqual(status["readiness"], "busy")
204 stopped = vm.call("computer_process", vm="test-vm", id=status["active_operation"]["id"], operation="stop")
205 self.assertTrue(stopped["stopping"])
206 vm.call("computer_vm", vm="test-vm", operation="reset")
207 self.assertLess(time.monotonic() - started, 2, "recovery controls waited behind long commands")
208 finally:
209 Host.release.set()
210 for future in held:
211 self.assertEqual(future.result()["exit"], 0)
186212 with self.assertRaisesRegex(computer.ComputerError, "granted"):
187213 vm.call("computer_shot", vm="other-vm")
188214
tools/computer-test.py+18
......@@ -226,6 +226,24 @@ class GuestCancellation(unittest.TestCase):
226226 self.assertEqual(status['capabilities'], [])
227227 work.assert_not_called()
228228
229 def test_control_snapshot_cannot_turn_into_slow_guest_setup_or_cleanup(self):
230 with computer.vms.computer_lock('fixture', 'exec'):
231 active = computer.vms.computer_activity('fixture')
232 operation = {'platform': 'windows', 'id': active['id'].removeprefix('operation:')}
233 computer.record_operation('fixture', operation)
234 def virsh(action, *args, **kwargs):
235 return 'running' if action == 'domstate' else '<domain/>'
236 with patch.object(computer.vms, 'virsh', virsh), patch.object(computer, 'worker') as worker, \
237 patch.object(computer, 'finish_operation') as cleanup:
238 status = computer.handle('fixture', {'action': 'status'}, active_snapshot=active)
239 self.assertEqual(status['readiness'], 'busy')
240 self.assertEqual(status['active_operation'], active)
241 stopped = computer.handle('fixture', {'action': 'process', 'id': active['id'], 'operation': 'stop'}, active_snapshot=active)
242 self.assertIsNone(stopped['running'])
243 self.assertTrue(stopped['recovery_required'])
244 worker.assert_not_called()
245 cleanup.assert_not_called()
246
229247 def test_active_operation_can_be_inspected_and_cancelled_during_execution(self):
230248 started = threading.Event()
231249 def virsh(action, *args, **kwargs):
tools/dashboard-host-test.py+87
......@@ -5,9 +5,12 @@ from pathlib import Path
55import sys
66import subprocess
77import os
8import pwd
89import socket
910import struct
11import tempfile
1012import threading
13import time
1114import unittest
1215from unittest.mock import patch
1316
......@@ -190,5 +193,89 @@ class Boundary(unittest.TestCase):
190193 command.assert_not_called()
191194
192195
196@unittest.skipUnless(sys.platform == 'linux', 'Linux Unix peer credentials')
197class Capacity(unittest.TestCase):
198 def test_recovery_controls_survive_full_ordinary_capacity(self):
199 program = '''
200from importlib import import_module
201import json, sys, time
202from pathlib import Path
203m = import_module('dashboard-host')
204root = Path(sys.argv[1])
205m.vms.DISKS = root / 'vms'
206for name in ['fixture', 'idle', 'managed-mac']:
207 (m.vms.DISKS / name).mkdir(parents=True)
208class Host:
209 def __init__(self, pool): pass
210 def handle(self, request, *, active_snapshot=None):
211 if request['operation'] == 'fixture.long':
212 (root / ('started-' + str(request['index']))).touch()
213 while not (root / 'release').exists(): time.sleep(.01)
214 return {'operation': request['operation'], 'active_snapshot': active_snapshot}
215m.Host = Host
216m.vms.mac_vms.CONFIG = {'name': 'managed-mac'}
217with m.vms.computer_lock('fixture', 'exec') as active, m.vms.computer_lock('managed-mac', 'exec'):
218 (root / 'active.json').write_text(json.dumps(active))
219 m.main()
220'''
221 with tempfile.TemporaryDirectory() as directory:
222 root = Path(directory)
223 path = root / 'host.sock'
224 environment = {**os.environ, 'SNOWGLOBE_POOL': 'fixture',
225 'SNOWGLOBE_DASHBOARD_USER': pwd.getpwuid(os.getuid()).pw_name,
226 'SNOWGLOBE_HOST_SOCKET': str(path)}
227 environment.pop('NOTIFY_SOCKET', None)
228 process = subprocess.Popen([sys.executable, '-c', program, str(root)], cwd=Path(__file__).parent, env=environment,
229 stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
230 held = []
231 def connect(request):
232 connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
233 connection.settimeout(2)
234 try:
235 connection.connect(str(path))
236 connection.sendall(json.dumps(request).encode() + b'\n')
237 except BaseException:
238 connection.close()
239 raise
240 return connection
241 def call(request):
242 with connect(request) as connection, connection.makefile('rb') as incoming:
243 length = struct.unpack('!I', incoming.read(4))[0]
244 return json.loads(incoming.read(length))
245 try:
246 deadline = time.monotonic() + 5
247 while not path.exists():
248 self.assertIsNone(process.poll())
249 self.assertLess(time.monotonic(), deadline)
250 time.sleep(.01)
251 for index in range(host_module.WORKERS):
252 held.append(connect({'operation': 'fixture.long', 'index': index}))
253 while len(list(root.glob('started-*'))) != host_module.WORKERS:
254 self.assertLess(time.monotonic(), deadline)
255 time.sleep(.01)
256 active = json.loads((root / 'active.json').read_text())
257 for request in [
258 {'operation': 'vm.owner', 'payload': {'name': 'fixture'}}, {'operation': 'vm.computers'},
259 *({'operation': 'vm.act', 'payload': {'name': 'fixture', 'action': action}} for action in ['reset', 'destroy']),
260 *({'operation': 'vm.computer', 'payload': {'name': 'fixture', 'request': request}} for request in [
261 {'action': 'status'}, {'action': 'process', 'id': active['id'], 'operation': 'status'},
262 {'action': 'process', 'id': active['id'], 'operation': 'stop'}]),
263 ]:
264 self.assertEqual(call(request)['value']['operation'], request['operation'])
265 for request in [
266 {'operation': 'host.sample'}, {'operation': 'vm.act', 'payload': {'name': 'fixture', 'action': 'start'}},
267 {'operation': 'vm.computer', 'payload': {'name': 'idle', 'request': {'action': 'status'}}},
268 {'operation': 'vm.computer', 'payload': {'name': 'managed-mac', 'request': {'action': 'status'}}},
269 {'operation': 'vm.computer', 'payload': {'name': 'idle', 'request': {'action': 'process', 'id': active['id'], 'operation': 'stop'}}},
270 ]:
271 self.assertEqual(call(request)['status'], 503)
272 finally:
273 (root / 'release').touch()
274 for connection in held:
275 connection.close()
276 process.terminate()
277 process.communicate(timeout=5)
278
279
193280if __name__ == "__main__":
194281 unittest.main()
tools/dashboard-host.py+42-11
......@@ -40,6 +40,9 @@ TEXT_FIELDS = {"name", "compression", "mountpoint", "origin", "mounted"}
4040PROTOCOL = json.loads((Path(__file__).resolve().parent.parent / "dashboard/src/host-protocol.json").read_text())
4141MAX_REQUEST = PROTOCOL['request']
4242MAX_RESPONSE = PROTOCOL['response']
43WORKERS = 12
44CONNECTIONS = WORKERS + 4
45WORK_SLOTS = threading.BoundedSemaphore(WORKERS)
4346STREAM_SLOTS = threading.BoundedSemaphore(4)
4447GUEST_SLOTS = threading.BoundedSemaphore(4)
4548CLONE_SLOTS = threading.BoundedSemaphore(2)
......@@ -74,7 +77,7 @@ class Host:
7477 raise Rejected("Choose a snapshot from the list.")
7578 return value
7679
77 def handle(self, request):
80 def handle(self, request, *, active_snapshot=None):
7881 if not isinstance(request, dict):
7982 raise Rejected("Choose a supported host operation.")
8083 operation = request.get("operation")
......@@ -206,7 +209,7 @@ class Host:
206209 raise Rejected(str(error)) from error
207210 if action == "computer":
208211 try:
209 return importlib.import_module("vm-computer").handle(payload["name"], payload["request"])
212 return importlib.import_module("vm-computer").handle(payload["name"], payload["request"], active_snapshot=active_snapshot)
210213 except (BlockingIOError, InterruptedError) as error:
211214 raise dashboard_runs.Error(409, str(error) or "Another computer operation is running. Read computer_status before retrying.") from None
212215 except (TimeoutError, subprocess.TimeoutExpired):
......@@ -485,7 +488,7 @@ def stream_clone(connection, plan):
485488
486489
487490def serve_connection(connection, host, allowed_uid, allowed_operations=None):
488 with connection:
491 with connection, contextlib.ExitStack() as resources:
489492 connection.settimeout(65)
490493 _, uid, _ = struct.unpack("3i", connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12))
491494 if uid != allowed_uid:
......@@ -510,19 +513,47 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None):
510513 or request["operation"] not in allowed_operations
511514 ):
512515 raise dashboard_runs.Error(403, "This preview cannot change Snow Globe. Use production for host controls.")
513 if isinstance(request, dict) and request.get("operation") in {"vm.console", "vm.serial", "vm.guest", "vm.upload"}:
514 streams = GUEST_SLOTS if request["operation"] == "vm.guest" else STREAM_SLOTS
516 operation = request.get("operation") if isinstance(request, dict) else None
517 operation = operation if isinstance(operation, str) else None
518 active_snapshot = None
519 control = operation == "vm.computers"
520 if operation in {"vm.owner", "vm.act", "vm.computer"}:
521 payload = request.get("payload")
522 try:
523 vms.validate(operation[3:], payload)
524 except ValueError as error:
525 raise Rejected(str(error)) from error
526 if operation == "vm.owner":
527 control = True
528 elif operation == "vm.act":
529 control = payload["action"] in {"reset", "destroy"}
530 else:
531 action = payload["request"]
532 identity = action.get("id")
533 operation_handle = action.get("action") == "process" and isinstance(identity, str) and identity.startswith("operation:")
534 control = operation_handle and action.get("operation") == "status"
535 if (action.get("action") == "status" or operation_handle and action.get("operation") == "stop") and (
536 not vms.mac_vms.CONFIG or payload["name"] != vms.mac_vms.CONFIG["name"]
537 ):
538 active_snapshot = vms.computer_activity(payload["name"])
539 control = active_snapshot is not None
540 if not control:
541 if not WORK_SLOTS.acquire(blocking=False):
542 raise dashboard_runs.Error(503, "The host is busy. Wait for an operation to finish, then retry.")
543 resources.callback(WORK_SLOTS.release)
544 if operation in {"vm.console", "vm.serial", "vm.guest", "vm.upload"}:
545 streams = GUEST_SLOTS if operation == "vm.guest" else STREAM_SLOTS
515546 if not streams.acquire(blocking=False):
516547 raise Rejected("Four VM connections or uploads are open. Close one and try again.")
517548 try:
518 if request["operation"] in {"vm.console", "vm.serial", "vm.guest"}:
549 if operation in {"vm.console", "vm.serial", "vm.guest"}:
519550 stream_console(connection, host, request)
520551 else:
521552 stream_upload(connection, request)
522553 return
523554 finally:
524555 streams.release()
525 if isinstance(request, dict) and request.get("operation") == "shale.clone":
556 if operation == "shale.clone":
526557 if not CLONE_SLOTS.acquire(blocking=False):
527558 raise dashboard_runs.Error(503, "Two Knife clones are running. Try again shortly.")
528559 try:
......@@ -530,7 +561,7 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None):
530561 return
531562 finally:
532563 CLONE_SLOTS.release()
533 response = {"value": host.handle(request)}
564 response = {"value": host.handle(request, active_snapshot=active_snapshot) if active_snapshot is not None else host.handle(request)}
534565 except Rejected as error:
535566 response = {"error": str(error), "status": 400}
536567 except dashboard_runs.Error as error:
......@@ -558,13 +589,13 @@ def main():
558589 with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener:
559590 listener.bind(str(socket_path))
560591 socket_path.chmod(0o660)
561 listener.listen(8)
592 listener.listen(CONNECTIONS)
562593 if address := os.environ.get("NOTIFY_SOCKET"):
563594 with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as notification:
564595 notification.connect("\0" + address[1:] if address.startswith("@") else address)
565596 notification.sendall(b"READY=1")
566 slots = threading.BoundedSemaphore(16)
567 with concurrent.futures.ThreadPoolExecutor(max_workers=12) as workers:
597 slots = threading.BoundedSemaphore(CONNECTIONS)
598 with concurrent.futures.ThreadPoolExecutor(max_workers=CONNECTIONS) as workers:
568599 while True:
569600 connection, _ = listener.accept()
570601 if not slots.acquire(blocking=False):
tools/dashboard-stage-test.py+12
......@@ -80,6 +80,18 @@ class PreviewBoundary(unittest.TestCase):
8080 self.assertEqual(host.handle(request), "real snapshot")
8181 handler.assert_called_once_with(request)
8282
83 def test_busy_observation_keeps_preview_vm_access_checks(self):
84 host = stage.PreviewHost({'clone': 'globe/staging/fixture', 'computer': 'fixture'})
85 request = {'operation': 'vm.computer', 'payload': {'name': 'fixture', 'request': {'action': 'status'}}}
86 active = {'id': 'operation:' + 'a' * 32, 'action': 'exec'}
87 with patch.object(boundary.Host, 'handle', return_value={'readiness': 'busy'}) as call:
88 self.assertEqual(host.handle(request, active_snapshot=active), {'readiness': 'busy'})
89 call.assert_called_once_with(request, active_snapshot=active)
90 call.reset_mock()
91 with self.assertRaises(boundary.dashboard_runs.Error):
92 host.handle({**request, 'payload': {**request['payload'], 'name': 'personal'}}, active_snapshot=active)
93 call.assert_not_called()
94
8395 def test_preview_vm_creation_and_ownership_stay_in_its_account(self):
8496 with tempfile.TemporaryDirectory() as directory:
8597 root = Path(directory)
tools/dashboard-stage.py+7-6
......@@ -1,5 +1,6 @@
11#!/usr/bin/env python3
22import concurrent.futures
3from contextlib import closing
34from importlib import import_module
45import json
56import os
......@@ -35,11 +36,11 @@ class PreviewHost(boundary.Host):
3536 super().__init__(saved["clone"].split("/", 1)[0])
3637 self.stage = saved
3738
38 def handle(self, request):
39 def handle(self, request, *, active_snapshot=None):
3940 operation = request.get("operation") if isinstance(request, dict) else None
4041 owner = None
4142 if self.stage.get("computer") and self.stage.get("mount") and operation in {"vm.computer", "vm.owner", "vm.access", "vm.console", "vm.guest", "vm.domains", "vm.computers", "vm.ai", "vm.act"}:
42 with sqlite3.connect(f"file:{self.stage['mount']}/data/accounts.sqlite?mode=ro", uri=True) as accounts:
43 with closing(sqlite3.connect(f"file:{self.stage['mount']}/data/accounts.sqlite?mode=ro", uri=True)) as accounts:
4344 owner = accounts.execute("SELECT id FROM users WHERE username='preview'").fetchone()
4445 if not owner:
4546 raise ValueError("The preview account is unavailable.")
......@@ -75,7 +76,7 @@ class PreviewHost(boundary.Host):
7576 ).stdout)
7677 mounted["filesystems"][0]["target"] = "/srv"
7778 return mounted
78 result = super().handle(request)
79 result = super().handle(request, active_snapshot=active_snapshot) if active_snapshot is not None else super().handle(request)
7980 if owner and operation in {"vm.owner", "vm.domains", "vm.computers"}:
8081 if operation == "vm.owner":
8182 result = {**result, "owner": owner[0], "ownerOnly": True}
......@@ -106,9 +107,9 @@ def serve(stage):
106107 listener.bind(str(path))
107108 os.chown(path, 0, saved["uid"])
108109 path.chmod(0o660)
109 listener.listen(8)
110 slots = threading.BoundedSemaphore(16)
111 with concurrent.futures.ThreadPoolExecutor(max_workers=12) as workers:
110 listener.listen(boundary.CONNECTIONS)
111 slots = threading.BoundedSemaphore(boundary.CONNECTIONS)
112 with concurrent.futures.ThreadPoolExecutor(max_workers=boundary.CONNECTIONS) as workers:
112113 while True:
113114 connection, _ = listener.accept()
114115 if not slots.acquire(blocking=False):
tools/vm-computer.py+13-11
......@@ -517,7 +517,8 @@ Move-Item -LiteralPath 'WORKER.new.exe' -Destination 'WORKER'
517517
518518
519519
520def handle(name, request):
520def handle(name, request, *, active_snapshot=None):
521 """Admission snapshots cannot start guest work on reserved control workers."""
521522 validate(request)
522523 vms.ensure_name(name)
523524 if request['action'] == 'process' and request['id'].startswith('operation:'):
......@@ -541,7 +542,7 @@ def handle(name, request):
541542 return {'running': False}
542543 if 'operation:' + unfinished['id'].removeprefix('snowglobe-computer-request-') != identity:
543544 return {'running': False}
544 if request['operation'] == 'status':
545 if request['operation'] == 'status' or active_snapshot is not None:
545546 return {'running': None, 'recovery_required': True, 'id': identity}
546547 with vms.computer_lock(name, 'cancel', identity):
547548 try:
......@@ -559,14 +560,15 @@ def handle(name, request):
559560 operation_token = None
560561 try:
561562 with contextlib.ExitStack() as scope:
562 busy = False
563 try:
564 active = scope.enter_context(vms.computer_lock(name, request['action']))
565 operation_token = OPERATION.set((name, active['id']))
566 except BlockingIOError:
567 if request['action'] not in ('status', 'shot'):
568 raise BlockingIOError('Another computer operation is running. Read computer_status before retrying.') from None
569 busy = True
563 busy = active_snapshot is not None
564 if not busy:
565 try:
566 active = scope.enter_context(vms.computer_lock(name, request['action']))
567 operation_token = OPERATION.set((name, active['id']))
568 except BlockingIOError:
569 if request['action'] not in ('status', 'shot'):
570 raise BlockingIOError('Another computer operation is running. Read computer_status before retrying.') from None
571 busy = True
570572 root = ET.fromstring(vms.virsh('dumpxml',name,timeout=remaining(15)))
571573 power = vms.virsh('domstate',name,timeout=remaining(15)).strip()
572574 platform = root.findtext(f'metadata/{vms.NS}vm/{vms.NS}platform','linux')
......@@ -583,7 +585,7 @@ def handle(name, request):
583585 if request['action'] == 'shot':
584586 return asyncio.run(desktop(name))
585587 if busy:
586 active = vms.computer_activity(name)
588 active = active_snapshot or vms.computer_activity(name)
587589 return {**status, 'readiness': 'busy', 'capabilities': ['shot'], 'active_operation': active}
588590 try:
589591 with os.fdopen(vms.open_regular(vms.DISKS / name / 'computer-operation.json')) as incoming: