authoravatarclover caruso<git@paperclover.net>2026-11-17 00:10:51-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-22 00:44:02-08:00
log2a5ae6ab62a20d20e9077bde2718caf8c50fce89
tree288529eadda1f7f3656124a0d75ac3d0286b4b62
parent6586db88bc31255e09f30e9ff799c62057736d5d
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Reserve broker capacity for VM inspection and recovery

Keep twelve ordinary host workers and reserve four connection workers for VM metadata, busy status, active-operation inspection/cancellation and forced power controls. Preserve admission snapshots so a finishing command cannot turn a reserved request into slow guest setup or cleanup. Match the listener backlog to the sixteen-connection bound and preserve preview VM isolation. The Rust client permits recovery calls while its four ordinary slots are occupied. Real Linux Unix-peer load testing, preview access checks, 43 computer protocol tests, 28 VM boundary tests, 61 Rust tests and the complete OAuth client integration pass. Assisted-by: gpt-6.1-sol

8 files changed, 220 insertions(+), 36 deletions(-)

dashboard/src/host.rs+15-8
...@@ -16,14 +16,21 @@ pub async fn sample(app: Arc<App>) -> Result<Arc<Document>> {...@@ -16,14 +16,21 @@ pub async fn sample(app: Arc<App>) -> Result<Arc<Document>> {
16}16}
1717
18pub async fn call(request: Value) -> Result<Value> {18pub async fn call(request: Value) -> Result<Value> {
19 let _slot = tokio::time::timeout(Duration::from_secs(70), SLOTS.acquire())19 let computer = &request["payload"]["request"];
20 .await20 let control = matches!(request["operation"].as_str(), Some("vm.owner" | "vm.computers"))
21 .map_err(|_| {21 || (request["operation"] == "vm.act"
22 Error::new(22 && matches!(request["payload"]["action"].as_str(), Some("reset" | "destroy")))
23 504,23 || (request["operation"] == "vm.computer"
24 "The host operation is taking too long. Check its logs, then retry.",24 && (matches!(computer["action"].as_str(), Some("status" | "shot"))
25 )25 || (computer["action"] == "process"
26 })??;26 && computer["id"].as_str().is_some_and(|id| id.starts_with("operation:")))));
27 let _slot = if control {
28 None
29 } else {
30 Some(tokio::time::timeout(Duration::from_secs(70), SLOTS.acquire())
31 .await
32 .map_err(|_| Error::new(504, "The host operation is taking too long. Check its logs, then retry."))??)
33 };
27 open(request).await.map(|(value, _)| value)34 open(request).await.map(|(value, _)| value)
28}35}
2936
tools/computer-client-test.py+26
...@@ -15,6 +15,7 @@ import tempfile...@@ -15,6 +15,7 @@ import tempfile
15import threading15import threading
16import time16import time
17import unittest17import unittest
18from concurrent.futures import ThreadPoolExecutor
18from pathlib import Path19from pathlib import Path
19from urllib.parse import urlsplit20from urllib.parse import urlsplit
20from urllib.request import Request21from urllib.request import Request
...@@ -31,6 +32,8 @@ class Host(socketserver.StreamRequestHandler):...@@ -31,6 +32,8 @@ class Host(socketserver.StreamRequestHandler):
31 files = {}32 files = {}
32 power_actions = []33 power_actions = []
33 writes = []34 writes = []
35 started = threading.Semaphore(0)
36 release = threading.Event()
3437
35 def handle(self):38 def handle(self):
36 line = self.rfile.readline()39 line = self.rfile.readline()
...@@ -60,7 +63,14 @@ class Host(socketserver.StreamRequestHandler):...@@ -60,7 +63,14 @@ class Host(socketserver.StreamRequestHandler):
60 raise KeyError(operation)63 raise KeyError(operation)
61 request, files = payload["request"], self.files64 request, files = payload["request"], self.files
62 if request["action"] == "exec":65 if request["action"] == "exec":
66 if request["script"].startswith("held-"):
67 self.started.release()
68 self.release.wait(10)
63 return {"exit": 0, "stdout": "ran " + request["script"], "png": PNG}69 return {"exit": 0, "stdout": "ran " + request["script"], "png": PNG}
70 if request["action"] == "status":
71 return {"readiness": "busy", "active_operation": {"id": "operation:" + "a" * 32}}
72 if request["action"] == "process":
73 return {"running": True, "stopping": request["operation"] == "stop"}
64 if request["action"] == "shot":74 if request["action"] == "shot":
65 return {"png": PNG}75 return {"png": PNG}
66 operation, path = request["operation"], request["path"]76 operation, path = request["operation"], request["path"]
...@@ -183,6 +193,22 @@ class Client(unittest.TestCase):...@@ -183,6 +193,22 @@ class Client(unittest.TestCase):
183 self.assertEqual(vm.call("computer_vm", vm="test-vm", operation=operation),193 self.assertEqual(vm.call("computer_vm", vm="test-vm", operation=operation),
184 {"id":"test-vm", "requested":operation})194 {"id":"test-vm", "requested":operation})
185 self.assertEqual(Host.power_actions, ["destroy", "reset"])195 self.assertEqual(Host.power_actions, ["destroy", "reset"])
196 with ThreadPoolExecutor(max_workers=4) as pool:
197 held = [pool.submit(vm.call, "computer_exec", vm="test-vm", script=f"held-{index}") for index in range(4)]
198 try:
199 for _ in held:
200 self.assertTrue(Host.started.acquire(timeout=3))
201 started = time.monotonic()
202 status = vm.call("computer_status", vm="test-vm")
203 self.assertEqual(status["readiness"], "busy")
204 stopped = vm.call("computer_process", vm="test-vm", id=status["active_operation"]["id"], operation="stop")
205 self.assertTrue(stopped["stopping"])
206 vm.call("computer_vm", vm="test-vm", operation="reset")
207 self.assertLess(time.monotonic() - started, 2, "recovery controls waited behind long commands")
208 finally:
209 Host.release.set()
210 for future in held:
211 self.assertEqual(future.result()["exit"], 0)
186 with self.assertRaisesRegex(computer.ComputerError, "granted"):212 with self.assertRaisesRegex(computer.ComputerError, "granted"):
187 vm.call("computer_shot", vm="other-vm")213 vm.call("computer_shot", vm="other-vm")
188214
tools/computer-test.py+18
...@@ -226,6 +226,24 @@ class GuestCancellation(unittest.TestCase):...@@ -226,6 +226,24 @@ class GuestCancellation(unittest.TestCase):
226 self.assertEqual(status['capabilities'], [])226 self.assertEqual(status['capabilities'], [])
227 work.assert_not_called()227 work.assert_not_called()
228228
229 def test_control_snapshot_cannot_turn_into_slow_guest_setup_or_cleanup(self):
230 with computer.vms.computer_lock('fixture', 'exec'):
231 active = computer.vms.computer_activity('fixture')
232 operation = {'platform': 'windows', 'id': active['id'].removeprefix('operation:')}
233 computer.record_operation('fixture', operation)
234 def virsh(action, *args, **kwargs):
235 return 'running' if action == 'domstate' else '<domain/>'
236 with patch.object(computer.vms, 'virsh', virsh), patch.object(computer, 'worker') as worker, \
237 patch.object(computer, 'finish_operation') as cleanup:
238 status = computer.handle('fixture', {'action': 'status'}, active_snapshot=active)
239 self.assertEqual(status['readiness'], 'busy')
240 self.assertEqual(status['active_operation'], active)
241 stopped = computer.handle('fixture', {'action': 'process', 'id': active['id'], 'operation': 'stop'}, active_snapshot=active)
242 self.assertIsNone(stopped['running'])
243 self.assertTrue(stopped['recovery_required'])
244 worker.assert_not_called()
245 cleanup.assert_not_called()
246
229 def test_active_operation_can_be_inspected_and_cancelled_during_execution(self):247 def test_active_operation_can_be_inspected_and_cancelled_during_execution(self):
230 started = threading.Event()248 started = threading.Event()
231 def virsh(action, *args, **kwargs):249 def virsh(action, *args, **kwargs):
tools/dashboard-host-test.py+87
...@@ -5,9 +5,12 @@ from pathlib import Path...@@ -5,9 +5,12 @@ from pathlib import Path
5import sys5import sys
6import subprocess6import subprocess
7import os7import os
8import pwd
8import socket9import socket
9import struct10import struct
11import tempfile
10import threading12import threading
13import time
11import unittest14import unittest
12from unittest.mock import patch15from unittest.mock import patch
1316
...@@ -190,5 +193,89 @@ class Boundary(unittest.TestCase):...@@ -190,5 +193,89 @@ class Boundary(unittest.TestCase):
190 command.assert_not_called()193 command.assert_not_called()
191194
192195
196@unittest.skipUnless(sys.platform == 'linux', 'Linux Unix peer credentials')
197class Capacity(unittest.TestCase):
198 def test_recovery_controls_survive_full_ordinary_capacity(self):
199 program = '''
200from importlib import import_module
201import json, sys, time
202from pathlib import Path
203m = import_module('dashboard-host')
204root = Path(sys.argv[1])
205m.vms.DISKS = root / 'vms'
206for name in ['fixture', 'idle', 'managed-mac']:
207 (m.vms.DISKS / name).mkdir(parents=True)
208class Host:
209 def __init__(self, pool): pass
210 def handle(self, request, *, active_snapshot=None):
211 if request['operation'] == 'fixture.long':
212 (root / ('started-' + str(request['index']))).touch()
213 while not (root / 'release').exists(): time.sleep(.01)
214 return {'operation': request['operation'], 'active_snapshot': active_snapshot}
215m.Host = Host
216m.vms.mac_vms.CONFIG = {'name': 'managed-mac'}
217with m.vms.computer_lock('fixture', 'exec') as active, m.vms.computer_lock('managed-mac', 'exec'):
218 (root / 'active.json').write_text(json.dumps(active))
219 m.main()
220'''
221 with tempfile.TemporaryDirectory() as directory:
222 root = Path(directory)
223 path = root / 'host.sock'
224 environment = {**os.environ, 'SNOWGLOBE_POOL': 'fixture',
225 'SNOWGLOBE_DASHBOARD_USER': pwd.getpwuid(os.getuid()).pw_name,
226 'SNOWGLOBE_HOST_SOCKET': str(path)}
227 environment.pop('NOTIFY_SOCKET', None)
228 process = subprocess.Popen([sys.executable, '-c', program, str(root)], cwd=Path(__file__).parent, env=environment,
229 stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
230 held = []
231 def connect(request):
232 connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
233 connection.settimeout(2)
234 try:
235 connection.connect(str(path))
236 connection.sendall(json.dumps(request).encode() + b'\n')
237 except BaseException:
238 connection.close()
239 raise
240 return connection
241 def call(request):
242 with connect(request) as connection, connection.makefile('rb') as incoming:
243 length = struct.unpack('!I', incoming.read(4))[0]
244 return json.loads(incoming.read(length))
245 try:
246 deadline = time.monotonic() + 5
247 while not path.exists():
248 self.assertIsNone(process.poll())
249 self.assertLess(time.monotonic(), deadline)
250 time.sleep(.01)
251 for index in range(host_module.WORKERS):
252 held.append(connect({'operation': 'fixture.long', 'index': index}))
253 while len(list(root.glob('started-*'))) != host_module.WORKERS:
254 self.assertLess(time.monotonic(), deadline)
255 time.sleep(.01)
256 active = json.loads((root / 'active.json').read_text())
257 for request in [
258 {'operation': 'vm.owner', 'payload': {'name': 'fixture'}}, {'operation': 'vm.computers'},
259 *({'operation': 'vm.act', 'payload': {'name': 'fixture', 'action': action}} for action in ['reset', 'destroy']),
260 *({'operation': 'vm.computer', 'payload': {'name': 'fixture', 'request': request}} for request in [
261 {'action': 'status'}, {'action': 'process', 'id': active['id'], 'operation': 'status'},
262 {'action': 'process', 'id': active['id'], 'operation': 'stop'}]),
263 ]:
264 self.assertEqual(call(request)['value']['operation'], request['operation'])
265 for request in [
266 {'operation': 'host.sample'}, {'operation': 'vm.act', 'payload': {'name': 'fixture', 'action': 'start'}},
267 {'operation': 'vm.computer', 'payload': {'name': 'idle', 'request': {'action': 'status'}}},
268 {'operation': 'vm.computer', 'payload': {'name': 'managed-mac', 'request': {'action': 'status'}}},
269 {'operation': 'vm.computer', 'payload': {'name': 'idle', 'request': {'action': 'process', 'id': active['id'], 'operation': 'stop'}}},
270 ]:
271 self.assertEqual(call(request)['status'], 503)
272 finally:
273 (root / 'release').touch()
274 for connection in held:
275 connection.close()
276 process.terminate()
277 process.communicate(timeout=5)
278
279
193if __name__ == "__main__":280if __name__ == "__main__":
194 unittest.main()281 unittest.main()
tools/dashboard-host.py+42-11
...@@ -40,6 +40,9 @@ TEXT_FIELDS = {"name", "compression", "mountpoint", "origin", "mounted"}...@@ -40,6 +40,9 @@ TEXT_FIELDS = {"name", "compression", "mountpoint", "origin", "mounted"}
40PROTOCOL = json.loads((Path(__file__).resolve().parent.parent / "dashboard/src/host-protocol.json").read_text())40PROTOCOL = json.loads((Path(__file__).resolve().parent.parent / "dashboard/src/host-protocol.json").read_text())
41MAX_REQUEST = PROTOCOL['request']41MAX_REQUEST = PROTOCOL['request']
42MAX_RESPONSE = PROTOCOL['response']42MAX_RESPONSE = PROTOCOL['response']
43WORKERS = 12
44CONNECTIONS = WORKERS + 4
45WORK_SLOTS = threading.BoundedSemaphore(WORKERS)
43STREAM_SLOTS = threading.BoundedSemaphore(4)46STREAM_SLOTS = threading.BoundedSemaphore(4)
44GUEST_SLOTS = threading.BoundedSemaphore(4)47GUEST_SLOTS = threading.BoundedSemaphore(4)
45CLONE_SLOTS = threading.BoundedSemaphore(2)48CLONE_SLOTS = threading.BoundedSemaphore(2)
...@@ -74,7 +77,7 @@ class Host:...@@ -74,7 +77,7 @@ class Host:
74 raise Rejected("Choose a snapshot from the list.")77 raise Rejected("Choose a snapshot from the list.")
75 return value78 return value
7679
77 def handle(self, request):80 def handle(self, request, *, active_snapshot=None):
78 if not isinstance(request, dict):81 if not isinstance(request, dict):
79 raise Rejected("Choose a supported host operation.")82 raise Rejected("Choose a supported host operation.")
80 operation = request.get("operation")83 operation = request.get("operation")
...@@ -206,7 +209,7 @@ class Host:...@@ -206,7 +209,7 @@ class Host:
206 raise Rejected(str(error)) from error209 raise Rejected(str(error)) from error
207 if action == "computer":210 if action == "computer":
208 try:211 try:
209 return importlib.import_module("vm-computer").handle(payload["name"], payload["request"])212 return importlib.import_module("vm-computer").handle(payload["name"], payload["request"], active_snapshot=active_snapshot)
210 except (BlockingIOError, InterruptedError) as error:213 except (BlockingIOError, InterruptedError) as error:
211 raise dashboard_runs.Error(409, str(error) or "Another computer operation is running. Read computer_status before retrying.") from None214 raise dashboard_runs.Error(409, str(error) or "Another computer operation is running. Read computer_status before retrying.") from None
212 except (TimeoutError, subprocess.TimeoutExpired):215 except (TimeoutError, subprocess.TimeoutExpired):
...@@ -485,7 +488,7 @@ def stream_clone(connection, plan):...@@ -485,7 +488,7 @@ def stream_clone(connection, plan):
485488
486489
487def serve_connection(connection, host, allowed_uid, allowed_operations=None):490def serve_connection(connection, host, allowed_uid, allowed_operations=None):
488 with connection:491 with connection, contextlib.ExitStack() as resources:
489 connection.settimeout(65)492 connection.settimeout(65)
490 _, uid, _ = struct.unpack("3i", connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12))493 _, uid, _ = struct.unpack("3i", connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12))
491 if uid != allowed_uid:494 if uid != allowed_uid:
...@@ -510,19 +513,47 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None):...@@ -510,19 +513,47 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None):
510 or request["operation"] not in allowed_operations513 or request["operation"] not in allowed_operations
511 ):514 ):
512 raise dashboard_runs.Error(403, "This preview cannot change Snow Globe. Use production for host controls.")515 raise dashboard_runs.Error(403, "This preview cannot change Snow Globe. Use production for host controls.")
513 if isinstance(request, dict) and request.get("operation") in {"vm.console", "vm.serial", "vm.guest", "vm.upload"}:516 operation = request.get("operation") if isinstance(request, dict) else None
514 streams = GUEST_SLOTS if request["operation"] == "vm.guest" else STREAM_SLOTS517 operation = operation if isinstance(operation, str) else None
518 active_snapshot = None
519 control = operation == "vm.computers"
520 if operation in {"vm.owner", "vm.act", "vm.computer"}:
521 payload = request.get("payload")
522 try:
523 vms.validate(operation[3:], payload)
524 except ValueError as error:
525 raise Rejected(str(error)) from error
526 if operation == "vm.owner":
527 control = True
528 elif operation == "vm.act":
529 control = payload["action"] in {"reset", "destroy"}
530 else:
531 action = payload["request"]
532 identity = action.get("id")
533 operation_handle = action.get("action") == "process" and isinstance(identity, str) and identity.startswith("operation:")
534 control = operation_handle and action.get("operation") == "status"
535 if (action.get("action") == "status" or operation_handle and action.get("operation") == "stop") and (
536 not vms.mac_vms.CONFIG or payload["name"] != vms.mac_vms.CONFIG["name"]
537 ):
538 active_snapshot = vms.computer_activity(payload["name"])
539 control = active_snapshot is not None
540 if not control:
541 if not WORK_SLOTS.acquire(blocking=False):
542 raise dashboard_runs.Error(503, "The host is busy. Wait for an operation to finish, then retry.")
543 resources.callback(WORK_SLOTS.release)
544 if operation in {"vm.console", "vm.serial", "vm.guest", "vm.upload"}:
545 streams = GUEST_SLOTS if operation == "vm.guest" else STREAM_SLOTS
515 if not streams.acquire(blocking=False):546 if not streams.acquire(blocking=False):
516 raise Rejected("Four VM connections or uploads are open. Close one and try again.")547 raise Rejected("Four VM connections or uploads are open. Close one and try again.")
517 try:548 try:
518 if request["operation"] in {"vm.console", "vm.serial", "vm.guest"}:549 if operation in {"vm.console", "vm.serial", "vm.guest"}:
519 stream_console(connection, host, request)550 stream_console(connection, host, request)
520 else:551 else:
521 stream_upload(connection, request)552 stream_upload(connection, request)
522 return553 return
523 finally:554 finally:
524 streams.release()555 streams.release()
525 if isinstance(request, dict) and request.get("operation") == "shale.clone":556 if operation == "shale.clone":
526 if not CLONE_SLOTS.acquire(blocking=False):557 if not CLONE_SLOTS.acquire(blocking=False):
527 raise dashboard_runs.Error(503, "Two Knife clones are running. Try again shortly.")558 raise dashboard_runs.Error(503, "Two Knife clones are running. Try again shortly.")
528 try:559 try:
...@@ -530,7 +561,7 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None):...@@ -530,7 +561,7 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None):
530 return561 return
531 finally:562 finally:
532 CLONE_SLOTS.release()563 CLONE_SLOTS.release()
533 response = {"value": host.handle(request)}564 response = {"value": host.handle(request, active_snapshot=active_snapshot) if active_snapshot is not None else host.handle(request)}
534 except Rejected as error:565 except Rejected as error:
535 response = {"error": str(error), "status": 400}566 response = {"error": str(error), "status": 400}
536 except dashboard_runs.Error as error:567 except dashboard_runs.Error as error:
...@@ -558,13 +589,13 @@ def main():...@@ -558,13 +589,13 @@ def main():
558 with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener:589 with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener:
559 listener.bind(str(socket_path))590 listener.bind(str(socket_path))
560 socket_path.chmod(0o660)591 socket_path.chmod(0o660)
561 listener.listen(8)592 listener.listen(CONNECTIONS)
562 if address := os.environ.get("NOTIFY_SOCKET"):593 if address := os.environ.get("NOTIFY_SOCKET"):
563 with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as notification:594 with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as notification:
564 notification.connect("\0" + address[1:] if address.startswith("@") else address)595 notification.connect("\0" + address[1:] if address.startswith("@") else address)
565 notification.sendall(b"READY=1")596 notification.sendall(b"READY=1")
566 slots = threading.BoundedSemaphore(16)597 slots = threading.BoundedSemaphore(CONNECTIONS)
567 with concurrent.futures.ThreadPoolExecutor(max_workers=12) as workers:598 with concurrent.futures.ThreadPoolExecutor(max_workers=CONNECTIONS) as workers:
568 while True:599 while True:
569 connection, _ = listener.accept()600 connection, _ = listener.accept()
570 if not slots.acquire(blocking=False):601 if not slots.acquire(blocking=False):
tools/dashboard-stage-test.py+12
...@@ -80,6 +80,18 @@ class PreviewBoundary(unittest.TestCase):...@@ -80,6 +80,18 @@ class PreviewBoundary(unittest.TestCase):
80 self.assertEqual(host.handle(request), "real snapshot")80 self.assertEqual(host.handle(request), "real snapshot")
81 handler.assert_called_once_with(request)81 handler.assert_called_once_with(request)
8282
83 def test_busy_observation_keeps_preview_vm_access_checks(self):
84 host = stage.PreviewHost({'clone': 'globe/staging/fixture', 'computer': 'fixture'})
85 request = {'operation': 'vm.computer', 'payload': {'name': 'fixture', 'request': {'action': 'status'}}}
86 active = {'id': 'operation:' + 'a' * 32, 'action': 'exec'}
87 with patch.object(boundary.Host, 'handle', return_value={'readiness': 'busy'}) as call:
88 self.assertEqual(host.handle(request, active_snapshot=active), {'readiness': 'busy'})
89 call.assert_called_once_with(request, active_snapshot=active)
90 call.reset_mock()
91 with self.assertRaises(boundary.dashboard_runs.Error):
92 host.handle({**request, 'payload': {**request['payload'], 'name': 'personal'}}, active_snapshot=active)
93 call.assert_not_called()
94
83 def test_preview_vm_creation_and_ownership_stay_in_its_account(self):95 def test_preview_vm_creation_and_ownership_stay_in_its_account(self):
84 with tempfile.TemporaryDirectory() as directory:96 with tempfile.TemporaryDirectory() as directory:
85 root = Path(directory)97 root = Path(directory)
tools/dashboard-stage.py+7-6
...@@ -1,5 +1,6 @@...@@ -1,5 +1,6 @@
1#!/usr/bin/env python31#!/usr/bin/env python3
2import concurrent.futures2import concurrent.futures
3from contextlib import closing
3from importlib import import_module4from importlib import import_module
4import json5import json
5import os6import os
...@@ -35,11 +36,11 @@ class PreviewHost(boundary.Host):...@@ -35,11 +36,11 @@ class PreviewHost(boundary.Host):
35 super().__init__(saved["clone"].split("/", 1)[0])36 super().__init__(saved["clone"].split("/", 1)[0])
36 self.stage = saved37 self.stage = saved
3738
38 def handle(self, request):39 def handle(self, request, *, active_snapshot=None):
39 operation = request.get("operation") if isinstance(request, dict) else None40 operation = request.get("operation") if isinstance(request, dict) else None
40 owner = None41 owner = None
41 if self.stage.get("computer") and self.stage.get("mount") and operation in {"vm.computer", "vm.owner", "vm.access", "vm.console", "vm.guest", "vm.domains", "vm.computers", "vm.ai", "vm.act"}:42 if self.stage.get("computer") and self.stage.get("mount") and operation in {"vm.computer", "vm.owner", "vm.access", "vm.console", "vm.guest", "vm.domains", "vm.computers", "vm.ai", "vm.act"}:
42 with sqlite3.connect(f"file:{self.stage['mount']}/data/accounts.sqlite?mode=ro", uri=True) as accounts:43 with closing(sqlite3.connect(f"file:{self.stage['mount']}/data/accounts.sqlite?mode=ro", uri=True)) as accounts:
43 owner = accounts.execute("SELECT id FROM users WHERE username='preview'").fetchone()44 owner = accounts.execute("SELECT id FROM users WHERE username='preview'").fetchone()
44 if not owner:45 if not owner:
45 raise ValueError("The preview account is unavailable.")46 raise ValueError("The preview account is unavailable.")
...@@ -75,7 +76,7 @@ class PreviewHost(boundary.Host):...@@ -75,7 +76,7 @@ class PreviewHost(boundary.Host):
75 ).stdout)76 ).stdout)
76 mounted["filesystems"][0]["target"] = "/srv"77 mounted["filesystems"][0]["target"] = "/srv"
77 return mounted78 return mounted
78 result = super().handle(request)79 result = super().handle(request, active_snapshot=active_snapshot) if active_snapshot is not None else super().handle(request)
79 if owner and operation in {"vm.owner", "vm.domains", "vm.computers"}:80 if owner and operation in {"vm.owner", "vm.domains", "vm.computers"}:
80 if operation == "vm.owner":81 if operation == "vm.owner":
81 result = {**result, "owner": owner[0], "ownerOnly": True}82 result = {**result, "owner": owner[0], "ownerOnly": True}
...@@ -106,9 +107,9 @@ def serve(stage):...@@ -106,9 +107,9 @@ def serve(stage):
106 listener.bind(str(path))107 listener.bind(str(path))
107 os.chown(path, 0, saved["uid"])108 os.chown(path, 0, saved["uid"])
108 path.chmod(0o660)109 path.chmod(0o660)
109 listener.listen(8)110 listener.listen(boundary.CONNECTIONS)
110 slots = threading.BoundedSemaphore(16)111 slots = threading.BoundedSemaphore(boundary.CONNECTIONS)
111 with concurrent.futures.ThreadPoolExecutor(max_workers=12) as workers:112 with concurrent.futures.ThreadPoolExecutor(max_workers=boundary.CONNECTIONS) as workers:
112 while True:113 while True:
113 connection, _ = listener.accept()114 connection, _ = listener.accept()
114 if not slots.acquire(blocking=False):115 if not slots.acquire(blocking=False):
tools/vm-computer.py+13-11
...@@ -517,7 +517,8 @@ Move-Item -LiteralPath 'WORKER.new.exe' -Destination 'WORKER'...@@ -517,7 +517,8 @@ Move-Item -LiteralPath 'WORKER.new.exe' -Destination 'WORKER'
517517
518518
519519
520def handle(name, request):520def handle(name, request, *, active_snapshot=None):
521 """Admission snapshots cannot start guest work on reserved control workers."""
521 validate(request)522 validate(request)
522 vms.ensure_name(name)523 vms.ensure_name(name)
523 if request['action'] == 'process' and request['id'].startswith('operation:'):524 if request['action'] == 'process' and request['id'].startswith('operation:'):
...@@ -541,7 +542,7 @@ def handle(name, request):...@@ -541,7 +542,7 @@ def handle(name, request):
541 return {'running': False}542 return {'running': False}
542 if 'operation:' + unfinished['id'].removeprefix('snowglobe-computer-request-') != identity:543 if 'operation:' + unfinished['id'].removeprefix('snowglobe-computer-request-') != identity:
543 return {'running': False}544 return {'running': False}
544 if request['operation'] == 'status':545 if request['operation'] == 'status' or active_snapshot is not None:
545 return {'running': None, 'recovery_required': True, 'id': identity}546 return {'running': None, 'recovery_required': True, 'id': identity}
546 with vms.computer_lock(name, 'cancel', identity):547 with vms.computer_lock(name, 'cancel', identity):
547 try:548 try:
...@@ -559,14 +560,15 @@ def handle(name, request):...@@ -559,14 +560,15 @@ def handle(name, request):
559 operation_token = None560 operation_token = None
560 try:561 try:
561 with contextlib.ExitStack() as scope:562 with contextlib.ExitStack() as scope:
562 busy = False563 busy = active_snapshot is not None
563 try:564 if not busy:
564 active = scope.enter_context(vms.computer_lock(name, request['action']))565 try:
565 operation_token = OPERATION.set((name, active['id']))566 active = scope.enter_context(vms.computer_lock(name, request['action']))
566 except BlockingIOError:567 operation_token = OPERATION.set((name, active['id']))
567 if request['action'] not in ('status', 'shot'):568 except BlockingIOError:
568 raise BlockingIOError('Another computer operation is running. Read computer_status before retrying.') from None569 if request['action'] not in ('status', 'shot'):
569 busy = True570 raise BlockingIOError('Another computer operation is running. Read computer_status before retrying.') from None
571 busy = True
570 root = ET.fromstring(vms.virsh('dumpxml',name,timeout=remaining(15)))572 root = ET.fromstring(vms.virsh('dumpxml',name,timeout=remaining(15)))
571 power = vms.virsh('domstate',name,timeout=remaining(15)).strip()573 power = vms.virsh('domstate',name,timeout=remaining(15)).strip()
572 platform = root.findtext(f'metadata/{vms.NS}vm/{vms.NS}platform','linux')574 platform = root.findtext(f'metadata/{vms.NS}vm/{vms.NS}platform','linux')
...@@ -583,7 +585,7 @@ def handle(name, request):...@@ -583,7 +585,7 @@ def handle(name, request):
583 if request['action'] == 'shot':585 if request['action'] == 'shot':
584 return asyncio.run(desktop(name))586 return asyncio.run(desktop(name))
585 if busy:587 if busy:
586 active = vms.computer_activity(name)588 active = active_snapshot or vms.computer_activity(name)
587 return {**status, 'readiness': 'busy', 'capabilities': ['shot'], 'active_operation': active}589 return {**status, 'readiness': 'busy', 'capabilities': ['shot'], 'active_operation': active}
588 try:590 try:
589 with os.fdopen(vms.open_regular(vms.DISKS / name / 'computer-operation.json')) as incoming:591 with os.fdopen(vms.open_regular(vms.DISKS / name / 'computer-operation.json')) as incoming: