| author | |
| committer | |
| log | 2a5ae6ab62a20d20e9077bde2718caf8c50fce89 |
| tree | 288529eadda1f7f3656124a0d75ac3d0286b4b62 |
| parent | 6586db88bc31255e09f30e9ff799c62057736d5d |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Keep twelve ordinary host workers and reserve four connection workers for VM metadata, busy status, active-operation inspection/cancellation and forced power controls. Preserve admission snapshots so a finishing command cannot turn a reserved request into slow guest setup or cleanup. Match the listener backlog to the sixteen-connection bound and preserve preview VM isolation.
The Rust client permits recovery calls while its four ordinary slots are occupied. Real Linux Unix-peer load testing, preview access checks, 43 computer protocol tests, 28 VM boundary tests, 61 Rust tests and the complete OAuth client integration pass.
Assisted-by: gpt-6.1-sol8 files changed, 220 insertions(+), 36 deletions(-)
dashboard/src/host.rs+15-8| ... | ... | @@ -16,14 +16,21 @@ pub async fn sample(app: Arc<App>) -> Result<Arc<Document>> { |
| 16 | 16 | } |
| 17 | 17 | |
| 18 | 18 | pub async fn call(request: Value) -> Result<Value> { |
| 19 | let _slot = tokio::time::timeout(Duration::from_secs(70), SLOTS.acquire()) | |
| 20 | .await | |
| 21 | .map_err(|_| { | |
| 22 | Error::new( | |
| 23 | 504, | |
| 24 | "The host operation is taking too long. Check its logs, then retry.", | |
| 25 | ) | |
| 26 | })??; | |
| 19 | let computer = &request["payload"]["request"]; | |
| 20 | let control = matches!(request["operation"].as_str(), Some("vm.owner" | "vm.computers")) | |
| 21 | || (request["operation"] == "vm.act" | |
| 22 | && matches!(request["payload"]["action"].as_str(), Some("reset" | "destroy"))) | |
| 23 | || (request["operation"] == "vm.computer" | |
| 24 | && (matches!(computer["action"].as_str(), Some("status" | "shot")) | |
| 25 | || (computer["action"] == "process" | |
| 26 | && computer["id"].as_str().is_some_and(|id| id.starts_with("operation:"))))); | |
| 27 | let _slot = if control { | |
| 28 | None | |
| 29 | } else { | |
| 30 | Some(tokio::time::timeout(Duration::from_secs(70), SLOTS.acquire()) | |
| 31 | .await | |
| 32 | .map_err(|_| Error::new(504, "The host operation is taking too long. Check its logs, then retry."))??) | |
| 33 | }; | |
| 27 | 34 | open(request).await.map(|(value, _)| value) |
| 28 | 35 | } |
| 29 | 36 |
tools/computer-client-test.py+26| ... | ... | @@ -15,6 +15,7 @@ import tempfile |
| 15 | 15 | import threading |
| 16 | 16 | import time |
| 17 | 17 | import unittest |
| 18 | from concurrent.futures import ThreadPoolExecutor | |
| 18 | 19 | from pathlib import Path |
| 19 | 20 | from urllib.parse import urlsplit |
| 20 | 21 | from urllib.request import Request |
| ... | ... | @@ -31,6 +32,8 @@ class Host(socketserver.StreamRequestHandler): |
| 31 | 32 | files = {} |
| 32 | 33 | power_actions = [] |
| 33 | 34 | writes = [] |
| 35 | started = threading.Semaphore(0) | |
| 36 | release = threading.Event() | |
| 34 | 37 | |
| 35 | 38 | def handle(self): |
| 36 | 39 | line = self.rfile.readline() |
| ... | ... | @@ -60,7 +63,14 @@ class Host(socketserver.StreamRequestHandler): |
| 60 | 63 | raise KeyError(operation) |
| 61 | 64 | request, files = payload["request"], self.files |
| 62 | 65 | if request["action"] == "exec": |
| 66 | if request["script"].startswith("held-"): | |
| 67 | self.started.release() | |
| 68 | self.release.wait(10) | |
| 63 | 69 | return {"exit": 0, "stdout": "ran " + request["script"], "png": PNG} |
| 70 | if request["action"] == "status": | |
| 71 | return {"readiness": "busy", "active_operation": {"id": "operation:" + "a" * 32}} | |
| 72 | if request["action"] == "process": | |
| 73 | return {"running": True, "stopping": request["operation"] == "stop"} | |
| 64 | 74 | if request["action"] == "shot": |
| 65 | 75 | return {"png": PNG} |
| 66 | 76 | operation, path = request["operation"], request["path"] |
| ... | ... | @@ -183,6 +193,22 @@ class Client(unittest.TestCase): |
| 183 | 193 | self.assertEqual(vm.call("computer_vm", vm="test-vm", operation=operation), |
| 184 | 194 | {"id":"test-vm", "requested":operation}) |
| 185 | 195 | self.assertEqual(Host.power_actions, ["destroy", "reset"]) |
| 196 | with ThreadPoolExecutor(max_workers=4) as pool: | |
| 197 | held = [pool.submit(vm.call, "computer_exec", vm="test-vm", script=f"held-{index}") for index in range(4)] | |
| 198 | try: | |
| 199 | for _ in held: | |
| 200 | self.assertTrue(Host.started.acquire(timeout=3)) | |
| 201 | started = time.monotonic() | |
| 202 | status = vm.call("computer_status", vm="test-vm") | |
| 203 | self.assertEqual(status["readiness"], "busy") | |
| 204 | stopped = vm.call("computer_process", vm="test-vm", id=status["active_operation"]["id"], operation="stop") | |
| 205 | self.assertTrue(stopped["stopping"]) | |
| 206 | vm.call("computer_vm", vm="test-vm", operation="reset") | |
| 207 | self.assertLess(time.monotonic() - started, 2, "recovery controls waited behind long commands") | |
| 208 | finally: | |
| 209 | Host.release.set() | |
| 210 | for future in held: | |
| 211 | self.assertEqual(future.result()["exit"], 0) | |
| 186 | 212 | with self.assertRaisesRegex(computer.ComputerError, "granted"): |
| 187 | 213 | vm.call("computer_shot", vm="other-vm") |
| 188 | 214 |
tools/computer-test.py+18| ... | ... | @@ -226,6 +226,24 @@ class GuestCancellation(unittest.TestCase): |
| 226 | 226 | self.assertEqual(status['capabilities'], []) |
| 227 | 227 | work.assert_not_called() |
| 228 | 228 | |
| 229 | def test_control_snapshot_cannot_turn_into_slow_guest_setup_or_cleanup(self): | |
| 230 | with computer.vms.computer_lock('fixture', 'exec'): | |
| 231 | active = computer.vms.computer_activity('fixture') | |
| 232 | operation = {'platform': 'windows', 'id': active['id'].removeprefix('operation:')} | |
| 233 | computer.record_operation('fixture', operation) | |
| 234 | def virsh(action, *args, **kwargs): | |
| 235 | return 'running' if action == 'domstate' else '<domain/>' | |
| 236 | with patch.object(computer.vms, 'virsh', virsh), patch.object(computer, 'worker') as worker, \ | |
| 237 | patch.object(computer, 'finish_operation') as cleanup: | |
| 238 | status = computer.handle('fixture', {'action': 'status'}, active_snapshot=active) | |
| 239 | self.assertEqual(status['readiness'], 'busy') | |
| 240 | self.assertEqual(status['active_operation'], active) | |
| 241 | stopped = computer.handle('fixture', {'action': 'process', 'id': active['id'], 'operation': 'stop'}, active_snapshot=active) | |
| 242 | self.assertIsNone(stopped['running']) | |
| 243 | self.assertTrue(stopped['recovery_required']) | |
| 244 | worker.assert_not_called() | |
| 245 | cleanup.assert_not_called() | |
| 246 | ||
| 229 | 247 | def test_active_operation_can_be_inspected_and_cancelled_during_execution(self): |
| 230 | 248 | started = threading.Event() |
| 231 | 249 | def virsh(action, *args, **kwargs): |
tools/dashboard-host-test.py+87| ... | ... | @@ -5,9 +5,12 @@ from pathlib import Path |
| 5 | 5 | import sys |
| 6 | 6 | import subprocess |
| 7 | 7 | import os |
| 8 | import pwd | |
| 8 | 9 | import socket |
| 9 | 10 | import struct |
| 11 | import tempfile | |
| 10 | 12 | import threading |
| 13 | import time | |
| 11 | 14 | import unittest |
| 12 | 15 | from unittest.mock import patch |
| 13 | 16 | |
| ... | ... | @@ -190,5 +193,89 @@ class Boundary(unittest.TestCase): |
| 190 | 193 | command.assert_not_called() |
| 191 | 194 | |
| 192 | 195 | |
| 196 | @unittest.skipUnless(sys.platform == 'linux', 'Linux Unix peer credentials') | |
| 197 | class Capacity(unittest.TestCase): | |
| 198 | def test_recovery_controls_survive_full_ordinary_capacity(self): | |
| 199 | program = ''' | |
| 200 | from importlib import import_module | |
| 201 | import json, sys, time | |
| 202 | from pathlib import Path | |
| 203 | m = import_module('dashboard-host') | |
| 204 | root = Path(sys.argv[1]) | |
| 205 | m.vms.DISKS = root / 'vms' | |
| 206 | for name in ['fixture', 'idle', 'managed-mac']: | |
| 207 | (m.vms.DISKS / name).mkdir(parents=True) | |
| 208 | class Host: | |
| 209 | def __init__(self, pool): pass | |
| 210 | def handle(self, request, *, active_snapshot=None): | |
| 211 | if request['operation'] == 'fixture.long': | |
| 212 | (root / ('started-' + str(request['index']))).touch() | |
| 213 | while not (root / 'release').exists(): time.sleep(.01) | |
| 214 | return {'operation': request['operation'], 'active_snapshot': active_snapshot} | |
| 215 | m.Host = Host | |
| 216 | m.vms.mac_vms.CONFIG = {'name': 'managed-mac'} | |
| 217 | with m.vms.computer_lock('fixture', 'exec') as active, m.vms.computer_lock('managed-mac', 'exec'): | |
| 218 | (root / 'active.json').write_text(json.dumps(active)) | |
| 219 | m.main() | |
| 220 | ''' | |
| 221 | with tempfile.TemporaryDirectory() as directory: | |
| 222 | root = Path(directory) | |
| 223 | path = root / 'host.sock' | |
| 224 | environment = {**os.environ, 'SNOWGLOBE_POOL': 'fixture', | |
| 225 | 'SNOWGLOBE_DASHBOARD_USER': pwd.getpwuid(os.getuid()).pw_name, | |
| 226 | 'SNOWGLOBE_HOST_SOCKET': str(path)} | |
| 227 | environment.pop('NOTIFY_SOCKET', None) | |
| 228 | process = subprocess.Popen([sys.executable, '-c', program, str(root)], cwd=Path(__file__).parent, env=environment, | |
| 229 | stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) | |
| 230 | held = [] | |
| 231 | def connect(request): | |
| 232 | connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) | |
| 233 | connection.settimeout(2) | |
| 234 | try: | |
| 235 | connection.connect(str(path)) | |
| 236 | connection.sendall(json.dumps(request).encode() + b'\n') | |
| 237 | except BaseException: | |
| 238 | connection.close() | |
| 239 | raise | |
| 240 | return connection | |
| 241 | def call(request): | |
| 242 | with connect(request) as connection, connection.makefile('rb') as incoming: | |
| 243 | length = struct.unpack('!I', incoming.read(4))[0] | |
| 244 | return json.loads(incoming.read(length)) | |
| 245 | try: | |
| 246 | deadline = time.monotonic() + 5 | |
| 247 | while not path.exists(): | |
| 248 | self.assertIsNone(process.poll()) | |
| 249 | self.assertLess(time.monotonic(), deadline) | |
| 250 | time.sleep(.01) | |
| 251 | for index in range(host_module.WORKERS): | |
| 252 | held.append(connect({'operation': 'fixture.long', 'index': index})) | |
| 253 | while len(list(root.glob('started-*'))) != host_module.WORKERS: | |
| 254 | self.assertLess(time.monotonic(), deadline) | |
| 255 | time.sleep(.01) | |
| 256 | active = json.loads((root / 'active.json').read_text()) | |
| 257 | for request in [ | |
| 258 | {'operation': 'vm.owner', 'payload': {'name': 'fixture'}}, {'operation': 'vm.computers'}, | |
| 259 | *({'operation': 'vm.act', 'payload': {'name': 'fixture', 'action': action}} for action in ['reset', 'destroy']), | |
| 260 | *({'operation': 'vm.computer', 'payload': {'name': 'fixture', 'request': request}} for request in [ | |
| 261 | {'action': 'status'}, {'action': 'process', 'id': active['id'], 'operation': 'status'}, | |
| 262 | {'action': 'process', 'id': active['id'], 'operation': 'stop'}]), | |
| 263 | ]: | |
| 264 | self.assertEqual(call(request)['value']['operation'], request['operation']) | |
| 265 | for request in [ | |
| 266 | {'operation': 'host.sample'}, {'operation': 'vm.act', 'payload': {'name': 'fixture', 'action': 'start'}}, | |
| 267 | {'operation': 'vm.computer', 'payload': {'name': 'idle', 'request': {'action': 'status'}}}, | |
| 268 | {'operation': 'vm.computer', 'payload': {'name': 'managed-mac', 'request': {'action': 'status'}}}, | |
| 269 | {'operation': 'vm.computer', 'payload': {'name': 'idle', 'request': {'action': 'process', 'id': active['id'], 'operation': 'stop'}}}, | |
| 270 | ]: | |
| 271 | self.assertEqual(call(request)['status'], 503) | |
| 272 | finally: | |
| 273 | (root / 'release').touch() | |
| 274 | for connection in held: | |
| 275 | connection.close() | |
| 276 | process.terminate() | |
| 277 | process.communicate(timeout=5) | |
| 278 | ||
| 279 | ||
| 193 | 280 | if __name__ == "__main__": |
| 194 | 281 | unittest.main() |
tools/dashboard-host.py+42-11| ... | ... | @@ -40,6 +40,9 @@ TEXT_FIELDS = {"name", "compression", "mountpoint", "origin", "mounted"} |
| 40 | 40 | PROTOCOL = json.loads((Path(__file__).resolve().parent.parent / "dashboard/src/host-protocol.json").read_text()) |
| 41 | 41 | MAX_REQUEST = PROTOCOL['request'] |
| 42 | 42 | MAX_RESPONSE = PROTOCOL['response'] |
| 43 | WORKERS = 12 | |
| 44 | CONNECTIONS = WORKERS + 4 | |
| 45 | WORK_SLOTS = threading.BoundedSemaphore(WORKERS) | |
| 43 | 46 | STREAM_SLOTS = threading.BoundedSemaphore(4) |
| 44 | 47 | GUEST_SLOTS = threading.BoundedSemaphore(4) |
| 45 | 48 | CLONE_SLOTS = threading.BoundedSemaphore(2) |
| ... | ... | @@ -74,7 +77,7 @@ class Host: |
| 74 | 77 | raise Rejected("Choose a snapshot from the list.") |
| 75 | 78 | return value |
| 76 | 79 | |
| 77 | def handle(self, request): | |
| 80 | def handle(self, request, *, active_snapshot=None): | |
| 78 | 81 | if not isinstance(request, dict): |
| 79 | 82 | raise Rejected("Choose a supported host operation.") |
| 80 | 83 | operation = request.get("operation") |
| ... | ... | @@ -206,7 +209,7 @@ class Host: |
| 206 | 209 | raise Rejected(str(error)) from error |
| 207 | 210 | if action == "computer": |
| 208 | 211 | try: |
| 209 | return importlib.import_module("vm-computer").handle(payload["name"], payload["request"]) | |
| 212 | return importlib.import_module("vm-computer").handle(payload["name"], payload["request"], active_snapshot=active_snapshot) | |
| 210 | 213 | except (BlockingIOError, InterruptedError) as error: |
| 211 | 214 | raise dashboard_runs.Error(409, str(error) or "Another computer operation is running. Read computer_status before retrying.") from None |
| 212 | 215 | except (TimeoutError, subprocess.TimeoutExpired): |
| ... | ... | @@ -485,7 +488,7 @@ def stream_clone(connection, plan): |
| 485 | 488 | |
| 486 | 489 | |
| 487 | 490 | def serve_connection(connection, host, allowed_uid, allowed_operations=None): |
| 488 | with connection: | |
| 491 | with connection, contextlib.ExitStack() as resources: | |
| 489 | 492 | connection.settimeout(65) |
| 490 | 493 | _, uid, _ = struct.unpack("3i", connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12)) |
| 491 | 494 | if uid != allowed_uid: |
| ... | ... | @@ -510,19 +513,47 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None): |
| 510 | 513 | or request["operation"] not in allowed_operations |
| 511 | 514 | ): |
| 512 | 515 | raise dashboard_runs.Error(403, "This preview cannot change Snow Globe. Use production for host controls.") |
| 513 | if isinstance(request, dict) and request.get("operation") in {"vm.console", "vm.serial", "vm.guest", "vm.upload"}: | |
| 514 | streams = GUEST_SLOTS if request["operation"] == "vm.guest" else STREAM_SLOTS | |
| 516 | operation = request.get("operation") if isinstance(request, dict) else None | |
| 517 | operation = operation if isinstance(operation, str) else None | |
| 518 | active_snapshot = None | |
| 519 | control = operation == "vm.computers" | |
| 520 | if operation in {"vm.owner", "vm.act", "vm.computer"}: | |
| 521 | payload = request.get("payload") | |
| 522 | try: | |
| 523 | vms.validate(operation[3:], payload) | |
| 524 | except ValueError as error: | |
| 525 | raise Rejected(str(error)) from error | |
| 526 | if operation == "vm.owner": | |
| 527 | control = True | |
| 528 | elif operation == "vm.act": | |
| 529 | control = payload["action"] in {"reset", "destroy"} | |
| 530 | else: | |
| 531 | action = payload["request"] | |
| 532 | identity = action.get("id") | |
| 533 | operation_handle = action.get("action") == "process" and isinstance(identity, str) and identity.startswith("operation:") | |
| 534 | control = operation_handle and action.get("operation") == "status" | |
| 535 | if (action.get("action") == "status" or operation_handle and action.get("operation") == "stop") and ( | |
| 536 | not vms.mac_vms.CONFIG or payload["name"] != vms.mac_vms.CONFIG["name"] | |
| 537 | ): | |
| 538 | active_snapshot = vms.computer_activity(payload["name"]) | |
| 539 | control = active_snapshot is not None | |
| 540 | if not control: | |
| 541 | if not WORK_SLOTS.acquire(blocking=False): | |
| 542 | raise dashboard_runs.Error(503, "The host is busy. Wait for an operation to finish, then retry.") | |
| 543 | resources.callback(WORK_SLOTS.release) | |
| 544 | if operation in {"vm.console", "vm.serial", "vm.guest", "vm.upload"}: | |
| 545 | streams = GUEST_SLOTS if operation == "vm.guest" else STREAM_SLOTS | |
| 515 | 546 | if not streams.acquire(blocking=False): |
| 516 | 547 | raise Rejected("Four VM connections or uploads are open. Close one and try again.") |
| 517 | 548 | try: |
| 518 | if request["operation"] in {"vm.console", "vm.serial", "vm.guest"}: | |
| 549 | if operation in {"vm.console", "vm.serial", "vm.guest"}: | |
| 519 | 550 | stream_console(connection, host, request) |
| 520 | 551 | else: |
| 521 | 552 | stream_upload(connection, request) |
| 522 | 553 | return |
| 523 | 554 | finally: |
| 524 | 555 | streams.release() |
| 525 | if isinstance(request, dict) and request.get("operation") == "shale.clone": | |
| 556 | if operation == "shale.clone": | |
| 526 | 557 | if not CLONE_SLOTS.acquire(blocking=False): |
| 527 | 558 | raise dashboard_runs.Error(503, "Two Knife clones are running. Try again shortly.") |
| 528 | 559 | try: |
| ... | ... | @@ -530,7 +561,7 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None): |
| 530 | 561 | return |
| 531 | 562 | finally: |
| 532 | 563 | CLONE_SLOTS.release() |
| 533 | response = {"value": host.handle(request)} | |
| 564 | response = {"value": host.handle(request, active_snapshot=active_snapshot) if active_snapshot is not None else host.handle(request)} | |
| 534 | 565 | except Rejected as error: |
| 535 | 566 | response = {"error": str(error), "status": 400} |
| 536 | 567 | except dashboard_runs.Error as error: |
| ... | ... | @@ -558,13 +589,13 @@ def main(): |
| 558 | 589 | with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener: |
| 559 | 590 | listener.bind(str(socket_path)) |
| 560 | 591 | socket_path.chmod(0o660) |
| 561 | listener.listen(8) | |
| 592 | listener.listen(CONNECTIONS) | |
| 562 | 593 | if address := os.environ.get("NOTIFY_SOCKET"): |
| 563 | 594 | with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as notification: |
| 564 | 595 | notification.connect("\0" + address[1:] if address.startswith("@") else address) |
| 565 | 596 | notification.sendall(b"READY=1") |
| 566 | slots = threading.BoundedSemaphore(16) | |
| 567 | with concurrent.futures.ThreadPoolExecutor(max_workers=12) as workers: | |
| 597 | slots = threading.BoundedSemaphore(CONNECTIONS) | |
| 598 | with concurrent.futures.ThreadPoolExecutor(max_workers=CONNECTIONS) as workers: | |
| 568 | 599 | while True: |
| 569 | 600 | connection, _ = listener.accept() |
| 570 | 601 | if not slots.acquire(blocking=False): |
tools/dashboard-stage-test.py+12| ... | ... | @@ -80,6 +80,18 @@ class PreviewBoundary(unittest.TestCase): |
| 80 | 80 | self.assertEqual(host.handle(request), "real snapshot") |
| 81 | 81 | handler.assert_called_once_with(request) |
| 82 | 82 | |
| 83 | def test_busy_observation_keeps_preview_vm_access_checks(self): | |
| 84 | host = stage.PreviewHost({'clone': 'globe/staging/fixture', 'computer': 'fixture'}) | |
| 85 | request = {'operation': 'vm.computer', 'payload': {'name': 'fixture', 'request': {'action': 'status'}}} | |
| 86 | active = {'id': 'operation:' + 'a' * 32, 'action': 'exec'} | |
| 87 | with patch.object(boundary.Host, 'handle', return_value={'readiness': 'busy'}) as call: | |
| 88 | self.assertEqual(host.handle(request, active_snapshot=active), {'readiness': 'busy'}) | |
| 89 | call.assert_called_once_with(request, active_snapshot=active) | |
| 90 | call.reset_mock() | |
| 91 | with self.assertRaises(boundary.dashboard_runs.Error): | |
| 92 | host.handle({**request, 'payload': {**request['payload'], 'name': 'personal'}}, active_snapshot=active) | |
| 93 | call.assert_not_called() | |
| 94 | ||
| 83 | 95 | def test_preview_vm_creation_and_ownership_stay_in_its_account(self): |
| 84 | 96 | with tempfile.TemporaryDirectory() as directory: |
| 85 | 97 | root = Path(directory) |
tools/dashboard-stage.py+7-6| ... | ... | @@ -1,5 +1,6 @@ |
| 1 | 1 | #!/usr/bin/env python3 |
| 2 | 2 | import concurrent.futures |
| 3 | from contextlib import closing | |
| 3 | 4 | from importlib import import_module |
| 4 | 5 | import json |
| 5 | 6 | import os |
| ... | ... | @@ -35,11 +36,11 @@ class PreviewHost(boundary.Host): |
| 35 | 36 | super().__init__(saved["clone"].split("/", 1)[0]) |
| 36 | 37 | self.stage = saved |
| 37 | 38 | |
| 38 | def handle(self, request): | |
| 39 | def handle(self, request, *, active_snapshot=None): | |
| 39 | 40 | operation = request.get("operation") if isinstance(request, dict) else None |
| 40 | 41 | owner = None |
| 41 | 42 | if self.stage.get("computer") and self.stage.get("mount") and operation in {"vm.computer", "vm.owner", "vm.access", "vm.console", "vm.guest", "vm.domains", "vm.computers", "vm.ai", "vm.act"}: |
| 42 | with sqlite3.connect(f"file:{self.stage['mount']}/data/accounts.sqlite?mode=ro", uri=True) as accounts: | |
| 43 | with closing(sqlite3.connect(f"file:{self.stage['mount']}/data/accounts.sqlite?mode=ro", uri=True)) as accounts: | |
| 43 | 44 | owner = accounts.execute("SELECT id FROM users WHERE username='preview'").fetchone() |
| 44 | 45 | if not owner: |
| 45 | 46 | raise ValueError("The preview account is unavailable.") |
| ... | ... | @@ -75,7 +76,7 @@ class PreviewHost(boundary.Host): |
| 75 | 76 | ).stdout) |
| 76 | 77 | mounted["filesystems"][0]["target"] = "/srv" |
| 77 | 78 | return mounted |
| 78 | result = super().handle(request) | |
| 79 | result = super().handle(request, active_snapshot=active_snapshot) if active_snapshot is not None else super().handle(request) | |
| 79 | 80 | if owner and operation in {"vm.owner", "vm.domains", "vm.computers"}: |
| 80 | 81 | if operation == "vm.owner": |
| 81 | 82 | result = {**result, "owner": owner[0], "ownerOnly": True} |
| ... | ... | @@ -106,9 +107,9 @@ def serve(stage): |
| 106 | 107 | listener.bind(str(path)) |
| 107 | 108 | os.chown(path, 0, saved["uid"]) |
| 108 | 109 | path.chmod(0o660) |
| 109 | listener.listen(8) | |
| 110 | slots = threading.BoundedSemaphore(16) | |
| 111 | with concurrent.futures.ThreadPoolExecutor(max_workers=12) as workers: | |
| 110 | listener.listen(boundary.CONNECTIONS) | |
| 111 | slots = threading.BoundedSemaphore(boundary.CONNECTIONS) | |
| 112 | with concurrent.futures.ThreadPoolExecutor(max_workers=boundary.CONNECTIONS) as workers: | |
| 112 | 113 | while True: |
| 113 | 114 | connection, _ = listener.accept() |
| 114 | 115 | if not slots.acquire(blocking=False): |
tools/vm-computer.py+13-11| ... | ... | @@ -517,7 +517,8 @@ Move-Item -LiteralPath 'WORKER.new.exe' -Destination 'WORKER' |
| 517 | 517 | |
| 518 | 518 | |
| 519 | 519 | |
| 520 | def handle(name, request): | |
| 520 | def handle(name, request, *, active_snapshot=None): | |
| 521 | """Admission snapshots cannot start guest work on reserved control workers.""" | |
| 521 | 522 | validate(request) |
| 522 | 523 | vms.ensure_name(name) |
| 523 | 524 | if request['action'] == 'process' and request['id'].startswith('operation:'): |
| ... | ... | @@ -541,7 +542,7 @@ def handle(name, request): |
| 541 | 542 | return {'running': False} |
| 542 | 543 | if 'operation:' + unfinished['id'].removeprefix('snowglobe-computer-request-') != identity: |
| 543 | 544 | return {'running': False} |
| 544 | if request['operation'] == 'status': | |
| 545 | if request['operation'] == 'status' or active_snapshot is not None: | |
| 545 | 546 | return {'running': None, 'recovery_required': True, 'id': identity} |
| 546 | 547 | with vms.computer_lock(name, 'cancel', identity): |
| 547 | 548 | try: |
| ... | ... | @@ -559,14 +560,15 @@ def handle(name, request): |
| 559 | 560 | operation_token = None |
| 560 | 561 | try: |
| 561 | 562 | with contextlib.ExitStack() as scope: |
| 562 | busy = False | |
| 563 | try: | |
| 564 | active = scope.enter_context(vms.computer_lock(name, request['action'])) | |
| 565 | operation_token = OPERATION.set((name, active['id'])) | |
| 566 | except BlockingIOError: | |
| 567 | if request['action'] not in ('status', 'shot'): | |
| 568 | raise BlockingIOError('Another computer operation is running. Read computer_status before retrying.') from None | |
| 569 | busy = True | |
| 563 | busy = active_snapshot is not None | |
| 564 | if not busy: | |
| 565 | try: | |
| 566 | active = scope.enter_context(vms.computer_lock(name, request['action'])) | |
| 567 | operation_token = OPERATION.set((name, active['id'])) | |
| 568 | except BlockingIOError: | |
| 569 | if request['action'] not in ('status', 'shot'): | |
| 570 | raise BlockingIOError('Another computer operation is running. Read computer_status before retrying.') from None | |
| 571 | busy = True | |
| 570 | 572 | root = ET.fromstring(vms.virsh('dumpxml',name,timeout=remaining(15))) |
| 571 | 573 | power = vms.virsh('domstate',name,timeout=remaining(15)).strip() |
| 572 | 574 | platform = root.findtext(f'metadata/{vms.NS}vm/{vms.NS}platform','linux') |
| ... | ... | @@ -583,7 +585,7 @@ def handle(name, request): |
| 583 | 585 | if request['action'] == 'shot': |
| 584 | 586 | return asyncio.run(desktop(name)) |
| 585 | 587 | if busy: |
| 586 | active = vms.computer_activity(name) | |
| 588 | active = active_snapshot or vms.computer_activity(name) | |
| 587 | 589 | return {**status, 'readiness': 'busy', 'capabilities': ['shot'], 'active_operation': active} |
| 588 | 590 | try: |
| 589 | 591 | with os.fdopen(vms.open_regular(vms.DISKS / name / 'computer-operation.json')) as incoming: |