| author | |
| committer | |
| log | 2a5ae6ab62a20d20e9077bde2718caf8c50fce89 |
| tree | 288529eadda1f7f3656124a0d75ac3d0286b4b62 |
| parent | 6586db88bc31255e09f30e9ff799c62057736d5d |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Keep twelve ordinary host workers and reserve four connection workers for VM metadata, busy status, active-operation inspection/cancellation and forced power controls. Preserve admission snapshots so a finishing command cannot turn a reserved request into slow guest setup or cleanup. Match the listener backlog to the sixteen-connection bound and preserve preview VM isolation.
The Rust client permits recovery calls while its four ordinary slots are occupied. Real Linux Unix-peer load testing, preview access checks, 43 computer protocol tests, 28 VM boundary tests, 61 Rust tests and the complete OAuth client integration pass.
Assisted-by: gpt-6.1-sol8 files changed, 220 insertions(+), 36 deletions(-)
dashboard/src/host.rs+15-8| ... | @@ -16,14 +16,21 @@ pub async fn sample(app: Arc<App>) -> Result<Arc<Document>> { | ... | @@ -16,14 +16,21 @@ pub async fn sample(app: Arc<App>) -> Result<Arc<Document>> { |
| 16 | } | 16 | } |
| 17 | 17 | ||
| 18 | pub async fn call(request: Value) -> Result<Value> { | 18 | pub async fn call(request: Value) -> Result<Value> { |
| 19 | let _slot = tokio::time::timeout(Duration::from_secs(70), SLOTS.acquire()) | 19 | let computer = &request["payload"]["request"]; |
| 20 | .await | 20 | let control = matches!(request["operation"].as_str(), Some("vm.owner" | "vm.computers")) |
| 21 | .map_err(|_| { | 21 | || (request["operation"] == "vm.act" |
| 22 | Error::new( | 22 | && matches!(request["payload"]["action"].as_str(), Some("reset" | "destroy"))) |
| 23 | 504, | 23 | || (request["operation"] == "vm.computer" |
| 24 | "The host operation is taking too long. Check its logs, then retry.", | 24 | && (matches!(computer["action"].as_str(), Some("status" | "shot")) |
| 25 | ) | 25 | || (computer["action"] == "process" |
| 26 | })??; | 26 | && computer["id"].as_str().is_some_and(|id| id.starts_with("operation:"))))); |
| 27 | let _slot = if control { | ||
| 28 | None | ||
| 29 | } else { | ||
| 30 | Some(tokio::time::timeout(Duration::from_secs(70), SLOTS.acquire()) | ||
| 31 | .await | ||
| 32 | .map_err(|_| Error::new(504, "The host operation is taking too long. Check its logs, then retry."))??) | ||
| 33 | }; | ||
| 27 | open(request).await.map(|(value, _)| value) | 34 | open(request).await.map(|(value, _)| value) |
| 28 | } | 35 | } |
| 29 | 36 |
tools/computer-client-test.py+26| ... | @@ -15,6 +15,7 @@ import tempfile | ... | @@ -15,6 +15,7 @@ import tempfile |
| 15 | import threading | 15 | import threading |
| 16 | import time | 16 | import time |
| 17 | import unittest | 17 | import unittest |
| 18 | from concurrent.futures import ThreadPoolExecutor | ||
| 18 | from pathlib import Path | 19 | from pathlib import Path |
| 19 | from urllib.parse import urlsplit | 20 | from urllib.parse import urlsplit |
| 20 | from urllib.request import Request | 21 | from urllib.request import Request |
| ... | @@ -31,6 +32,8 @@ class Host(socketserver.StreamRequestHandler): | ... | @@ -31,6 +32,8 @@ class Host(socketserver.StreamRequestHandler): |
| 31 | files = {} | 32 | files = {} |
| 32 | power_actions = [] | 33 | power_actions = [] |
| 33 | writes = [] | 34 | writes = [] |
| 35 | started = threading.Semaphore(0) | ||
| 36 | release = threading.Event() | ||
| 34 | 37 | ||
| 35 | def handle(self): | 38 | def handle(self): |
| 36 | line = self.rfile.readline() | 39 | line = self.rfile.readline() |
| ... | @@ -60,7 +63,14 @@ class Host(socketserver.StreamRequestHandler): | ... | @@ -60,7 +63,14 @@ class Host(socketserver.StreamRequestHandler): |
| 60 | raise KeyError(operation) | 63 | raise KeyError(operation) |
| 61 | request, files = payload["request"], self.files | 64 | request, files = payload["request"], self.files |
| 62 | if request["action"] == "exec": | 65 | if request["action"] == "exec": |
| 66 | if request["script"].startswith("held-"): | ||
| 67 | self.started.release() | ||
| 68 | self.release.wait(10) | ||
| 63 | return {"exit": 0, "stdout": "ran " + request["script"], "png": PNG} | 69 | return {"exit": 0, "stdout": "ran " + request["script"], "png": PNG} |
| 70 | if request["action"] == "status": | ||
| 71 | return {"readiness": "busy", "active_operation": {"id": "operation:" + "a" * 32}} | ||
| 72 | if request["action"] == "process": | ||
| 73 | return {"running": True, "stopping": request["operation"] == "stop"} | ||
| 64 | if request["action"] == "shot": | 74 | if request["action"] == "shot": |
| 65 | return {"png": PNG} | 75 | return {"png": PNG} |
| 66 | operation, path = request["operation"], request["path"] | 76 | operation, path = request["operation"], request["path"] |
| ... | @@ -183,6 +193,22 @@ class Client(unittest.TestCase): | ... | @@ -183,6 +193,22 @@ class Client(unittest.TestCase): |
| 183 | self.assertEqual(vm.call("computer_vm", vm="test-vm", operation=operation), | 193 | self.assertEqual(vm.call("computer_vm", vm="test-vm", operation=operation), |
| 184 | {"id":"test-vm", "requested":operation}) | 194 | {"id":"test-vm", "requested":operation}) |
| 185 | self.assertEqual(Host.power_actions, ["destroy", "reset"]) | 195 | self.assertEqual(Host.power_actions, ["destroy", "reset"]) |
| 196 | with ThreadPoolExecutor(max_workers=4) as pool: | ||
| 197 | held = [pool.submit(vm.call, "computer_exec", vm="test-vm", script=f"held-{index}") for index in range(4)] | ||
| 198 | try: | ||
| 199 | for _ in held: | ||
| 200 | self.assertTrue(Host.started.acquire(timeout=3)) | ||
| 201 | started = time.monotonic() | ||
| 202 | status = vm.call("computer_status", vm="test-vm") | ||
| 203 | self.assertEqual(status["readiness"], "busy") | ||
| 204 | stopped = vm.call("computer_process", vm="test-vm", id=status["active_operation"]["id"], operation="stop") | ||
| 205 | self.assertTrue(stopped["stopping"]) | ||
| 206 | vm.call("computer_vm", vm="test-vm", operation="reset") | ||
| 207 | self.assertLess(time.monotonic() - started, 2, "recovery controls waited behind long commands") | ||
| 208 | finally: | ||
| 209 | Host.release.set() | ||
| 210 | for future in held: | ||
| 211 | self.assertEqual(future.result()["exit"], 0) | ||
| 186 | with self.assertRaisesRegex(computer.ComputerError, "granted"): | 212 | with self.assertRaisesRegex(computer.ComputerError, "granted"): |
| 187 | vm.call("computer_shot", vm="other-vm") | 213 | vm.call("computer_shot", vm="other-vm") |
| 188 | 214 |
tools/computer-test.py+18| ... | @@ -226,6 +226,24 @@ class GuestCancellation(unittest.TestCase): | ... | @@ -226,6 +226,24 @@ class GuestCancellation(unittest.TestCase): |
| 226 | self.assertEqual(status['capabilities'], []) | 226 | self.assertEqual(status['capabilities'], []) |
| 227 | work.assert_not_called() | 227 | work.assert_not_called() |
| 228 | 228 | ||
| 229 | def test_control_snapshot_cannot_turn_into_slow_guest_setup_or_cleanup(self): | ||
| 230 | with computer.vms.computer_lock('fixture', 'exec'): | ||
| 231 | active = computer.vms.computer_activity('fixture') | ||
| 232 | operation = {'platform': 'windows', 'id': active['id'].removeprefix('operation:')} | ||
| 233 | computer.record_operation('fixture', operation) | ||
| 234 | def virsh(action, *args, **kwargs): | ||
| 235 | return 'running' if action == 'domstate' else '<domain/>' | ||
| 236 | with patch.object(computer.vms, 'virsh', virsh), patch.object(computer, 'worker') as worker, \ | ||
| 237 | patch.object(computer, 'finish_operation') as cleanup: | ||
| 238 | status = computer.handle('fixture', {'action': 'status'}, active_snapshot=active) | ||
| 239 | self.assertEqual(status['readiness'], 'busy') | ||
| 240 | self.assertEqual(status['active_operation'], active) | ||
| 241 | stopped = computer.handle('fixture', {'action': 'process', 'id': active['id'], 'operation': 'stop'}, active_snapshot=active) | ||
| 242 | self.assertIsNone(stopped['running']) | ||
| 243 | self.assertTrue(stopped['recovery_required']) | ||
| 244 | worker.assert_not_called() | ||
| 245 | cleanup.assert_not_called() | ||
| 246 | |||
| 229 | def test_active_operation_can_be_inspected_and_cancelled_during_execution(self): | 247 | def test_active_operation_can_be_inspected_and_cancelled_during_execution(self): |
| 230 | started = threading.Event() | 248 | started = threading.Event() |
| 231 | def virsh(action, *args, **kwargs): | 249 | def virsh(action, *args, **kwargs): |
tools/dashboard-host-test.py+87| ... | @@ -5,9 +5,12 @@ from pathlib import Path | ... | @@ -5,9 +5,12 @@ from pathlib import Path |
| 5 | import sys | 5 | import sys |
| 6 | import subprocess | 6 | import subprocess |
| 7 | import os | 7 | import os |
| 8 | import pwd | ||
| 8 | import socket | 9 | import socket |
| 9 | import struct | 10 | import struct |
| 11 | import tempfile | ||
| 10 | import threading | 12 | import threading |
| 13 | import time | ||
| 11 | import unittest | 14 | import unittest |
| 12 | from unittest.mock import patch | 15 | from unittest.mock import patch |
| 13 | 16 | ||
| ... | @@ -190,5 +193,89 @@ class Boundary(unittest.TestCase): | ... | @@ -190,5 +193,89 @@ class Boundary(unittest.TestCase): |
| 190 | command.assert_not_called() | 193 | command.assert_not_called() |
| 191 | 194 | ||
| 192 | 195 | ||
| 196 | @unittest.skipUnless(sys.platform == 'linux', 'Linux Unix peer credentials') | ||
| 197 | class Capacity(unittest.TestCase): | ||
| 198 | def test_recovery_controls_survive_full_ordinary_capacity(self): | ||
| 199 | program = ''' | ||
| 200 | from importlib import import_module | ||
| 201 | import json, sys, time | ||
| 202 | from pathlib import Path | ||
| 203 | m = import_module('dashboard-host') | ||
| 204 | root = Path(sys.argv[1]) | ||
| 205 | m.vms.DISKS = root / 'vms' | ||
| 206 | for name in ['fixture', 'idle', 'managed-mac']: | ||
| 207 | (m.vms.DISKS / name).mkdir(parents=True) | ||
| 208 | class Host: | ||
| 209 | def __init__(self, pool): pass | ||
| 210 | def handle(self, request, *, active_snapshot=None): | ||
| 211 | if request['operation'] == 'fixture.long': | ||
| 212 | (root / ('started-' + str(request['index']))).touch() | ||
| 213 | while not (root / 'release').exists(): time.sleep(.01) | ||
| 214 | return {'operation': request['operation'], 'active_snapshot': active_snapshot} | ||
| 215 | m.Host = Host | ||
| 216 | m.vms.mac_vms.CONFIG = {'name': 'managed-mac'} | ||
| 217 | with m.vms.computer_lock('fixture', 'exec') as active, m.vms.computer_lock('managed-mac', 'exec'): | ||
| 218 | (root / 'active.json').write_text(json.dumps(active)) | ||
| 219 | m.main() | ||
| 220 | ''' | ||
| 221 | with tempfile.TemporaryDirectory() as directory: | ||
| 222 | root = Path(directory) | ||
| 223 | path = root / 'host.sock' | ||
| 224 | environment = {**os.environ, 'SNOWGLOBE_POOL': 'fixture', | ||
| 225 | 'SNOWGLOBE_DASHBOARD_USER': pwd.getpwuid(os.getuid()).pw_name, | ||
| 226 | 'SNOWGLOBE_HOST_SOCKET': str(path)} | ||
| 227 | environment.pop('NOTIFY_SOCKET', None) | ||
| 228 | process = subprocess.Popen([sys.executable, '-c', program, str(root)], cwd=Path(__file__).parent, env=environment, | ||
| 229 | stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) | ||
| 230 | held = [] | ||
| 231 | def connect(request): | ||
| 232 | connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) | ||
| 233 | connection.settimeout(2) | ||
| 234 | try: | ||
| 235 | connection.connect(str(path)) | ||
| 236 | connection.sendall(json.dumps(request).encode() + b'\n') | ||
| 237 | except BaseException: | ||
| 238 | connection.close() | ||
| 239 | raise | ||
| 240 | return connection | ||
| 241 | def call(request): | ||
| 242 | with connect(request) as connection, connection.makefile('rb') as incoming: | ||
| 243 | length = struct.unpack('!I', incoming.read(4))[0] | ||
| 244 | return json.loads(incoming.read(length)) | ||
| 245 | try: | ||
| 246 | deadline = time.monotonic() + 5 | ||
| 247 | while not path.exists(): | ||
| 248 | self.assertIsNone(process.poll()) | ||
| 249 | self.assertLess(time.monotonic(), deadline) | ||
| 250 | time.sleep(.01) | ||
| 251 | for index in range(host_module.WORKERS): | ||
| 252 | held.append(connect({'operation': 'fixture.long', 'index': index})) | ||
| 253 | while len(list(root.glob('started-*'))) != host_module.WORKERS: | ||
| 254 | self.assertLess(time.monotonic(), deadline) | ||
| 255 | time.sleep(.01) | ||
| 256 | active = json.loads((root / 'active.json').read_text()) | ||
| 257 | for request in [ | ||
| 258 | {'operation': 'vm.owner', 'payload': {'name': 'fixture'}}, {'operation': 'vm.computers'}, | ||
| 259 | *({'operation': 'vm.act', 'payload': {'name': 'fixture', 'action': action}} for action in ['reset', 'destroy']), | ||
| 260 | *({'operation': 'vm.computer', 'payload': {'name': 'fixture', 'request': request}} for request in [ | ||
| 261 | {'action': 'status'}, {'action': 'process', 'id': active['id'], 'operation': 'status'}, | ||
| 262 | {'action': 'process', 'id': active['id'], 'operation': 'stop'}]), | ||
| 263 | ]: | ||
| 264 | self.assertEqual(call(request)['value']['operation'], request['operation']) | ||
| 265 | for request in [ | ||
| 266 | {'operation': 'host.sample'}, {'operation': 'vm.act', 'payload': {'name': 'fixture', 'action': 'start'}}, | ||
| 267 | {'operation': 'vm.computer', 'payload': {'name': 'idle', 'request': {'action': 'status'}}}, | ||
| 268 | {'operation': 'vm.computer', 'payload': {'name': 'managed-mac', 'request': {'action': 'status'}}}, | ||
| 269 | {'operation': 'vm.computer', 'payload': {'name': 'idle', 'request': {'action': 'process', 'id': active['id'], 'operation': 'stop'}}}, | ||
| 270 | ]: | ||
| 271 | self.assertEqual(call(request)['status'], 503) | ||
| 272 | finally: | ||
| 273 | (root / 'release').touch() | ||
| 274 | for connection in held: | ||
| 275 | connection.close() | ||
| 276 | process.terminate() | ||
| 277 | process.communicate(timeout=5) | ||
| 278 | |||
| 279 | |||
| 193 | if __name__ == "__main__": | 280 | if __name__ == "__main__": |
| 194 | unittest.main() | 281 | unittest.main() |
tools/dashboard-host.py+42-11| ... | @@ -40,6 +40,9 @@ TEXT_FIELDS = {"name", "compression", "mountpoint", "origin", "mounted"} | ... | @@ -40,6 +40,9 @@ TEXT_FIELDS = {"name", "compression", "mountpoint", "origin", "mounted"} |
| 40 | PROTOCOL = json.loads((Path(__file__).resolve().parent.parent / "dashboard/src/host-protocol.json").read_text()) | 40 | PROTOCOL = json.loads((Path(__file__).resolve().parent.parent / "dashboard/src/host-protocol.json").read_text()) |
| 41 | MAX_REQUEST = PROTOCOL['request'] | 41 | MAX_REQUEST = PROTOCOL['request'] |
| 42 | MAX_RESPONSE = PROTOCOL['response'] | 42 | MAX_RESPONSE = PROTOCOL['response'] |
| 43 | WORKERS = 12 | ||
| 44 | CONNECTIONS = WORKERS + 4 | ||
| 45 | WORK_SLOTS = threading.BoundedSemaphore(WORKERS) | ||
| 43 | STREAM_SLOTS = threading.BoundedSemaphore(4) | 46 | STREAM_SLOTS = threading.BoundedSemaphore(4) |
| 44 | GUEST_SLOTS = threading.BoundedSemaphore(4) | 47 | GUEST_SLOTS = threading.BoundedSemaphore(4) |
| 45 | CLONE_SLOTS = threading.BoundedSemaphore(2) | 48 | CLONE_SLOTS = threading.BoundedSemaphore(2) |
| ... | @@ -74,7 +77,7 @@ class Host: | ... | @@ -74,7 +77,7 @@ class Host: |
| 74 | raise Rejected("Choose a snapshot from the list.") | 77 | raise Rejected("Choose a snapshot from the list.") |
| 75 | return value | 78 | return value |
| 76 | 79 | ||
| 77 | def handle(self, request): | 80 | def handle(self, request, *, active_snapshot=None): |
| 78 | if not isinstance(request, dict): | 81 | if not isinstance(request, dict): |
| 79 | raise Rejected("Choose a supported host operation.") | 82 | raise Rejected("Choose a supported host operation.") |
| 80 | operation = request.get("operation") | 83 | operation = request.get("operation") |
| ... | @@ -206,7 +209,7 @@ class Host: | ... | @@ -206,7 +209,7 @@ class Host: |
| 206 | raise Rejected(str(error)) from error | 209 | raise Rejected(str(error)) from error |
| 207 | if action == "computer": | 210 | if action == "computer": |
| 208 | try: | 211 | try: |
| 209 | return importlib.import_module("vm-computer").handle(payload["name"], payload["request"]) | 212 | return importlib.import_module("vm-computer").handle(payload["name"], payload["request"], active_snapshot=active_snapshot) |
| 210 | except (BlockingIOError, InterruptedError) as error: | 213 | except (BlockingIOError, InterruptedError) as error: |
| 211 | raise dashboard_runs.Error(409, str(error) or "Another computer operation is running. Read computer_status before retrying.") from None | 214 | raise dashboard_runs.Error(409, str(error) or "Another computer operation is running. Read computer_status before retrying.") from None |
| 212 | except (TimeoutError, subprocess.TimeoutExpired): | 215 | except (TimeoutError, subprocess.TimeoutExpired): |
| ... | @@ -485,7 +488,7 @@ def stream_clone(connection, plan): | ... | @@ -485,7 +488,7 @@ def stream_clone(connection, plan): |
| 485 | 488 | ||
| 486 | 489 | ||
| 487 | def serve_connection(connection, host, allowed_uid, allowed_operations=None): | 490 | def serve_connection(connection, host, allowed_uid, allowed_operations=None): |
| 488 | with connection: | 491 | with connection, contextlib.ExitStack() as resources: |
| 489 | connection.settimeout(65) | 492 | connection.settimeout(65) |
| 490 | _, uid, _ = struct.unpack("3i", connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12)) | 493 | _, uid, _ = struct.unpack("3i", connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12)) |
| 491 | if uid != allowed_uid: | 494 | if uid != allowed_uid: |
| ... | @@ -510,19 +513,47 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None): | ... | @@ -510,19 +513,47 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None): |
| 510 | or request["operation"] not in allowed_operations | 513 | or request["operation"] not in allowed_operations |
| 511 | ): | 514 | ): |
| 512 | raise dashboard_runs.Error(403, "This preview cannot change Snow Globe. Use production for host controls.") | 515 | raise dashboard_runs.Error(403, "This preview cannot change Snow Globe. Use production for host controls.") |
| 513 | if isinstance(request, dict) and request.get("operation") in {"vm.console", "vm.serial", "vm.guest", "vm.upload"}: | 516 | operation = request.get("operation") if isinstance(request, dict) else None |
| 514 | streams = GUEST_SLOTS if request["operation"] == "vm.guest" else STREAM_SLOTS | 517 | operation = operation if isinstance(operation, str) else None |
| 518 | active_snapshot = None | ||
| 519 | control = operation == "vm.computers" | ||
| 520 | if operation in {"vm.owner", "vm.act", "vm.computer"}: | ||
| 521 | payload = request.get("payload") | ||
| 522 | try: | ||
| 523 | vms.validate(operation[3:], payload) | ||
| 524 | except ValueError as error: | ||
| 525 | raise Rejected(str(error)) from error | ||
| 526 | if operation == "vm.owner": | ||
| 527 | control = True | ||
| 528 | elif operation == "vm.act": | ||
| 529 | control = payload["action"] in {"reset", "destroy"} | ||
| 530 | else: | ||
| 531 | action = payload["request"] | ||
| 532 | identity = action.get("id") | ||
| 533 | operation_handle = action.get("action") == "process" and isinstance(identity, str) and identity.startswith("operation:") | ||
| 534 | control = operation_handle and action.get("operation") == "status" | ||
| 535 | if (action.get("action") == "status" or operation_handle and action.get("operation") == "stop") and ( | ||
| 536 | not vms.mac_vms.CONFIG or payload["name"] != vms.mac_vms.CONFIG["name"] | ||
| 537 | ): | ||
| 538 | active_snapshot = vms.computer_activity(payload["name"]) | ||
| 539 | control = active_snapshot is not None | ||
| 540 | if not control: | ||
| 541 | if not WORK_SLOTS.acquire(blocking=False): | ||
| 542 | raise dashboard_runs.Error(503, "The host is busy. Wait for an operation to finish, then retry.") | ||
| 543 | resources.callback(WORK_SLOTS.release) | ||
| 544 | if operation in {"vm.console", "vm.serial", "vm.guest", "vm.upload"}: | ||
| 545 | streams = GUEST_SLOTS if operation == "vm.guest" else STREAM_SLOTS | ||
| 515 | if not streams.acquire(blocking=False): | 546 | if not streams.acquire(blocking=False): |
| 516 | raise Rejected("Four VM connections or uploads are open. Close one and try again.") | 547 | raise Rejected("Four VM connections or uploads are open. Close one and try again.") |
| 517 | try: | 548 | try: |
| 518 | if request["operation"] in {"vm.console", "vm.serial", "vm.guest"}: | 549 | if operation in {"vm.console", "vm.serial", "vm.guest"}: |
| 519 | stream_console(connection, host, request) | 550 | stream_console(connection, host, request) |
| 520 | else: | 551 | else: |
| 521 | stream_upload(connection, request) | 552 | stream_upload(connection, request) |
| 522 | return | 553 | return |
| 523 | finally: | 554 | finally: |
| 524 | streams.release() | 555 | streams.release() |
| 525 | if isinstance(request, dict) and request.get("operation") == "shale.clone": | 556 | if operation == "shale.clone": |
| 526 | if not CLONE_SLOTS.acquire(blocking=False): | 557 | if not CLONE_SLOTS.acquire(blocking=False): |
| 527 | raise dashboard_runs.Error(503, "Two Knife clones are running. Try again shortly.") | 558 | raise dashboard_runs.Error(503, "Two Knife clones are running. Try again shortly.") |
| 528 | try: | 559 | try: |
| ... | @@ -530,7 +561,7 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None): | ... | @@ -530,7 +561,7 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None): |
| 530 | return | 561 | return |
| 531 | finally: | 562 | finally: |
| 532 | CLONE_SLOTS.release() | 563 | CLONE_SLOTS.release() |
| 533 | response = {"value": host.handle(request)} | 564 | response = {"value": host.handle(request, active_snapshot=active_snapshot) if active_snapshot is not None else host.handle(request)} |
| 534 | except Rejected as error: | 565 | except Rejected as error: |
| 535 | response = {"error": str(error), "status": 400} | 566 | response = {"error": str(error), "status": 400} |
| 536 | except dashboard_runs.Error as error: | 567 | except dashboard_runs.Error as error: |
| ... | @@ -558,13 +589,13 @@ def main(): | ... | @@ -558,13 +589,13 @@ def main(): |
| 558 | with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener: | 589 | with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener: |
| 559 | listener.bind(str(socket_path)) | 590 | listener.bind(str(socket_path)) |
| 560 | socket_path.chmod(0o660) | 591 | socket_path.chmod(0o660) |
| 561 | listener.listen(8) | 592 | listener.listen(CONNECTIONS) |
| 562 | if address := os.environ.get("NOTIFY_SOCKET"): | 593 | if address := os.environ.get("NOTIFY_SOCKET"): |
| 563 | with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as notification: | 594 | with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as notification: |
| 564 | notification.connect("\0" + address[1:] if address.startswith("@") else address) | 595 | notification.connect("\0" + address[1:] if address.startswith("@") else address) |
| 565 | notification.sendall(b"READY=1") | 596 | notification.sendall(b"READY=1") |
| 566 | slots = threading.BoundedSemaphore(16) | 597 | slots = threading.BoundedSemaphore(CONNECTIONS) |
| 567 | with concurrent.futures.ThreadPoolExecutor(max_workers=12) as workers: | 598 | with concurrent.futures.ThreadPoolExecutor(max_workers=CONNECTIONS) as workers: |
| 568 | while True: | 599 | while True: |
| 569 | connection, _ = listener.accept() | 600 | connection, _ = listener.accept() |
| 570 | if not slots.acquire(blocking=False): | 601 | if not slots.acquire(blocking=False): |
tools/dashboard-stage-test.py+12| ... | @@ -80,6 +80,18 @@ class PreviewBoundary(unittest.TestCase): | ... | @@ -80,6 +80,18 @@ class PreviewBoundary(unittest.TestCase): |
| 80 | self.assertEqual(host.handle(request), "real snapshot") | 80 | self.assertEqual(host.handle(request), "real snapshot") |
| 81 | handler.assert_called_once_with(request) | 81 | handler.assert_called_once_with(request) |
| 82 | 82 | ||
| 83 | def test_busy_observation_keeps_preview_vm_access_checks(self): | ||
| 84 | host = stage.PreviewHost({'clone': 'globe/staging/fixture', 'computer': 'fixture'}) | ||
| 85 | request = {'operation': 'vm.computer', 'payload': {'name': 'fixture', 'request': {'action': 'status'}}} | ||
| 86 | active = {'id': 'operation:' + 'a' * 32, 'action': 'exec'} | ||
| 87 | with patch.object(boundary.Host, 'handle', return_value={'readiness': 'busy'}) as call: | ||
| 88 | self.assertEqual(host.handle(request, active_snapshot=active), {'readiness': 'busy'}) | ||
| 89 | call.assert_called_once_with(request, active_snapshot=active) | ||
| 90 | call.reset_mock() | ||
| 91 | with self.assertRaises(boundary.dashboard_runs.Error): | ||
| 92 | host.handle({**request, 'payload': {**request['payload'], 'name': 'personal'}}, active_snapshot=active) | ||
| 93 | call.assert_not_called() | ||
| 94 | |||
| 83 | def test_preview_vm_creation_and_ownership_stay_in_its_account(self): | 95 | def test_preview_vm_creation_and_ownership_stay_in_its_account(self): |
| 84 | with tempfile.TemporaryDirectory() as directory: | 96 | with tempfile.TemporaryDirectory() as directory: |
| 85 | root = Path(directory) | 97 | root = Path(directory) |
tools/dashboard-stage.py+7-6| ... | @@ -1,5 +1,6 @@ | ... | @@ -1,5 +1,6 @@ |
| 1 | #!/usr/bin/env python3 | 1 | #!/usr/bin/env python3 |
| 2 | import concurrent.futures | 2 | import concurrent.futures |
| 3 | from contextlib import closing | ||
| 3 | from importlib import import_module | 4 | from importlib import import_module |
| 4 | import json | 5 | import json |
| 5 | import os | 6 | import os |
| ... | @@ -35,11 +36,11 @@ class PreviewHost(boundary.Host): | ... | @@ -35,11 +36,11 @@ class PreviewHost(boundary.Host): |
| 35 | super().__init__(saved["clone"].split("/", 1)[0]) | 36 | super().__init__(saved["clone"].split("/", 1)[0]) |
| 36 | self.stage = saved | 37 | self.stage = saved |
| 37 | 38 | ||
| 38 | def handle(self, request): | 39 | def handle(self, request, *, active_snapshot=None): |
| 39 | operation = request.get("operation") if isinstance(request, dict) else None | 40 | operation = request.get("operation") if isinstance(request, dict) else None |
| 40 | owner = None | 41 | owner = None |
| 41 | if self.stage.get("computer") and self.stage.get("mount") and operation in {"vm.computer", "vm.owner", "vm.access", "vm.console", "vm.guest", "vm.domains", "vm.computers", "vm.ai", "vm.act"}: | 42 | if self.stage.get("computer") and self.stage.get("mount") and operation in {"vm.computer", "vm.owner", "vm.access", "vm.console", "vm.guest", "vm.domains", "vm.computers", "vm.ai", "vm.act"}: |
| 42 | with sqlite3.connect(f"file:{self.stage['mount']}/data/accounts.sqlite?mode=ro", uri=True) as accounts: | 43 | with closing(sqlite3.connect(f"file:{self.stage['mount']}/data/accounts.sqlite?mode=ro", uri=True)) as accounts: |
| 43 | owner = accounts.execute("SELECT id FROM users WHERE username='preview'").fetchone() | 44 | owner = accounts.execute("SELECT id FROM users WHERE username='preview'").fetchone() |
| 44 | if not owner: | 45 | if not owner: |
| 45 | raise ValueError("The preview account is unavailable.") | 46 | raise ValueError("The preview account is unavailable.") |
| ... | @@ -75,7 +76,7 @@ class PreviewHost(boundary.Host): | ... | @@ -75,7 +76,7 @@ class PreviewHost(boundary.Host): |
| 75 | ).stdout) | 76 | ).stdout) |
| 76 | mounted["filesystems"][0]["target"] = "/srv" | 77 | mounted["filesystems"][0]["target"] = "/srv" |
| 77 | return mounted | 78 | return mounted |
| 78 | result = super().handle(request) | 79 | result = super().handle(request, active_snapshot=active_snapshot) if active_snapshot is not None else super().handle(request) |
| 79 | if owner and operation in {"vm.owner", "vm.domains", "vm.computers"}: | 80 | if owner and operation in {"vm.owner", "vm.domains", "vm.computers"}: |
| 80 | if operation == "vm.owner": | 81 | if operation == "vm.owner": |
| 81 | result = {**result, "owner": owner[0], "ownerOnly": True} | 82 | result = {**result, "owner": owner[0], "ownerOnly": True} |
| ... | @@ -106,9 +107,9 @@ def serve(stage): | ... | @@ -106,9 +107,9 @@ def serve(stage): |
| 106 | listener.bind(str(path)) | 107 | listener.bind(str(path)) |
| 107 | os.chown(path, 0, saved["uid"]) | 108 | os.chown(path, 0, saved["uid"]) |
| 108 | path.chmod(0o660) | 109 | path.chmod(0o660) |
| 109 | listener.listen(8) | 110 | listener.listen(boundary.CONNECTIONS) |
| 110 | slots = threading.BoundedSemaphore(16) | 111 | slots = threading.BoundedSemaphore(boundary.CONNECTIONS) |
| 111 | with concurrent.futures.ThreadPoolExecutor(max_workers=12) as workers: | 112 | with concurrent.futures.ThreadPoolExecutor(max_workers=boundary.CONNECTIONS) as workers: |
| 112 | while True: | 113 | while True: |
| 113 | connection, _ = listener.accept() | 114 | connection, _ = listener.accept() |
| 114 | if not slots.acquire(blocking=False): | 115 | if not slots.acquire(blocking=False): |
tools/vm-computer.py+13-11| ... | @@ -517,7 +517,8 @@ Move-Item -LiteralPath 'WORKER.new.exe' -Destination 'WORKER' | ... | @@ -517,7 +517,8 @@ Move-Item -LiteralPath 'WORKER.new.exe' -Destination 'WORKER' |
| 517 | 517 | ||
| 518 | 518 | ||
| 519 | 519 | ||
| 520 | def handle(name, request): | 520 | def handle(name, request, *, active_snapshot=None): |
| 521 | """Admission snapshots cannot start guest work on reserved control workers.""" | ||
| 521 | validate(request) | 522 | validate(request) |
| 522 | vms.ensure_name(name) | 523 | vms.ensure_name(name) |
| 523 | if request['action'] == 'process' and request['id'].startswith('operation:'): | 524 | if request['action'] == 'process' and request['id'].startswith('operation:'): |
| ... | @@ -541,7 +542,7 @@ def handle(name, request): | ... | @@ -541,7 +542,7 @@ def handle(name, request): |
| 541 | return {'running': False} | 542 | return {'running': False} |
| 542 | if 'operation:' + unfinished['id'].removeprefix('snowglobe-computer-request-') != identity: | 543 | if 'operation:' + unfinished['id'].removeprefix('snowglobe-computer-request-') != identity: |
| 543 | return {'running': False} | 544 | return {'running': False} |
| 544 | if request['operation'] == 'status': | 545 | if request['operation'] == 'status' or active_snapshot is not None: |
| 545 | return {'running': None, 'recovery_required': True, 'id': identity} | 546 | return {'running': None, 'recovery_required': True, 'id': identity} |
| 546 | with vms.computer_lock(name, 'cancel', identity): | 547 | with vms.computer_lock(name, 'cancel', identity): |
| 547 | try: | 548 | try: |
| ... | @@ -559,14 +560,15 @@ def handle(name, request): | ... | @@ -559,14 +560,15 @@ def handle(name, request): |
| 559 | operation_token = None | 560 | operation_token = None |
| 560 | try: | 561 | try: |
| 561 | with contextlib.ExitStack() as scope: | 562 | with contextlib.ExitStack() as scope: |
| 562 | busy = False | 563 | busy = active_snapshot is not None |
| 563 | try: | 564 | if not busy: |
| 564 | active = scope.enter_context(vms.computer_lock(name, request['action'])) | 565 | try: |
| 565 | operation_token = OPERATION.set((name, active['id'])) | 566 | active = scope.enter_context(vms.computer_lock(name, request['action'])) |
| 566 | except BlockingIOError: | 567 | operation_token = OPERATION.set((name, active['id'])) |
| 567 | if request['action'] not in ('status', 'shot'): | 568 | except BlockingIOError: |
| 568 | raise BlockingIOError('Another computer operation is running. Read computer_status before retrying.') from None | 569 | if request['action'] not in ('status', 'shot'): |
| 569 | busy = True | 570 | raise BlockingIOError('Another computer operation is running. Read computer_status before retrying.') from None |
| 571 | busy = True | ||
| 570 | root = ET.fromstring(vms.virsh('dumpxml',name,timeout=remaining(15))) | 572 | root = ET.fromstring(vms.virsh('dumpxml',name,timeout=remaining(15))) |
| 571 | power = vms.virsh('domstate',name,timeout=remaining(15)).strip() | 573 | power = vms.virsh('domstate',name,timeout=remaining(15)).strip() |
| 572 | platform = root.findtext(f'metadata/{vms.NS}vm/{vms.NS}platform','linux') | 574 | platform = root.findtext(f'metadata/{vms.NS}vm/{vms.NS}platform','linux') |
| ... | @@ -583,7 +585,7 @@ def handle(name, request): | ... | @@ -583,7 +585,7 @@ def handle(name, request): |
| 583 | if request['action'] == 'shot': | 585 | if request['action'] == 'shot': |
| 584 | return asyncio.run(desktop(name)) | 586 | return asyncio.run(desktop(name)) |
| 585 | if busy: | 587 | if busy: |
| 586 | active = vms.computer_activity(name) | 588 | active = active_snapshot or vms.computer_activity(name) |
| 587 | return {**status, 'readiness': 'busy', 'capabilities': ['shot'], 'active_operation': active} | 589 | return {**status, 'readiness': 'busy', 'capabilities': ['shot'], 'active_operation': active} |
| 588 | try: | 590 | try: |
| 589 | with os.fdopen(vms.open_regular(vms.DISKS / name / 'computer-operation.json')) as incoming: | 591 | with os.fdopen(vms.open_regular(vms.DISKS / name / 'computer-operation.json')) as incoming: |