diff --git a/dashboard/src/host.rs b/dashboard/src/host.rs index 892cce35b63f989cc7b83be95621c8a88498aecc..6be684de5cfd03c4987760f724aaf93148eab6c8 100644 --- a/dashboard/src/host.rs +++ b/dashboard/src/host.rs @@ -16,14 +16,21 @@ pub async fn sample(app: Arc) -> Result> { } pub async fn call(request: Value) -> Result { - let _slot = tokio::time::timeout(Duration::from_secs(70), SLOTS.acquire()) - .await - .map_err(|_| { - Error::new( - 504, - "The host operation is taking too long. Check its logs, then retry.", - ) - })??; + let computer = &request["payload"]["request"]; + let control = matches!(request["operation"].as_str(), Some("vm.owner" | "vm.computers")) + || (request["operation"] == "vm.act" + && matches!(request["payload"]["action"].as_str(), Some("reset" | "destroy"))) + || (request["operation"] == "vm.computer" + && (matches!(computer["action"].as_str(), Some("status" | "shot")) + || (computer["action"] == "process" + && computer["id"].as_str().is_some_and(|id| id.starts_with("operation:"))))); + let _slot = if control { + None + } else { + Some(tokio::time::timeout(Duration::from_secs(70), SLOTS.acquire()) + .await + .map_err(|_| Error::new(504, "The host operation is taking too long. Check its logs, then retry."))??) + }; open(request).await.map(|(value, _)| value) } diff --git a/tools/computer-client-test.py b/tools/computer-client-test.py index 0958859bfa22a6129566772e7b2286adcaf1ca5c..c22be0b72c9d79928541bc239837f7b1b00e7fa5 100644 --- a/tools/computer-client-test.py +++ b/tools/computer-client-test.py @@ -15,6 +15,7 @@ import tempfile import threading import time import unittest +from concurrent.futures import ThreadPoolExecutor from pathlib import Path from urllib.parse import urlsplit from urllib.request import Request @@ -31,6 +32,8 @@ class Host(socketserver.StreamRequestHandler): files = {} power_actions = [] writes = [] + started = threading.Semaphore(0) + release = threading.Event() def handle(self): line = self.rfile.readline() @@ -60,7 +63,14 @@ class Host(socketserver.StreamRequestHandler): raise KeyError(operation) request, files = payload["request"], self.files if request["action"] == "exec": + if request["script"].startswith("held-"): + self.started.release() + self.release.wait(10) return {"exit": 0, "stdout": "ran " + request["script"], "png": PNG} + if request["action"] == "status": + return {"readiness": "busy", "active_operation": {"id": "operation:" + "a" * 32}} + if request["action"] == "process": + return {"running": True, "stopping": request["operation"] == "stop"} if request["action"] == "shot": return {"png": PNG} operation, path = request["operation"], request["path"] @@ -183,6 +193,22 @@ class Client(unittest.TestCase): self.assertEqual(vm.call("computer_vm", vm="test-vm", operation=operation), {"id":"test-vm", "requested":operation}) self.assertEqual(Host.power_actions, ["destroy", "reset"]) + with ThreadPoolExecutor(max_workers=4) as pool: + held = [pool.submit(vm.call, "computer_exec", vm="test-vm", script=f"held-{index}") for index in range(4)] + try: + for _ in held: + self.assertTrue(Host.started.acquire(timeout=3)) + started = time.monotonic() + status = vm.call("computer_status", vm="test-vm") + self.assertEqual(status["readiness"], "busy") + stopped = vm.call("computer_process", vm="test-vm", id=status["active_operation"]["id"], operation="stop") + self.assertTrue(stopped["stopping"]) + vm.call("computer_vm", vm="test-vm", operation="reset") + self.assertLess(time.monotonic() - started, 2, "recovery controls waited behind long commands") + finally: + Host.release.set() + for future in held: + self.assertEqual(future.result()["exit"], 0) with self.assertRaisesRegex(computer.ComputerError, "granted"): vm.call("computer_shot", vm="other-vm") diff --git a/tools/computer-test.py b/tools/computer-test.py index 6ad5fda2aab8161f9870b789b4253530702149b6..1100bc3ed49edca715f57371cd05666720042813 100644 --- a/tools/computer-test.py +++ b/tools/computer-test.py @@ -226,6 +226,24 @@ class GuestCancellation(unittest.TestCase): self.assertEqual(status['capabilities'], []) work.assert_not_called() + def test_control_snapshot_cannot_turn_into_slow_guest_setup_or_cleanup(self): + with computer.vms.computer_lock('fixture', 'exec'): + active = computer.vms.computer_activity('fixture') + operation = {'platform': 'windows', 'id': active['id'].removeprefix('operation:')} + computer.record_operation('fixture', operation) + def virsh(action, *args, **kwargs): + return 'running' if action == 'domstate' else '' + with patch.object(computer.vms, 'virsh', virsh), patch.object(computer, 'worker') as worker, \ + patch.object(computer, 'finish_operation') as cleanup: + status = computer.handle('fixture', {'action': 'status'}, active_snapshot=active) + self.assertEqual(status['readiness'], 'busy') + self.assertEqual(status['active_operation'], active) + stopped = computer.handle('fixture', {'action': 'process', 'id': active['id'], 'operation': 'stop'}, active_snapshot=active) + self.assertIsNone(stopped['running']) + self.assertTrue(stopped['recovery_required']) + worker.assert_not_called() + cleanup.assert_not_called() + def test_active_operation_can_be_inspected_and_cancelled_during_execution(self): started = threading.Event() def virsh(action, *args, **kwargs): diff --git a/tools/dashboard-host-test.py b/tools/dashboard-host-test.py index 3d9fd67e2c3a08d0ef1b396f55398288d157c2bf..91721f32c5875951b3e767e30087a160714a4774 100644 --- a/tools/dashboard-host-test.py +++ b/tools/dashboard-host-test.py @@ -5,9 +5,12 @@ from pathlib import Path import sys import subprocess import os +import pwd import socket import struct +import tempfile import threading +import time import unittest from unittest.mock import patch @@ -190,5 +193,89 @@ class Boundary(unittest.TestCase): command.assert_not_called() +@unittest.skipUnless(sys.platform == 'linux', 'Linux Unix peer credentials') +class Capacity(unittest.TestCase): + def test_recovery_controls_survive_full_ordinary_capacity(self): + program = ''' +from importlib import import_module +import json, sys, time +from pathlib import Path +m = import_module('dashboard-host') +root = Path(sys.argv[1]) +m.vms.DISKS = root / 'vms' +for name in ['fixture', 'idle', 'managed-mac']: + (m.vms.DISKS / name).mkdir(parents=True) +class Host: + def __init__(self, pool): pass + def handle(self, request, *, active_snapshot=None): + if request['operation'] == 'fixture.long': + (root / ('started-' + str(request['index']))).touch() + while not (root / 'release').exists(): time.sleep(.01) + return {'operation': request['operation'], 'active_snapshot': active_snapshot} +m.Host = Host +m.vms.mac_vms.CONFIG = {'name': 'managed-mac'} +with m.vms.computer_lock('fixture', 'exec') as active, m.vms.computer_lock('managed-mac', 'exec'): + (root / 'active.json').write_text(json.dumps(active)) + m.main() +''' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path = root / 'host.sock' + environment = {**os.environ, 'SNOWGLOBE_POOL': 'fixture', + 'SNOWGLOBE_DASHBOARD_USER': pwd.getpwuid(os.getuid()).pw_name, + 'SNOWGLOBE_HOST_SOCKET': str(path)} + environment.pop('NOTIFY_SOCKET', None) + process = subprocess.Popen([sys.executable, '-c', program, str(root)], cwd=Path(__file__).parent, env=environment, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + held = [] + def connect(request): + connection = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + connection.settimeout(2) + try: + connection.connect(str(path)) + connection.sendall(json.dumps(request).encode() + b'\n') + except BaseException: + connection.close() + raise + return connection + def call(request): + with connect(request) as connection, connection.makefile('rb') as incoming: + length = struct.unpack('!I', incoming.read(4))[0] + return json.loads(incoming.read(length)) + try: + deadline = time.monotonic() + 5 + while not path.exists(): + self.assertIsNone(process.poll()) + self.assertLess(time.monotonic(), deadline) + time.sleep(.01) + for index in range(host_module.WORKERS): + held.append(connect({'operation': 'fixture.long', 'index': index})) + while len(list(root.glob('started-*'))) != host_module.WORKERS: + self.assertLess(time.monotonic(), deadline) + time.sleep(.01) + active = json.loads((root / 'active.json').read_text()) + for request in [ + {'operation': 'vm.owner', 'payload': {'name': 'fixture'}}, {'operation': 'vm.computers'}, + *({'operation': 'vm.act', 'payload': {'name': 'fixture', 'action': action}} for action in ['reset', 'destroy']), + *({'operation': 'vm.computer', 'payload': {'name': 'fixture', 'request': request}} for request in [ + {'action': 'status'}, {'action': 'process', 'id': active['id'], 'operation': 'status'}, + {'action': 'process', 'id': active['id'], 'operation': 'stop'}]), + ]: + self.assertEqual(call(request)['value']['operation'], request['operation']) + for request in [ + {'operation': 'host.sample'}, {'operation': 'vm.act', 'payload': {'name': 'fixture', 'action': 'start'}}, + {'operation': 'vm.computer', 'payload': {'name': 'idle', 'request': {'action': 'status'}}}, + {'operation': 'vm.computer', 'payload': {'name': 'managed-mac', 'request': {'action': 'status'}}}, + {'operation': 'vm.computer', 'payload': {'name': 'idle', 'request': {'action': 'process', 'id': active['id'], 'operation': 'stop'}}}, + ]: + self.assertEqual(call(request)['status'], 503) + finally: + (root / 'release').touch() + for connection in held: + connection.close() + process.terminate() + process.communicate(timeout=5) + + if __name__ == "__main__": unittest.main() diff --git a/tools/dashboard-host.py b/tools/dashboard-host.py index a9e55c18a9baf3b89b045c2579e0a8bb094b9b2c..a150cc43d5b4aa25a0cf4531eeabfe3013cab157 100644 --- a/tools/dashboard-host.py +++ b/tools/dashboard-host.py @@ -40,6 +40,9 @@ TEXT_FIELDS = {"name", "compression", "mountpoint", "origin", "mounted"} PROTOCOL = json.loads((Path(__file__).resolve().parent.parent / "dashboard/src/host-protocol.json").read_text()) MAX_REQUEST = PROTOCOL['request'] MAX_RESPONSE = PROTOCOL['response'] +WORKERS = 12 +CONNECTIONS = WORKERS + 4 +WORK_SLOTS = threading.BoundedSemaphore(WORKERS) STREAM_SLOTS = threading.BoundedSemaphore(4) GUEST_SLOTS = threading.BoundedSemaphore(4) CLONE_SLOTS = threading.BoundedSemaphore(2) @@ -74,7 +77,7 @@ class Host: raise Rejected("Choose a snapshot from the list.") return value - def handle(self, request): + def handle(self, request, *, active_snapshot=None): if not isinstance(request, dict): raise Rejected("Choose a supported host operation.") operation = request.get("operation") @@ -206,7 +209,7 @@ class Host: raise Rejected(str(error)) from error if action == "computer": try: - return importlib.import_module("vm-computer").handle(payload["name"], payload["request"]) + return importlib.import_module("vm-computer").handle(payload["name"], payload["request"], active_snapshot=active_snapshot) except (BlockingIOError, InterruptedError) as error: raise dashboard_runs.Error(409, str(error) or "Another computer operation is running. Read computer_status before retrying.") from None except (TimeoutError, subprocess.TimeoutExpired): @@ -485,7 +488,7 @@ def stream_clone(connection, plan): def serve_connection(connection, host, allowed_uid, allowed_operations=None): - with connection: + with connection, contextlib.ExitStack() as resources: connection.settimeout(65) _, uid, _ = struct.unpack("3i", connection.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12)) if uid != allowed_uid: @@ -510,19 +513,47 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None): or request["operation"] not in allowed_operations ): raise dashboard_runs.Error(403, "This preview cannot change Snow Globe. Use production for host controls.") - if isinstance(request, dict) and request.get("operation") in {"vm.console", "vm.serial", "vm.guest", "vm.upload"}: - streams = GUEST_SLOTS if request["operation"] == "vm.guest" else STREAM_SLOTS + operation = request.get("operation") if isinstance(request, dict) else None + operation = operation if isinstance(operation, str) else None + active_snapshot = None + control = operation == "vm.computers" + if operation in {"vm.owner", "vm.act", "vm.computer"}: + payload = request.get("payload") + try: + vms.validate(operation[3:], payload) + except ValueError as error: + raise Rejected(str(error)) from error + if operation == "vm.owner": + control = True + elif operation == "vm.act": + control = payload["action"] in {"reset", "destroy"} + else: + action = payload["request"] + identity = action.get("id") + operation_handle = action.get("action") == "process" and isinstance(identity, str) and identity.startswith("operation:") + control = operation_handle and action.get("operation") == "status" + if (action.get("action") == "status" or operation_handle and action.get("operation") == "stop") and ( + not vms.mac_vms.CONFIG or payload["name"] != vms.mac_vms.CONFIG["name"] + ): + active_snapshot = vms.computer_activity(payload["name"]) + control = active_snapshot is not None + if not control: + if not WORK_SLOTS.acquire(blocking=False): + raise dashboard_runs.Error(503, "The host is busy. Wait for an operation to finish, then retry.") + resources.callback(WORK_SLOTS.release) + if operation in {"vm.console", "vm.serial", "vm.guest", "vm.upload"}: + streams = GUEST_SLOTS if operation == "vm.guest" else STREAM_SLOTS if not streams.acquire(blocking=False): raise Rejected("Four VM connections or uploads are open. Close one and try again.") try: - if request["operation"] in {"vm.console", "vm.serial", "vm.guest"}: + if operation in {"vm.console", "vm.serial", "vm.guest"}: stream_console(connection, host, request) else: stream_upload(connection, request) return finally: streams.release() - if isinstance(request, dict) and request.get("operation") == "shale.clone": + if operation == "shale.clone": if not CLONE_SLOTS.acquire(blocking=False): raise dashboard_runs.Error(503, "Two Knife clones are running. Try again shortly.") try: @@ -530,7 +561,7 @@ def serve_connection(connection, host, allowed_uid, allowed_operations=None): return finally: CLONE_SLOTS.release() - response = {"value": host.handle(request)} + response = {"value": host.handle(request, active_snapshot=active_snapshot) if active_snapshot is not None else host.handle(request)} except Rejected as error: response = {"error": str(error), "status": 400} except dashboard_runs.Error as error: @@ -558,13 +589,13 @@ def main(): with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener: listener.bind(str(socket_path)) socket_path.chmod(0o660) - listener.listen(8) + listener.listen(CONNECTIONS) if address := os.environ.get("NOTIFY_SOCKET"): with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as notification: notification.connect("\0" + address[1:] if address.startswith("@") else address) notification.sendall(b"READY=1") - slots = threading.BoundedSemaphore(16) - with concurrent.futures.ThreadPoolExecutor(max_workers=12) as workers: + slots = threading.BoundedSemaphore(CONNECTIONS) + with concurrent.futures.ThreadPoolExecutor(max_workers=CONNECTIONS) as workers: while True: connection, _ = listener.accept() if not slots.acquire(blocking=False): diff --git a/tools/dashboard-stage-test.py b/tools/dashboard-stage-test.py index e8cd25f8d2cf8e9abd2bdbecb4ca06ef0ba20379..a3c34a7f84c2bd029feb4cf6f39fc32b1be8db34 100644 --- a/tools/dashboard-stage-test.py +++ b/tools/dashboard-stage-test.py @@ -80,6 +80,18 @@ class PreviewBoundary(unittest.TestCase): self.assertEqual(host.handle(request), "real snapshot") handler.assert_called_once_with(request) + def test_busy_observation_keeps_preview_vm_access_checks(self): + host = stage.PreviewHost({'clone': 'globe/staging/fixture', 'computer': 'fixture'}) + request = {'operation': 'vm.computer', 'payload': {'name': 'fixture', 'request': {'action': 'status'}}} + active = {'id': 'operation:' + 'a' * 32, 'action': 'exec'} + with patch.object(boundary.Host, 'handle', return_value={'readiness': 'busy'}) as call: + self.assertEqual(host.handle(request, active_snapshot=active), {'readiness': 'busy'}) + call.assert_called_once_with(request, active_snapshot=active) + call.reset_mock() + with self.assertRaises(boundary.dashboard_runs.Error): + host.handle({**request, 'payload': {**request['payload'], 'name': 'personal'}}, active_snapshot=active) + call.assert_not_called() + def test_preview_vm_creation_and_ownership_stay_in_its_account(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/tools/dashboard-stage.py b/tools/dashboard-stage.py index fea28b22bf53fec95b68f23b9b5cc75932ba621d..c7545ee3f39cabaf9c370710d66b2fd4649c3924 100644 --- a/tools/dashboard-stage.py +++ b/tools/dashboard-stage.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import concurrent.futures +from contextlib import closing from importlib import import_module import json import os @@ -35,11 +36,11 @@ class PreviewHost(boundary.Host): super().__init__(saved["clone"].split("/", 1)[0]) self.stage = saved - def handle(self, request): + def handle(self, request, *, active_snapshot=None): operation = request.get("operation") if isinstance(request, dict) else None owner = None if self.stage.get("computer") and self.stage.get("mount") and operation in {"vm.computer", "vm.owner", "vm.access", "vm.console", "vm.guest", "vm.domains", "vm.computers", "vm.ai", "vm.act"}: - with sqlite3.connect(f"file:{self.stage['mount']}/data/accounts.sqlite?mode=ro", uri=True) as accounts: + with closing(sqlite3.connect(f"file:{self.stage['mount']}/data/accounts.sqlite?mode=ro", uri=True)) as accounts: owner = accounts.execute("SELECT id FROM users WHERE username='preview'").fetchone() if not owner: raise ValueError("The preview account is unavailable.") @@ -75,7 +76,7 @@ class PreviewHost(boundary.Host): ).stdout) mounted["filesystems"][0]["target"] = "/srv" return mounted - result = super().handle(request) + result = super().handle(request, active_snapshot=active_snapshot) if active_snapshot is not None else super().handle(request) if owner and operation in {"vm.owner", "vm.domains", "vm.computers"}: if operation == "vm.owner": result = {**result, "owner": owner[0], "ownerOnly": True} @@ -106,9 +107,9 @@ def serve(stage): listener.bind(str(path)) os.chown(path, 0, saved["uid"]) path.chmod(0o660) - listener.listen(8) - slots = threading.BoundedSemaphore(16) - with concurrent.futures.ThreadPoolExecutor(max_workers=12) as workers: + listener.listen(boundary.CONNECTIONS) + slots = threading.BoundedSemaphore(boundary.CONNECTIONS) + with concurrent.futures.ThreadPoolExecutor(max_workers=boundary.CONNECTIONS) as workers: while True: connection, _ = listener.accept() if not slots.acquire(blocking=False): diff --git a/tools/vm-computer.py b/tools/vm-computer.py index 8649d6481a9a40e0c8a3ff5f884682daa860ed8e..669a90170780d70169f2fc5a5ec444d4c76583f0 100644 --- a/tools/vm-computer.py +++ b/tools/vm-computer.py @@ -517,7 +517,8 @@ Move-Item -LiteralPath 'WORKER.new.exe' -Destination 'WORKER' -def handle(name, request): +def handle(name, request, *, active_snapshot=None): + """Admission snapshots cannot start guest work on reserved control workers.""" validate(request) vms.ensure_name(name) if request['action'] == 'process' and request['id'].startswith('operation:'): @@ -541,7 +542,7 @@ def handle(name, request): return {'running': False} if 'operation:' + unfinished['id'].removeprefix('snowglobe-computer-request-') != identity: return {'running': False} - if request['operation'] == 'status': + if request['operation'] == 'status' or active_snapshot is not None: return {'running': None, 'recovery_required': True, 'id': identity} with vms.computer_lock(name, 'cancel', identity): try: @@ -559,14 +560,15 @@ def handle(name, request): operation_token = None try: with contextlib.ExitStack() as scope: - busy = False - try: - active = scope.enter_context(vms.computer_lock(name, request['action'])) - operation_token = OPERATION.set((name, active['id'])) - except BlockingIOError: - if request['action'] not in ('status', 'shot'): - raise BlockingIOError('Another computer operation is running. Read computer_status before retrying.') from None - busy = True + busy = active_snapshot is not None + if not busy: + try: + active = scope.enter_context(vms.computer_lock(name, request['action'])) + operation_token = OPERATION.set((name, active['id'])) + except BlockingIOError: + if request['action'] not in ('status', 'shot'): + raise BlockingIOError('Another computer operation is running. Read computer_status before retrying.') from None + busy = True root = ET.fromstring(vms.virsh('dumpxml',name,timeout=remaining(15))) power = vms.virsh('domstate',name,timeout=remaining(15)).strip() platform = root.findtext(f'metadata/{vms.NS}vm/{vms.NS}platform','linux') @@ -583,7 +585,7 @@ def handle(name, request): if request['action'] == 'shot': return asyncio.run(desktop(name)) if busy: - active = vms.computer_activity(name) + active = active_snapshot or vms.computer_activity(name) return {**status, 'readiness': 'busy', 'capabilities': ['shot'], 'active_operation': active} try: with os.fdopen(vms.open_regular(vms.DISKS / name / 'computer-operation.json')) as incoming: