| 1 | #!/usr/bin/env python3 |
| 2 | """computer.py against a local dashboard whose host socket is a stub with one VM.""" |
| 3 | import argparse |
| 4 | import base64 |
| 5 | import http.client |
| 6 | import json |
| 7 | import os |
| 8 | import socket |
| 9 | import socketserver |
| 10 | import sqlite3 |
| 11 | import struct |
| 12 | import subprocess |
| 13 | import sys |
| 14 | import tempfile |
| 15 | import threading |
| 16 | import time |
| 17 | import unittest |
| 18 | from concurrent.futures import ThreadPoolExecutor |
| 19 | from pathlib import Path |
| 20 | from urllib.parse import urlsplit |
| 21 | from urllib.request import Request |
| 22 | |
| 23 | sys.path.insert(0, str(Path(__file__).parent)) |
| 24 | import computer # noqa: E402 |
| 25 | |
| 26 | PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Wl6qWQAAAAASUVORK5CYII=" |
| 27 | HOST = "snowglobe.paperclover.net" |
| 28 | |
| 29 | |
| 30 | class Host(socketserver.StreamRequestHandler): |
| 31 | owner = None |
| 32 | files = {} |
| 33 | power_actions = [] |
| 34 | writes = [] |
| 35 | reads = [] |
| 36 | chunk_bytes = None |
| 37 | started = threading.Semaphore(0) |
| 38 | release = threading.Event() |
| 39 | |
| 40 | def handle(self): |
| 41 | line = self.rfile.readline() |
| 42 | if not line: |
| 43 | return |
| 44 | request = json.loads(line) |
| 45 | try: |
| 46 | reply = {"value": self.answer(request["operation"], request.get("payload") or {})} |
| 47 | except KeyError as missing: |
| 48 | reply = {"error": f"missing {missing}", "status": 404} |
| 49 | except BlockingIOError as error: |
| 50 | reply = {"error": str(error), "status": 409} |
| 51 | except TimeoutError as error: |
| 52 | reply = {"error": str(error), "status": 504} |
| 53 | except ValueError as error: |
| 54 | reply = {"error": str(error), "status": 400} |
| 55 | body = json.dumps(reply).encode() |
| 56 | try: |
| 57 | self.wfile.write(struct.pack(">I", len(body)) + body) |
| 58 | except BrokenPipeError: |
| 59 | pass # The dashboard can drop a request before its reply. |
| 60 | |
| 61 | def answer(self, operation, payload): |
| 62 | if operation == "vm.computers": |
| 63 | return [{"name": "test-vm", "owner": self.owner, "capabilities": ["computer"], "platform": "linux", |
| 64 | "os": "Fedora", "arch": "x86_64", "state": "running", "aiOwned": False}] |
| 65 | if operation == "vm.owner": |
| 66 | return {"owner": self.owner} |
| 67 | if operation == "vm.act": |
| 68 | self.power_actions.append(payload["action"]) |
| 69 | return None |
| 70 | if operation != "vm.computer": |
| 71 | raise KeyError(operation) |
| 72 | request, files = payload["request"], self.files |
| 73 | if request["action"] == "exec": |
| 74 | if request["script"].startswith("held-"): |
| 75 | self.started.release() |
| 76 | self.release.wait(10) |
| 77 | return {"exit": 0, "stdout": "ran " + request["script"], "png": PNG} |
| 78 | if request["action"] == "status": |
| 79 | return {"readiness": "busy", "active_operation": {"id": "operation:" + "a" * 32}} |
| 80 | if request["action"] == "process": |
| 81 | return {"running": True, "stopping": request["operation"] == "stop"} |
| 82 | if request["action"] == "shot": |
| 83 | return {"png": PNG} |
| 84 | operation, path = request["operation"], request["path"] |
| 85 | if operation == "stat": |
| 86 | return {"directory": True} if path.endswith("/") else {"directory": False, "size": len(files[path])} |
| 87 | if operation == "write": |
| 88 | data = base64.b64decode(request["data"]) |
| 89 | self.writes.append((path, len(data))) |
| 90 | files[path] = files.get(path, b"")[:request["offset"]] + data |
| 91 | elif operation == "commit": |
| 92 | files[request["destination"]] = files.pop(path) |
| 93 | elif operation == "read": |
| 94 | self.reads.append((path, request["length"])) |
| 95 | data = files[path][request["offset"]:request["offset"] + request["length"]] |
| 96 | reply = {"data": base64.b64encode(data).decode()} |
| 97 | if self.chunk_bytes is not None: |
| 98 | reply["chunk_bytes"] = self.chunk_bytes |
| 99 | return reply |
| 100 | elif operation == "remove": |
| 101 | time.sleep(.05) |
| 102 | files.pop(path, None) |
| 103 | return {"chunk_bytes": self.chunk_bytes} if operation == "write" and self.chunk_bytes is not None else {} |
| 104 | |
| 105 | |
| 106 | class Client(unittest.TestCase): |
| 107 | @classmethod |
| 108 | def setUpClass(cls): |
| 109 | cls.temporary = tempfile.TemporaryDirectory(prefix="computer-client-") |
| 110 | data = Path(cls.temporary.name).resolve() |
| 111 | proof = os.urandom(32).hex() |
| 112 | (data / "proof").write_text(proof) |
| 113 | with socket.socket() as available: |
| 114 | available.bind(("127.0.0.1", 0)) |
| 115 | port = available.getsockname()[1] |
| 116 | cls.host = socketserver.ThreadingUnixStreamServer(str(data / "host.sock"), Host) |
| 117 | threading.Thread(target=cls.host.serve_forever, daemon=True).start() |
| 118 | environment = {**os.environ, "SNOWGLOBE_DOMAIN": "paperclover.net", "SNOWGLOBE_DATA_DIR": str(data), |
| 119 | "SNOWGLOBE_PUBLIC_ORIGIN": "https://" + HOST, "SNOWGLOBE_AUTH_RP_ID": "auth.paperclover.net", |
| 120 | "SNOWGLOBE_FILE_ORIGIN": "https://file.paperclover.net", "SNOWGLOBE_AUTH_REQUIRED": "1", |
| 121 | "SNOWGLOBE_WEB_DIR": str(Path("dashboard/dist").resolve()), "PORT": str(port), |
| 122 | "SNOWGLOBE_PROXY_TOKEN_FILE": str(data / "proof"), "SNOWGLOBE_HOST_SOCKET": str(data / "host.sock")} |
| 123 | account = json.loads(subprocess.run([BINARY, "--preview-account"], env=environment, check=True, |
| 124 | capture_output=True, text=True).stdout) |
| 125 | with sqlite3.connect(data / "accounts.sqlite") as db: |
| 126 | Host.owner = db.execute("SELECT id FROM users WHERE username='preview'").fetchone()[0] |
| 127 | cls.log = data / "server.log" |
| 128 | with cls.log.open("wb") as log: |
| 129 | cls.server = subprocess.Popen([BINARY], env=environment, stdout=log, stderr=subprocess.STDOUT) |
| 130 | deadline = time.monotonic() + 30 |
| 131 | while True: |
| 132 | try: |
| 133 | socket.create_connection(("127.0.0.1", port), timeout=0.1).close() |
| 134 | break |
| 135 | except OSError: |
| 136 | assert cls.server.poll() is None and time.monotonic() < deadline, cls.log.read_text()[-2000:] |
| 137 | time.sleep(0.05) |
| 138 | cls.headers = {"Host": HOST, "Snowglobe-Proxy-Token": proof, "X-Snowglobe-Client-IP": "127.0.0.1"} |
| 139 | cls.port = port |
| 140 | real = computer.urlopen |
| 141 | |
| 142 | def local(request, timeout=None): |
| 143 | request = Request(request) if isinstance(request, str) else request |
| 144 | url = urlsplit(request.full_url) |
| 145 | assert url.netloc == HOST, request.full_url |
| 146 | request.full_url = f"http://127.0.0.1:{port}{url.path}" + (f"?{url.query}" if url.query else "") |
| 147 | for key, value in cls.headers.items(): |
| 148 | request.add_header(key, value) |
| 149 | return real(request, timeout=timeout) |
| 150 | |
| 151 | computer.urlopen = local |
| 152 | computer.ORIGIN, computer.RESOURCE = f"https://{HOST}/", f"https://{HOST}/mcp/computer" |
| 153 | computer.STATE = data / "client/state.json" |
| 154 | cls.cookies = {} |
| 155 | csrf = cls.request("/auth/status")["csrf"] |
| 156 | cls.request("/auth/password", "POST", {"csrf": csrf, "username": account["username"], |
| 157 | "password": account["password"], "next": "/"}) |
| 158 | |
| 159 | @classmethod |
| 160 | def tearDownClass(cls): |
| 161 | cls.server.terminate() |
| 162 | cls.server.wait(timeout=10) |
| 163 | cls.host.shutdown() |
| 164 | cls.host.server_close() |
| 165 | cls.temporary.cleanup() |
| 166 | |
| 167 | @classmethod |
| 168 | def request(cls, path, method="GET", body=None): |
| 169 | headers = {**cls.headers, "Cookie": "; ".join(f"{k}={v}" for k, v in cls.cookies.items())} |
| 170 | if body is not None: |
| 171 | headers.update({"Origin": "https://" + HOST, "Content-Type": "application/json"}) |
| 172 | connection = http.client.HTTPConnection("127.0.0.1", cls.port, timeout=30) |
| 173 | connection.request(method, path, json.dumps(body) if body is not None else None, headers) |
| 174 | response = connection.getresponse() |
| 175 | content = response.read() |
| 176 | connection.close() |
| 177 | for cookie in response.headers.get_all("set-cookie") or []: |
| 178 | key, value = cookie.split(";", 1)[0].split("=", 1) |
| 179 | cls.cookies[key] = value |
| 180 | assert response.status in (200, 302), (path, response.status, content[:300]) |
| 181 | return response.headers["location"] if response.status == 302 else json.loads(content) |
| 182 | |
| 183 | def connect(self): |
| 184 | link = urlsplit(computer.login()) |
| 185 | pending = self.request(f"{link.path}?{link.query}").removeprefix("/connect/") |
| 186 | self.assertEqual(self.request(f"/api/mcp/consent/{pending}")["scopes"], computer.SCOPES.split()) |
| 187 | return self.request(f"/api/mcp/consent/{pending}", "POST", {"resources": ["test-vm"]})["redirect"] |
| 188 | |
| 189 | def test_end_to_end(self): |
| 190 | with self.assertRaisesRegex(computer.ComputerError, "login"): |
| 191 | computer.Computer().call("computer_list") |
| 192 | redirected = self.connect() |
| 193 | with self.assertRaisesRegex(computer.ComputerError, "different login"): |
| 194 | computer.login(redirected.replace("state=", "state=x")) |
| 195 | redirected = self.connect() |
| 196 | self.assertIsNone(computer.login(redirected)) |
| 197 | self.assertEqual(computer.STATE.stat().st_mode & 0o777, 0o600) |
| 198 | |
| 199 | vm = computer.Computer() |
| 200 | self.assertIn("computer_exec", {tool["name"] for tool in vm.tools()}) |
| 201 | self.assertEqual([c["id"] for c in vm.call("computer_list")["computers"]], ["test-vm"]) |
| 202 | ran = vm.call("computer_exec", vm="test-vm", script="uname") |
| 203 | self.assertEqual((ran["stdout"], ran["png"]), ("ran uname", base64.b64decode(PNG))) |
| 204 | for operation in ["force_off", "reset"]: |
| 205 | self.assertEqual(vm.call("computer_vm", vm="test-vm", operation=operation), |
| 206 | {"id":"test-vm", "requested":operation}) |
| 207 | self.assertEqual(Host.power_actions, ["destroy", "reset"]) |
| 208 | with ThreadPoolExecutor(max_workers=4) as pool: |
| 209 | held = [pool.submit(vm.call, "computer_exec", vm="test-vm", script=f"held-{index}") for index in range(4)] |
| 210 | try: |
| 211 | for _ in held: |
| 212 | self.assertTrue(Host.started.acquire(timeout=3)) |
| 213 | started = time.monotonic() |
| 214 | status = vm.call("computer_status", vm="test-vm") |
| 215 | self.assertEqual(status["readiness"], "busy") |
| 216 | stopped = vm.call("computer_process", vm="test-vm", id=status["active_operation"]["id"], operation="stop") |
| 217 | self.assertTrue(stopped["stopping"]) |
| 218 | vm.call("computer_vm", vm="test-vm", operation="reset") |
| 219 | self.assertLess(time.monotonic() - started, 2, "recovery controls waited behind long commands") |
| 220 | finally: |
| 221 | Host.release.set() |
| 222 | for future in held: |
| 223 | self.assertEqual(future.result()["exit"], 0) |
| 224 | with self.assertRaisesRegex(computer.ComputerError, "granted"): |
| 225 | vm.call("computer_shot", vm="other-vm") |
| 226 | |
| 227 | source = Path(self.temporary.name) / "payload.bin" |
| 228 | source.write_bytes(os.urandom(200_000)) |
| 229 | self.assertEqual(vm.upload("test-vm", source, "/tmp/payload.bin")["bytes"], 200_000) |
| 230 | vm.download("test-vm", "/tmp/payload.bin", source.with_suffix(".back")) |
| 231 | self.assertEqual(source.with_suffix(".back").read_bytes(), source.read_bytes()) |
| 232 | |
| 233 | data = source.read_bytes() |
| 234 | for declared, content, expected in [(len(data), data, 200), (0, b"", 200), |
| 235 | (len(data) - 1, data, 413), (len(data) + 1, data, 400)]: |
| 236 | destination = f"/tmp/fragmented-{declared}" |
| 237 | if expected != 200: |
| 238 | Host.files[destination] = b"retained" |
| 239 | ticket = vm.call("computer_upload", vm="test-vm", path=destination, size=declared, overwrite=True) |
| 240 | def fragments(): |
| 241 | for offset in range(0, len(content), 997): |
| 242 | yield content[offset:offset + 997] |
| 243 | time.sleep(.001) |
| 244 | connection = http.client.HTTPConnection("127.0.0.1", self.port, timeout=30) |
| 245 | connection.request("PUT", urlsplit(ticket["url"]).path, fragments(), self.headers, encode_chunked=True) |
| 246 | response = connection.getresponse() |
| 247 | body = response.read() |
| 248 | connection.close() |
| 249 | self.assertEqual(response.status, expected, body) |
| 250 | if expected == 200: |
| 251 | self.assertEqual(Host.files[destination], content) |
| 252 | sizes = [size for path, size in Host.writes if path.startswith(destination + ".snowglobe-")] |
| 253 | self.assertEqual(len(sizes), max(1, (len(content) + 65535) // 65536)) |
| 254 | self.assertEqual(sum(sizes), len(content)) |
| 255 | else: |
| 256 | self.assertEqual(Host.files[destination], b"retained") |
| 257 | self.assertFalse(any(path.startswith(destination + ".snowglobe-") for path in Host.files), |
| 258 | "upload returned before removing its temporary file") |
| 259 | |
| 260 | for index, cap in enumerate([2**20, 0, -1, True, "1048576", 2**20 + 1, 1.5, 2**100]): |
| 261 | Host.chunk_bytes = cap |
| 262 | content = os.urandom(2**21 + 17 if index == 0 else 200_000) |
| 263 | source.write_bytes(content) |
| 264 | destination = f"/tmp/negotiated-{index}" |
| 265 | Host.writes.clear() |
| 266 | Host.reads.clear() |
| 267 | vm.upload("test-vm", source, destination) |
| 268 | back = source.with_suffix(".negotiated") |
| 269 | vm.download("test-vm", destination, back) |
| 270 | self.assertEqual(back.read_bytes(), content) |
| 271 | expected = [65536, 2**20, len(content) - 65536 - 2**20] if index == 0 else [65536, 65536, 65536, 3392] |
| 272 | self.assertEqual([size for _, size in Host.writes], expected) |
| 273 | self.assertEqual([size for _, size in Host.reads], expected) |
| 274 | Host.chunk_bytes = None |
| 275 | |
| 276 | self.assertIn("png", vm.call("computer_shot", vm="test-vm")) |
| 277 | |
| 278 | state = json.loads(computer.STATE.read_text()) |
| 279 | computer.STATE.write_text(json.dumps({**state, "access_token": "x" * 43})) |
| 280 | self.assertTrue(computer.Computer().call("computer_list")["computers"]) |
| 281 | |
| 282 | # Concurrent renewals must not replay a refresh token, which would revoke the connection. |
| 283 | state = json.loads(computer.STATE.read_text()) |
| 284 | computer.STATE.write_text(json.dumps({**state, "expires_at": 0})) |
| 285 | results = [] |
| 286 | threads = [threading.Thread(target=lambda: results.append(computer.Computer().call("computer_list"))) |
| 287 | for _ in range(6)] |
| 288 | for thread in threads: |
| 289 | thread.start() |
| 290 | for thread in threads: |
| 291 | thread.join() |
| 292 | self.assertEqual(len(results), 6) |
| 293 | self.assertNotEqual(json.loads(computer.STATE.read_text())["refresh_token"], state["refresh_token"]) |
| 294 | self.assertTrue(computer.Computer().call("computer_list")["computers"]) |
| 295 | |
| 296 | |
| 297 | if __name__ == "__main__": |
| 298 | parser = argparse.ArgumentParser() |
| 299 | parser.add_argument("--binary", type=Path, default=Path("dashboard/target/debug/home-dashboard")) |
| 300 | args, rest = parser.parse_known_args() |
| 301 | BINARY = str(args.binary.resolve()) |
| 302 | unittest.main(argv=[sys.argv[0], *rest]) |