| 1 | #!/usr/bin/env python3 |
| 2 | import concurrent.futures |
| 3 | from contextlib import closing |
| 4 | from importlib import import_module |
| 5 | import json |
| 6 | import os |
| 7 | from pathlib import Path |
| 8 | import re |
| 9 | import secrets |
| 10 | import shlex |
| 11 | import shutil |
| 12 | import socket |
| 13 | import sqlite3 |
| 14 | import subprocess |
| 15 | import sys |
| 16 | import tempfile |
| 17 | import threading |
| 18 | import time |
| 19 | |
| 20 | import snowglobe |
| 21 | |
| 22 | |
| 23 | OPERATIONS = frozenset({ |
| 24 | "host.sample", "host.usage", "storage.mounts", "storage.datasets", "storage.pool", |
| 25 | "storage.snapshots", "vm.node", "vm.domains", "vm.computers", "vm.library", "deploy.current", "deploy.main", "deploy.history", |
| 26 | "deploy.stages", "deploy.managed", "deploy.release", "deploy.last", |
| 27 | "files.snapshot", "files.snapshots", "files.discard", |
| 28 | }) |
| 29 | STAGE_ID = re.compile(r"dashboard-preview-[0-9a-f]{8}\Z") |
| 30 | boundary = import_module("dashboard-host") |
| 31 | |
| 32 | |
| 33 | class PreviewHost(boundary.Host): |
| 34 | def __init__(self, saved): |
| 35 | super().__init__(saved["clone"].split("/", 1)[0]) |
| 36 | self.stage = saved |
| 37 | |
| 38 | def handle(self, request, *, active_snapshot=None): |
| 39 | operation = request.get("operation") if isinstance(request, dict) else None |
| 40 | owner = None |
| 41 | if self.stage.get("computer") and self.stage.get("mount") and operation in {"vm.computer", "vm.owner", "vm.access", "vm.console", "vm.guest", "vm.domains", "vm.computers", "vm.ai", "vm.act"}: |
| 42 | with closing(sqlite3.connect(f"file:{self.stage['mount']}/data/accounts.sqlite?mode=ro", uri=True)) as accounts: |
| 43 | owner = accounts.execute("SELECT id FROM users WHERE username='preview'").fetchone() |
| 44 | if not owner: |
| 45 | raise ValueError("The preview account is unavailable.") |
| 46 | if operation == "vm.ai": |
| 47 | payload = request.get("payload") |
| 48 | if not owner or not isinstance(payload, dict) or payload.get("owner") != owner[0]: |
| 49 | raise boundary.dashboard_runs.Error(403, "Test VMs must belong to this preview account.") |
| 50 | if operation in {"vm.computer", "vm.owner", "vm.access", "vm.act", "vm.console", "vm.guest"}: |
| 51 | payload = request.get("payload") |
| 52 | if not self.stage.get("computer") or not isinstance(payload, dict): |
| 53 | raise boundary.dashboard_runs.Error(403, "Choose the VM explicitly attached to this preview.") |
| 54 | if payload.get("name") not in self.stage["computer"].split(","): |
| 55 | actual = super().handle({"operation": "vm.owner", "payload": {"name": payload.get("name")}}) if owner else {} |
| 56 | if not owner or actual.get("owner") != owner[0] or actual.get("ownerOnly") is not True: |
| 57 | raise boundary.dashboard_runs.Error(403, "Choose a VM attached to or created by this preview.") |
| 58 | if operation in {"files.snapshot", "files.snapshots", "files.discard"}: |
| 59 | datasets = request.get("datasets") if operation != "files.discard" else [request.get("dataset")] |
| 60 | if not isinstance(datasets, list) or datasets != [self.stage["clone"] + "/files"]: |
| 61 | raise boundary.dashboard_runs.Error(403, "File changes are limited to this preview's files.") |
| 62 | if operation == "storage.mounts" and set(request) == {"operation"}: |
| 63 | mounted = json.loads(snowglobe.command( |
| 64 | "findmnt", "--json", "-o", "SOURCE,TARGET,FSTYPE,OPTIONS", "--mountpoint", self.stage["mount"] + "/files", capture=True, |
| 65 | ).stdout) |
| 66 | mounted["filesystems"][0]["target"] = "/srv" |
| 67 | return mounted |
| 68 | result = super().handle(request, active_snapshot=active_snapshot) if active_snapshot is not None else super().handle(request) |
| 69 | if owner and operation in {"vm.owner", "vm.domains", "vm.computers"}: |
| 70 | if operation == "vm.owner": |
| 71 | result = {**result, "owner": owner[0], "ownerOnly": True} |
| 72 | else: |
| 73 | result = [{**vm, "owner": owner[0], "ownerOnly": True} if vm["name"] in self.stage["computer"].split(",") else vm for vm in result] |
| 74 | return result |
| 75 | |
| 76 | |
| 77 | def metadata(stage): |
| 78 | if not STAGE_ID.fullmatch(stage): |
| 79 | raise ValueError("invalid dashboard stage ID") |
| 80 | saved = json.loads((snowglobe.STATE / "stages" / f"{stage}.json").read_text()) |
| 81 | if saved.get("sourceId") != "dashboard": |
| 82 | raise ValueError("not a dashboard stage") |
| 83 | return saved |
| 84 | |
| 85 | |
| 86 | def serve(stage): |
| 87 | saved = metadata(stage) |
| 88 | host = PreviewHost(saved) |
| 89 | runtime = Path("/run/snowglobe-preview") / stage |
| 90 | runtime.mkdir(parents=True, exist_ok=True) |
| 91 | os.chown(runtime, 0, saved["uid"]) |
| 92 | runtime.chmod(0o750) |
| 93 | path = runtime / "host.sock" |
| 94 | path.unlink(missing_ok=True) |
| 95 | with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener: |
| 96 | listener.bind(str(path)) |
| 97 | os.chown(path, 0, saved["uid"]) |
| 98 | path.chmod(0o660) |
| 99 | listener.listen(boundary.CONNECTIONS) |
| 100 | slots = threading.BoundedSemaphore(boundary.CONNECTIONS) |
| 101 | with concurrent.futures.ThreadPoolExecutor(max_workers=boundary.CONNECTIONS) as workers: |
| 102 | while True: |
| 103 | connection, _ = listener.accept() |
| 104 | if not slots.acquire(blocking=False): |
| 105 | connection.close() |
| 106 | continue |
| 107 | operations = OPERATIONS | {"vm.computer", "vm.owner", "vm.access", "vm.ai", "vm.act", "vm.console", "vm.guest"} if saved.get("computer") else OPERATIONS |
| 108 | work = workers.submit(boundary.serve_connection, connection, host, saved["uid"], operations) |
| 109 | work.add_done_callback(lambda _: slots.release()) |
| 110 | |
| 111 | |
| 112 | def stage(computer=None): |
| 113 | snowglobe.token() |
| 114 | site = json.loads(snowglobe.command("pkl", "eval", "-f", "json", str(snowglobe.REPO / "config/site.pkl"), capture=True).stdout) |
| 115 | site["hostRoot"] = str(Path(site["root"]) / "prod/dashboard") |
| 116 | site["stagingRoot"] = str(Path(site["root"]) / "staging") |
| 117 | snowglobe.check_staging_pool(site) |
| 118 | directory = snowglobe.STATE / "stages" |
| 119 | previous = [path for path in directory.glob("*.json") if json.loads(path.read_text()).get("sourceId") == "dashboard"] |
| 120 | if len(previous) > 1: |
| 121 | raise ValueError("multiple dashboard stages exist; destroy one first") |
| 122 | created = not previous |
| 123 | identity = "dashboard-preview-" + secrets.token_hex(4) if created else previous[0].stem |
| 124 | hostname = identity + "." + site["domain"] |
| 125 | root = Path(site["stagingRoot"]) / identity |
| 126 | saved = {"sourceId": "dashboard", "tasks": ["app"], "inputs": {}, "mount": str(root), |
| 127 | "clone": site["pool"] + "/staging/" + identity, "uid": snowglobe.identity("dashboard"), |
| 128 | "stageIsolation": "fresh"} if created else metadata(identity) |
| 129 | if computer: |
| 130 | saved["computer"] = computer |
| 131 | host_environment = shlex.split(snowglobe.command("systemctl", "show", "snowglobe-host", "--property=Environment", "--value", capture=True).stdout) |
| 132 | host_path = next(value for value in host_environment if value.startswith("PATH=")) |
| 133 | mac = None |
| 134 | if saved.get("computer"): |
| 135 | configured = next((value.partition("=")[2] for value in host_environment if value.startswith("SNOWGLOBE_MAC_VM=")), "null") |
| 136 | mac = json.loads(configured) |
| 137 | selected = saved["computer"].split(",") |
| 138 | for name in selected: |
| 139 | boundary.vms.ensure_name(name) |
| 140 | if not mac or all(entry["name"] != name for entry in mac["vms"]): |
| 141 | snowglobe.command("virsh", "domuuid", name, capture=True) |
| 142 | if mac: |
| 143 | mac["vms"] = [entry for entry in mac["vms"] if entry["name"] in selected] |
| 144 | if mac and not mac["vms"]: |
| 145 | mac = None |
| 146 | nix = ["nix", "--extra-experimental-features", "nix-command flakes"] |
| 147 | package = "path:" + str(snowglobe.REPO) + "#dashboard-image" |
| 148 | image = subprocess.run([*nix, "eval", "--raw", package, "--apply", 'image: "${image.imageName}:${image.imageTag}"'], |
| 149 | stdout=subprocess.PIPE, text=True, check=True).stdout.strip() |
| 150 | if subprocess.run(["podman", "image", "exists", image], capture_output=True).returncode: |
| 151 | with tempfile.TemporaryDirectory(prefix="dashboard-image-", dir=site["stagingRoot"]) as temporary: |
| 152 | built = subprocess.run([*nix, "build", package + ".stream", "--option", "build-dir", temporary, |
| 153 | "--out-link", str(Path(temporary) / "stream"), "--print-out-paths"], |
| 154 | stdout=subprocess.PIPE, text=True, check=True).stdout.strip() |
| 155 | environment = {**os.environ, "TMPDIR": temporary} |
| 156 | with subprocess.Popen([built], stdout=subprocess.PIPE, env=environment) as stream: |
| 157 | try: |
| 158 | subprocess.run(["podman", "load", "--quiet"], stdin=stream.stdout, env=environment, check=True) |
| 159 | finally: |
| 160 | stream.stdout.close() |
| 161 | if stream.wait(): |
| 162 | raise RuntimeError("dashboard image stream failed") |
| 163 | env = { |
| 164 | "SNOWGLOBE_DOMAIN": site["domain"], "SNOWGLOBE_PUBLIC_ORIGIN": "https://" + hostname, |
| 165 | "SNOWGLOBE_FILE_ORIGIN": "https://files-" + hostname, "SNOWGLOBE_AUTH_RP_ID": hostname, |
| 166 | "SNOWGLOBE_AUTH_REQUIRED": "1", "SNOWGLOBE_PROXY_TOKEN_FILE": "/run/secrets/proxy.token", "SNOWGLOBE_DATA_DIR": "/data", |
| 167 | "SNOWGLOBE_PREVIEW": "1", "SNOWGLOBE_METRICS": "follow", |
| 168 | "SNOWGLOBE_LISTEN_ADDRESS": "127.0.0.1", "PORT": "${NOMAD_PORT_http}", |
| 169 | "SNOWGLOBE_NOMAD_TOKEN_FILE": "/run/secrets/nomad.token", "NOMAD_ADDR": "http://127.0.0.1:4646", |
| 170 | "SNOWGLOBE_HOST_SOCKET": "/run/snowglobe-host/host.sock", "SNOWGLOBE_NODE_NAME": site["nodeName"], |
| 171 | "SNOWGLOBE_STORE_ROOT": "/srv", "SNOWGLOBE_MEDIA_ROOT": "/srv/Media", |
| 172 | "SNOWGLOBE_FILES_WRITABLE": "1", "SNOWGLOBE_CLOVER_READ_ONLY": "false", "SNOWGLOBE_MEDIA_READ_ONLY": "false", |
| 173 | "HOME": "/data", "XDG_CACHE_HOME": "/data/cache", |
| 174 | } |
| 175 | try: |
| 176 | if created: |
| 177 | snowglobe.command("zfs", "create", "-o", "mountpoint=" + str(root), saved["clone"]) |
| 178 | saved["external"] = {"files": {"clone": saved["clone"] + "/files"}} |
| 179 | directory.mkdir(parents=True, exist_ok=True) |
| 180 | snowglobe.write_private(directory / f"{identity}.json", json.dumps(saved)) |
| 181 | snowglobe.command("zfs", "create", "-o", "mountpoint=" + str(root / "files"), saved["clone"] + "/files") |
| 182 | for path in (root / "data", root / "files", root / "files/Media"): |
| 183 | path.mkdir(parents=True, exist_ok=True) |
| 184 | os.chown(path, saved["uid"], saved["uid"]) |
| 185 | path.chmod(0o750) |
| 186 | fixture = root / "files/preview.txt" |
| 187 | fixture.write_text("Disposable Snow Globe dashboard preview.\n") |
| 188 | fixture.chmod(0o644) |
| 189 | proof = root / "proxy.token" |
| 190 | if not proof.exists(): |
| 191 | snowglobe.write_private(proof, secrets.token_hex(32) + "\n") |
| 192 | os.chown(proof, saved["uid"], saved["uid"]) |
| 193 | proof.chmod(0o400) |
| 194 | directory.mkdir(parents=True, exist_ok=True) |
| 195 | saved.update(release=snowglobe.REPO.name, ready=False) |
| 196 | snowglobe.write_private(directory / f"{identity}.json", json.dumps(saved)) |
| 197 | if created: |
| 198 | policy = 'namespace "default" { capabilities = ["list-jobs", "read-job"] }\nnode { policy = "read" }\n' |
| 199 | snowglobe.command("nomad", "acl", "policy", "apply", identity, "-", input=policy) |
| 200 | token = json.loads(snowglobe.command("nomad", "acl", "token", "create", "-policy=" + identity, |
| 201 | "-name=" + identity, "-json", capture=True).stdout) |
| 202 | saved["tokenAccessor"] = token["AccessorID"] |
| 203 | snowglobe.write_private(directory / f"{identity}.json", json.dumps(saved)) |
| 204 | snowglobe.write_private(root / "nomad.token", token["SecretID"] + "\n") |
| 205 | os.chown(root / "nomad.token", saved["uid"], saved["uid"]) |
| 206 | imported = snowglobe.command("podman", "run", "--rm", "--network=none", "--read-only", |
| 207 | "--user=" + str(saved["uid"]), "--cap-drop=ALL", "--security-opt=no-new-privileges", |
| 208 | "--entrypoint=/bin/home-dashboard", |
| 209 | "--volume=" + str(root / "data") + ":/data:rw", |
| 210 | "--volume=" + str(proof) + ":/run/secrets/proxy.token:ro", |
| 211 | *["--env=" + key + "=" + value for key, value in env.items()], image, "--preview-account", capture=True) |
| 212 | snowglobe.write_private(root / "access.json", imported.stdout) |
| 213 | runtime = "/run/snowglobe-preview/" + identity |
| 214 | computer_environment = [] |
| 215 | if saved.get("computer"): |
| 216 | ahk = snowglobe.command(*nix, "build", "--out-link", str(root / "computer-ahk"), "--print-out-paths", "path:" + str(snowglobe.REPO) + "#computer-ahk", capture=True).stdout.strip() |
| 217 | computer_environment.append("--setenv=SNOWGLOBE_COMPUTER_AHK=" + ahk) |
| 218 | if mac: |
| 219 | with sqlite3.connect(f"file:{root / 'data/accounts.sqlite'}?mode=ro", uri=True) as accounts: |
| 220 | owner = accounts.execute("SELECT id FROM users WHERE username='preview'").fetchone() |
| 221 | if not owner: |
| 222 | raise ValueError("The preview account is unavailable.") |
| 223 | for entry in mac["vms"]: |
| 224 | entry["owner"] = owner[0] |
| 225 | computer_environment.append("--setenv=SNOWGLOBE_MAC_VM=" + json.dumps(mac)) |
| 226 | subprocess.run(["systemctl", "stop", "snowglobe-preview-screen-" + identity], capture_output=True) |
| 227 | subprocess.run(["systemctl", "stop", "snowglobe-preview-" + identity], capture_output=True) |
| 228 | snowglobe.command("systemd-run", "--collect", "--quiet", "--unit=snowglobe-preview-" + identity, |
| 229 | "--property=Restart=on-failure", "--property=MemoryMax=" + str(boundary.PROTOCOL["memory"]), "--property=TasksMax=32", |
| 230 | "--setenv=" + host_path, "--setenv=SNOWGLOBE_DOMAIN=" + site["domain"], |
| 231 | *computer_environment, |
| 232 | sys.executable, str(Path(__file__).resolve()), "--serve", identity) |
| 233 | for _ in range(100): |
| 234 | if Path(runtime, "host.sock").exists(): |
| 235 | break |
| 236 | time.sleep(.1) |
| 237 | else: |
| 238 | raise RuntimeError("dashboard preview host socket did not start") |
| 239 | screen_volumes = [] |
| 240 | if saved.get("computer"): |
| 241 | screen = snowglobe.command(*nix, "build", "--out-link", str(root / "vm-screen"), "--print-out-paths", "path:" + str(snowglobe.REPO) + "#vm-screen", capture=True).stdout.strip() |
| 242 | screen_runtime = "/run/snowglobe-preview-screen-" + identity |
| 243 | launcher = f"import os; os.chown({screen_runtime!r}, {saved['uid']}, {saved['uid']}); os.setgroups([]); os.setgid({saved['uid']}); os.setuid({saved['uid']}); os.execv({screen + '/bin/snowglobe-vm-screen'!r}, [{screen + '/bin/snowglobe-vm-screen'!r}])" |
| 244 | snowglobe.command("systemd-run", "--collect", "--quiet", "--unit=snowglobe-preview-screen-" + identity, |
| 245 | "--property=RuntimeDirectory=snowglobe-preview-screen-" + identity, "--property=RuntimeDirectoryMode=0700", |
| 246 | "--property=RuntimeDirectoryPreserve=yes", |
| 247 | "--property=Restart=on-failure", "--property=MemoryMax=2G", "--property=TasksMax=128", |
| 248 | "--property=NoNewPrivileges=yes", "--property=ProtectSystem=strict", "--property=ProtectHome=yes", "--property=PrivateTmp=yes", |
| 249 | "--property=DevicePolicy=closed", |
| 250 | "--setenv=SNOWGLOBE_HOST_SOCKET=" + runtime + "/host.sock", |
| 251 | "--setenv=SNOWGLOBE_VM_SCREEN_SOCKET=" + screen_runtime + "/screen.sock", |
| 252 | "--setenv=XDG_RUNTIME_DIR=" + screen_runtime, |
| 253 | "--setenv=XDG_CACHE_HOME=" + screen_runtime + "/cache", |
| 254 | sys.executable, "-c", launcher) |
| 255 | for _ in range(100): |
| 256 | if Path(screen_runtime, "screen.sock").exists(): |
| 257 | break |
| 258 | time.sleep(.1) |
| 259 | else: |
| 260 | raise RuntimeError("dashboard preview screen socket did not start") |
| 261 | screen_volumes.append(screen_runtime + ":/run/snowglobe-vm-screen:ro") |
| 262 | job = {"ID": identity, "Name": identity, "Type": "service", "Datacenters": ["clover"], |
| 263 | "Meta": {"snowglobe_service": "dashboard", "snowglobe_name": "Snow Globe preview", "snowglobe_hostname": hostname}, |
| 264 | "TaskGroups": [{"Name": "app", "Count": 1, "Networks": [{"Mode": "host", "DynamicPorts": [{"Label": "http", "HostNetwork": "loopback"}]}], |
| 265 | "Services": [{"Name": identity, "Provider": "nomad", "PortLabel": "http", "TaskName": "app", "Tags": ["caddy-host=" + hostname], |
| 266 | "Checks": [{"Name": "Dashboard listener", "Type": "tcp", "Interval": 10000000000, "Timeout": 5000000000}]}], |
| 267 | "Tasks": [{"Name": "app", "Driver": "podman", "User": str(saved["uid"]) + ":" + str(saved["uid"]), "Env": env, |
| 268 | "Config": {"image": image, "network_mode": "host", "cap_drop": ["ALL"], "security_opt": ["no-new-privileges"], |
| 269 | "volumes": [str(root / "data") + ":/data:rw", str(root / "files") + ":/srv:rw,rslave", runtime + ":/run/snowglobe-host:ro", |
| 270 | str(root / "nomad.token") + ":/run/secrets/nomad.token:ro", |
| 271 | str(proof) + ":/run/secrets/proxy.token:ro", *screen_volumes]}, |
| 272 | "Resources": {"CPU": 1000, "MemoryMB": 2048}}]}]} |
| 273 | snowglobe.command("nomad", "job", "run", "-json", "-", input=json.dumps(job)) |
| 274 | snowglobe.check_http(hostname, "/sign-in", site["tlsInternal"]) |
| 275 | saved["ready"] = True |
| 276 | snowglobe.write_private(directory / f"{identity}.json", json.dumps(saved)) |
| 277 | except BaseException: |
| 278 | subprocess.run(["systemctl", "stop", "snowglobe-preview-screen-" + identity], capture_output=True) |
| 279 | if created and (directory / f"{identity}.json").exists(): |
| 280 | destroy(identity) |
| 281 | raise |
| 282 | print(f"stage={identity}\nrelease={saved['release']}\nhost={hostname}\ndataset={saved['clone']}\naccess={root / 'access.json'}") |
| 283 | |
| 284 | |
| 285 | def destroy(identity): |
| 286 | snowglobe.token() |
| 287 | saved = metadata(identity) |
| 288 | subprocess.run(["systemctl", "stop", "snowglobe-preview-screen-" + identity], capture_output=True) |
| 289 | subprocess.run(["systemctl", "stop", "snowglobe-preview-" + identity], capture_output=True) |
| 290 | if saved.get("tokenAccessor"): |
| 291 | revoked = subprocess.run(["nomad", "acl", "token", "delete", saved["tokenAccessor"]], capture_output=True, text=True) |
| 292 | if revoked.returncode and "400 (Cannot delete nonexistent tokens:" not in revoked.stderr: |
| 293 | raise RuntimeError(revoked.stderr.strip()) |
| 294 | snowglobe.command("nomad", "acl", "policy", "delete", identity, capture=True) |
| 295 | datasets = set(snowglobe.command("zfs", "list", "-H", "-o", "name", capture=True).stdout.splitlines()) |
| 296 | saved["external"] = {name: item for name, item in saved.get("external", {}).items() if item["clone"] in datasets} |
| 297 | if saved.get("external", {}).get("files", {}).get("clone") == saved["clone"] + "/files": |
| 298 | snapshots = snowglobe.command("zfs", "list", "-H", "-t", "snapshot", "-r", "-o", "name", saved["clone"] + "/files", capture=True).stdout.splitlines() |
| 299 | for snapshot in snapshots: |
| 300 | if snapshot.startswith(saved["clone"] + "/files@"): |
| 301 | snowglobe.command("zfs", "destroy", snapshot) |
| 302 | if saved["clone"] not in datasets: |
| 303 | del saved["clone"] |
| 304 | snowglobe.destroy_stage(identity, saved, {}) |
| 305 | shutil.rmtree(Path("/run/snowglobe-preview") / identity, ignore_errors=True) |
| 306 | shutil.rmtree(Path("/run") / ("snowglobe-preview-screen-" + identity), ignore_errors=True) |
| 307 | |
| 308 | |
| 309 | if __name__ == "__main__": |
| 310 | if len(sys.argv) == 3 and sys.argv[1] == "--serve": |
| 311 | serve(sys.argv[2]) |
| 312 | else: |
| 313 | raise SystemExit("Expected --serve STAGE") |