1#!/usr/bin/env python3
2import concurrent.futures
3from contextlib import closing
4from importlib import import_module
5import json
6import os
7from pathlib import Path
8import re
9import secrets
10import shlex
11import shutil
12import socket
13import sqlite3
14import subprocess
15import sys
16import tempfile
17import threading
18import time
19
20import snowglobe
21
22
23OPERATIONS = frozenset({
24 "host.sample", "host.usage", "storage.mounts", "storage.datasets", "storage.pool",
25 "storage.snapshots", "vm.node", "vm.domains", "vm.computers", "vm.library", "deploy.current", "deploy.main", "deploy.history",
26 "deploy.stages", "deploy.managed", "deploy.release", "deploy.last",
27 "files.snapshot", "files.snapshots", "files.discard",
28})
29STAGE_ID = re.compile(r"dashboard-preview-[0-9a-f]{8}\Z")
30boundary = import_module("dashboard-host")
31
32
33class PreviewHost(boundary.Host):
34 def __init__(self, saved):
35 super().__init__(saved["clone"].split("/", 1)[0])
36 self.stage = saved
37
38 def handle(self, request, *, active_snapshot=None):
39 operation = request.get("operation") if isinstance(request, dict) else None
40 owner = None
41 if self.stage.get("computer") and self.stage.get("mount") and operation in {"vm.computer", "vm.owner", "vm.access", "vm.console", "vm.guest", "vm.domains", "vm.computers", "vm.ai", "vm.act"}:
42 with closing(sqlite3.connect(f"file:{self.stage['mount']}/data/accounts.sqlite?mode=ro", uri=True)) as accounts:
43 owner = accounts.execute("SELECT id FROM users WHERE username='preview'").fetchone()
44 if not owner:
45 raise ValueError("The preview account is unavailable.")
46 if operation == "vm.ai":
47 payload = request.get("payload")
48 if not owner or not isinstance(payload, dict) or payload.get("owner") != owner[0]:
49 raise boundary.dashboard_runs.Error(403, "Test VMs must belong to this preview account.")
50 if operation in {"vm.computer", "vm.owner", "vm.access", "vm.act", "vm.console", "vm.guest"}:
51 payload = request.get("payload")
52 if not self.stage.get("computer") or not isinstance(payload, dict):
53 raise boundary.dashboard_runs.Error(403, "Choose the VM explicitly attached to this preview.")
54 if payload.get("name") not in self.stage["computer"].split(","):
55 actual = super().handle({"operation": "vm.owner", "payload": {"name": payload.get("name")}}) if owner else {}
56 if not owner or actual.get("owner") != owner[0] or actual.get("ownerOnly") is not True:
57 raise boundary.dashboard_runs.Error(403, "Choose a VM attached to or created by this preview.")
58 if operation in {"files.snapshot", "files.snapshots", "files.discard"}:
59 datasets = request.get("datasets") if operation != "files.discard" else [request.get("dataset")]
60 if not isinstance(datasets, list) or datasets != [self.stage["clone"] + "/files"]:
61 raise boundary.dashboard_runs.Error(403, "File changes are limited to this preview's files.")
62 if operation == "storage.mounts" and set(request) == {"operation"}:
63 mounted = json.loads(snowglobe.command(
64 "findmnt", "--json", "-o", "SOURCE,TARGET,FSTYPE,OPTIONS", "--mountpoint", self.stage["mount"] + "/files", capture=True,
65 ).stdout)
66 mounted["filesystems"][0]["target"] = "/srv"
67 return mounted
68 result = super().handle(request, active_snapshot=active_snapshot) if active_snapshot is not None else super().handle(request)
69 if owner and operation in {"vm.owner", "vm.domains", "vm.computers"}:
70 if operation == "vm.owner":
71 result = {**result, "owner": owner[0], "ownerOnly": True}
72 else:
73 result = [{**vm, "owner": owner[0], "ownerOnly": True} if vm["name"] in self.stage["computer"].split(",") else vm for vm in result]
74 return result
75
76
77def metadata(stage):
78 if not STAGE_ID.fullmatch(stage):
79 raise ValueError("invalid dashboard stage ID")
80 saved = json.loads((snowglobe.STATE / "stages" / f"{stage}.json").read_text())
81 if saved.get("sourceId") != "dashboard":
82 raise ValueError("not a dashboard stage")
83 return saved
84
85
86def serve(stage):
87 saved = metadata(stage)
88 host = PreviewHost(saved)
89 runtime = Path("/run/snowglobe-preview") / stage
90 runtime.mkdir(parents=True, exist_ok=True)
91 os.chown(runtime, 0, saved["uid"])
92 runtime.chmod(0o750)
93 path = runtime / "host.sock"
94 path.unlink(missing_ok=True)
95 with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener:
96 listener.bind(str(path))
97 os.chown(path, 0, saved["uid"])
98 path.chmod(0o660)
99 listener.listen(boundary.CONNECTIONS)
100 slots = threading.BoundedSemaphore(boundary.CONNECTIONS)
101 with concurrent.futures.ThreadPoolExecutor(max_workers=boundary.CONNECTIONS) as workers:
102 while True:
103 connection, _ = listener.accept()
104 if not slots.acquire(blocking=False):
105 connection.close()
106 continue
107 operations = OPERATIONS | {"vm.computer", "vm.owner", "vm.access", "vm.ai", "vm.act", "vm.console", "vm.guest"} if saved.get("computer") else OPERATIONS
108 work = workers.submit(boundary.serve_connection, connection, host, saved["uid"], operations)
109 work.add_done_callback(lambda _: slots.release())
110
111
112def stage(computer=None):
113 snowglobe.token()
114 site = json.loads(snowglobe.command("pkl", "eval", "-f", "json", str(snowglobe.REPO / "config/site.pkl"), capture=True).stdout)
115 site["hostRoot"] = str(Path(site["root"]) / "prod/dashboard")
116 site["stagingRoot"] = str(Path(site["root"]) / "staging")
117 snowglobe.check_staging_pool(site)
118 directory = snowglobe.STATE / "stages"
119 previous = [path for path in directory.glob("*.json") if json.loads(path.read_text()).get("sourceId") == "dashboard"]
120 if len(previous) > 1:
121 raise ValueError("multiple dashboard stages exist; destroy one first")
122 created = not previous
123 identity = "dashboard-preview-" + secrets.token_hex(4) if created else previous[0].stem
124 hostname = identity + "." + site["domain"]
125 root = Path(site["stagingRoot"]) / identity
126 saved = {"sourceId": "dashboard", "tasks": ["app"], "inputs": {}, "mount": str(root),
127 "clone": site["pool"] + "/staging/" + identity, "uid": snowglobe.identity("dashboard"),
128 "stageIsolation": "fresh"} if created else metadata(identity)
129 if computer:
130 saved["computer"] = computer
131 host_environment = shlex.split(snowglobe.command("systemctl", "show", "snowglobe-host", "--property=Environment", "--value", capture=True).stdout)
132 host_path = next(value for value in host_environment if value.startswith("PATH="))
133 mac = None
134 if saved.get("computer"):
135 configured = next((value.partition("=")[2] for value in host_environment if value.startswith("SNOWGLOBE_MAC_VM=")), "null")
136 mac = json.loads(configured)
137 selected = saved["computer"].split(",")
138 for name in selected:
139 boundary.vms.ensure_name(name)
140 if not mac or all(entry["name"] != name for entry in mac["vms"]):
141 snowglobe.command("virsh", "domuuid", name, capture=True)
142 if mac:
143 mac["vms"] = [entry for entry in mac["vms"] if entry["name"] in selected]
144 if mac and not mac["vms"]:
145 mac = None
146 nix = ["nix", "--extra-experimental-features", "nix-command flakes"]
147 package = "path:" + str(snowglobe.REPO) + "#dashboard-image"
148 image = subprocess.run([*nix, "eval", "--raw", package, "--apply", 'image: "${image.imageName}:${image.imageTag}"'],
149 stdout=subprocess.PIPE, text=True, check=True).stdout.strip()
150 if subprocess.run(["podman", "image", "exists", image], capture_output=True).returncode:
151 with tempfile.TemporaryDirectory(prefix="dashboard-image-", dir=site["stagingRoot"]) as temporary:
152 built = subprocess.run([*nix, "build", package + ".stream", "--option", "build-dir", temporary,
153 "--out-link", str(Path(temporary) / "stream"), "--print-out-paths"],
154 stdout=subprocess.PIPE, text=True, check=True).stdout.strip()
155 environment = {**os.environ, "TMPDIR": temporary}
156 with subprocess.Popen([built], stdout=subprocess.PIPE, env=environment) as stream:
157 try:
158 subprocess.run(["podman", "load", "--quiet"], stdin=stream.stdout, env=environment, check=True)
159 finally:
160 stream.stdout.close()
161 if stream.wait():
162 raise RuntimeError("dashboard image stream failed")
163 env = {
164 "SNOWGLOBE_DOMAIN": site["domain"], "SNOWGLOBE_PUBLIC_ORIGIN": "https://" + hostname,
165 "SNOWGLOBE_FILE_ORIGIN": "https://files-" + hostname, "SNOWGLOBE_AUTH_RP_ID": hostname,
166 "SNOWGLOBE_AUTH_REQUIRED": "1", "SNOWGLOBE_PROXY_TOKEN_FILE": "/run/secrets/proxy.token", "SNOWGLOBE_DATA_DIR": "/data",
167 "SNOWGLOBE_PREVIEW": "1", "SNOWGLOBE_METRICS": "follow",
168 "SNOWGLOBE_LISTEN_ADDRESS": "127.0.0.1", "PORT": "${NOMAD_PORT_http}",
169 "SNOWGLOBE_NOMAD_TOKEN_FILE": "/run/secrets/nomad.token", "NOMAD_ADDR": "http://127.0.0.1:4646",
170 "SNOWGLOBE_HOST_SOCKET": "/run/snowglobe-host/host.sock", "SNOWGLOBE_NODE_NAME": site["nodeName"],
171 "SNOWGLOBE_STORE_ROOT": "/srv", "SNOWGLOBE_MEDIA_ROOT": "/srv/Media",
172 "SNOWGLOBE_FILES_WRITABLE": "1", "SNOWGLOBE_CLOVER_READ_ONLY": "false", "SNOWGLOBE_MEDIA_READ_ONLY": "false",
173 "HOME": "/data", "XDG_CACHE_HOME": "/data/cache",
174 }
175 try:
176 if created:
177 snowglobe.command("zfs", "create", "-o", "mountpoint=" + str(root), saved["clone"])
178 saved["external"] = {"files": {"clone": saved["clone"] + "/files"}}
179 directory.mkdir(parents=True, exist_ok=True)
180 snowglobe.write_private(directory / f"{identity}.json", json.dumps(saved))
181 snowglobe.command("zfs", "create", "-o", "mountpoint=" + str(root / "files"), saved["clone"] + "/files")
182 for path in (root / "data", root / "files", root / "files/Media"):
183 path.mkdir(parents=True, exist_ok=True)
184 os.chown(path, saved["uid"], saved["uid"])
185 path.chmod(0o750)
186 fixture = root / "files/preview.txt"
187 fixture.write_text("Disposable Snow Globe dashboard preview.\n")
188 fixture.chmod(0o644)
189 proof = root / "proxy.token"
190 if not proof.exists():
191 snowglobe.write_private(proof, secrets.token_hex(32) + "\n")
192 os.chown(proof, saved["uid"], saved["uid"])
193 proof.chmod(0o400)
194 directory.mkdir(parents=True, exist_ok=True)
195 saved.update(release=snowglobe.REPO.name, ready=False)
196 snowglobe.write_private(directory / f"{identity}.json", json.dumps(saved))
197 if created:
198 policy = 'namespace "default" { capabilities = ["list-jobs", "read-job"] }\nnode { policy = "read" }\n'
199 snowglobe.command("nomad", "acl", "policy", "apply", identity, "-", input=policy)
200 token = json.loads(snowglobe.command("nomad", "acl", "token", "create", "-policy=" + identity,
201 "-name=" + identity, "-json", capture=True).stdout)
202 saved["tokenAccessor"] = token["AccessorID"]
203 snowglobe.write_private(directory / f"{identity}.json", json.dumps(saved))
204 snowglobe.write_private(root / "nomad.token", token["SecretID"] + "\n")
205 os.chown(root / "nomad.token", saved["uid"], saved["uid"])
206 imported = snowglobe.command("podman", "run", "--rm", "--network=none", "--read-only",
207 "--user=" + str(saved["uid"]), "--cap-drop=ALL", "--security-opt=no-new-privileges",
208 "--entrypoint=/bin/home-dashboard",
209 "--volume=" + str(root / "data") + ":/data:rw",
210 "--volume=" + str(proof) + ":/run/secrets/proxy.token:ro",
211 *["--env=" + key + "=" + value for key, value in env.items()], image, "--preview-account", capture=True)
212 snowglobe.write_private(root / "access.json", imported.stdout)
213 runtime = "/run/snowglobe-preview/" + identity
214 computer_environment = []
215 if saved.get("computer"):
216 ahk = snowglobe.command(*nix, "build", "--out-link", str(root / "computer-ahk"), "--print-out-paths", "path:" + str(snowglobe.REPO) + "#computer-ahk", capture=True).stdout.strip()
217 computer_environment.append("--setenv=SNOWGLOBE_COMPUTER_AHK=" + ahk)
218 if mac:
219 with sqlite3.connect(f"file:{root / 'data/accounts.sqlite'}?mode=ro", uri=True) as accounts:
220 owner = accounts.execute("SELECT id FROM users WHERE username='preview'").fetchone()
221 if not owner:
222 raise ValueError("The preview account is unavailable.")
223 for entry in mac["vms"]:
224 entry["owner"] = owner[0]
225 computer_environment.append("--setenv=SNOWGLOBE_MAC_VM=" + json.dumps(mac))
226 subprocess.run(["systemctl", "stop", "snowglobe-preview-screen-" + identity], capture_output=True)
227 subprocess.run(["systemctl", "stop", "snowglobe-preview-" + identity], capture_output=True)
228 snowglobe.command("systemd-run", "--collect", "--quiet", "--unit=snowglobe-preview-" + identity,
229 "--property=Restart=on-failure", "--property=MemoryMax=" + str(boundary.PROTOCOL["memory"]), "--property=TasksMax=32",
230 "--setenv=" + host_path, "--setenv=SNOWGLOBE_DOMAIN=" + site["domain"],
231 *computer_environment,
232 sys.executable, str(Path(__file__).resolve()), "--serve", identity)
233 for _ in range(100):
234 if Path(runtime, "host.sock").exists():
235 break
236 time.sleep(.1)
237 else:
238 raise RuntimeError("dashboard preview host socket did not start")
239 screen_volumes = []
240 if saved.get("computer"):
241 screen = snowglobe.command(*nix, "build", "--out-link", str(root / "vm-screen"), "--print-out-paths", "path:" + str(snowglobe.REPO) + "#vm-screen", capture=True).stdout.strip()
242 screen_runtime = "/run/snowglobe-preview-screen-" + identity
243 launcher = f"import os; os.chown({screen_runtime!r}, {saved['uid']}, {saved['uid']}); os.setgroups([]); os.setgid({saved['uid']}); os.setuid({saved['uid']}); os.execv({screen + '/bin/snowglobe-vm-screen'!r}, [{screen + '/bin/snowglobe-vm-screen'!r}])"
244 snowglobe.command("systemd-run", "--collect", "--quiet", "--unit=snowglobe-preview-screen-" + identity,
245 "--property=RuntimeDirectory=snowglobe-preview-screen-" + identity, "--property=RuntimeDirectoryMode=0700",
246 "--property=RuntimeDirectoryPreserve=yes",
247 "--property=Restart=on-failure", "--property=MemoryMax=2G", "--property=TasksMax=128",
248 "--property=NoNewPrivileges=yes", "--property=ProtectSystem=strict", "--property=ProtectHome=yes", "--property=PrivateTmp=yes",
249 "--property=DevicePolicy=closed",
250 "--setenv=SNOWGLOBE_HOST_SOCKET=" + runtime + "/host.sock",
251 "--setenv=SNOWGLOBE_VM_SCREEN_SOCKET=" + screen_runtime + "/screen.sock",
252 "--setenv=XDG_RUNTIME_DIR=" + screen_runtime,
253 "--setenv=XDG_CACHE_HOME=" + screen_runtime + "/cache",
254 sys.executable, "-c", launcher)
255 for _ in range(100):
256 if Path(screen_runtime, "screen.sock").exists():
257 break
258 time.sleep(.1)
259 else:
260 raise RuntimeError("dashboard preview screen socket did not start")
261 screen_volumes.append(screen_runtime + ":/run/snowglobe-vm-screen:ro")
262 job = {"ID": identity, "Name": identity, "Type": "service", "Datacenters": ["clover"],
263 "Meta": {"snowglobe_service": "dashboard", "snowglobe_name": "Snow Globe preview", "snowglobe_hostname": hostname},
264 "TaskGroups": [{"Name": "app", "Count": 1, "Networks": [{"Mode": "host", "DynamicPorts": [{"Label": "http", "HostNetwork": "loopback"}]}],
265 "Services": [{"Name": identity, "Provider": "nomad", "PortLabel": "http", "TaskName": "app", "Tags": ["caddy-host=" + hostname],
266 "Checks": [{"Name": "Dashboard listener", "Type": "tcp", "Interval": 10000000000, "Timeout": 5000000000}]}],
267 "Tasks": [{"Name": "app", "Driver": "podman", "User": str(saved["uid"]) + ":" + str(saved["uid"]), "Env": env,
268 "Config": {"image": image, "network_mode": "host", "cap_drop": ["ALL"], "security_opt": ["no-new-privileges"],
269 "volumes": [str(root / "data") + ":/data:rw", str(root / "files") + ":/srv:rw,rslave", runtime + ":/run/snowglobe-host:ro",
270 str(root / "nomad.token") + ":/run/secrets/nomad.token:ro",
271 str(proof) + ":/run/secrets/proxy.token:ro", *screen_volumes]},
272 "Resources": {"CPU": 1000, "MemoryMB": 2048}}]}]}
273 snowglobe.command("nomad", "job", "run", "-json", "-", input=json.dumps(job))
274 snowglobe.check_http(hostname, "/sign-in", site["tlsInternal"])
275 saved["ready"] = True
276 snowglobe.write_private(directory / f"{identity}.json", json.dumps(saved))
277 except BaseException:
278 subprocess.run(["systemctl", "stop", "snowglobe-preview-screen-" + identity], capture_output=True)
279 if created and (directory / f"{identity}.json").exists():
280 destroy(identity)
281 raise
282 print(f"stage={identity}\nrelease={saved['release']}\nhost={hostname}\ndataset={saved['clone']}\naccess={root / 'access.json'}")
283
284
285def destroy(identity):
286 snowglobe.token()
287 saved = metadata(identity)
288 subprocess.run(["systemctl", "stop", "snowglobe-preview-screen-" + identity], capture_output=True)
289 subprocess.run(["systemctl", "stop", "snowglobe-preview-" + identity], capture_output=True)
290 if saved.get("tokenAccessor"):
291 revoked = subprocess.run(["nomad", "acl", "token", "delete", saved["tokenAccessor"]], capture_output=True, text=True)
292 if revoked.returncode and "400 (Cannot delete nonexistent tokens:" not in revoked.stderr:
293 raise RuntimeError(revoked.stderr.strip())
294 snowglobe.command("nomad", "acl", "policy", "delete", identity, capture=True)
295 datasets = set(snowglobe.command("zfs", "list", "-H", "-o", "name", capture=True).stdout.splitlines())
296 saved["external"] = {name: item for name, item in saved.get("external", {}).items() if item["clone"] in datasets}
297 if saved.get("external", {}).get("files", {}).get("clone") == saved["clone"] + "/files":
298 snapshots = snowglobe.command("zfs", "list", "-H", "-t", "snapshot", "-r", "-o", "name", saved["clone"] + "/files", capture=True).stdout.splitlines()
299 for snapshot in snapshots:
300 if snapshot.startswith(saved["clone"] + "/files@"):
301 snowglobe.command("zfs", "destroy", snapshot)
302 if saved["clone"] not in datasets:
303 del saved["clone"]
304 snowglobe.destroy_stage(identity, saved, {})
305 shutil.rmtree(Path("/run/snowglobe-preview") / identity, ignore_errors=True)
306 shutil.rmtree(Path("/run") / ("snowglobe-preview-screen-" + identity), ignore_errors=True)
307
308
309if __name__ == "__main__":
310 if len(sys.argv) == 3 and sys.argv[1] == "--serve":
311 serve(sys.argv[2])
312 else:
313 raise SystemExit("Expected --serve STAGE")