authoravatarclover caruso<git@paperclover.net>2026-11-01 22:19:03-08:00
committeravatarclover caruso<git@paperclover.net>2026-11-12 16:08:55-08:00
logdf4161ba6e1bae0f7c7596554bde1ccfe49ea94d
tree60ab485b2cfae8a17c940be301cd4cffb338bdca
parentffc3896c9d5ee7c48a19bfa615c002d2c9de7077
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Enable the Snow Globe GPU tensor unlock

Build pinned gpu-unlock firmware and an initrd module for the RTX 3090 before the NVIDIA driver loads. Add host boot preparation that preserves the existing default until promotion, plus repeatable inference and tensor benchmarks. Verified two isolated cold boots, a real generation-198 boot with the MAC-bound network intact, all service routes, authenticated inference, and 45 before/after benchmark samples per phase. Keep the installed kernel and NVIDIA driver versions unchanged. Assisted-by: gpt-6.1-sol

12 files changed, 445 insertions(+), 18 deletions(-)

nixos/configuration.nix+1-1
...@@ -47,7 +47,7 @@ let...@@ -47,7 +47,7 @@ let
47 });47 });
48in48in
49{49{
50 nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "nomad" "nvidia-x11" "nvidia-kernel-modules" ];50 nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "nomad" "nvidia-x11" "nvidia-kernel-modules" "gpu-unlock-firmware" ];
5151
52 networking.firewall = {52 networking.firewall = {
53 enable = true;53 enable = true;
nixos/gpu-unlock.nix created+52
...@@ -0,0 +1,52 @@
1{ config, lib, pkgs, ... }:
2let
3 source = pkgs.fetchzip {
4 url = "https://github.com/Zagitta/gpu-unlock/archive/7eb51bccae0a0b7e599e084d971170e9a1e82d0c.tar.gz";
5 hash = "sha256-1KD/J1S1uyI5GGoTwQ1cU/sXeEKlG54exSvi6U+j73M=";
6 };
7 nvidiaVersion = "610.43.03";
8 nvidiaSource = pkgs.fetchurl {
9 url = "https://download.nvidia.com/XFree86/NVIDIA-kernel-module-source/NVIDIA-kernel-module-source-${nvidiaVersion}.tar.xz";
10 hash = "sha256-fhGJI8eiPtw2EU1jJzpG4+BOmvmGlaQiA+esLf6fwdw=";
11 };
12 nvidiaInstaller = pkgs.fetchurl {
13 url = "https://download.nvidia.com/XFree86/Linux-x86_64/${nvidiaVersion}/NVIDIA-Linux-x86_64-${nvidiaVersion}-no-compat32.run";
14 hash = "sha256-4IRFHbcwpt8p09dPZVdJxNI248lF99lLUgXoQxj+L5A=";
15 };
16 firmware = pkgs.runCommand "gpu-unlock-firmware" {
17 nativeBuildInputs = [ pkgs.python3 pkgs.zstd ];
18 meta.license = lib.licenses.unfreeRedistributableFirmware;
19 } ''
20 tar -xf ${nvidiaSource}
21 python3 ${source}/firmware/extract_gsp.py ${nvidiaInstaller} assets
22 python3 ${source}/firmware/extract_boot_assets.py NVIDIA-kernel-module-source-${nvidiaVersion} assets
23 mkdir -p $out/lib/firmware/gpu-unlock/${nvidiaVersion} $out/share/licenses/gpu-unlock
24 install -m444 assets/*.bin $out/lib/firmware/gpu-unlock/${nvidiaVersion}/
25 install -m444 assets/NVIDIA-LICENSE ${source}/firmware/NOTICE.md $out/share/licenses/gpu-unlock/
26 '';
27 kernel = config.boot.kernelPackages.kernel;
28 module = kernel.stdenv.mkDerivation {
29 pname = "gpu-unlock";
30 version = "2.0.0";
31 src = source;
32 nativeBuildInputs = kernel.moduleBuildDependencies;
33 hardeningDisable = [ "pic" ];
34 buildPhase = ''
35 runHook preBuild
36 make -C ${kernel.dev}/lib/modules/${kernel.modDirVersion}/build M="$PWD/kernel/gpu_unlock" modules
37 runHook postBuild
38 '';
39 installPhase = ''
40 runHook preInstall
41 install -Dm444 kernel/gpu_unlock/gpu_unlock.ko $out/lib/modules/${kernel.modDirVersion}/extra/gpu_unlock.ko
42 runHook postInstall
43 '';
44 meta.license = lib.licenses.gpl2Only;
45 };
46in
47{
48 boot.extraModulePackages = [ module ];
49 hardware.firmware = [ firmware ];
50 boot.initrd.kernelModules = [ "gpu_unlock" ];
51 boot.kernelParams = [ "gpu_unlock.targets=0000:09:00.0" ];
52}
nixos/snowglobe.nix+1-1
...@@ -15,7 +15,7 @@ let...@@ -15,7 +15,7 @@ let
15 };15 };
16in16in
17{17{
18 imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ./knife-linux.nix ./mac-vm.nix ./zfs-read-cache.nix ];18 imports = [ ./snowglobe-network.nix ./mac-studio-sandbox.nix ./knife-linux.nix ./mac-vm.nix ./zfs-read-cache.nix ./gpu-unlock.nix ];
1919
20 networking.hostName = "snowglobe";20 networking.hostName = "snowglobe";
21 environment.etc."machine-info".text = ''21 environment.etc."machine-info".text = ''
nixos/tests/snowglobe-network.nix created+41
...@@ -0,0 +1,41 @@
1{ pkgs }:
2pkgs.testers.runNixOSTest {
3 name = "snowglobe-network";
4 globalTimeout = 300;
5 nodes.machine = { config, lib, ... }: {
6 imports = [ ../snowglobe-network.nix ../gpu-unlock.nix ];
7 boot.initrd.systemd.enable = true;
8 boot.initrd.kernelModules = [ "virtio_net" ];
9 networking.usePredictableInterfaceNames = lib.mkForce true;
10 services.openssh.enable = true;
11 virtualisation = {
12 memorySize = 1024;
13 cores = 2;
14 vlans = [];
15 qemu.networkingOptions = lib.mkForce [
16 "-netdev user,id=ethernet,net=10.0.0.0/24,host=10.0.0.2"
17 "-device virtio-net-pci,netdev=ethernet,mac=${config.systemd.network.links."10-snowglobe-ethernet".matchConfig.MACAddress},addr=0x5"
18 ];
19 };
20 };
21 testScript = ''
22 import json
23 start_all()
24 for boot in range(2):
25 machine.wait_for_unit("network-addresses-enp4s0.service")
26 machine.wait_for_unit("sshd.service")
27 address = json.loads(machine.succeed("ip -j -4 address show dev enp4s0"))[0]
28 assert {item["local"] for item in address["addr_info"]} == {"10.0.0.1", "192.168.0.1"}, address
29 route = json.loads(machine.succeed("ip -j route show default"))
30 assert len(route) == 1 and route[0]["gateway"] == "10.0.0.2" and route[0]["dev"] == "enp4s0", route
31 properties = machine.succeed("udevadm info -q property /sys/class/net/enp4s0")
32 assert "ID_NET_LINK_FILE=/etc/systemd/network/10-snowglobe-ethernet.link" in properties, properties
33 machine.succeed("ping -c 1 -W 2 10.0.0.2")
34 machine.succeed("test -d /sys/module/gpu_unlock")
35 machine.succeed("test ! -d /sys/module/nvidia")
36 print(f"Boot {boot}: MAC naming, both addresses, default gateway, SSH, initrd GPU module passed")
37 if boot == 0:
38 machine.shutdown()
39 machine.start()
40 '';
41}
tools/benchmark-gpu-unlock.py created+222
...@@ -0,0 +1,222 @@
1#!/usr/bin/env python3
2"""Emit repeatable GPU-unlock measurements using synthetic inputs only."""
3import argparse
4import ctypes as ct
5from datetime import datetime, timezone
6import hashlib
7import http.client
8import importlib.metadata
9import json
10import os
11from pathlib import Path
12import random
13import time
14
15
16def emit(**record):
17 print(json.dumps(record), flush=True)
18
19
20def request(path, data=None):
21 connection = http.client.HTTPConnection("127.0.0.1", 8081, timeout=600)
22 connection.request("GET" if data is None else "POST", path,
23 None if data is None else json.dumps(data), {
24 "Authorization": "Bearer " + os.environ["VLLM_API_KEY"],
25 "Content-Type": "application/json",
26 })
27 response = connection.getresponse()
28 if response.status != 200:
29 response.read()
30 connection.close()
31 raise RuntimeError(f"HTTP {response.status} at {path}")
32 return connection, response
33
34
35def metrics():
36 connection, response = request("/metrics")
37 try:
38 values = {}
39 for line in response.read().decode().splitlines():
40 if line.startswith("vllm:") and "_bucket{" not in line:
41 name = line.split("{", 1)[0].split(" ", 1)[0]
42 values[name] = values.get(name, 0) + float(line.rsplit(" ", 1)[1])
43 return values
44 finally:
45 connection.close()
46
47
48def idle():
49 values = metrics()
50 if values["vllm:num_requests_running"] or values["vllm:num_requests_waiting"]:
51 raise RuntimeError("Inference endpoint is busy")
52 return values
53
54
55def inference(args):
56 connection, response = request("/tokenize", {
57 "model": "qwen3.8-27b",
58 "prompt": "Synthetic benchmark: blue green yellow red apple pear stone river cloud mountain 0123456789.\n",
59 })
60 vocabulary = json.loads(response.read())["tokens"]
61 connection.close()
62 for length in args.lengths:
63 for trial in range(args.repeats):
64 prompt = random.Random(15027 + length + trial).choices(vocabulary, k=length)
65 before = idle()
66 started = time.perf_counter()
67 connection, response = request("/v1/completions", {
68 "model": "qwen3.8-27b", "prompt": prompt,
69 "max_tokens": args.output_tokens, "ignore_eos": True,
70 "temperature": 0, "seed": 15027,
71 "cache_salt": f"gpu-unlock-{args.phase}-{length}-{trial}",
72 "stream": True, "stream_options": {"include_usage": True},
73 })
74 first = None
75 usage = None
76 digest = hashlib.sha256()
77 chunks = 0
78 try:
79 for line in response:
80 if not line.startswith(b"data: ") or line.strip() == b"data: [DONE]":
81 continue
82 event = json.loads(line[6:])
83 for choice in event.get("choices", []):
84 content = choice.get("text", "")
85 if content:
86 if first is None:
87 first = time.perf_counter()
88 chunks += 1
89 digest.update(content.encode())
90 if event.get("usage"):
91 usage = event["usage"]
92 finally:
93 connection.close()
94 elapsed = time.perf_counter() - started
95 after = metrics()
96 delta = {name: value - before.get(name, 0) for name, value in after.items()}
97 valid = (usage is not None and usage["prompt_tokens"] == length
98 and usage["completion_tokens"] == args.output_tokens
99 and delta.get("vllm:request_success_total") == 1
100 and delta.get("vllm:prefix_cache_hits_total", 0) == 0)
101 emit(kind="inference", phase=args.phase, prompt_tokens=length, trial=trial,
102 output_tokens=None if usage is None else usage["completion_tokens"],
103 wall_seconds=elapsed, ttft_seconds=None if first is None else first - started,
104 prefill_seconds=delta.get("vllm:request_prefill_time_seconds_sum"),
105 decode_seconds=delta.get("vllm:request_decode_time_seconds_sum"),
106 queue_seconds=delta.get("vllm:request_queue_time_seconds_sum"),
107 output_sha256=digest.hexdigest(), stream_chunks=chunks, valid=valid,
108 counters={name: value for name, value in delta.items()
109 if name.endswith(("_sum", "_total")) and value})
110 if not valid:
111 raise RuntimeError("Invalid measurement: concurrent work, cached input, or incomplete output")
112
113
114def tensor(args):
115 # ctypes avoids a second PyTorch allocator beside the resident 24 GiB model.
116 cuda = ct.CDLL("libcudart.so.13")
117 libraries = Path(importlib.metadata.distribution("torch").locate_file("nvidia"))
118 blas = ct.CDLL(str(next(libraries.rglob("libcublas.so.13"))))
119 cuda.cudaMalloc.argtypes = [ct.POINTER(ct.c_void_p), ct.c_size_t]
120 cuda.cudaFree.argtypes = [ct.c_void_p]
121 cuda.cudaMemset.argtypes = [ct.c_void_p, ct.c_int, ct.c_size_t]
122 cuda.cudaMemGetInfo.argtypes = [ct.POINTER(ct.c_size_t), ct.POINTER(ct.c_size_t)]
123 cuda.cudaEventCreate.argtypes = [ct.POINTER(ct.c_void_p)]
124 cuda.cudaEventRecord.argtypes = [ct.c_void_p, ct.c_void_p]
125 cuda.cudaEventSynchronize.argtypes = [ct.c_void_p]
126 cuda.cudaEventElapsedTime.argtypes = [ct.POINTER(ct.c_float), ct.c_void_p, ct.c_void_p]
127 cuda.cudaEventDestroy.argtypes = [ct.c_void_p]
128 blas.cublasCreate_v2.argtypes = [ct.POINTER(ct.c_void_p)]
129 blas.cublasDestroy_v2.argtypes = [ct.c_void_p]
130 blas.cublasSetMathMode.argtypes = [ct.c_void_p, ct.c_int]
131 blas.cublasGemmEx.argtypes = [ct.c_void_p, ct.c_int, ct.c_int, ct.c_int, ct.c_int,
132 ct.c_int, ct.c_void_p, ct.c_void_p, ct.c_int, ct.c_int,
133 ct.c_void_p, ct.c_int, ct.c_int, ct.c_void_p, ct.c_void_p,
134 ct.c_int, ct.c_int, ct.c_int, ct.c_int]
135
136 def checked(function, *values):
137 status = function(*values)
138 if status:
139 raise RuntimeError(f"{function.__name__}: status {status}")
140
141 idle()
142 handle = ct.c_void_p()
143 checked(blas.cublasCreate_v2, ct.byref(handle))
144 start, end = ct.c_void_p(), ct.c_void_p()
145 try:
146 checked(cuda.cudaEventCreate, ct.byref(start))
147 checked(cuda.cudaEventCreate, ct.byref(end))
148 checked(blas.cublasSetMathMode, handle, 16)
149 for dtype, input_type, output_type, element_bytes, compute in (
150 ("bf16_fp32", 14, 0, 2, 68), ("fp16_fp32", 2, 0, 2, 68),
151 ("tf32_fp32", 0, 0, 4, 77), ("fp32", 0, 0, 4, 68),
152 ("int8_int32", 3, 10, 1, 72),
153 ):
154 for size in args.sizes:
155 idle()
156 free, total = ct.c_size_t(), ct.c_size_t()
157 checked(cuda.cudaMemGetInfo, ct.byref(free), ct.byref(total))
158 needed = size * size * (element_bytes * 2 + 4)
159 if free.value < needed + 64 * 1024**2:
160 emit(kind="tensor", phase=args.phase, dtype=dtype, size=size,
161 skipped="GPU memory reserve", free_bytes=free.value, needed_bytes=needed)
162 continue
163 buffers = []
164 try:
165 for byte_count in (size * size * element_bytes,) * 2 + (size * size * 4,):
166 pointer = ct.c_void_p()
167 checked(cuda.cudaMalloc, ct.byref(pointer), byte_count)
168 buffers.append(pointer)
169 checked(cuda.cudaMemset, pointer, 60, byte_count)
170 scalar = ct.c_int if output_type == 10 else ct.c_float
171 alpha, beta = scalar(1), scalar(0)
172
173 def multiply():
174 checked(blas.cublasGemmEx, handle, 0, 0, size, size, size,
175 ct.byref(alpha), buffers[0], input_type, size,
176 buffers[1], input_type, size, ct.byref(beta),
177 buffers[2], output_type, size, compute, 99)
178
179 for _ in range(10):
180 multiply()
181 checked(cuda.cudaDeviceSynchronize)
182 for trial in range(args.repeats):
183 checked(cuda.cudaEventRecord, start, None)
184 for _ in range(100):
185 multiply()
186 checked(cuda.cudaEventRecord, end, None)
187 checked(cuda.cudaEventSynchronize, end)
188 milliseconds = ct.c_float()
189 checked(cuda.cudaEventElapsedTime, ct.byref(milliseconds), start, end)
190 seconds = milliseconds.value / 1000 / 100
191 emit(kind="tensor", phase=args.phase, dtype=dtype, size=size,
192 trial=trial, seconds=seconds, tera_ops=2 * size**3 / seconds / 1e12)
193 finally:
194 for pointer in buffers:
195 checked(cuda.cudaFree, pointer)
196 finally:
197 if start:
198 checked(cuda.cudaEventDestroy, start)
199 if end:
200 checked(cuda.cudaEventDestroy, end)
201 checked(blas.cublasDestroy_v2, handle)
202
203
204if __name__ == "__main__":
205 parser = argparse.ArgumentParser(description=__doc__)
206 parser.add_argument("kind", choices=("inference", "tensor"))
207 parser.add_argument("--phase", required=True)
208 parser.add_argument("--repeats", type=int, default=3)
209 parser.add_argument("--lengths", type=int, nargs="+", default=[1024, 4096, 16000, 64000, 148000])
210 parser.add_argument("--output-tokens", type=int, default=256)
211 parser.add_argument("--sizes", type=int, nargs="+", default=[1024, 2048])
212 args = parser.parse_args()
213 emit(kind="metadata", phase=args.phase, benchmark=args.kind,
214 utc=datetime.now(timezone.utc).isoformat(),
215 script_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
216 model_config_sha256=hashlib.sha256(Path("/models/config.json").read_bytes()).hexdigest(),
217 versions={name: importlib.metadata.version(name) for name in ("vllm", "torch", "flashinfer-python")},
218 arguments=vars(args))
219 if args.kind == "inference":
220 inference(args)
221 else:
222 tensor(args)
tools/deploy-test.py+10
...@@ -13,6 +13,16 @@ import deploy...@@ -13,6 +13,16 @@ import deploy
13import release13import release
1414
1515
16class HostStage(unittest.TestCase):
17 def test_prepares_working_host_without_publishing_or_deploying_services(self):
18 with patch("sys.argv", ["deploy.py", "stage", "host"]), patch.object(
19 deploy, "upload", return_value="0123456789abcdef",
20 ) as upload, patch.object(deploy, "run_logged") as run:
21 deploy.main()
22 upload.assert_called_once_with()
23 run.assert_called_once_with("stage", "host", "python3 /opt/snowglobe/releases/0123456789abcdef/tools/release.py prepare-boot 0123456789abcdef")
24
25
16class MainExport(unittest.TestCase):26class MainExport(unittest.TestCase):
17 def test_production_exports_main_while_staging_exports_working_files(self):27 def test_production_exports_main_while_staging_exports_working_files(self):
18 with tempfile.TemporaryDirectory(prefix="snowglobe-main-export-") as temporary:28 with tempfile.TemporaryDirectory(prefix="snowglobe-main-export-") as temporary:
tools/deploy.py+5-2
...@@ -187,8 +187,8 @@ def main():...@@ -187,8 +187,8 @@ def main():
187 parser.error("rollback target must be a release ID")187 parser.error("rollback target must be a release ID")
188 if args.mode in {"publish", "prod", "history", "backups", "bootstrap"} and args.target:188 if args.mode in {"publish", "prod", "history", "backups", "bootstrap"} and args.target:
189 parser.error(f"{args.mode} takes no target")189 parser.error(f"{args.mode} takes no target")
190 if args.env and args.mode != "stage":190 if args.env and (args.mode != "stage" or args.target == "host"):
191 parser.error("--env is available only for stage")191 parser.error("--env requires a service or dashboard stage")
192 if bool(args.file) != (args.mode == "secrets"):192 if bool(args.file) != (args.mode == "secrets"):
193 parser.error("--file is required only for secrets")193 parser.error("--file is required only for secrets")
194 if args.key and args.mode != "secrets":194 if args.key and args.mode != "secrets":
...@@ -212,6 +212,9 @@ def main():...@@ -212,6 +212,9 @@ def main():
212 manager = f"python3 {REMOTE}/release.py"212 manager = f"python3 {REMOTE}/release.py"
213 if args.mode == "stage":213 if args.mode == "stage":
214 release = upload()214 release = upload()
215 if args.target == "host":
216 run_logged("stage", "host", f"python3 {REMOTE}/releases/{release}/tools/release.py prepare-boot {release}")
217 return
215 overrides = "".join(f" --env {shlex.quote(value)}" for value in args.env)218 overrides = "".join(f" --env {shlex.quote(value)}" for value in args.env)
216 if args.computer:219 if args.computer:
217 overrides += " --computer " + shlex.quote(args.computer)220 overrides += " --computer " + shlex.quote(args.computer)
tools/gpu-unlock.md created+35
...@@ -0,0 +1,35 @@
1# GPU tensor unlock
2
3[The host module](../nixos/gpu-unlock.nix) builds the pinned upstream kernel
4module and extracts its NVIDIA firmware without executing the driver installer.
5It runs in the initrd before the normal NVIDIA driver. The installed driver
6and kernel stay on the repository's existing versions. The target is the
7current GPU's PCI address; a different address is skipped by upstream.
8
9`python3 tools/deploy.py stage host` builds and installs a candidate boot
10generation from an immutable uploaded release. It preserves the previous
11default boot entry and leaves the running host, services, and published release
12unchanged. It prints the candidate entry and fallback. Only after reboot
13approval, select the printed entry with `bootctl set-oneshot <entry>` and
14reboot. A subsequent boot uses the previous default again. Promote the reviewed
15source through the normal main deployment workflow when retaining the change.
16After promotion, clear the trial's EFI default with `bootctl set-default ''`
17so NixOS's `loader.conf` selects the production generation on subsequent boots.
18
19Before rebooting, inspect the candidate and fallback initrds for the MAC-bound
20Ethernet link rule, and verify both static addresses and the gateway in the
21built system. The [isolated boot test](../nixos/tests/snowglobe-network.nix)
22exercises the same network module and GPU module in a disposable VM.
23
24Run [the benchmark](benchmark-gpu-unlock.py) with the existing inference image's
25Python and environment, using `inference --phase before-<run>` and
26`tensor --phase before-<run>`. Repeat with `after-<run>` after the approved boot.
27Keep the script, image, model, serving settings, lengths, and seeds identical.
28Each context length has three uncached requests with a fixed output budget;
29measurements include server prefill/decode time, streamed first-token latency,
30speculation counters, and output hashes. Tensor tests use CUDA events and
31FP32 accumulation, plus FP32 and integer controls, while retaining GPU memory
32for the resident model. The script rejects busy inference and records skipped
33tensor sizes if memory is insufficient. Save stdout with host/GPU metadata
34and clock, power, and temperature telemetry. These are synthetic performance
35probes, not model-quality evaluations.
tools/mac-agent-vm.md+4-1
...@@ -42,7 +42,10 @@ and listener shutdown. This test uses the existing guest without changing its da...@@ -42,7 +42,10 @@ and listener shutdown. This test uses the existing guest without changing its da
4242
43`start` resumes a suspended VM or boots a stopped VM, then waits for commands43`start` resumes a suspended VM or boots a stopped VM, then waits for commands
44to work through the logged-in guest agent. `stop` requests a graceful44to work through the logged-in guest agent. `stop` requests a graceful
45guest shutdown and waits for it to finish. `suspend` waits for Tart's RAM45guest shutdown and waits for it to finish. `start` launches Tart through
46`ssh localhost` (needs Remote Login and this user's key) because closing an
47agent session kills every process it spawned, even in a new session; launchd
48jobs cannot read the VM volume (TCC). `suspend` waits for Tart's RAM
46snapshot; it writes memory to disk and is not an instant CPU pause. A running49snapshot; it writes memory to disk and is not an instant CPU pause. A running
47VM holds a process-scoped idle-sleep assertion, so locking the host does not50VM holds a process-scoped idle-sleep assertion, so locking the host does not
48cause idle sleep. Host login, FileVault, and explicit sleep remain host policy.51cause idle sleep. Host login, FileVault, and explicit sleep remain host policy.
tools/mac-agent-vm.py+10-12
...@@ -10,6 +10,7 @@ import os...@@ -10,6 +10,7 @@ import os
10from pathlib import Path10from pathlib import Path
11import select11import select
12import secrets12import secrets
13import shlex
13import signal14import signal
14import socket15import socket
15import socketserver16import socketserver
...@@ -247,20 +248,17 @@ def main():...@@ -247,20 +248,17 @@ def main():
247 if next(row for row in rows if row["Name"] == NAME and row["Source"] == "local")["Running"]:248 if next(row for row in rows if row["Name"] == NAME and row["Source"] == "local")["Running"]:
248 print("GUI-agent VM is already running.")249 print("GUI-agent VM is already running.")
249 return250 return
250 with (PRIVATE / "run.log").open("w") as log:251 # Closing an agent session kills every process it ever spawned, so sshd must own the VM.
251 os.chmod(PRIVATE / "run.log", 0o600)252 run = shlex.join(["nohup", "/usr/bin/env", f"TART_HOME={ROOT / 'state'}", "/usr/bin/caffeinate", "-i",
252 process = subprocess.Popen([str(TART), "run", NAME, "--no-graphics", "--no-audio",253 str(TART), "run", NAME, "--no-graphics", "--no-audio", "--no-usb-accessories",
253 "--no-usb-accessories", "--no-clipboard", "--suspendable",254 "--no-clipboard", "--suspendable", "--root-disk-opts=sync=full"])
254 "--root-disk-opts=sync=full"],255 (PRIVATE / "run.log").write_text("")
255 env={**os.environ, "TART_HOME": str(ROOT / "state")},256 os.chmod(PRIVATE / "run.log", 0o600)
256 stdin=subprocess.DEVNULL, stdout=log, stderr=log,257 subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "localhost",
257 start_new_session=True)258 f"{run} >{shlex.quote(str(PRIVATE / 'run.log'))} 2>&1 </dev/null &"], check=True)
258 subprocess.Popen(["/usr/bin/caffeinate", "-i", "-w", str(process.pid)],
259 stdin=subprocess.DEVNULL, stdout=log, stderr=log,
260 start_new_session=True)
261 deadline = time.monotonic() + 90259 deadline = time.monotonic() + 90
262 while time.monotonic() < deadline:260 while time.monotonic() < deadline:
263 if process.poll() is not None:261 if subprocess.run(["pgrep", "-f", f"tart run {NAME}"], capture_output=True).returncode:
264 raise ValueError("VM startup failed; inspect the private run log locally.")262 raise ValueError("VM startup failed; inspect the private run log locally.")
265 try:263 try:
266 tart("exec", NAME, "/usr/bin/true", capture_output=True, timeout=5)264 tart("exec", NAME, "/usr/bin/true", capture_output=True, timeout=5)
tools/release-host-test.py+39
...@@ -97,5 +97,44 @@ class ActivationEnvironment(unittest.TestCase):...@@ -97,5 +97,44 @@ class ActivationEnvironment(unittest.TestCase):
97 self.assertLess(rename, deploy)97 self.assertLess(rename, deploy)
9898
9999
100class BootPreparation(unittest.TestCase):
101 def test_installs_a_boot_entry_and_preserves_the_default_without_activation(self):
102 entries = [
103 [{"id": "previous.conf", "isDefault": True}],
104 [{"id": "candidate.conf", "options": "init=/candidate/init loglevel=4"}],
105 ]
106 calls = []
107
108 def run(argv, **kwargs):
109 calls.append(argv)
110 return subprocess.CompletedProcess(argv, 0, json.dumps(entries.pop(0)) if argv[:2] == ["bootctl", "list"] else "")
111
112 with patch.object(release, "check_release", return_value=Path("/candidate")), patch.object(
113 release, "host_configuration", return_value="snowglobe",
114 ), patch.object(release.subprocess, "run", side_effect=run), patch.object(
115 Path, "resolve", return_value=Path("/candidate"),
116 ):
117 release.prepare_boot("0123456789abcdef")
118 self.assertEqual(calls, [
119 ["bootctl", "list", "--json=short"],
120 ["nixos-rebuild", "boot", "--flake", "path:/candidate#snowglobe"],
121 ["bootctl", "set-default", "previous.conf"],
122 ["bootctl", "list", "--json=short"],
123 ])
124
125 def test_failed_build_restores_the_default_without_selecting_a_candidate(self):
126 with patch.object(release, "check_release", return_value=Path("/candidate")), patch.object(
127 release, "host_configuration", return_value="snowglobe",
128 ), patch.object(release.subprocess, "run", side_effect=[
129 subprocess.CompletedProcess([], 0, '[{"id":"previous.conf","isDefault":true}]'),
130 subprocess.CalledProcessError(1, "nixos-rebuild"),
131 subprocess.CompletedProcess([], 0),
132 ]) as command:
133 with self.assertRaises(subprocess.CalledProcessError):
134 release.prepare_boot("0123456789abcdef")
135 self.assertEqual(command.call_args.args[0], ["bootctl", "set-default", "previous.conf"])
136 self.assertEqual(command.call_count, 3)
137
138
100if __name__ == "__main__":139if __name__ == "__main__":
101 unittest.main()140 unittest.main()
tools/release.py+25-1
...@@ -140,6 +140,25 @@ def host_configuration(path):...@@ -140,6 +140,25 @@ def host_configuration(path):
140 return name140 return name
141141
142142
143def prepare_boot(release):
144 path = check_release(release)
145 configuration = host_configuration(path)
146 entries = json.loads(subprocess.run(
147 ["bootctl", "list", "--json=short"], check=True, capture_output=True, text=True,
148 ).stdout)
149 fallback = next(entry["id"] for entry in entries if entry["isDefault"])
150 try:
151 subprocess.run(["nixos-rebuild", "boot", "--flake", f"path:{path.resolve()}#{configuration}"], check=True)
152 finally:
153 subprocess.run(["bootctl", "set-default", fallback], check=True)
154 system = Path("/nix/var/nix/profiles/system").resolve()
155 entries = json.loads(subprocess.run(
156 ["bootctl", "list", "--json=short"], check=True, capture_output=True, text=True,
157 ).stdout)
158 candidate = next(entry["id"] for entry in entries if f"init={system}/init" in entry.get("options", "").split())
159 print(json.dumps({"boot_prepared": release, "entry": candidate, "fallback": fallback, "system": str(system)}))
160
161
143def activate(release, initial=False):162def activate(release, initial=False):
144 path = check_release(release)163 path = check_release(release)
145 script = path / "tools/snowglobe.py"164 script = path / "tools/snowglobe.py"
...@@ -219,7 +238,7 @@ def activate(release, initial=False):...@@ -219,7 +238,7 @@ def activate(release, initial=False):
219238
220def main():239def main():
221 parser = argparse.ArgumentParser(description="Deploy main or roll back a home server release")240 parser = argparse.ArgumentParser(description="Deploy main or roll back a home server release")
222 parser.add_argument("mode", choices=["publish", "deploy", "rollback", "history", "bootstrap", "verify", "stage-destroy"])241 parser.add_argument("mode", choices=["publish", "deploy", "rollback", "history", "bootstrap", "verify", "stage-destroy", "prepare-boot"])
223 parser.add_argument("target", nargs="?")242 parser.add_argument("target", nargs="?")
224 args = parser.parse_args()243 args = parser.parse_args()
225 STATE.mkdir(parents=True, exist_ok=True)244 STATE.mkdir(parents=True, exist_ok=True)
...@@ -249,6 +268,11 @@ def main():...@@ -249,6 +268,11 @@ def main():
249 return268 return
250 with (STATE / "release.lock").open("w") as lock:269 with (STATE / "release.lock").open("w") as lock:
251 fcntl.flock(lock, fcntl.LOCK_EX)270 fcntl.flock(lock, fcntl.LOCK_EX)
271 if args.mode == "prepare-boot":
272 if not args.target:
273 parser.error("prepare-boot requires a release ID")
274 prepare_boot(args.target)
275 return
252 if args.mode == "publish":276 if args.mode == "publish":
253 if not args.target:277 if not args.target:
254 parser.error("publish requires a main release ID")278 parser.error("publish requires a main release ID")