1#!/usr/bin/env python3
2import argparse
3import json
4import os
5from pathlib import Path
6import signal
7import subprocess
8import time
9import urllib.request
10
11
12def main():
13 parser = argparse.ArgumentParser()
14 parser.add_argument("--machine-id", required=True)
15 parser.add_argument("--image", required=True)
16 parser.add_argument("--models", type=Path, required=True)
17 parser.add_argument("--root", type=Path, required=True)
18 parser.add_argument("--name", required=True)
19 parser.add_argument("--program", choices=["experiment.py", "train.py", "evaluate.py", "test_steering.py"], default="experiment.py")
20 parser.add_argument("--source", type=Path, default=Path(__file__).parent)
21 parser.add_argument("experiment_args", nargs=argparse.REMAINDER)
22 args = parser.parse_args()
23 if args.experiment_args[:1] == ["--"]:
24 args.experiment_args = args.experiment_args[1:]
25 os.environ["PATH"] = "/run/current-system/sw/bin:" + os.environ.get("PATH", "")
26 if Path("/etc/machine-id").read_text().strip() != args.machine_id:
27 raise ValueError("Wrong host; inference was not touched")
28 args.root.mkdir(mode=0o700, parents=True, exist_ok=True)
29 token = Path("/var/lib/snowglobe/nomad.token").read_text().strip()
30
31 def job():
32 return json.load(urllib.request.urlopen(urllib.request.Request(
33 "http://127.0.0.1:4646/v1/job/local-ai", headers={"X-Nomad-Token": token}), timeout=10))
34
35 initial = job()
36 group = next(g for g in initial["TaskGroups"] if g["Name"] == "app")
37 if group["Count"] != 1:
38 raise ValueError("Expected one active inference allocation; inference was not touched")
39 environment = {**os.environ, "NOMAD_TOKEN": token, "NOMAD_ADDR": "http://127.0.0.1:4646"}
40 state = {"machine_id": args.machine_id, "job": initial["ID"], "job_version": initial["Version"],
41 "original_count": group["Count"], "image": args.image, "models": str(args.models),
42 "program": args.program, "source": str(args.source), "arguments": args.experiment_args,
43 "started": time.time(), "phase": "preparing"}
44
45 def record(phase, **fields):
46 state.update(phase=phase, updated=time.time(), **fields)
47 pending = args.root / "state.pending"
48 pending.write_text(json.dumps(state, indent=2) + "\n")
49 pending.replace(args.root / "state.json")
50 print(json.dumps({"phase": phase, **fields}), flush=True)
51
52 def scale(count):
53 deadline = time.monotonic() + 600
54 while True:
55 current = job()
56 current_group = next(g for g in current["TaskGroups"] if g["Name"] == "app")
57 if current_group["Count"] == count:
58 return
59 deployments = json.load(urllib.request.urlopen(urllib.request.Request(
60 "http://127.0.0.1:4646/v1/job/local-ai/deployments",
61 headers={"X-Nomad-Token": token}), timeout=10))
62 if not any(d["Status"] in {"running", "pending", "initializing"} for d in deployments):
63 break
64 if time.monotonic() >= deadline:
65 raise TimeoutError("Inference deployment did not settle before scaling")
66 time.sleep(5)
67 subprocess.run(["nomad", "job", "scale", "-detach", f"-check-index={current['JobModifyIndex']}",
68 "local-ai", "app", str(count)],
69 env=environment, check=True)
70
71 def stop(signum, frame):
72 raise InterruptedError(f"Received signal {signum}")
73
74 signal.signal(signal.SIGTERM, stop)
75 signal.signal(signal.SIGINT, stop)
76 work = args.root / "work"
77 work.mkdir(mode=0o700, exist_ok=True)
78 os.chown(work, 65534, 65534)
79 record("pausing")
80 try:
81 scale(0)
82 deadline = time.monotonic() + 180
83 while True:
84 used = int(subprocess.check_output([
85 "nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"], text=True).strip())
86 if used < 256:
87 break
88 if time.monotonic() >= deadline:
89 raise TimeoutError(f"GPU still has {used} MiB allocated")
90 time.sleep(2)
91 record("running", gpu_initial_mib=used)
92 command = [
93 "podman", "run", "--rm", "--name=" + args.name,
94 "--network=none", "--read-only", "--user=65534:65534",
95 "--group-add=" + str(args.models.stat().st_gid), "--cap-drop=ALL",
96 "--security-opt=no-new-privileges", "--pids-limit=1024", "--memory=34g",
97 "--memory-swap=34g", "--shm-size=2g", "--tmpfs=/tmp:rw,size=2g,mode=1777",
98 "--device=/dev/nvidia0", "--device=/dev/nvidiactl", "--device=/dev/nvidia-uvm",
99 "--volume=" + str(args.models) + ":/models:ro",
100 "--volume=" + str(args.source) + ":/experiment:ro",
101 "--volume=" + str(work) + ":/work:rw",
102 "--volume=/run/opengl-driver/lib:/nvidia/lib:ro", "--volume=/nix/store:/nix/store:ro",
103 "--env=LD_LIBRARY_PATH=/nvidia/lib:/usr/local/cuda/lib64",
104 "--env=HOME=/work", "--env=XDG_CACHE_HOME=/work/cache", "--env=HF_HUB_OFFLINE=1",
105 "--env=VLLM_NO_USAGE_STATS=1", "--env=DO_NOT_TRACK=1", "--env=PYTHONUNBUFFERED=1",
106 "--env=PYTHONDONTWRITEBYTECODE=1", "--env=PYTHONPATH=/experiment:/app/steering",
107 "--env=PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False",
108 "--env=FLASHINFER_DISABLE_VERSION_CHECK=1", "--env=VLLM_USE_FLASHINFER_SAMPLER=0",
109 "--env=VLLM_ALLOW_INSECURE_SERIALIZATION=1", "--env=VLLM_HOST_IP=127.0.0.1",
110 "--entrypoint=/app/venv/bin/python", args.image, "/experiment/" + args.program,
111 *args.experiment_args,
112 ]
113 with (args.root / "run.log").open("ab", buffering=0) as log:
114 result = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT)
115 if result.returncode:
116 raise RuntimeError(f"Experiment exited with status {result.returncode}; see run.log")
117 record("experiment_finished")
118 except BaseException as error:
119 record("failed", error=str(error))
120 raise
121 finally:
122 try:
123 subprocess.run(["podman", "stop", "--time=15", args.name], capture_output=True)
124 finally:
125 record("restoring", experiment_phase=state["phase"])
126 scale(group["Count"])
127 record("restored")
128
129
130if __name__ == "__main__":
131 main()