diff --git a/nixos/knife-linux.nix b/nixos/knife-linux.nix index ad3ba40192bbcfabcab8c028b97cfdb2aeedccee..8fe52be1fed3b258d3ed1f64eb9bf5ed60ce4558 100644 --- a/nixos/knife-linux.nix +++ b/nixos/knife-linux.nix @@ -39,7 +39,7 @@ in run "run --rm --replace --name=knife-linux --pull=never" "--sdnotify=conmon --cgroups=no-conmon --log-driver=journald --init --hostname=knife-linux" "--user=${uid}:${gid} --umask=0077 --cap-drop=ALL --security-opt=no-new-privileges" - "--memory=32g --pids-limit=32768 --shm-size=2g --network=podman" + "--memory=96g --pids-limit=32768 --shm-size=2g --network=podman" "--volume=${root}/home:/home/clo" "--volume=${sandbox}:/Volumes/Code" "--volume=/opt/studio-project-store/current/bin/jj-real:/usr/local/bin/jj:ro" @@ -65,4 +65,16 @@ in RestartSec = 5; }; }; + + systemd.services.knife-linux-oom = { + description = "Keep the Knife harnesses out of the OOM killer"; + wantedBy = [ "knife-linux.service" ]; + bindsTo = [ "knife-linux.service" ]; + after = [ "knife-linux.service" ]; + serviceConfig = { + ExecStart = "${pkgs.python3}/bin/python3 ${../tools/knife-linux-oom.py} ${run}"; + Restart = "always"; + RestartSec = 1; + }; + }; } diff --git a/tools/knife-linux-oom.py b/tools/knife-linux-oom.py new file mode 100644 index 0000000000000000000000000000000000000000..e6b239fa8d1723d27277247c40abbc8248de8a7c --- /dev/null +++ b/tools/knife-linux-oom.py @@ -0,0 +1,80 @@ +"""Exempts the Knife, Claude, and Codex harnesses in knife-linux from the OOM killer. + +Children inherit oom_score_adj at fork, so each exec outside the harness set gets a +killable score back from the kernel's process-event connector before it can grow. +""" + +import errno +import os +import re +import socket +import struct +import subprocess +import sys + +HARNESS = re.compile( + r"/run/podman-init|/home/clo/\.local/share/knife-server/.*" + r"|/home/clo/\.knife/tools/(claude/[^/]+/claude|codex/[^/]+/bin/codex)" +) +PROTECTED, WORK = -1000, 800 +NETLINK_CONNECTOR, CN_IDX_PROC, PROC_CN_MCAST_LISTEN, PROC_EVENT_EXEC = 11, 1, 1, 2 + + +def adjust(pid, scope): + try: + if not open(f"/proc/{pid}/cgroup").read().startswith(f"0::{scope}/"): + return + argv = open(f"/proc/{pid}/cmdline", "rb").read().split(b"\0") + names = [os.readlink(f"/proc/{pid}/exe"), *(a.decode(errors="replace") for a in argv[1:2])] + path = f"/proc/{pid}/oom_score_adj" + current = int(open(path).read()) + if any(HARNESS.fullmatch(name) for name in names): + wanted = PROTECTED + elif current < 0: + wanted = WORK + else: + return + if current != wanted: + open(path, "w").write(str(wanted)) + except (FileNotFoundError, ProcessLookupError, PermissionError): + pass + + +def scan(scope): + for directory, _, files in os.walk(f"/sys/fs/cgroup{scope}"): + if "cgroup.procs" in files: + for pid in open(f"{directory}/cgroup.procs").read().split(): + adjust(pid, scope) + + +def main(): + podman = sys.argv[1] + scope = subprocess.run( + [podman, "inspect", "--format", "{{.State.CgroupPath}}", "knife-linux"], + check=True, capture_output=True, text=True, + ).stdout.strip() + sock = socket.socket(socket.AF_NETLINK, socket.SOCK_DGRAM, NETLINK_CONNECTOR) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 8 << 20) + sock.bind((0, CN_IDX_PROC)) + listen = struct.pack("=IIIIHHI", CN_IDX_PROC, 1, 0, 0, 4, 0, PROC_CN_MCAST_LISTEN) + sock.send(struct.pack("=IHHII", 16 + len(listen), 3, 0, 0, 0) + listen) + scan(scope) + print(f"guarding {scope}", flush=True) + while True: + try: + data = sock.recv(65536) + except OSError as error: + if error.errno != errno.ENOBUFS: + raise + scan(scope) + continue + offset = 0 + while offset + 16 <= len(data): + length = struct.unpack_from("=I", data, offset)[0] + what = struct.unpack_from("=I", data, offset + 36)[0] + if what == PROC_EVENT_EXEC: + adjust(struct.unpack_from("=I", data, offset + 56)[0], scope) + offset += (length + 3) & ~3 or len(data) + + +main() diff --git a/tools/knife-linux.md b/tools/knife-linux.md index eab5f0f33a880e735cc14285d070b0ba31c95669..7c22806f4f24700c5dcb3eb4e71a93a3f312ce3c 100644 --- a/tools/knife-linux.md +++ b/tools/knife-linux.md @@ -4,11 +4,13 @@ agents directly beside ZFS. [knife-linux.nix](../nixos/knife-linux.nix) builds it from [knife-linux.Containerfile](../nixos/knife-linux.Containerfile) on each start and runs it as the `knife-linux` system service: uid 3000 (`clo`), no -capabilities, the 16 threads of the second CCD to itself, a 32 GiB memory limit, `/dev/kvm`, and Podman's NAT network. +capabilities, the 16 threads of the second CCD to itself, a 96 GiB memory limit, `/dev/kvm`, and Podman's NAT network. Knife listens on container loopback port 3773; devices reach it through the mesh as `snowglobe` (device `4b85ee48…`). User namespaces work inside, but Podman's seccomp filter blocks the `chroot` Chromium-based sandboxes need, so Chromium, Electron, and T3's browser run with their own sandbox off inside the container. +[knife-linux-oom.py](knife-linux-oom.py) exempts the Knife server, Claude, and +Codex from the OOM killer; whatever they run stays killable at score 800. Deploys never restart the container: restarting stops Knife and every running agent. Image and unit changes apply at the next manual