| author | |
| committer | |
| log | dc8464162f442c4e404063c93a5221ae4d56b122 |
| tree | 95fc095f8df48e9aa23e298528ac534488738fd6 |
| parent | 3164185b1168ea91cb83b28c2ff62555315780ff |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
A 30 GB test binary filled the 32 GiB limit and the kernel took Codex with it.
A host service now watches exec events in the container: the Knife server,
Claude, and Codex get -1000, and anything they start drops back to 800.
Assisted-by: claude-opus-5.53 files changed, 96 insertions(+), 2 deletions(-)
nixos/knife-linux.nix+13-1| ... | ... | @@ -39,7 +39,7 @@ in |
| 39 | 39 | run "run --rm --replace --name=knife-linux --pull=never" |
| 40 | 40 | "--sdnotify=conmon --cgroups=no-conmon --log-driver=journald --init --hostname=knife-linux" |
| 41 | 41 | "--user=${uid}:${gid} --umask=0077 --cap-drop=ALL --security-opt=no-new-privileges" |
| 42 | "--memory=32g --pids-limit=32768 --shm-size=2g --network=podman" | |
| 42 | "--memory=96g --pids-limit=32768 --shm-size=2g --network=podman" | |
| 43 | 43 | "--volume=${root}/home:/home/clo" |
| 44 | 44 | "--volume=${sandbox}:/Volumes/Code" |
| 45 | 45 | "--volume=/opt/studio-project-store/current/bin/jj-real:/usr/local/bin/jj:ro" |
| ... | ... | @@ -65,4 +65,16 @@ in |
| 65 | 65 | RestartSec = 5; |
| 66 | 66 | }; |
| 67 | 67 | }; |
| 68 | ||
| 69 | systemd.services.knife-linux-oom = { | |
| 70 | description = "Keep the Knife harnesses out of the OOM killer"; | |
| 71 | wantedBy = [ "knife-linux.service" ]; | |
| 72 | bindsTo = [ "knife-linux.service" ]; | |
| 73 | after = [ "knife-linux.service" ]; | |
| 74 | serviceConfig = { | |
| 75 | ExecStart = "${pkgs.python3}/bin/python3 ${../tools/knife-linux-oom.py} ${run}"; | |
| 76 | Restart = "always"; | |
| 77 | RestartSec = 1; | |
| 78 | }; | |
| 79 | }; | |
| 68 | 80 | } |
tools/knife-linux-oom.py created+80| ... | ... | @@ -0,0 +1,80 @@ |
| 1 | """Exempts the Knife, Claude, and Codex harnesses in knife-linux from the OOM killer. | |
| 2 | ||
| 3 | Children inherit oom_score_adj at fork, so each exec outside the harness set gets a | |
| 4 | killable score back from the kernel's process-event connector before it can grow. | |
| 5 | """ | |
| 6 | ||
| 7 | import errno | |
| 8 | import os | |
| 9 | import re | |
| 10 | import socket | |
| 11 | import struct | |
| 12 | import subprocess | |
| 13 | import sys | |
| 14 | ||
| 15 | HARNESS = re.compile( | |
| 16 | r"/run/podman-init|/home/clo/\.local/share/knife-server/.*" | |
| 17 | r"|/home/clo/\.knife/tools/(claude/[^/]+/claude|codex/[^/]+/bin/codex)" | |
| 18 | ) | |
| 19 | PROTECTED, WORK = -1000, 800 | |
| 20 | NETLINK_CONNECTOR, CN_IDX_PROC, PROC_CN_MCAST_LISTEN, PROC_EVENT_EXEC = 11, 1, 1, 2 | |
| 21 | ||
| 22 | ||
| 23 | def adjust(pid, scope): | |
| 24 | try: | |
| 25 | if not open(f"/proc/{pid}/cgroup").read().startswith(f"0::{scope}/"): | |
| 26 | return | |
| 27 | argv = open(f"/proc/{pid}/cmdline", "rb").read().split(b"\0") | |
| 28 | names = [os.readlink(f"/proc/{pid}/exe"), *(a.decode(errors="replace") for a in argv[1:2])] | |
| 29 | path = f"/proc/{pid}/oom_score_adj" | |
| 30 | current = int(open(path).read()) | |
| 31 | if any(HARNESS.fullmatch(name) for name in names): | |
| 32 | wanted = PROTECTED | |
| 33 | elif current < 0: | |
| 34 | wanted = WORK | |
| 35 | else: | |
| 36 | return | |
| 37 | if current != wanted: | |
| 38 | open(path, "w").write(str(wanted)) | |
| 39 | except (FileNotFoundError, ProcessLookupError, PermissionError): | |
| 40 | pass | |
| 41 | ||
| 42 | ||
| 43 | def scan(scope): | |
| 44 | for directory, _, files in os.walk(f"/sys/fs/cgroup{scope}"): | |
| 45 | if "cgroup.procs" in files: | |
| 46 | for pid in open(f"{directory}/cgroup.procs").read().split(): | |
| 47 | adjust(pid, scope) | |
| 48 | ||
| 49 | ||
| 50 | def main(): | |
| 51 | podman = sys.argv[1] | |
| 52 | scope = subprocess.run( | |
| 53 | [podman, "inspect", "--format", "{{.State.CgroupPath}}", "knife-linux"], | |
| 54 | check=True, capture_output=True, text=True, | |
| 55 | ).stdout.strip() | |
| 56 | sock = socket.socket(socket.AF_NETLINK, socket.SOCK_DGRAM, NETLINK_CONNECTOR) | |
| 57 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 8 << 20) | |
| 58 | sock.bind((0, CN_IDX_PROC)) | |
| 59 | listen = struct.pack("=IIIIHHI", CN_IDX_PROC, 1, 0, 0, 4, 0, PROC_CN_MCAST_LISTEN) | |
| 60 | sock.send(struct.pack("=IHHII", 16 + len(listen), 3, 0, 0, 0) + listen) | |
| 61 | scan(scope) | |
| 62 | print(f"guarding {scope}", flush=True) | |
| 63 | while True: | |
| 64 | try: | |
| 65 | data = sock.recv(65536) | |
| 66 | except OSError as error: | |
| 67 | if error.errno != errno.ENOBUFS: | |
| 68 | raise | |
| 69 | scan(scope) | |
| 70 | continue | |
| 71 | offset = 0 | |
| 72 | while offset + 16 <= len(data): | |
| 73 | length = struct.unpack_from("=I", data, offset)[0] | |
| 74 | what = struct.unpack_from("=I", data, offset + 36)[0] | |
| 75 | if what == PROC_EVENT_EXEC: | |
| 76 | adjust(struct.unpack_from("=I", data, offset + 56)[0], scope) | |
| 77 | offset += (length + 3) & ~3 or len(data) | |
| 78 | ||
| 79 | ||
| 80 | main() |
tools/knife-linux.md+3-1| ... | ... | @@ -4,11 +4,13 @@ |
| 4 | 4 | agents directly beside ZFS. [knife-linux.nix](../nixos/knife-linux.nix) builds it |
| 5 | 5 | from [knife-linux.Containerfile](../nixos/knife-linux.Containerfile) on each start |
| 6 | 6 | and runs it as the `knife-linux` system service: uid 3000 (`clo`), no |
| 7 | capabilities, the 16 threads of the second CCD to itself, a 32 GiB memory limit, `/dev/kvm`, and Podman's NAT network. | |
| 7 | capabilities, the 16 threads of the second CCD to itself, a 96 GiB memory limit, `/dev/kvm`, and Podman's NAT network. | |
| 8 | 8 | Knife listens on container loopback port 3773; devices reach it through the mesh |
| 9 | 9 | as `snowglobe` (device `4b85ee48…`). User namespaces work inside, but Podman's |
| 10 | 10 | seccomp filter blocks the `chroot` Chromium-based sandboxes need, so Chromium, |
| 11 | 11 | Electron, and T3's browser run with their own sandbox off inside the container. |
| 12 | [knife-linux-oom.py](knife-linux-oom.py) exempts the Knife server, Claude, and | |
| 13 | Codex from the OOM killer; whatever they run stays killable at score 800. | |
| 12 | 14 | |
| 13 | 15 | Deploys never restart the container: restarting stops Knife and every running |
| 14 | 16 | agent. Image and unit changes apply at the next manual |