authoravatarclover caruso<git@paperclover.net>2026-10-23 18:18:08-07:00
committeravatarclover caruso<git@paperclover.net>2026-10-24 19:01:23-07:00
log225f9e709874838f85d8a55d28e270cd44a1b624
treef8988b28009cd0b7ca7706cd1b1c3eab35f731c9
parent98cae3f5303586df4613ce13f6456f86e5a12bf6
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU

Give the Knife container its own CCD

The weight-only setup still left agent builds and host services contending for the same cores. Assisted-by: claude-opus-5.5

2 files changed, 13 insertions(+), 3 deletions(-)

nixos/knife-linux.nix+12-2
...@@ -15,6 +15,7 @@ let...@@ -15,6 +15,7 @@ let
15 exec ${pkgs.podman}/bin/podman --root=${root}/containers/storage --runroot=/run/knife-linux/containers "$@"15 exec ${pkgs.podman}/bin/podman --root=${root}/containers/storage --runroot=/run/knife-linux/containers "$@"
16 '';16 '';
17 run = "${podman}/bin/knife-linux-podman";17 run = "${podman}/bin/knife-linux-podman";
18 ccd = "8-15,24-31";
18in19in
19{20{
20 environment.systemPackages = [ podman ];21 environment.systemPackages = [ podman ];
...@@ -38,14 +39,23 @@ in...@@ -38,14 +39,23 @@ in
38 run "run --rm --replace --name=knife-linux --pull=never"39 run "run --rm --replace --name=knife-linux --pull=never"
39 "--sdnotify=conmon --cgroups=no-conmon --log-driver=journald --init --hostname=knife-linux"40 "--sdnotify=conmon --cgroups=no-conmon --log-driver=journald --init --hostname=knife-linux"
40 "--user=${uid}:${gid} --umask=0077 --cap-drop=ALL --security-opt=no-new-privileges"41 "--user=${uid}:${gid} --umask=0077 --cap-drop=ALL --security-opt=no-new-privileges"
41 # A low weight rather than a CPU quota: an exhausted quota freezes the Knife server too.42 "--memory=32g --pids-limit=32768 --shm-size=2g --network=podman"
42 "--memory=32g --cpu-shares=1024 --pids-limit=32768 --shm-size=2g --network=podman"
43 "--volume=${root}/home:/home/clo"43 "--volume=${root}/home:/home/clo"
44 "--volume=${sandbox}:/Volumes/Code"44 "--volume=${sandbox}:/Volumes/Code"
45 "--volume=/opt/studio-project-store/current/bin/jj-real:/usr/local/bin/jj:ro"45 "--volume=/opt/studio-project-store/current/bin/jj-real:/usr/local/bin/jj:ro"
46 "--device=/dev/kvm"46 "--device=/dev/kvm"
47 "localhost/knife-linux"47 "localhost/knife-linux"
48 ];48 ];
49 # Agents get the second CCD to themselves and the host keeps the first. A partition
50 # rather than a CPU quota: an exhausted quota freezes the Knife server with the builds.
51 ExecStartPost = "-" + pkgs.writeShellScript "knife-linux-cpus" ''
52 set -e
53 scope=/sys/fs/cgroup$(${run} inspect --format '{{.State.CgroupPath}}' knife-linux)
54 echo ${ccd} > /sys/fs/cgroup/machine.slice/cpuset.cpus.exclusive
55 echo ${ccd} > $scope/cpuset.cpus.exclusive
56 echo ${ccd} > $scope/cpuset.cpus
57 echo root > $scope/cpuset.cpus.partition
58 '';
49 ExecStop = "${run} stop --ignore --time=90 knife-linux";59 ExecStop = "${run} stop --ignore --time=90 knife-linux";
50 ExecStopPost = "${run} rm --ignore --force knife-linux";60 ExecStopPost = "${run} rm --ignore --force knife-linux";
51 KillMode = "mixed";61 KillMode = "mixed";
tools/knife-linux.md+1-1
...@@ -4,7 +4,7 @@...@@ -4,7 +4,7 @@
4agents directly beside ZFS. [knife-linux.nix](../nixos/knife-linux.nix) builds it4agents directly beside ZFS. [knife-linux.nix](../nixos/knife-linux.nix) builds it
5from [knife-linux.Containerfile](../nixos/knife-linux.Containerfile) on each start5from [knife-linux.Containerfile](../nixos/knife-linux.Containerfile) on each start
6and runs it as the `knife-linux` system service: uid 3000 (`clo`), no6and runs it as the `knife-linux` system service: uid 3000 (`clo`), no
7capabilities, a low CPU weight with no core cap, a 32 GiB memory limit, `/dev/kvm`, and Podman's NAT network.7capabilities, the 16 threads of the second CCD to itself, a 32 GiB memory limit, `/dev/kvm`, and Podman's NAT network.
8Knife listens on container loopback port 3773; devices reach it through the mesh8Knife listens on container loopback port 3773; devices reach it through the mesh
9as `snowglobe` (device `4b85ee48…`). User namespaces work inside, but Podman's9as `snowglobe` (device `4b85ee48…`). User namespaces work inside, but Podman's
10seccomp filter blocks the `chroot` Chromium-based sandboxes need, so Chromium,10seccomp filter blocks the `chroot` Chromium-based sandboxes need, so Chromium,