1{ config, lib, pkgs, ... }:
2let
3 root = "/srv/knife-linux";
4 uid = toString config.users.users.clo.uid;
5 gid = toString config.users.groups.chloe.gid;
6 engine = pkgs.writeText "knife-linux-containers.conf" ''
7 [engine]
8 tmp_dir = "/run/knife-linux/libpod"
9 lock_type = "file"
10 '';
11 # A second rootful store must not share Nomad's lock segment or state directory.
12 podman = pkgs.writeShellScriptBin "knife-linux-podman" ''
13 export CONTAINERS_CONF_OVERRIDE=${engine}
14 exec ${pkgs.podman}/bin/podman --root=${root}/containers/storage --runroot=/run/knife-linux/containers "$@"
15 '';
16 run = "${podman}/bin/knife-linux-podman";
17 cpus = "3-15,19-31";
18 checkWork = pkgs.writeShellScript "check-knife-work" ''
19 test "$(${pkgs.util-linux}/bin/findmnt -n -o SOURCE --mountpoint /srv/work)" = work
20 '';
21in
22{
23 environment.systemPackages = [ podman ];
24
25 systemd.services.knife-linux-storage = {
26 requires = [ "zfs-import-work.service" "dev-tpmrm0.device" ];
27 after = [ "zfs-import-work.service" "zfs-mount.service" "dev-tpmrm0.device" ];
28 before = [ "knife-linux.service" ];
29 serviceConfig = {
30 Type = "oneshot";
31 RemainAfterExit = true;
32 LoadCredentialEncrypted = "zfs-clover:/etc/credstore.encrypted/zfs-clover";
33 };
34 script = ''
35 test "$(${pkgs.zfs}/bin/zfs get -H -o value encryptionroot work/knife-linux)" = work/knife-linux
36 test "$(${pkgs.zfs}/bin/zfs get -H -o value mountpoint work/knife-linux)" = ${root}
37 if test "$(${pkgs.zfs}/bin/zfs get -H -o value keystatus work/knife-linux)" = unavailable; then
38 ${pkgs.zfs}/bin/zfs load-key -L "file://$CREDENTIALS_DIRECTORY/zfs-clover" work/knife-linux
39 fi
40 if test "$(${pkgs.zfs}/bin/zfs get -H -o value mounted work/knife-linux)" != yes; then
41 ${pkgs.zfs}/bin/zfs mount work/knife-linux
42 fi
43 test "$(${pkgs.util-linux}/bin/findmnt -n -o SOURCE --mountpoint ${root})" = work/knife-linux
44 '';
45 };
46
47 systemd.services.knife-linux = {
48 description = "The Knife Linux agent container";
49 wantedBy = [ "multi-user.target" ];
50 wants = [ "network-online.target" ];
51 after = [ "network-online.target" "knife-linux-storage.service" ];
52 requires = [ "knife-linux-storage.service" ];
53 # Restarting stops every running agent, so changes wait for a manual restart.
54 restartIfChanged = false;
55 serviceConfig = {
56 Type = "notify";
57 NotifyAccess = "all";
58 ExecStartPre = [
59 (pkgs.writeShellScript "check-knife-root" ''
60 test "$(${pkgs.util-linux}/bin/findmnt -n -o SOURCE --mountpoint ${root})" = work/knife-linux
61 '')
62 checkWork
63 "${run} build --quiet --build-arg=UID=${uid} --build-arg=GID=${gid} --tag=localhost/knife-linux --file=${./knife-linux.Containerfile} /var/empty"
64 ];
65 ExecStart = lib.concatStringsSep " " [
66 run "run --rm --replace --name=knife-linux --pull=never"
67 "--sdnotify=conmon --cgroups=no-conmon --log-driver=journald --init --hostname=knife-linux"
68 "--user=${uid}:${gid} --umask=0077 --cap-drop=ALL --security-opt=no-new-privileges"
69 "--memory=96g --pids-limit=32768 --shm-size=2g --network=podman"
70 "--volume=${root}/home:/home/clo"
71 "--volume=/srv/work:/work"
72 "--volume=/opt/studio-project-store/current/bin/jj-real:/usr/local/bin/jj:ro"
73 "--device=/dev/kvm"
74 "localhost/knife-linux"
75 ];
76 # Agents get 13 cores to themselves and the host keeps three. A partition
77 # rather than a CPU quota: an exhausted quota freezes the Knife server with the builds.
78 ExecStartPost = "-" + pkgs.writeShellScript "knife-linux-cpus" ''
79 set -e
80 scope=/sys/fs/cgroup$(${run} inspect --format '{{.State.CgroupPath}}' knife-linux)
81 echo ${cpus} > /sys/fs/cgroup/machine.slice/cpuset.cpus.exclusive
82 echo ${cpus} > $scope/cpuset.cpus.exclusive
83 echo ${cpus} > $scope/cpuset.cpus
84 echo root > $scope/cpuset.cpus.partition
85 '';
86 ExecStop = "${run} stop --ignore --time=90 knife-linux";
87 ExecStopPost = "${run} rm --ignore --force knife-linux";
88 KillMode = "mixed";
89 TimeoutStartSec = "30min";
90 TimeoutStopSec = 120;
91 Restart = "always";
92 RestartSec = 5;
93 };
94 };
95
96 systemd.services.knife-linux-oom = {
97 description = "Protect Knife harnesses and prioritize the server over agent work";
98 wantedBy = [ "knife-linux.service" ];
99 bindsTo = [ "knife-linux.service" ];
100 after = [ "knife-linux.service" ];
101 serviceConfig = {
102 ExecStart = "${pkgs.python3}/bin/python3 ${../tools/knife-linux-oom.py} ${run}";
103 Restart = "always";
104 RestartSec = 1;
105 };
106 };
107}