| 1 | { config, lib, pkgs, ... }: |
| 2 | let |
| 3 | root = "/srv/knife-linux"; |
| 4 | uid = toString config.users.users.clo.uid; |
| 5 | gid = toString config.users.groups.chloe.gid; |
| 6 | engine = pkgs.writeText "knife-linux-containers.conf" '' |
| 7 | [engine] |
| 8 | tmp_dir = "/run/knife-linux/libpod" |
| 9 | lock_type = "file" |
| 10 | ''; |
| 11 | # A second rootful store must not share Nomad's lock segment or state directory. |
| 12 | podman = pkgs.writeShellScriptBin "knife-linux-podman" '' |
| 13 | export CONTAINERS_CONF_OVERRIDE=${engine} |
| 14 | exec ${pkgs.podman}/bin/podman --root=${root}/containers/storage --runroot=/run/knife-linux/containers "$@" |
| 15 | ''; |
| 16 | run = "${podman}/bin/knife-linux-podman"; |
| 17 | cpus = "3-15,19-31"; |
| 18 | checkWork = pkgs.writeShellScript "check-knife-work" '' |
| 19 | test "$(${pkgs.util-linux}/bin/findmnt -n -o SOURCE --mountpoint /srv/work)" = work |
| 20 | ''; |
| 21 | in |
| 22 | { |
| 23 | environment.systemPackages = [ podman ]; |
| 24 | |
| 25 | systemd.services.knife-linux-storage = { |
| 26 | requires = [ "zfs-import-work.service" "dev-tpmrm0.device" ]; |
| 27 | after = [ "zfs-import-work.service" "zfs-mount.service" "dev-tpmrm0.device" ]; |
| 28 | before = [ "knife-linux.service" ]; |
| 29 | serviceConfig = { |
| 30 | Type = "oneshot"; |
| 31 | RemainAfterExit = true; |
| 32 | LoadCredentialEncrypted = "zfs-clover:/etc/credstore.encrypted/zfs-clover"; |
| 33 | }; |
| 34 | script = '' |
| 35 | test "$(${pkgs.zfs}/bin/zfs get -H -o value encryptionroot work/knife-linux)" = work/knife-linux |
| 36 | test "$(${pkgs.zfs}/bin/zfs get -H -o value mountpoint work/knife-linux)" = ${root} |
| 37 | if test "$(${pkgs.zfs}/bin/zfs get -H -o value keystatus work/knife-linux)" = unavailable; then |
| 38 | ${pkgs.zfs}/bin/zfs load-key -L "file://$CREDENTIALS_DIRECTORY/zfs-clover" work/knife-linux |
| 39 | fi |
| 40 | if test "$(${pkgs.zfs}/bin/zfs get -H -o value mounted work/knife-linux)" != yes; then |
| 41 | ${pkgs.zfs}/bin/zfs mount work/knife-linux |
| 42 | fi |
| 43 | test "$(${pkgs.util-linux}/bin/findmnt -n -o SOURCE --mountpoint ${root})" = work/knife-linux |
| 44 | ''; |
| 45 | }; |
| 46 | |
| 47 | systemd.services.knife-linux = { |
| 48 | description = "The Knife Linux agent container"; |
| 49 | wantedBy = [ "multi-user.target" ]; |
| 50 | wants = [ "network-online.target" ]; |
| 51 | after = [ "network-online.target" "knife-linux-storage.service" ]; |
| 52 | requires = [ "knife-linux-storage.service" ]; |
| 53 | # Restarting stops every running agent, so changes wait for a manual restart. |
| 54 | restartIfChanged = false; |
| 55 | serviceConfig = { |
| 56 | Type = "notify"; |
| 57 | NotifyAccess = "all"; |
| 58 | ExecStartPre = [ |
| 59 | (pkgs.writeShellScript "check-knife-root" '' |
| 60 | test "$(${pkgs.util-linux}/bin/findmnt -n -o SOURCE --mountpoint ${root})" = work/knife-linux |
| 61 | '') |
| 62 | checkWork |
| 63 | "${run} build --quiet --build-arg=UID=${uid} --build-arg=GID=${gid} --tag=localhost/knife-linux --file=${./knife-linux.Containerfile} /var/empty" |
| 64 | ]; |
| 65 | ExecStart = lib.concatStringsSep " " [ |
| 66 | run "run --rm --replace --name=knife-linux --pull=never" |
| 67 | "--sdnotify=conmon --cgroups=no-conmon --log-driver=journald --init --hostname=knife-linux" |
| 68 | "--user=${uid}:${gid} --umask=0077 --cap-drop=ALL --security-opt=no-new-privileges" |
| 69 | "--memory=96g --pids-limit=32768 --shm-size=2g --network=podman" |
| 70 | "--volume=${root}/home:/home/clo" |
| 71 | "--volume=/srv/work:/work" |
| 72 | "--volume=/opt/studio-project-store/current/bin/jj-real:/usr/local/bin/jj:ro" |
| 73 | "--device=/dev/kvm" |
| 74 | "localhost/knife-linux" |
| 75 | ]; |
| 76 | # Agents get 13 cores to themselves and the host keeps three. A partition |
| 77 | # rather than a CPU quota: an exhausted quota freezes the Knife server with the builds. |
| 78 | ExecStartPost = "-" + pkgs.writeShellScript "knife-linux-cpus" '' |
| 79 | set -e |
| 80 | scope=/sys/fs/cgroup$(${run} inspect --format '{{.State.CgroupPath}}' knife-linux) |
| 81 | echo ${cpus} > /sys/fs/cgroup/machine.slice/cpuset.cpus.exclusive |
| 82 | echo ${cpus} > $scope/cpuset.cpus.exclusive |
| 83 | echo ${cpus} > $scope/cpuset.cpus |
| 84 | echo root > $scope/cpuset.cpus.partition |
| 85 | ''; |
| 86 | ExecStop = "${run} stop --ignore --time=90 knife-linux"; |
| 87 | ExecStopPost = "${run} rm --ignore --force knife-linux"; |
| 88 | KillMode = "mixed"; |
| 89 | TimeoutStartSec = "30min"; |
| 90 | TimeoutStopSec = 120; |
| 91 | Restart = "always"; |
| 92 | RestartSec = 5; |
| 93 | }; |
| 94 | }; |
| 95 | |
| 96 | systemd.services.knife-linux-oom = { |
| 97 | description = "Protect Knife harnesses and prioritize the server over agent work"; |
| 98 | wantedBy = [ "knife-linux.service" ]; |
| 99 | bindsTo = [ "knife-linux.service" ]; |
| 100 | after = [ "knife-linux.service" ]; |
| 101 | serviceConfig = { |
| 102 | ExecStart = "${pkgs.python3}/bin/python3 ${../tools/knife-linux-oom.py} ${run}"; |
| 103 | Restart = "always"; |
| 104 | RestartSec = 1; |
| 105 | }; |
| 106 | }; |
| 107 | } |