1{ config, lib, pkgs, ... }:
2let
3 dashboard = pkgs.callPackage ./dashboard.nix { };
4 dashboardPort = "7072";
5 internalPort = 8448;
6 proxyToken = "/var/lib/studio/dashboard-proxy.token";
7 hostTools = lib.fileset.toSource {
8 root = ../.;
9 fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../tools/vms.py ../tools/dashboard-run.py ../tools/release.py ../guest/protocol.json ];
10 };
11 nativePkl = pkgs.callPackage ./pkl.nix { };
12 screenPython = pkgs.python3.withPackages (packages: [ packages.pygobject3 packages.gst-python ]);
13 screenPlugins = (with pkgs.gst_all_1; [ (lib.getLib gstreamer) gst-plugins-base gst-plugins-good gst-plugins-bad gst-plugins-ugly ]) ++ [ (lib.getLib pkgs.libnice) ];
14 secureOvmf = pkgs.OVMF.override {
15 secureBoot = true;
16 msVarsTemplate = true;
17 tpmSupport = true;
18 };
19in
20{
21 nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "nomad" "nvidia-x11" "nvidia-kernel-modules" ];
22
23 networking.firewall = {
24 enable = true;
25 allowedTCPPorts = [ 22 ];
26 allowedUDPPortRanges = [{ from = 50000; to = 50031; }];
27 interfaces.podman0.allowedTCPPorts = [ 443 internalPort 10428 15432 18428 ];
28 interfaces.podman0.allowedTCPPortRanges = [{ from = 20000; to = 32000; }];
29 interfaces.tailscale0.allowedTCPPorts = [ 80 443 445 8081 8082 ];
30 };
31
32 services.openssh.enable = true;
33 services.caddy = {
34 enable = true;
35 package = pkgs.caddy.withPlugins {
36 plugins = [ "github.com/caddyserver/replace-response@v0.0.0-20250618171559-80962887e4c6" ];
37 hash = "sha256-2tcvl9ZDpe7SAy3tZCPNwmw0KoIIakqggVaRtVAg9R0=";
38 };
39 extraConfig = "import /var/lib/caddy/routes.caddy";
40 virtualHosts = {
41 ":8082".extraConfig = "reverse_proxy 127.0.0.1:4646";
42 };
43 };
44 systemd.services.caddy.environment = {
45 OTEL_SERVICE_NAME = "studio-caddy";
46 OTEL_EXPORTER_OTLP_TRACES_ENDPOINT = "http://127.0.0.1:10428/insert/opentelemetry/v1/traces";
47 OTEL_TRACES_SAMPLER = "parentbased_traceidratio";
48 OTEL_TRACES_SAMPLER_ARG = "0.25";
49 };
50
51 boot.supportedFilesystems = [ "zfs" ];
52 virtualisation.podman.enable = true;
53 virtualisation.libvirtd.enable = true;
54 virtualisation.libvirtd.qemu.swtpm.enable = true;
55 systemd.services.podman.environment.LOGGING = "--log-level=warn";
56
57 services.nomad = {
58 enable = true;
59 enableDocker = false;
60 dropPrivileges = false;
61 extraSettingsPlugins = [ pkgs.nomad-driver-podman ];
62 extraPackages = [ pkgs.cni-plugins ];
63 settings = {
64 datacenter = "clover";
65 bind_addr = "0.0.0.0";
66 addresses.http = "127.0.0.1";
67 advertise.http = "127.0.0.1";
68 server = {
69 enabled = true;
70 bootstrap_expect = 1;
71 };
72 acl.enabled = true;
73 client = {
74 enabled = true;
75 host_network.loopback.cidr = "127.0.0.1/8";
76 };
77 plugin.nomad-driver-podman.config.socket_path = "unix:///run/podman/podman.sock";
78 telemetry.collection_interval = "15s";
79 ui.enabled = true;
80 };
81 };
82
83 systemd.services.studio-log-shipper = {
84 wantedBy = [ "multi-user.target" ];
85 after = [ "nomad.service" ];
86 wants = [ "nomad.service" ];
87 unitConfig.ConditionPathExists = "/opt/studio/current/tools/log-shipper.py";
88 path = [ pkgs.systemd ];
89 serviceConfig = {
90 ExecStart = "${pkgs.python3}/bin/python3 /opt/studio/current/tools/log-shipper.py";
91 Restart = "always";
92 RestartSec = 5;
93 };
94 };
95
96 systemd.services.nomad = {
97 after = [ "podman.socket" ];
98 requires = [ "podman.socket" ];
99 };
100
101 systemd.services.studio-router = {
102 wantedBy = [ "multi-user.target" ];
103 wants = [ "nomad.service" "caddy.service" ];
104 after = [ "nomad.service" "caddy.service" ];
105 unitConfig.ConditionPathExists = "/var/lib/studio/router.token";
106 environment = {
107 STUDIO_DOMAIN = config.environment.variables.STUDIO_DOMAIN;
108 STUDIO_DASHBOARD_PORT = dashboardPort;
109 STUDIO_INTERNAL_PORT = toString internalPort;
110 STUDIO_PROXY_TOKEN_FILE = proxyToken;
111 };
112 serviceConfig = {
113 ExecStart = "${pkgs.python3}/bin/python3 ${../tools/router.py}";
114 Restart = "always";
115 RestartSec = 5;
116 };
117 };
118
119 users.groups.studio-dashboard = { };
120 users.users.studio-dashboard = {
121 isSystemUser = true;
122 group = "studio-dashboard";
123 };
124 systemd.services.studio-vm-screen = {
125 wantedBy = [ "multi-user.target" ];
126 wants = [ "studio-host.service" ];
127 after = [ "studio-host.service" ];
128 environment = {
129 GST_PLUGIN_SYSTEM_PATH_1_0 = lib.makeSearchPath "lib/gstreamer-1.0" screenPlugins;
130 GI_TYPELIB_PATH = lib.makeSearchPath "lib/girepository-1.0" (map lib.getLib ([ pkgs.gst_all_1.gstreamer ] ++ screenPlugins));
131 LD_LIBRARY_PATH = "/run/opengl-driver/lib";
132 STUDIO_VM_GUEST_PROTOCOL = toString ../guest/protocol.json;
133 };
134 serviceConfig = {
135 ExecStart = "${screenPython}/bin/python3 ${../tools/vm-screen.py}";
136 User = "studio-dashboard";
137 Group = "studio-dashboard";
138 RuntimeDirectory = "studio-vm-screen";
139 RuntimeDirectoryMode = "0700";
140 RuntimeDirectoryPreserve = "yes";
141 UMask = "0077";
142 ProtectSystem = "strict";
143 ProtectHome = true;
144 PrivateTmp = true;
145 NoNewPrivileges = true;
146 CapabilityBoundingSet = "";
147 MemoryMax = "2G";
148 TasksMax = 128;
149 DevicePolicy = "closed";
150 DeviceAllow = lib.optional (builtins.elem "nvidia" config.services.xserver.videoDrivers) "char-nvidia* rw";
151 RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" "AF_NETLINK" ];
152 Restart = "always";
153 RestartSec = 2;
154 };
155 };
156 systemd.services.studio-host = {
157 wantedBy = [ "multi-user.target" ];
158 wants = [ "podman.socket" ];
159 after = [ "podman.socket" ];
160 path = [ pkgs.zfs pkgs.util-linux pkgs.python3 pkgs.libvirt pkgs.qemu pkgs.cdrkit pkgs.openssl pkgs.podman pkgs.nomad pkgs.systemd pkgs.rsync pkgs.nixos-rebuild nativePkl ]
161 ++ lib.optional (builtins.elem "nvidia" config.services.xserver.videoDrivers) (lib.getBin config.hardware.nvidia.package);
162 environment = (lib.filterAttrs (name: _: lib.hasPrefix "STUDIO_" name) config.environment.variables) // {
163 STUDIO_POOL = lib.attrByPath [ "STUDIO_POOL" ] "studio-demo" config.environment.variables;
164 STUDIO_DASHBOARD_USER = "studio-dashboard";
165 STUDIO_VM_ACCEL = lib.attrByPath [ "STUDIO_VM_ACCEL" ] "kvm" config.environment.variables;
166 };
167 serviceConfig = {
168 ExecStart = "${pkgs.python3}/bin/python3 ${hostTools}/tools/dashboard-host.py";
169 Type = "notify";
170 Group = "studio-dashboard";
171 StateDirectory = "studio/host";
172 StateDirectoryMode = "0700";
173 RuntimeDirectory = "studio-host";
174 RuntimeDirectoryMode = "0750";
175 RuntimeDirectoryPreserve = "yes";
176 UMask = "0077";
177 ProtectSystem = "strict";
178 ReadWritePaths = [ "/srv/vm" ] ++ lib.optionals (config.networking.hostName != "clover-demo") [ "/srv/clover/Media/vm" ];
179 ProtectHome = true;
180 PrivateTmp = true;
181 NoNewPrivileges = true;
182 CapabilityBoundingSet = [ "CAP_SYS_ADMIN" "CAP_DAC_READ_SEARCH" ];
183 MemoryMax = "256M";
184 TasksMax = 64;
185 DevicePolicy = "closed";
186 DeviceAllow = [ "/dev/zfs rw" "char-pts rw" ] ++ lib.optional (builtins.elem "nvidia" config.services.xserver.videoDrivers) "char-nvidia* rw";
187 RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" ];
188 IPAddressDeny = "any";
189 IPAddressAllow = [ "127.0.0.1/32" ];
190 Restart = "always";
191 RestartSec = 2;
192 };
193 };
194 systemd.services.studio-dashboard = let
195 environment = (lib.filterAttrs (name: _: lib.hasPrefix "STUDIO_" name) config.environment.variables) // {
196 STUDIO_DATA_DIR = "/data";
197 STUDIO_PROXY_TOKEN_FILE = "/run/secrets/dashboard-proxy.token";
198 STUDIO_NOMAD_TOKEN_FILE = "/run/secrets/dashboard-nomad.token";
199 STUDIO_CA_BUNDLE = "/run/secrets/ca-bundle.crt";
200 STUDIO_INDEX_DIR = "/data/index";
201 STUDIO_INTERNAL_URL = "https://dashboard.internal.${config.environment.variables.STUDIO_DOMAIN}:${toString internalPort}";
202 STUDIO_FILES_URL = "https://file.${config.environment.variables.STUDIO_DOMAIN}";
203 STUDIO_AUTH_REQUIRED = "1";
204 STUDIO_AUTH_RP_ID = "auth.${config.environment.variables.STUDIO_DOMAIN}";
205 STUDIO_FILE_ORIGIN = "https://file.${config.environment.variables.STUDIO_DOMAIN}";
206 STUDIO_PUBLIC_ORIGIN = "https://snowglobe.${config.environment.variables.STUDIO_DOMAIN}";
207 STUDIO_JELLYFIN_URL = "https://jelly.${config.environment.variables.STUDIO_DOMAIN}";
208 STUDIO_SHALE_URL = "https://shale.${config.environment.variables.STUDIO_DOMAIN}";
209 STUDIO_PUBLISHED_ROOT = "/srv/clover/Published";
210 STUDIO_DOMAIN = config.environment.variables.STUDIO_DOMAIN;
211 STUDIO_POOL = lib.attrByPath [ "STUDIO_POOL" ] "studio-demo" config.environment.variables;
212 STUDIO_NODE_NAME = config.networking.hostName;
213 STUDIO_MEDIA_READ_ONLY = lib.attrByPath [ "STUDIO_MEDIA_READ_ONLY" ] "false" config.environment.variables;
214 STUDIO_CLOVER_READ_ONLY = lib.attrByPath [ "STUDIO_CLOVER_READ_ONLY" ] "false" config.environment.variables;
215 };
216 in {
217 inherit environment;
218 wantedBy = [ "multi-user.target" ];
219 wants = [ "nomad.service" "studio-router.service" "studio-host.service" "studio-vm-screen.service" ];
220 after = [ "nomad.service" "studio-host.service" "studio-vm-screen.service" ];
221 unitConfig.ConditionPathExists = [ "/var/lib/studio/dashboard.token" "/var/lib/studio/ca-bundle.crt" "/opt/studio/current" ];
222 path = [ pkgs.podman pkgs.coreutils pkgs.curl pkgs.openssl pkgs.systemd ];
223 serviceConfig = {
224 Type = "notify";
225 NotifyAccess = "all";
226 ExecStart = pkgs.writeShellScript "studio-dashboard-start" ''
227 uid=$(id -u studio-dashboard)
228 gid=$(id -g studio-dashboard)
229 optional=()
230 if test -d /srv/prod/ytdl/data && test -d '/srv/clover/Documents/Config/Youtube Downloader'; then
231 optional+=(--volume=/srv/prod/ytdl/data:/srv/prod/ytdl/data:rw
232 --env=STUDIO_YT_STATE=/srv/prod/ytdl/data
233 '--env=STUDIO_YT_CONFIG=/srv/clover/Documents/Config/Youtube Downloader'
234 --env=STUDIO_YT_MEDIA=/srv/clover/Media)
235 fi
236 exec podman run --rm --replace --name=studio-dashboard --pull=never \
237 --sdnotify=conmon --cgroups=no-conmon --log-driver=journald \
238 --user="$uid:$gid" \
239 --group-add=${toString config.users.groups.chloe.gid} --network=podman \
240 --read-only --read-only-tmpfs=false --cap-drop=ALL --security-opt=no-new-privileges \
241 --memory=2g --cpus=4 --pids-limit=256 --cpu-shares=1024 \
242 --publish=127.0.0.1:${dashboardPort}:7072 \
243 --tmpfs=/tmp:rw,noexec,nosuid,nodev,size=256m,mode=1777 \
244 --tmpfs=/run:rw,noexec,nosuid,nodev,size=16m,mode=755 \
245 --env=HOME=/data --env=XDG_CACHE_HOME=/data/cache \
246 --volume=/var/lib/studio/dashboard:/data:rw \
247 --volume=/run/studio-host:/run/studio-host:ro \
248 --volume=/run/studio-vm-screen:/run/studio-vm-screen:ro \
249 --volume=${proxyToken}:/run/secrets/dashboard-proxy.token:ro \
250 --volume=/var/lib/studio/dashboard.token:/run/secrets/dashboard-nomad.token:ro \
251 --volume=/var/lib/studio/ca-bundle.crt:/run/secrets/ca-bundle.crt:ro \
252 --mount=type=bind,src=/srv/clover,dst=/srv/clover,bind-nonrecursive,bind-propagation=rslave,ro="$STUDIO_CLOVER_READ_ONLY" \
253 --mount=type=bind,src=/srv/clover/Media,dst=/srv/clover/Media,bind-nonrecursive,bind-propagation=rslave,ro="$STUDIO_MEDIA_READ_ONLY" \
254 ${lib.concatMapStringsSep " " (name: lib.escapeShellArg "--env=${name}") (builtins.attrNames environment)} \
255 ${lib.concatMapStringsSep " " (name: lib.escapeShellArg "--add-host=${name}.${config.environment.variables.STUDIO_DOMAIN}:host-gateway") [ "dashboard.internal" "auth" "jelly" "db" "shale" ]} \
256 "''${optional[@]}" ${lib.escapeShellArg "${dashboard.image.imageName}:${dashboard.image.imageTag}"}
257 '';
258 ExecStop = "${pkgs.podman}/bin/podman stop --ignore --time=8 studio-dashboard";
259 ExecStopPost = "${pkgs.podman}/bin/podman rm --ignore --force studio-dashboard";
260 CPUWeight = 1000;
261 KillMode = "mixed";
262 TimeoutStartSec = 180;
263 TimeoutStopSec = 30;
264 Restart = "always";
265 RestartSec = 5;
266 };
267 preStart = ''
268 umask 0077
269 if ! test -e ${proxyToken}; then
270 openssl rand -hex 32 > ${proxyToken}.pending
271 mv ${proxyToken}.pending ${proxyToken}
272 fi
273 chown root:studio-dashboard ${proxyToken} /var/lib/studio/dashboard.token
274 chmod 0640 ${proxyToken} /var/lib/studio/dashboard.token
275 chown -R studio-dashboard:studio-dashboard /var/lib/studio/dashboard
276 systemd-tmpfiles --create --prefix=/srv/prod/ytdl/data
277 if ! podman image exists ${lib.escapeShellArg "${dashboard.image.imageName}:${dashboard.image.imageTag}"}; then
278 podman load --quiet --input ${dashboard.image}
279 fi
280 '';
281 postStart = ''
282 for attempt in $(seq 1 30); do
283 if printf 'header = "Studio-Proxy-Token: %s"\n' "$(cat ${proxyToken})" | curl --config - --fail --silent http://127.0.0.1:${dashboardPort}/ >/dev/null; then
284 exit 0
285 fi
286 sleep 1
287 done
288 exit 1
289 '';
290 };
291
292 systemd.tmpfiles.rules = [
293 "d /srv/prod 0755 root root -"
294 "d /srv/staging 0755 root root -"
295 "d /srv/vm 0755 root root -"
296 "d /srv/logs 0770 clo chloe -"
297 "A+ /srv/prod/ytdl/data - - - - g:chloe:rwX,m::rwX,d:g:chloe:rwx,d:m::rwx"
298 "d /var/lib/studio 0700 root root -"
299 "d /var/lib/studio/dashboard 0700 studio-dashboard studio-dashboard -"
300 "d /var/lib/studio/routes 0700 root root -"
301 "d /var/lib/caddy/studio 0755 caddy caddy -"
302 "f /var/lib/caddy/routes.caddy 0640 caddy caddy -"
303 "d /var/lib/swtpm-localca 0750 tss tss -"
304 ];
305
306 environment.variables.STUDIO_NODE_NAME = config.networking.hostName;
307 environment.variables.STUDIO_DOMAIN = lib.mkDefault "studio.test";
308 environment.variables.STUDIO_VM_SECURE_OVMF_CODE = secureOvmf.firmware;
309 environment.variables.STUDIO_VM_SECURE_OVMF_VARS = secureOvmf.variablesMs;
310 environment.systemPackages = with pkgs; [ acl curl jq openssl python3 rsync zfs qemu ] ++ [ nativePkl ];
311}