From d16b9185f320c372908c7c4bdc6fba091e403b19 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Sun, 4 Oct 2026 21:12:07 -0700 Subject: [PATCH] Group Victoria and YTDL services and refine theme indicators Consolidate the collectors and YouTube tasks into two multi-container services. Preserve collector discovery names, scrape each container, and move the dashboard worker to the YTDL state and secrets. Use diff purple for recent Shale timestamps, keep visited links unchanged, and center status spinner rotation. Assisted-by: gpt-6.1-sol --- dashboard/server/youtube-worker.py | 2 +- dashboard/src/core.rs | 4 +- dashboard/src/telemetry.rs | 20 +++-- dashboard/src/youtube.rs | 2 +- dashboard/web/styles.css | 2 +- nixos/configuration.nix | 10 +-- service/keycloak/service.pkl | 2 +- service/pds/service.pkl | 2 +- service/shale/theme.css | 10 +-- .../{victoria-metrics => }/icon-dark.svg | 0 .../{victoria-metrics => }/icon-light.svg | 0 service/victoria/service.pkl | 56 ++++++++++++++ service/victoria/victoria-logs.pkl | 21 ------ service/victoria/victoria-metrics.pkl | 22 ------ service/victoria/victoria-traces.pkl | 22 ------ service/victoria/victoria-traces/icon.svg | 1 - service/youtube/yt-feed.pkl | 34 --------- service/youtube/ytdl-sub.pkl | 45 ------------ service/youtube/ytdl-sub/icon.svg | 1 - .../ytdl-sub => ytdl}/config/archive-loop.sh | 0 .../ytdl-sub => ytdl}/config/config.yaml | 0 service/{youtube/yt-feed => ytdl}/icon.svg | 0 service/ytdl/service.pkl | 73 +++++++++++++++++++ .../yt-feed => ytdl}/upscaler/Dockerfile | 0 .../yt-feed => ytdl}/upscaler/upscale.py | 0 tools/dashboard-unit-test.py | 18 ++--- tools/import-personal-files.py | 4 +- tools/import-yt-feed.sh | 34 ++++----- tools/personal-cutover.md | 4 +- tools/studio.py | 9 ++- tools/yt-feed-migration.md | 4 +- 31 files changed, 196 insertions(+), 206 deletions(-) rename service/victoria/{victoria-metrics => }/icon-dark.svg (100%) rename service/victoria/{victoria-metrics => }/icon-light.svg (100%) create mode 100644 service/victoria/service.pkl delete mode 100644 service/victoria/victoria-logs.pkl delete mode 100644 service/victoria/victoria-metrics.pkl delete mode 100644 service/victoria/victoria-traces.pkl delete mode 100644 service/victoria/victoria-traces/icon.svg delete mode 100644 service/youtube/yt-feed.pkl delete mode 100644 service/youtube/ytdl-sub.pkl delete mode 100644 service/youtube/ytdl-sub/icon.svg rename service/{youtube/ytdl-sub => ytdl}/config/archive-loop.sh (100%) rename service/{youtube/ytdl-sub => ytdl}/config/config.yaml (100%) rename service/{youtube/yt-feed => ytdl}/icon.svg (100%) create mode 100644 service/ytdl/service.pkl rename service/{youtube/yt-feed => ytdl}/upscaler/Dockerfile (100%) rename service/{youtube/yt-feed => ytdl}/upscaler/upscale.py (100%) diff --git a/dashboard/server/youtube-worker.py b/dashboard/server/youtube-worker.py index b4ba209fe5cc808eb2f2d2c2ee668f9d16e70b1a..edaf45de929efa026d12359f2b69c152cc36d2f7 100644 --- a/dashboard/server/youtube-worker.py +++ b/dashboard/server/youtube-worker.py @@ -21,7 +21,7 @@ os.umask(0o007) YT_CONFIG_DIR = os.environ.get("STUDIO_YT_CONFIG", "/srv/clover/Documents/Config/Youtube Downloader") FEED_CONFIG = os.path.join(YT_CONFIG_DIR, "feed.yaml") -STATE_DIR = os.environ.get("STUDIO_YT_STATE", "/srv/prod/yt-feed/data") +STATE_DIR = os.environ.get("STUDIO_YT_STATE", "/srv/prod/ytdl/data") INTERVAL = int(os.environ.get("CHECK_INTERVAL", "1800")) SMTP_HOST = os.environ.get("SMTP_HOST", "") SMTP_PORT = int(os.environ.get("SMTP_PORT", "465")) diff --git a/dashboard/src/core.rs b/dashboard/src/core.rs index 535723c74c5915a4e3ba5f12b3229d8d9941fe87..993e5a5fb2849c6de0593867b3d99f002938dea1 100644 --- a/dashboard/src/core.rs +++ b/dashboard/src/core.rs @@ -566,7 +566,7 @@ async fn service(app: Arc, id: &str) -> Result> { false }; let metrics = - if meta["studio_metrics_path"].as_str().is_some_and(|p| !p.is_empty()) || meta["studio_metrics_pushed"] == "true" { + if array(&found["job"]["TaskGroups"]).iter().flat_map(|g| array(&g["Services"])).any(|s| array(&s["Tags"]).iter().any(|tag| string(tag).starts_with("studio-metrics-path="))) || meta["studio_metrics_pushed"] == "true" { telemetry::metric_names(app.clone(), id) .await .unwrap_or(json!([])) @@ -587,7 +587,7 @@ async fn service(app: Arc, id: &str) -> Result> { let rows = host::call(json!({"operation":"storage.datasets"})).await?; Ok(json!(array(&rows).iter().map(|d| json!({"name":d["name"],"mountpoint":d["mountpoint"],"used":d["usedbydataset"],"snapshots":d["usedbysnapshots"]})).collect::>())) }).await?; - let logs = array(&all.value).iter().find(|s| s["id"] == "victoria-logs" && s["url"].is_string()).map(|s| json!({"app":{"id":s["id"],"name":s["name"],"icon":s["icon"]},"url":format!("{}/select/vmui/#/?query={}",string(&s["url"]),encoded(&format!("{{job=\"{id}\"}}")))})); + let logs = array(&all.value).iter().find(|s| s["id"] == "victoria" && s["url"].is_string()).map(|s| json!({"app":{"id":s["id"],"name":s["name"],"icon":s["icon"]},"url":format!("{}/select/vmui/#/?query={}",string(&s["url"]),encoded(&format!("{{job=\"{id}\"}}")))})); let fields = json!({"applicationTraces":application_traces,"applicationMetrics":metrics,"release":release,"deployedAt":number(&found["job"]["SubmitTime"])/1e9,"rollout":if array(&found["job"]["TaskGroups"]).iter().any(|g| number(&g["Update"]["Canary"]) > 0.0) {"overlapped"} else {"simple"},"containers":containers(found),"checks":checks_of(&json!(array(&found["allocs"]).iter().flat_map(|a| array(&a["home_checks"])).collect::>())),"requirements":requirements,"dependents":dependents,"secrets":secrets,"datasets":array(&datasets.value).iter().filter(|d| string(&d["name"]).ends_with(&format!("/prod/{id}"))).collect::>(),"restartsAcknowledged":ack[id],"logs":logs}); summary .as_object_mut() diff --git a/dashboard/src/telemetry.rs b/dashboard/src/telemetry.rs index 3149a14b43ee1e12184d5c43849d4023b15f7157..36977e4eb79125bb58d7b3efdd62a0ceafdf4b1c 100644 --- a/dashboard/src/telemetry.rs +++ b/dashboard/src/telemetry.rs @@ -822,15 +822,19 @@ pub fn start(app: Arc) { .as_object() .unwrap() .iter() - .filter_map(|(id, job)| { - job["job"]["Meta"]["studio_metrics_path"] - .as_str() - .filter(|p| !p.is_empty()) - .map(|p| (id.clone(), p.to_owned())) + .flat_map(|(id, job)| { + array(&job["job"]["TaskGroups"]).iter() + .flat_map(|group| array(&group["Services"])) + .flat_map(move |service| { + array(&service["Tags"]).iter().filter_map(move |tag| { + string(tag).strip_prefix("studio-metrics-path=") + .map(|path| (id.clone(), string(&service["Name"]).to_owned(), path.to_owned())) + }) + }) }) .collect(); stream::iter(scrapes) - .for_each_concurrent(4, |(id, path)| { + .for_each_concurrent(4, |(id, service, path)| { let app = app.clone(); let base = base.clone(); async move { @@ -838,7 +842,7 @@ pub fn start(app: Arc) { let response = app .request( Method::GET, - &format!("{}{path}", endpoint(app.clone(), &id).await?), + &format!("{}{path}", endpoint(app.clone(), &service).await?), )? .timeout(Duration::from_secs(5)) .send() @@ -848,7 +852,7 @@ pub fn start(app: Arc) { Method::POST, &format!( "{base}/api/v1/import/prometheus?{}", - params(&[("extra_label", format!("service={id}"))]) + params(&[("extra_label", format!("service={id}")), ("extra_label", format!("instance={service}"))]) ), )? .body(response.text().await?) diff --git a/dashboard/src/youtube.rs b/dashboard/src/youtube.rs index 65ec0c633ed3c090d50dbeec1e998a5f569187a0..e983300b47accc4dbef01cd26a9a098f6376a71e 100644 --- a/dashboard/src/youtube.rs +++ b/dashboard/src/youtube.rs @@ -42,7 +42,7 @@ impl Worker { ("SMTP_PASS", "smtp_pass"), ] { if let Ok(secret) = host::call( - json!({"operation":"deploy.secret.get","service":"yt-feed","key":name}), + json!({"operation":"deploy.secret.get","service":"ytdl","key":name}), ).await { if let Some(value) = secret.as_str() { command.env(key, value); diff --git a/dashboard/web/styles.css b/dashboard/web/styles.css index b4ce76d891c16cbcc0602281a5e8c1539b1d72fa..d306236776d4f08c2e5e6b00dcb735305cf01534 100644 --- a/dashboard/web/styles.css +++ b/dashboard/web/styles.css @@ -299,7 +299,7 @@ button { font: inherit; color: inherit; } .status.stopped { color: var(--stopped); } .status.working { color: var(--text-2); } .status:is(.deploying, .restarting, .starting) { color: var(--accent); } -.status:is(.deploying, .restarting, .starting, .working) { animation: spin 1.1s cubic-bezier(0.55, 0.15, 0.45, 0.85) infinite; } +.status:is(.deploying, .restarting, .starting, .working) { transform-box: view-box; transform-origin: center; animation: spin 1.1s cubic-bezier(0.55, 0.15, 0.45, 0.85) infinite; } @keyframes spin { to { transform: rotate(360deg); } } .status-label { display: inline-flex; align-items: center; gap: 6px; color: var(--text-2); } diff --git a/nixos/configuration.nix b/nixos/configuration.nix index 42451c6ca0f7e2cf7c683811c6afde1e30b9fb2c..19becc7e52524765eb473fa288b419e77ffe1921 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -183,9 +183,9 @@ in uid=$(id -u studio-dashboard) gid=$(id -g studio-dashboard) optional=() - if test -d /srv/prod/yt-feed/data && test -d '/srv/clover/Documents/Config/Youtube Downloader'; then - optional+=(--volume=/srv/prod/yt-feed/data:/srv/prod/yt-feed/data:rw - --env=STUDIO_YT_STATE=/srv/prod/yt-feed/data + if test -d /srv/prod/ytdl/data && test -d '/srv/clover/Documents/Config/Youtube Downloader'; then + optional+=(--volume=/srv/prod/ytdl/data:/srv/prod/ytdl/data:rw + --env=STUDIO_YT_STATE=/srv/prod/ytdl/data '--env=STUDIO_YT_CONFIG=/srv/clover/Documents/Config/Youtube Downloader' --env=STUDIO_YT_MEDIA=/srv/clover/Media) fi @@ -228,7 +228,7 @@ in chown root:studio-dashboard ${proxyToken} /var/lib/studio/dashboard.token chmod 0640 ${proxyToken} /var/lib/studio/dashboard.token chown -R studio-dashboard:studio-dashboard /var/lib/studio/dashboard - systemd-tmpfiles --create --prefix=/srv/prod/yt-feed/data + systemd-tmpfiles --create --prefix=/srv/prod/ytdl/data if ! podman image exists ${lib.escapeShellArg "${dashboard.image.imageName}:${dashboard.image.imageTag}"}; then podman load --quiet --input ${dashboard.image} fi @@ -249,7 +249,7 @@ in "d /srv/staging 0755 root root -" "d /srv/vm 0755 root root -" "d /srv/logs 0770 clo chloe -" - "A+ /srv/prod/yt-feed/data - - - - g:chloe:rwX,m::rwX,d:g:chloe:rwx,d:m::rwx" + "A+ /srv/prod/ytdl/data - - - - g:chloe:rwX,m::rwX,d:g:chloe:rwx,d:m::rwx" "d /var/lib/studio 0700 root root -" "d /var/lib/studio/dashboard 0700 studio-dashboard studio-dashboard -" "d /var/lib/studio/routes 0700 root root -" diff --git a/service/keycloak/service.pkl b/service/keycloak/service.pkl index 0f2e6ae5f517bf3c02e882a3ad87bcf0584ae666..f181428932eff8fcc22264009835214dd7f48ac8 100644 --- a/service/keycloak/service.pkl +++ b/service/keycloak/service.pkl @@ -17,7 +17,7 @@ local database = new postgres.Database { name = "keycloak_next" } meta { name = "Keycloak"; launcher = false } traceServiceName = module.id -dependsOn { "victoria-traces" } +dependsOn { "victoria" } healthyDeadline = "20m" setup = "configure.py" provide = "provide.py" diff --git a/service/pds/service.pkl b/service/pds/service.pkl index 1e75be090092594faf698e36e5010a0441862efc..9ae290032657c4eb0aaf3e2bc628deb7638a7aa9 100644 --- a/service/pds/service.pkl +++ b/service/pds/service.pkl @@ -9,7 +9,7 @@ traceServiceName = module.id metricsPushed = true rollout = "simple" stageIsolation = "fresh" -dependsOn { "victoria-metrics"; "victoria-traces" } +dependsOn { "victoria" } secrets = if (isolated) new { ["jwt_secret"] {} diff --git a/service/shale/theme.css b/service/shale/theme.css index b20577426dd78ce715c90685b67da0abd3459c9c..84c63161ba0935cf23f64cc093c68aa4f526d418 100644 --- a/service/shale/theme.css +++ b/service/shale/theme.css @@ -57,7 +57,7 @@ --theme-color-primary-darker: #74113a; --theme-body-background-color: #fffafd; --theme-link-color: var(--theme-color-primary); - --theme-link-visited-color: #88436c; + --theme-link-visited-color: var(--theme-link-color); --theme-link-hover-color: var(--theme-color-primary-dark); --theme-link-active-color: var(--theme-color-primary-darker); --theme-focus-color: var(--theme-color-primary-vivid); @@ -166,8 +166,8 @@ border-color: #d9bdca; } - :is(td, span).idleage:is(.seconds, .minutes, .hours) { - color: #216e1f; + :is(td, span).idleage:is(.seconds, .minutes, .hours, .days) { + color: var(--shale-color-cool-dark); } :is(td, span).idleage.weeks, @@ -242,7 +242,7 @@ --theme-text-color: #f7edf2; --theme-text-reverse-color: #21161c; --theme-link-color: #ff8fba; - --theme-link-visited-color: #d9a3c1; + --theme-link-visited-color: var(--theme-link-color); --theme-link-hover-color: #ffc0d8; --theme-link-active-color: #ffd7e6; --theme-focus-color: var(--theme-color-primary-vivid); @@ -356,7 +356,7 @@ } :is(td, span).idleage:is(.seconds, .minutes, .hours, .days) { - color: #90db9b; + color: var(--shale-color-cool-dark); } :is(td, span).idleage.weeks, diff --git a/service/victoria/victoria-metrics/icon-dark.svg b/service/victoria/icon-dark.svg similarity index 100% rename from service/victoria/victoria-metrics/icon-dark.svg rename to service/victoria/icon-dark.svg diff --git a/service/victoria/victoria-metrics/icon-light.svg b/service/victoria/icon-light.svg similarity index 100% rename from service/victoria/victoria-metrics/icon-light.svg rename to service/victoria/icon-light.svg diff --git a/service/victoria/service.pkl b/service/victoria/service.pkl new file mode 100644 index 0000000000000000000000000000000000000000..019449b6d64973361f470d924be2a2de80fd3833 --- /dev/null +++ b/service/victoria/service.pkl @@ -0,0 +1,56 @@ +amends "../../config/Service.pkl" + +meta { name = "Victoria"; tagline = "Metrics, logs, and traces" } + +containers { + ["logs"] { + image = "docker.io/victoriametrics/victoria-logs:v1.52.0" + cpu = 300 + memory = 512 + args { "-storageDataPath=/logs"; "-retentionPeriod=30d" } + + http { + containerPort = 9428 + subdomain = "logs" + authRole = "infra-admin" + checkPath = "/health" + metricsPath = "/metrics" + } + + volumes { ["/logs"] {} } + } + ["metrics"] { + image = "docker.io/victoriametrics/victoria-metrics:v1.152.0" + cpu = 300 + memory = 512 + args { "-storageDataPath=/metrics"; "-retentionPeriod=30d"; "-usePromCompatibleNaming" } + + http { + containerPort = 8428 + hostPort = 18428 + subdomain = "metrics" + authRole = "infra-admin" + checkPath = "/health" + metricsPath = "/metrics" + } + + volumes { ["/metrics"] {} } + } + ["traces"] { + image = "docker.io/victoriametrics/victoria-traces:v0.11.1" + cpu = 300 + memory = 512 + args { "-storageDataPath=/traces"; "-retentionPeriod=30d" } + + http { + containerPort = 10428 + hostPort = 10428 + subdomain = "traces" + authRole = "infra-admin" + checkPath = "/health" + metricsPath = "/metrics" + } + + volumes { ["/traces"] {} } + } +} diff --git a/service/victoria/victoria-logs.pkl b/service/victoria/victoria-logs.pkl deleted file mode 100644 index f09dfc6686c748b9bb65077164c34aa0fc6ffac6..0000000000000000000000000000000000000000 --- a/service/victoria/victoria-logs.pkl +++ /dev/null @@ -1,21 +0,0 @@ -amends "../../config/Service.pkl" - -meta { name = "VictoriaLogs"; tagline = "Searchable service and system logs" } -rollout = "simple" - -container { - image = "docker.io/victoriametrics/victoria-logs:v1.52.0" - cpu = 300 - memory = 512 - args { "-storageDataPath=/data"; "-retentionPeriod=30d" } - - http { - containerPort = 9428 - subdomain = "logs" - authRole = "infra-admin" - checkPath = "/health" - metricsPath = "/metrics" - } - - volumes { ["/data"] {} } -} diff --git a/service/victoria/victoria-metrics.pkl b/service/victoria/victoria-metrics.pkl deleted file mode 100644 index 91b5a8139198a20b2a1d60335b4bf5bf218a6fcc..0000000000000000000000000000000000000000 --- a/service/victoria/victoria-metrics.pkl +++ /dev/null @@ -1,22 +0,0 @@ -amends "../../config/Service.pkl" - -meta { name = "VictoriaMetrics"; tagline = "Metrics history" } -rollout = "simple" - -container { - image = "docker.io/victoriametrics/victoria-metrics:v1.152.0" - cpu = 300 - memory = 512 - args { "-storageDataPath=/data"; "-retentionPeriod=30d"; "-usePromCompatibleNaming" } - - http { - containerPort = 8428 - hostPort = 18428 - subdomain = "metrics" - authRole = "infra-admin" - checkPath = "/health" - metricsPath = "/metrics" - } - - volumes { ["/data"] {} } -} diff --git a/service/victoria/victoria-traces.pkl b/service/victoria/victoria-traces.pkl deleted file mode 100644 index f6c15c85d1e1e3e0537ba64cfb3d7060610ae851..0000000000000000000000000000000000000000 --- a/service/victoria/victoria-traces.pkl +++ /dev/null @@ -1,22 +0,0 @@ -amends "../../config/Service.pkl" - -meta { name = "VictoriaTraces"; tagline = "Request traces" } -rollout = "simple" - -container { - image = "docker.io/victoriametrics/victoria-traces:v0.11.1" - cpu = 300 - memory = 512 - args { "-storageDataPath=/data"; "-retentionPeriod=30d" } - - http { - containerPort = 10428 - hostPort = 10428 - subdomain = "traces" - authRole = "infra-admin" - checkPath = "/health" - metricsPath = "/metrics" - } - - volumes { ["/data"] {} } -} diff --git a/service/victoria/victoria-traces/icon.svg b/service/victoria/victoria-traces/icon.svg deleted file mode 100644 index 2d8bd82f456790a26b08434d25dcc1466c073c56..0000000000000000000000000000000000000000 --- a/service/victoria/victoria-traces/icon.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/service/youtube/yt-feed.pkl b/service/youtube/yt-feed.pkl deleted file mode 100644 index 08460b59c39213009908c1a73fc54abe31590ad6..0000000000000000000000000000000000000000 --- a/service/youtube/yt-feed.pkl +++ /dev/null @@ -1,34 +0,0 @@ -amends "../../config/Service.pkl" - -import "../../config/site.pkl" as site - -local passive = site.preview || site.mediaReadOnly - -meta { name = "YouTube Thumbnails" } - -requiredSecrets { - "smtp_host" - "smtp_user" - "smtp_pass" -} - -container { - build = "upscaler" - cpu = if (passive) 200 else 1000 - memory = if (passive) 512 else 2048 - entrypoint = if (passive) "/bin/sh" else null - args = if (passive) new { - "-c" - "python3 /app/upscale.py --check && exec python3 -m http.server 8899 --bind 0.0.0.0 --directory /tmp" - } else new {} - http = if (passive) new { containerPort = 8899 } else null - volumes { - ["/data"] {} - ["/media"] { src = site.mediaRoot; readOnly = site.mediaReadOnly } - } - env { - ["STATE_DIR"] = "/data" - ["INDEP_DIR"] = "/media/\(site.jellyfinFolders["Independent"])" - ["SR_THREADS"] = "4" - } -} diff --git a/service/youtube/ytdl-sub.pkl b/service/youtube/ytdl-sub.pkl deleted file mode 100644 index 0804d21a60d64bb1c75f4f9842a3c26f303a204f..0000000000000000000000000000000000000000 --- a/service/youtube/ytdl-sub.pkl +++ /dev/null @@ -1,45 +0,0 @@ -amends "../../config/Service.pkl" - -import "../../config/site.pkl" as site - -local passive = site.preview || site.mediaReadOnly - -meta { name = "YouTube Archiver" } - -container { - image = "ghcr.io/jmbannon/ytdl-sub@sha256:29f27bc2b405b303d9da07f11f53ba39f7575efb94f98fe4c2c65d40a3e83bdd" - entrypoint = "/bin/sh" - args { - "-c" - if (passive) - "ytdl-sub --config /config-yt/config.yaml inspect --level 1 /yt-config/subscriptions.yaml >/dev/null && exec python3 -m http.server 8899 --bind 0.0.0.0 --directory /tmp" - else - "sh /config-yt/archive-loop.sh" - } - cpu = 200 - memory = if (passive) 256 else 512 - - http = if (passive) new { - containerPort = 8899 - } else null - - volumes { - ["/config"] {} - ["/config-yt"] { config = "config" } - ["/yt-config"] { - src = "\(site.cloverRoot)/Documents/Config/Youtube Downloader" - readOnly = true - } - ["/media"] {} - when (!passive) { - ["/media/jellyfin/Independent"] { - src = "\(site.mediaRoot)/\(site.jellyfinFolders["Independent"])" - readOnly = site.mediaReadOnly - } - } - } - - env { - ["HOME"] = "/config" - } -} diff --git a/service/youtube/ytdl-sub/icon.svg b/service/youtube/ytdl-sub/icon.svg deleted file mode 100644 index 42cc80773151addabf812255573a1324952e55c5..0000000000000000000000000000000000000000 --- a/service/youtube/ytdl-sub/icon.svg +++ /dev/null @@ -1 +0,0 @@ -YouTube diff --git a/service/youtube/ytdl-sub/config/archive-loop.sh b/service/ytdl/config/archive-loop.sh similarity index 100% rename from service/youtube/ytdl-sub/config/archive-loop.sh rename to service/ytdl/config/archive-loop.sh diff --git a/service/youtube/ytdl-sub/config/config.yaml b/service/ytdl/config/config.yaml similarity index 100% rename from service/youtube/ytdl-sub/config/config.yaml rename to service/ytdl/config/config.yaml diff --git a/service/youtube/yt-feed/icon.svg b/service/ytdl/icon.svg similarity index 100% rename from service/youtube/yt-feed/icon.svg rename to service/ytdl/icon.svg diff --git a/service/ytdl/service.pkl b/service/ytdl/service.pkl new file mode 100644 index 0000000000000000000000000000000000000000..d68da5991d2c66c4319bd7557eb3878a4cfe68a2 --- /dev/null +++ b/service/ytdl/service.pkl @@ -0,0 +1,73 @@ +amends "../../config/Service.pkl" + +import "../../config/site.pkl" as site + +local passive = site.preview || site.mediaReadOnly + +meta { name = "YTDL" } + +requiredSecrets { + "smtp_host" + "smtp_user" + "smtp_pass" +} + +containers { + ["thumbnails"] { + build = "upscaler" + cpu = if (passive) 200 else 1000 + memory = if (passive) 512 else 2048 + entrypoint = if (passive) "/bin/sh" else null + args = if (passive) new { + "-c" + "python3 /app/upscale.py --check && exec python3 -m http.server 8899 --bind 0.0.0.0 --directory /tmp" + } else new {} + http = if (passive) new { containerPort = 8899 } else null + volumes { + ["/data"] {} + ["/media"] { src = site.mediaRoot; readOnly = site.mediaReadOnly } + } + env { + ["STATE_DIR"] = "/data" + ["INDEP_DIR"] = "/media/\(site.jellyfinFolders["Independent"])" + ["SR_THREADS"] = "4" + } + } + ["archiver"] { + image = "ghcr.io/jmbannon/ytdl-sub@sha256:29f27bc2b405b303d9da07f11f53ba39f7575efb94f98fe4c2c65d40a3e83bdd" + entrypoint = "/bin/sh" + args { + "-c" + if (passive) + "ytdl-sub --config /config-yt/config.yaml inspect --level 1 /yt-config/subscriptions.yaml >/dev/null && exec python3 -m http.server 8899 --bind 0.0.0.0 --directory /tmp" + else + "sh /config-yt/archive-loop.sh" + } + cpu = 200 + memory = if (passive) 256 else 512 + + http = if (passive) new { + containerPort = 8899 + } else null + + volumes { + ["/config"] {} + ["/config-yt"] { config = "config" } + ["/yt-config"] { + src = "\(site.cloverRoot)/Documents/Config/Youtube Downloader" + readOnly = true + } + ["/media"] {} + when (!passive) { + ["/media/jellyfin/Independent"] { + src = "\(site.mediaRoot)/\(site.jellyfinFolders["Independent"])" + readOnly = site.mediaReadOnly + } + } + } + + env { + ["HOME"] = "/config" + } + } +} diff --git a/service/youtube/yt-feed/upscaler/Dockerfile b/service/ytdl/upscaler/Dockerfile similarity index 100% rename from service/youtube/yt-feed/upscaler/Dockerfile rename to service/ytdl/upscaler/Dockerfile diff --git a/service/youtube/yt-feed/upscaler/upscale.py b/service/ytdl/upscaler/upscale.py similarity index 100% rename from service/youtube/yt-feed/upscaler/upscale.py rename to service/ytdl/upscaler/upscale.py diff --git a/tools/dashboard-unit-test.py b/tools/dashboard-unit-test.py index 1327eb769bc43fbe8cd56a3afccbba87e1489de8..e78a8340b338d8e6c07d76208ca03f9446d93935 100644 --- a/tools/dashboard-unit-test.py +++ b/tools/dashboard-unit-test.py @@ -42,7 +42,7 @@ def main(): for name in ["studio-dashboard", "studio-host"]} permissions = [rule for rule in json.loads(shell(*nix, "eval", base + ".systemd.tmpfiles.rules", "--offline", "--json", "--option", "eval-cache", "false")) - if rule.startswith("A+ /srv/prod/yt-feed/data ")] + if rule.startswith("A+ /srv/prod/ytdl/data ")] assert len(permissions) == 1, permissions scripts = re.findall(r"^Exec(?:Start|StartPre|StartPost)=(/nix/store/\S+)$", units["studio-dashboard"], re.M) assert len(scripts) == 3, scripts @@ -92,7 +92,7 @@ def main(): ca.write_bytes(Path("/var/lib/studio/ca-bundle.crt").read_bytes()) ca.chmod(0o444) acl = root / "permissions.conf" - acl.write_text(permissions[0].replace("/srv/prod/yt-feed/data", str(state)) + "\n") + acl.write_text(permissions[0].replace("/srv/prod/ytdl/data", str(state)) + "\n") with socket.socket() as reservation: reservation.bind(("127.0.0.1", 0)) port = reservation.getsockname()[1] @@ -129,7 +129,7 @@ def main(): "/var/lib/studio/dashboard.token": str(readonly), "/var/lib/studio/ca-bundle.crt": str(ca), "/var/lib/studio/dashboard": str(data), - "/srv/prod/yt-feed/data": str(state), + "/srv/prod/ytdl/data": str(state), "/srv/clover": str(library), "--prefix=" + str(state): "--prefix=" + str(state) + " " + str(acl), "--publish=127.0.0.1:7072:7072": f"--publish=127.0.0.1:{port}:7072", @@ -140,7 +140,7 @@ def main(): content = Path(source).read_text() # Only host mount sources move; container paths retain the generated unit's contract. for original, replacement in replacements.items(): - if original in ["/srv/prod/yt-feed/data", "/srv/clover"]: + if original in ["/srv/prod/ytdl/data", "/srv/clover"]: content = content.replace("test -d " + original, "test -d " + replacement) content = content.replace("test -d '" + original, "test -d '" + replacement) content = content.replace("src=" + original, "src=" + replacement) @@ -188,11 +188,11 @@ def main(): sources = {"/data": data, "/run/studio-host": Path("/run/" + host_unit), "/run/secrets/dashboard-proxy.token": token, "/run/secrets/dashboard-nomad.token": readonly, "/run/secrets/ca-bundle.crt": ca, "/srv/clover": library, - "/srv/clover/Media": library / "Media", "/srv/prod/yt-feed/data": state} + "/srv/clover/Media": library / "Media", "/srv/prod/ytdl/data": state} assert mounts.keys() == sources.keys(), mounts.keys() for target, source in sources.items(): assert mounts[target]["Source"] == str(source), mounts[target] - assert mounts[target]["RW"] == (target in ["/data", "/srv/clover", "/srv/prod/yt-feed/data"]), mounts[target] + assert mounts[target]["RW"] == (target in ["/data", "/srv/clover", "/srv/prod/ytdl/data"]), mounts[target] for namespace in ["net", "pid", "mnt"]: assert Path(f"/proc/{info['State']['Pid']}/ns/{namespace}").stat().st_ino != Path(f"/proc/self/ns/{namespace}").stat().st_ino status = shell("podman", "exec", container, "/bin/cat", "/proc/1/status") @@ -206,11 +206,11 @@ def main(): shell("podman", "exec", container, python, "-c", "import ssl; assert ssl.create_default_context().cert_store_stats()['x509_ca'] > 0") refused = "import socket; s=socket.socket(socket.AF_UNIX); s.connect('/run/studio-host/host.sock'); " + "\ntry:\n s.sendall(b'{\"operation\":\"host.sample\"}\\n'); assert s.recv(4) == b''\nexcept (BrokenPipeError, ConnectionResetError):\n pass\n" shell("podman", "exec", f"--user=65534:{account.pw_gid}", container, python, "-c", refused) - shell("podman", "exec", container, python, "-c", "from pathlib import Path; p=Path('/srv/prod/yt-feed/data'); (p/'existing.json').write_text('fixture'); (p/'nested/new.json').write_text('fixture')") + shell("podman", "exec", container, python, "-c", "from pathlib import Path; p=Path('/srv/prod/ytdl/data'); (p/'existing.json').write_text('fixture'); (p/'nested/new.json').write_text('fixture')") assert (state / "existing.json").stat().st_uid == 3118 and state.stat().st_uid == 3118 assert (state / "nested/new.json").exists() - shell("podman", "exec", "--user=3118:3000", container, python, "-c", "from pathlib import Path; p=Path('/srv/prod/yt-feed/data'); (p/'nested/new.json').open('a').write('service'); (p/'nested/service.json').write_text('service')") - shell("podman", "exec", container, python, "-c", "from pathlib import Path; p=Path('/srv/prod/yt-feed/data/nested/service.json'); assert p.read_text() == 'service'; p.open('a').write('dashboard')") + shell("podman", "exec", "--user=3118:3000", container, python, "-c", "from pathlib import Path; p=Path('/srv/prod/ytdl/data'); (p/'nested/new.json').open('a').write('service'); (p/'nested/service.json').write_text('service')") + shell("podman", "exec", container, python, "-c", "from pathlib import Path; p=Path('/srv/prod/ytdl/data/nested/service.json'); assert p.read_text() == 'service'; p.open('a').write('dashboard')") assert subprocess.run(["podman", "exec", container, "/bin/touch", "/srv/clover/Media/escaped"], capture_output=True).returncode != 0 def get(path): diff --git a/tools/import-personal-files.py b/tools/import-personal-files.py index 6a79958e21d7e50b5506e5f2c2e6d4fc0faa046e..aebe1e825e88b223bfcef734d61b13dc6179714d 100644 --- a/tools/import-personal-files.py +++ b/tools/import-personal-files.py @@ -11,7 +11,7 @@ import urllib.request parser = argparse.ArgumentParser(description="Copy retained app files on the new host before starting the service.") -parser.add_argument("service", choices=["tailscale", "ddns-updater", "copyparty", "ytdl-sub", "pds"]) +parser.add_argument("service", choices=["tailscale", "ddns-updater", "copyparty", "ytdl", "pds"]) def main(): @@ -51,7 +51,7 @@ def main(): "tailscale": ("tailscale", "var/lib/tailscale"), "ddns-updater": ("ddns-updater", "updater/data"), "copyparty": ("copyparty/copyparty", "cfg/copyparty"), - "ytdl-sub": ("ytdl-sub", "config"), + "ytdl": ("ytdl-sub", "config"), "pds": ("pds", "pds"), } source_name, target_name = directories[service] diff --git a/tools/import-yt-feed.sh b/tools/import-yt-feed.sh index 35e425523990ca1705b6d56ad2204754103854fc..938713bb79bcad6e7c1d218b2e719f5ebc3bf9ae 100644 --- a/tools/import-yt-feed.sh +++ b/tools/import-yt-feed.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash set -euo pipefail -instance=${1:-yt-feed} -[[ $instance == yt-feed || $instance =~ ^yt-feed-preview-[0-9a-f]{8}$ ]] || { - echo 'Expected yt-feed or its preview ID' >&2 +instance=${1:-ytdl} +[[ $instance == ytdl || $instance =~ ^ytdl-preview-[0-9a-f]{8}$ ]] || { + echo 'Expected ytdl or its preview ID' >&2 exit 1 } @@ -15,7 +15,7 @@ source_copy_host=$source_host source_rsh=ssh offline=false if [[ -n ${STUDIO_LEGACY_HANDOFF:-} ]]; then - [[ $instance == yt-feed && -n ${STUDIO_DEPLOY_HOST:-} ]] || { + [[ $instance == ytdl && -n ${STUDIO_DEPLOY_HOST:-} ]] || { echo 'Offline handoff requires production YouTube Triage and a target' >&2; exit 1; } sh "$(dirname "$0")/check-legacy-handoff.sh" "$STUDIO_LEGACY_HANDOFF" "$target_host" "$target_port" @@ -23,8 +23,8 @@ if [[ -n ${STUDIO_LEGACY_HANDOFF:-} ]]; then source_rsh="ssh -p $target_port" offline=true fi -if [[ $instance == yt-feed ]]; then - root=/srv/prod/yt-feed +if [[ $instance == ytdl ]]; then + root=/srv/prod/ytdl "${remote[@]}" 'test "$(findmnt -n -o FSTYPE --mountpoint /srv/clover/Media)" = zfs; test -d "/srv/clover/Media/Intake - Music"; test -d "/srv/clover/Media/Indie Shows"; test -d /srv/clover/Media/Videos/Independent' || { echo 'Mount Media with Intake - Music, Indie Shows, and Videos/Independent before importing YouTube Triage' >&2 exit 1 @@ -33,23 +33,23 @@ if [[ $instance == yt-feed ]]; then source_running=$(ssh "$source_host" "sudo -n docker inspect -f '{{.State.Running}}' yt-feed") [[ $source_running == false ]] || { echo 'Stop Zenith yt-feed before importing production data' >&2; exit 1; } fi - response=$("${remote[@]}" "curl -s -w '\n%{http_code}' -H \"X-Nomad-Token: \$(cat /var/lib/studio/nomad.token)\" http://127.0.0.1:4646/v1/job/yt-feed") + response=$("${remote[@]}" "curl -s -w '\n%{http_code}' -H \"X-Nomad-Token: \$(cat /var/lib/studio/nomad.token)\" http://127.0.0.1:4646/v1/job/ytdl") status=${response##*$'\n'} if [[ $status == 200 ]]; then stopped=$(python3 -c 'import json,sys; print(str(json.load(sys.stdin)["Stop"]).lower())' <<<"${response%$'\n'*}") - [[ $stopped == true ]] || { echo 'Stop production yt-feed before importing' >&2; exit 1; } + [[ $stopped == true ]] || { echo 'Stop production YTDL before importing' >&2; exit 1; } elif [[ $status != 404 ]]; then - echo "Could not verify production yt-feed job state: $status" >&2 + echo "Could not verify production YTDL job state: $status" >&2 exit 1 fi else root=/srv/staging/$instance source_id=$("${remote[@]}" "python3 -c 'import json; print(json.load(open(\"/var/lib/studio/stages/$instance.json\"))[\"sourceId\"])'") - [[ $source_id == yt-feed ]] || { echo 'Preview belongs to another service' >&2; exit 1; } + [[ $source_id == ytdl ]] || { echo 'Preview belongs to another service' >&2; exit 1; } fi dataset=$("${remote[@]}" "findmnt -n -o SOURCE --mountpoint $root") -[[ $dataset == */prod/yt-feed || $dataset == */staging/"$instance" ]] || { - echo "Expected a mounted yt-feed dataset at $root" >&2 +[[ $dataset == */prod/ytdl || $dataset == */staging/"$instance" ]] || { + echo "Expected a mounted YTDL dataset at $root" >&2 exit 1 } @@ -72,7 +72,7 @@ PY drift=$(rsync -rnc --delete --out-format='%n' -e "$source_rsh" "$source_copy_host:/mnt/storage1/apps/yt-feed/" "$scratch/") [[ -z $drift ]] || { echo 'Source changed during copy; retry the import' >&2; exit 1; } -if [[ $instance != yt-feed ]]; then +if [[ $instance != ytdl ]]; then "${remote[@]}" "NOMAD_TOKEN=\$(cat /var/lib/studio/nomad.token) nomad job stop -yes $instance" else "${remote[@]}" 'systemctl stop studio-dashboard' @@ -88,12 +88,12 @@ done snapshot="$dataset@before-yt-feed-import-$(date +%s)-$$" "${remote[@]}" "zfs snapshot $snapshot" rsync -a --delete -e "ssh -p $target_port" "$scratch/" "$target_host:$root/data/" -uid=$("${remote[@]}" "python3 -c 'import json; print(json.load(open(\"/var/lib/studio/identities.json\"))[\"yt-feed\"])'") +uid=$("${remote[@]}" "python3 -c 'import json; print(json.load(open(\"/var/lib/studio/identities.json\"))[\"ytdl\"])'") "${remote[@]}" "chown -R $uid:$uid $root/data" drift=$(rsync -rnc --delete --out-format='%n' -e "ssh -p $target_port" "$scratch/" "$target_host:$root/data/") [[ -z $drift ]] || { echo "Copied state differs from source; restore $snapshot" >&2; exit 1; } -if [[ $instance == yt-feed ]]; then +if [[ $instance == ytdl ]]; then if [[ $offline == false ]]; then [[ $(ssh "$source_host" "sudo -n docker inspect -f '{{.State.Running}}' yt-feed") == false ]] || { echo 'Zenith yt-feed restarted during import; leave the home server stopped' >&2 @@ -101,9 +101,9 @@ if [[ $instance == yt-feed ]]; then } fi "${remote[@]}" 'systemctl start studio-dashboard' - echo "Imported production yt-feed from Zenith. Previous dataset state: $snapshot" + echo "Imported production YTDL from Zenith. Previous dataset state: $snapshot" else - python3 "$(dirname "$0")/deploy.py" stage yt-feed + python3 "$(dirname "$0")/deploy.py" stage ytdl "${remote[@]}" "zfs destroy $snapshot" echo "Imported and tested $instance" fi diff --git a/tools/personal-cutover.md b/tools/personal-cutover.md index e943b2879464a239cfdaeedfdabdcf3f4e830fad..65ec21856cd79ab57f195bfb4c04981fd7460bfd 100644 --- a/tools/personal-cutover.md +++ b/tools/personal-cutover.md @@ -19,7 +19,7 @@ Provision managed service datasets and identities before import. Register a stop | PDS | `apps/pds/` → `/srv/prod/pds/pds/` | `ssh root@10.0.0.1 'python3 - pds' < tools/import-personal-files.py`; preserve original JWT/admin/PLC/mail secrets first. Checks every SQLite database, including actor stores, and copies actor keys/blocks. Refuses WAL files requiring a consistent SQLite backup. | | Sonarr / Radarr | `apps//` → `/srv/prod//config/` | `bash tools/import-arr.sh `; consistent SQLite backup, integrity and hash; excludes logs/PIDs. Configure current internal endpoints after startup. | | Jackett | `apps/jackett/` → `/srv/prod/jackett/config/Jackett/` | `sh tools/import-jackett.sh jackett`; retains API key and indexer JSON, verifies checksums. | -| YouTube state | `apps/yt-feed/` → `/srv/prod/yt-feed/data/` | `bash tools/import-yt-feed.sh yt-feed`; validates JSON and checksums; stops and restarts dashboard's worker around import. Import mail secrets before activating worker. | +| YouTube state | `apps/yt-feed/` → `/srv/prod/ytdl/data/` | `bash tools/import-yt-feed.sh ytdl`; validates JSON and checksums; stops and restarts dashboard's worker around import. Import mail secrets before activating worker. | | Clover DB | `/srv/clover/.zfs/snapshot//Documents/Config/paper clover/` → `/srv/prod/clover-source-of-truth/data/` | `STUDIO_MIGRATION_SNAPSHOT= bash tools/import-source-data.sh`; direct host-local tar stream, tree digest and SQLite integrity. Import original API key. | | Tailscale / DDNS / Copyparty / YouTube Archiver / PDS | Sources and mount destinations are defined in [import-personal-files.py](import-personal-files.py). | Run the host-side importer below once per app. No app starts; it checks stopped allocations, snapshots destination, copies, checksum-verifies, and assigns its UID. | @@ -27,7 +27,7 @@ Provision managed service datasets and identities before import. Register a stop ssh root@10.0.0.1 'python3 - tailscale' < tools/import-personal-files.py ssh root@10.0.0.1 'python3 - ddns-updater' < tools/import-personal-files.py ssh root@10.0.0.1 'python3 - copyparty' < tools/import-personal-files.py -ssh root@10.0.0.1 'python3 - ytdl-sub' < tools/import-personal-files.py +ssh root@10.0.0.1 'python3 - ytdl' < tools/import-personal-files.py ``` Tailscale must retain `tailscaled.state` to preserve its node identity; the copy remains root-owned and mode `0600`. The PDS importer copies every actor store, not only its three root SQLite files; `import-pds.sh` excludes nested SQLite files and is unsuitable for this offline cutover. DDNS retains update history; its provider configuration comes from imported Cloudflare secrets and the service's generated `CONFIG`. Copyparty's active state is the **nested** old `copyparty/copyparty/` directory, including `shares.db`, sessions, IdP database, and salts; it stays nested under the new `/cfg/copyparty`. The [container sets `XDG_CONFIG_HOME=/cfg`](https://github.com/9001/copyparty/blob/hovudstraum/scripts/docker/Dockerfile.ac), and Copyparty appends its app directory. The outer legacy policy config is not copied: `prepare.py` generates the current policy. Its share database contained 17 shares and three selected-file records at the October 5 UTC inspection. Per-volume histories already on Clover/Media stay on those datasets. diff --git a/tools/studio.py b/tools/studio.py index 95dfbc8c39a906b3c1333b545a46675b36646bb7..ae5adc61a7b216d3f19d31792269f8356cb8045b 100644 --- a/tools/studio.py +++ b/tools/studio.py @@ -238,7 +238,6 @@ def render(data, definitions): f" studio_hostname = {q(primary.get('hostname') or '')}", f" studio_launcher = {q(str(data['launcher']).lower())}", f" studio_auth_role = {q(data.get('access') or primary.get('authRole') or '')}", - f" studio_metrics_path = {q(primary.get('metricsPath') or '')}", f" studio_trace_service = {q(data.get('traceServiceName') or '')}", f" studio_metrics_pushed = {q(str(data['metricsPushed']).lower())}", f" studio_requires = {q(json.dumps(requirements))}", @@ -331,8 +330,11 @@ def render(data, definitions): lines += [" service {", f" name = {q(name if task_name == 'app' else name + '-' + task_name)}", ' provider = "nomad"', f" port = {q(ports[(task_name, kind)])}"] if len(data["containers"]) == 1: lines.append(f" task = {q(task_name)}") + tags = [] + if kind == "http" and endpoint.get("metricsPath"): + tags.append("studio-metrics-path=" + endpoint["metricsPath"]) if kind == "http" and endpoint.get("hostname"): - tags = ["caddy-host=" + host for host in [endpoint["hostname"], *endpoint["plainHostnames"]]] + tags += ["caddy-host=" + host for host in [endpoint["hostname"], *endpoint["plainHostnames"]]] if endpoint.get("authRole"): tags.append("caddy-auth-role=" + endpoint["authRole"]) if endpoint.get("userHeader"): @@ -341,6 +343,7 @@ def render(data, definitions): tags.append("caddy-real-ip=true") if endpoint["tlsInternal"]: tags.append("caddy-internal=true") + if tags: lines.append(f" tags = {q(tags)}") lines += [" check {", f" type = {q(kind)}"] if kind == "http": @@ -494,7 +497,7 @@ def provision(data): path.mkdir(parents=True, exist_ok=True) os.chown(path, data["uid"], gid) path.chmod(0o750) - if data["id"] == "yt-feed" and target == "/data": + if data["sourceId"] == "ytdl" and target == "/data": command("systemd-tmpfiles", "--create", "--prefix=" + str(path)) bundle = STATE / "ca-bundle.crt" if any( diff --git a/tools/yt-feed-migration.md b/tools/yt-feed-migration.md index a7db0d4af14908fee2c461e1b3133ec685bf4171..d142ac2592f1a59f73a2973163907eb8cb2600d4 100644 --- a/tools/yt-feed-migration.md +++ b/tools/yt-feed-migration.md @@ -2,8 +2,8 @@ The service keeps six JSON state files in `/mnt/storage1/apps/yt-feed` and editable `feed.yaml` and `subscriptions.yaml` in Clover's `Documents/Config/Youtube Downloader`. Its downloaded videos and upscaler output live in the existing Media dataset, so that dataset moves by ZFS rename as described in [media-cutover.md](media-cutover.md). -On 2026-09-26, all six JSON files in `yt-feed-preview-e87b1ca1` matched Zenith by SHA-256 and parsed as JSON. Both Clover configuration files had matching SHA-256 hashes in the VM. The preview Media mount was read-only. Globe now owns the review API and worker; Nomad's `yt-feed` job runs only the thumbnail upscaler. +On 2026-09-26, all six JSON files in `yt-feed-preview-e87b1ca1` matched Zenith by SHA-256 and parsed as JSON. Both Clover configuration files had matching SHA-256 hashes in the VM. The preview Media mount was read-only. Globe now owns the review API and worker; Nomad's `ytdl` job runs the thumbnail upscaler and archiver. -For production, set `STUDIO_DEPLOY_HOST` and `STUDIO_DEPLOY_PORT` for the new host, then run `bash tools/import-legacy-secrets.sh yt-feed MAILER_ADDRESS MAILER_USERNAME MAILER_PASSWORD`. The order matches `smtp_host`, `smtp_user`, and `smtp_pass` in `service/youtube/yt-feed.pkl`. Stop Zenith's `yt-feed` and the Snow Globe `yt-feed` job before running `bash tools/import-yt-feed.sh yt-feed`; the importer stops Globe's worker, snapshots the Snow Globe dataset, verifies the JSON copy, then restarts Globe. Clover's configuration and the Media dataset must be mounted at their final paths before import. Verify Globe's YouTube queue and one manual review action before cutover. +For production, set `STUDIO_DEPLOY_HOST` and `STUDIO_DEPLOY_PORT` for the new host, then run `bash tools/import-legacy-secrets.sh ytdl MAILER_ADDRESS MAILER_USERNAME MAILER_PASSWORD`. The order matches `smtp_host`, `smtp_user`, and `smtp_pass` in `service/ytdl/service.pkl`. Stop Zenith's `yt-feed` and the Snow Globe `ytdl` job before running `bash tools/import-yt-feed.sh ytdl`; the importer stops Globe's worker, snapshots the Snow Globe dataset, verifies the JSON copy, then restarts Globe. Clover's configuration and the Media dataset must be mounted at their final paths before import. Verify Globe's YouTube queue and one manual review action before cutover. For a same-machine OS replacement, set `STUDIO_LEGACY_HANDOFF` to the [offline handoff](legacy-handoff.md) directory while importing both secrets and JSON state. The production importer then reads the retained files from the mounted old apps dataset on the new host, without old Docker. -- 2.54.0