| author |  clover caruso <git@paperclover.net> 2026-10-04 21:12:07-07:00 |
| committer | clover caruso <git@paperclover.net> 2026-10-04 21:21:40-07:00 |
| log | d16b9185f320c372908c7c4bdc6fba091e403b19 |
| tree | 91caa8e4bd7c68d4a7ec49fb84e60bebcce10c78 |
| parent | db72677058d1cdf39ba27832e57fd7b2d7c355ed |
| signature | Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU |
Group Victoria and YTDL services and refine theme indicators
Consolidate the collectors and YouTube tasks into two multi-container services. Preserve collector discovery names, scrape each container, and move the dashboard worker to the YTDL state and secrets.
Use diff purple for recent Shale timestamps, keep visited links unchanged, and center status spinner rotation.
Assisted-by: gpt-6.1-sol38 files changed, 481 insertions(+), 491 deletions(-)
dashboard
server
web
nixos
service
keycloak
pds
shale
victoria
youtube
- yt-feed.pkl
yt-feed
- icon.svg
upscaler
- ytdl-sub.pkl
ytdl-sub
ytdl
dashboard/server/youtube-worker.py+1-1
| ... | ... | @@ -21,7 +21,7 @@ os.umask(0o007) |
| 21 | 21 | |
| 22 | 22 | YT_CONFIG_DIR = os.environ.get("STUDIO_YT_CONFIG", "/srv/clover/Documents/Config/Youtube Downloader") |
| 23 | 23 | FEED_CONFIG = os.path.join(YT_CONFIG_DIR, "feed.yaml") |
| 24 | STATE_DIR = os.environ.get("STUDIO_YT_STATE", "/srv/prod/yt-feed/data") | |
| 24 | STATE_DIR = os.environ.get("STUDIO_YT_STATE", "/srv/prod/ytdl/data") | |
| 25 | 25 | INTERVAL = int(os.environ.get("CHECK_INTERVAL", "1800")) |
| 26 | 26 | SMTP_HOST = os.environ.get("SMTP_HOST", "") |
| 27 | 27 | SMTP_PORT = int(os.environ.get("SMTP_PORT", "465")) |
dashboard/src/core.rs+2-2
| ... | ... | @@ -566,7 +566,7 @@ async fn service(app: Arc<App>, id: &str) -> Result<Arc<Document>> { |
| 566 | 566 | false |
| 567 | 567 | }; |
| 568 | 568 | let metrics = |
| 569 | if meta["studio_metrics_path"].as_str().is_some_and(|p| !p.is_empty()) || meta["studio_metrics_pushed"] == "true" { | |
| 569 | if array(&found["job"]["TaskGroups"]).iter().flat_map(|g| array(&g["Services"])).any(|s| array(&s["Tags"]).iter().any(|tag| string(tag).starts_with("studio-metrics-path="))) || meta["studio_metrics_pushed"] == "true" { | |
| 570 | 570 | telemetry::metric_names(app.clone(), id) |
| 571 | 571 | .await |
| 572 | 572 | .unwrap_or(json!([])) |
| ... | ... | @@ -587,7 +587,7 @@ async fn service(app: Arc<App>, id: &str) -> Result<Arc<Document>> { |
| 587 | 587 | let rows = host::call(json!({"operation":"storage.datasets"})).await?; |
| 588 | 588 | Ok(json!(array(&rows).iter().map(|d| json!({"name":d["name"],"mountpoint":d["mountpoint"],"used":d["usedbydataset"],"snapshots":d["usedbysnapshots"]})).collect::<Vec<_>>())) |
| 589 | 589 | }).await?; |
| 590 | let logs = array(&all.value).iter().find(|s| s["id"] == "victoria-logs" && s["url"].is_string()).map(|s| json!({"app":{"id":s["id"],"name":s["name"],"icon":s["icon"]},"url":format!("{}/select/vmui/#/?query={}",string(&s["url"]),encoded(&format!("{{job=\"{id}\"}}")))})); | |
| 590 | let logs = array(&all.value).iter().find(|s| s["id"] == "victoria" && s["url"].is_string()).map(|s| json!({"app":{"id":s["id"],"name":s["name"],"icon":s["icon"]},"url":format!("{}/select/vmui/#/?query={}",string(&s["url"]),encoded(&format!("{{job=\"{id}\"}}")))})); | |
| 591 | 591 | let fields = json!({"applicationTraces":application_traces,"applicationMetrics":metrics,"release":release,"deployedAt":number(&found["job"]["SubmitTime"])/1e9,"rollout":if array(&found["job"]["TaskGroups"]).iter().any(|g| number(&g["Update"]["Canary"]) > 0.0) {"overlapped"} else {"simple"},"containers":containers(found),"checks":checks_of(&json!(array(&found["allocs"]).iter().flat_map(|a| array(&a["home_checks"])).collect::<Vec<_>>())),"requirements":requirements,"dependents":dependents,"secrets":secrets,"datasets":array(&datasets.value).iter().filter(|d| string(&d["name"]).ends_with(&format!("/prod/{id}"))).collect::<Vec<_>>(),"restartsAcknowledged":ack[id],"logs":logs}); |
| 592 | 592 | summary |
| 593 | 593 | .as_object_mut() |
dashboard/src/telemetry.rs+12-8
| ... | ... | @@ -822,15 +822,19 @@ pub fn start(app: Arc<App>) { |
| 822 | 822 | .as_object() |
| 823 | 823 | .unwrap() |
| 824 | 824 | .iter() |
| 825 | .filter_map(|(id, job)| { | |
| 826 | job["job"]["Meta"]["studio_metrics_path"] | |
| 827 | .as_str() | |
| 828 | .filter(|p| !p.is_empty()) | |
| 829 | .map(|p| (id.clone(), p.to_owned())) | |
| 825 | .flat_map(|(id, job)| { | |
| 826 | array(&job["job"]["TaskGroups"]).iter() | |
| 827 | .flat_map(|group| array(&group["Services"])) | |
| 828 | .flat_map(move |service| { | |
| 829 | array(&service["Tags"]).iter().filter_map(move |tag| { | |
| 830 | string(tag).strip_prefix("studio-metrics-path=") | |
| 831 | .map(|path| (id.clone(), string(&service["Name"]).to_owned(), path.to_owned())) | |
| 832 | }) | |
| 833 | }) | |
| 830 | 834 | }) |
| 831 | 835 | .collect(); |
| 832 | 836 | stream::iter(scrapes) |
| 833 | .for_each_concurrent(4, |(id, path)| { | |
| 837 | .for_each_concurrent(4, |(id, service, path)| { | |
| 834 | 838 | let app = app.clone(); |
| 835 | 839 | let base = base.clone(); |
| 836 | 840 | async move { |
| ... | ... | @@ -838,7 +842,7 @@ pub fn start(app: Arc<App>) { |
| 838 | 842 | let response = app |
| 839 | 843 | .request( |
| 840 | 844 | Method::GET, |
| 841 | &format!("{}{path}", endpoint(app.clone(), &id).await?), | |
| 845 | &format!("{}{path}", endpoint(app.clone(), &service).await?), | |
| 842 | 846 | )? |
| 843 | 847 | .timeout(Duration::from_secs(5)) |
| 844 | 848 | .send() |
| ... | ... | @@ -848,7 +852,7 @@ pub fn start(app: Arc<App>) { |
| 848 | 852 | Method::POST, |
| 849 | 853 | &format!( |
| 850 | 854 | "{base}/api/v1/import/prometheus?{}", |
| 851 | params(&[("extra_label", format!("service={id}"))]) | |
| 855 | params(&[("extra_label", format!("service={id}")), ("extra_label", format!("instance={service}"))]) | |
| 852 | 856 | ), |
| 853 | 857 | )? |
| 854 | 858 | .body(response.text().await?) |
dashboard/src/youtube.rs+1-1
| ... | ... | @@ -42,7 +42,7 @@ impl Worker { |
| 42 | 42 | ("SMTP_PASS", "smtp_pass"), |
| 43 | 43 | ] { |
| 44 | 44 | if let Ok(secret) = host::call( |
| 45 | json!({"operation":"deploy.secret.get","service":"yt-feed","key":name}), | |
| 45 | json!({"operation":"deploy.secret.get","service":"ytdl","key":name}), | |
| 46 | 46 | ).await { |
| 47 | 47 | if let Some(value) = secret.as_str() { |
| 48 | 48 | command.env(key, value); |
dashboard/web/styles.css+1-1
| ... | ... | @@ -299,7 +299,7 @@ button { font: inherit; color: inherit; } |
| 299 | 299 | .status.stopped { color: var(--stopped); } |
| 300 | 300 | .status.working { color: var(--text-2); } |
| 301 | 301 | .status:is(.deploying, .restarting, .starting) { color: var(--accent); } |
| 302 | .status:is(.deploying, .restarting, .starting, .working) { animation: spin 1.1s cubic-bezier(0.55, 0.15, 0.45, 0.85) infinite; } | |
| 302 | .status:is(.deploying, .restarting, .starting, .working) { transform-box: view-box; transform-origin: center; animation: spin 1.1s cubic-bezier(0.55, 0.15, 0.45, 0.85) infinite; } | |
| 303 | 303 | @keyframes spin { to { transform: rotate(360deg); } } |
| 304 | 304 | |
| 305 | 305 | .status-label { display: inline-flex; align-items: center; gap: 6px; color: var(--text-2); } |
nixos/configuration.nix+5-5
| ... | ... | @@ -183,9 +183,9 @@ in |
| 183 | 183 | uid=$(id -u studio-dashboard) |
| 184 | 184 | gid=$(id -g studio-dashboard) |
| 185 | 185 | optional=() |
| 186 | if test -d /srv/prod/yt-feed/data && test -d '/srv/clover/Documents/Config/Youtube Downloader'; then | |
| 187 | optional+=(--volume=/srv/prod/yt-feed/data:/srv/prod/yt-feed/data:rw | |
| 188 | --env=STUDIO_YT_STATE=/srv/prod/yt-feed/data | |
| 186 | if test -d /srv/prod/ytdl/data && test -d '/srv/clover/Documents/Config/Youtube Downloader'; then | |
| 187 | optional+=(--volume=/srv/prod/ytdl/data:/srv/prod/ytdl/data:rw | |
| 188 | --env=STUDIO_YT_STATE=/srv/prod/ytdl/data | |
| 189 | 189 | '--env=STUDIO_YT_CONFIG=/srv/clover/Documents/Config/Youtube Downloader' |
| 190 | 190 | --env=STUDIO_YT_MEDIA=/srv/clover/Media) |
| 191 | 191 | fi |
| ... | ... | @@ -228,7 +228,7 @@ in |
| 228 | 228 | chown root:studio-dashboard ${proxyToken} /var/lib/studio/dashboard.token |
| 229 | 229 | chmod 0640 ${proxyToken} /var/lib/studio/dashboard.token |
| 230 | 230 | chown -R studio-dashboard:studio-dashboard /var/lib/studio/dashboard |
| 231 | systemd-tmpfiles --create --prefix=/srv/prod/yt-feed/data | |
| 231 | systemd-tmpfiles --create --prefix=/srv/prod/ytdl/data | |
| 232 | 232 | if ! podman image exists ${lib.escapeShellArg "${dashboard.image.imageName}:${dashboard.image.imageTag}"}; then |
| 233 | 233 | podman load --quiet --input ${dashboard.image} |
| 234 | 234 | fi |
| ... | ... | @@ -249,7 +249,7 @@ in |
| 249 | 249 | "d /srv/staging 0755 root root -" |
| 250 | 250 | "d /srv/vm 0755 root root -" |
| 251 | 251 | "d /srv/logs 0770 clo chloe -" |
| 252 | "A+ /srv/prod/yt-feed/data - - - - g:chloe:rwX,m::rwX,d:g:chloe:rwx,d:m::rwx" | |
| 252 | "A+ /srv/prod/ytdl/data - - - - g:chloe:rwX,m::rwX,d:g:chloe:rwx,d:m::rwx" | |
| 253 | 253 | "d /var/lib/studio 0700 root root -" |
| 254 | 254 | "d /var/lib/studio/dashboard 0700 studio-dashboard studio-dashboard -" |
| 255 | 255 | "d /var/lib/studio/routes 0700 root root -" |
service/keycloak/service.pkl+1-1
| ... | ... | @@ -17,7 +17,7 @@ local database = new postgres.Database { name = "keycloak_next" } |
| 17 | 17 | |
| 18 | 18 | meta { name = "Keycloak"; launcher = false } |
| 19 | 19 | traceServiceName = module.id |
| 20 | dependsOn { "victoria-traces" } | |
| 20 | dependsOn { "victoria" } | |
| 21 | 21 | healthyDeadline = "20m" |
| 22 | 22 | setup = "configure.py" |
| 23 | 23 | provide = "provide.py" |
service/pds/service.pkl+1-1
| ... | ... | @@ -9,7 +9,7 @@ traceServiceName = module.id |
| 9 | 9 | metricsPushed = true |
| 10 | 10 | rollout = "simple" |
| 11 | 11 | stageIsolation = "fresh" |
| 12 | dependsOn { "victoria-metrics"; "victoria-traces" } | |
| 12 | dependsOn { "victoria" } | |
| 13 | 13 | |
| 14 | 14 | secrets = if (isolated) new { |
| 15 | 15 | ["jwt_secret"] {} |
service/shale/theme.css+5-5
| ... | ... | @@ -57,7 +57,7 @@ |
| 57 | 57 | --theme-color-primary-darker: #74113a; |
| 58 | 58 | --theme-body-background-color: #fffafd; |
| 59 | 59 | --theme-link-color: var(--theme-color-primary); |
| 60 | --theme-link-visited-color: #88436c; | |
| 60 | --theme-link-visited-color: var(--theme-link-color); | |
| 61 | 61 | --theme-link-hover-color: var(--theme-color-primary-dark); |
| 62 | 62 | --theme-link-active-color: var(--theme-color-primary-darker); |
| 63 | 63 | --theme-focus-color: var(--theme-color-primary-vivid); |
| ... | ... | @@ -166,8 +166,8 @@ |
| 166 | 166 | border-color: #d9bdca; |
| 167 | 167 | } |
| 168 | 168 | |
| 169 | :is(td, span).idleage:is(.seconds, .minutes, .hours) { | |
| 170 | color: #216e1f; | |
| 169 | :is(td, span).idleage:is(.seconds, .minutes, .hours, .days) { | |
| 170 | color: var(--shale-color-cool-dark); | |
| 171 | 171 | } |
| 172 | 172 | |
| 173 | 173 | :is(td, span).idleage.weeks, |
| ... | ... | @@ -242,7 +242,7 @@ |
| 242 | 242 | --theme-text-color: #f7edf2; |
| 243 | 243 | --theme-text-reverse-color: #21161c; |
| 244 | 244 | --theme-link-color: #ff8fba; |
| 245 | --theme-link-visited-color: #d9a3c1; | |
| 245 | --theme-link-visited-color: var(--theme-link-color); | |
| 246 | 246 | --theme-link-hover-color: #ffc0d8; |
| 247 | 247 | --theme-link-active-color: #ffd7e6; |
| 248 | 248 | --theme-focus-color: var(--theme-color-primary-vivid); |
| ... | ... | @@ -356,7 +356,7 @@ |
| 356 | 356 | } |
| 357 | 357 | |
| 358 | 358 | :is(td, span).idleage:is(.seconds, .minutes, .hours, .days) { |
| 359 | color: #90db9b; | |
| 359 | color: var(--shale-color-cool-dark); | |
| 360 | 360 | } |
| 361 | 361 | |
| 362 | 362 | :is(td, span).idleage.weeks, |
service/victoria/icon-dark.svg created+1
| ... | ... | @@ -0,0 +1 @@ |
| 1 | <svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" viewBox="0 0 512 512"><path fill="#fafafa" d="m40.6 65.6 174.1 156.7c23.9 21.5 60.1 21.4 83.9-.1L471.4 65.6C506.7 32.8 408.6.2 256.8 0h-1.6C103.5.2 5.3 32.8 40.6 65.6m-7.2 130 181.3 163.2c23.9 21.5 60.1 21.4 83.9-.1l180.1-163.1v-71.2L289.4 295.8c-18.6 16.9-46.9 16.9-65.5.1L33.4 124.4zm181.3 300.3L43.8 342c-6.7-5.9-10.4-14.5-10.4-23.4v-57.1L224 433.1c18.6 16.8 46.9 16.8 65.5-.1l189.2-171.4v57.2c0 8.8-3.7 17.3-10.3 23.3L298.5 495.9c-23.7 21.4-59.9 21.5-83.8 0"/></svg> | |
| \ No newline at end of file |
service/victoria/icon-light.svg created+1
| ... | ... | @@ -0,0 +1 @@ |
| 1 | <svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" viewBox="0 0 512 512"><path d="m40.6 65.6 174.1 156.7c23.9 21.5 60.1 21.4 83.9-.1L471.4 65.6C506.7 32.8 408.6.2 256.8 0h-1.6C103.5.2 5.3 32.8 40.6 65.6m-7.2 130 181.3 163.2c23.9 21.5 60.1 21.4 83.9-.1l180.1-163.1v-71.2L289.4 295.8c-18.6 16.9-46.9 16.9-65.5.1L33.4 124.4zm181.3 300.3L43.8 342c-6.7-5.9-10.4-14.5-10.4-23.4v-57.1L224 433.1c18.6 16.8 46.9 16.8 65.5-.1l189.2-171.4v57.2c0 8.8-3.7 17.3-10.3 23.3L298.5 495.9c-23.7 21.4-59.9 21.5-83.8 0"/></svg> | |
| \ No newline at end of file |
service/victoria/service.pkl created+56
| ... | ... | @@ -0,0 +1,56 @@ |
| 1 | amends "../../config/Service.pkl" | |
| 2 | ||
| 3 | meta { name = "Victoria"; tagline = "Metrics, logs, and traces" } | |
| 4 | ||
| 5 | containers { | |
| 6 | ["logs"] { | |
| 7 | image = "docker.io/victoriametrics/victoria-logs:v1.52.0" | |
| 8 | cpu = 300 | |
| 9 | memory = 512 | |
| 10 | args { "-storageDataPath=/logs"; "-retentionPeriod=30d" } | |
| 11 | ||
| 12 | http { | |
| 13 | containerPort = 9428 | |
| 14 | subdomain = "logs" | |
| 15 | authRole = "infra-admin" | |
| 16 | checkPath = "/health" | |
| 17 | metricsPath = "/metrics" | |
| 18 | } | |
| 19 | ||
| 20 | volumes { ["/logs"] {} } | |
| 21 | } | |
| 22 | ["metrics"] { | |
| 23 | image = "docker.io/victoriametrics/victoria-metrics:v1.152.0" | |
| 24 | cpu = 300 | |
| 25 | memory = 512 | |
| 26 | args { "-storageDataPath=/metrics"; "-retentionPeriod=30d"; "-usePromCompatibleNaming" } | |
| 27 | ||
| 28 | http { | |
| 29 | containerPort = 8428 | |
| 30 | hostPort = 18428 | |
| 31 | subdomain = "metrics" | |
| 32 | authRole = "infra-admin" | |
| 33 | checkPath = "/health" | |
| 34 | metricsPath = "/metrics" | |
| 35 | } | |
| 36 | ||
| 37 | volumes { ["/metrics"] {} } | |
| 38 | } | |
| 39 | ["traces"] { | |
| 40 | image = "docker.io/victoriametrics/victoria-traces:v0.11.1" | |
| 41 | cpu = 300 | |
| 42 | memory = 512 | |
| 43 | args { "-storageDataPath=/traces"; "-retentionPeriod=30d" } | |
| 44 | ||
| 45 | http { | |
| 46 | containerPort = 10428 | |
| 47 | hostPort = 10428 | |
| 48 | subdomain = "traces" | |
| 49 | authRole = "infra-admin" | |
| 50 | checkPath = "/health" | |
| 51 | metricsPath = "/metrics" | |
| 52 | } | |
| 53 | ||
| 54 | volumes { ["/traces"] {} } | |
| 55 | } | |
| 56 | } |
service/victoria/victoria-logs.pkl deleted-21
| ... | ... | @@ -1,21 +0,0 @@ |
| 1 | amends "../../config/Service.pkl" | |
| 2 | ||
| 3 | meta { name = "VictoriaLogs"; tagline = "Searchable service and system logs" } | |
| 4 | rollout = "simple" | |
| 5 | ||
| 6 | container { | |
| 7 | image = "docker.io/victoriametrics/victoria-logs:v1.52.0" | |
| 8 | cpu = 300 | |
| 9 | memory = 512 | |
| 10 | args { "-storageDataPath=/data"; "-retentionPeriod=30d" } | |
| 11 | ||
| 12 | http { | |
| 13 | containerPort = 9428 | |
| 14 | subdomain = "logs" | |
| 15 | authRole = "infra-admin" | |
| 16 | checkPath = "/health" | |
| 17 | metricsPath = "/metrics" | |
| 18 | } | |
| 19 | ||
| 20 | volumes { ["/data"] {} } | |
| 21 | } |
service/victoria/victoria-metrics.pkl deleted-22
| ... | ... | @@ -1,22 +0,0 @@ |
| 1 | amends "../../config/Service.pkl" | |
| 2 | ||
| 3 | meta { name = "VictoriaMetrics"; tagline = "Metrics history" } | |
| 4 | rollout = "simple" | |
| 5 | ||
| 6 | container { | |
| 7 | image = "docker.io/victoriametrics/victoria-metrics:v1.152.0" | |
| 8 | cpu = 300 | |
| 9 | memory = 512 | |
| 10 | args { "-storageDataPath=/data"; "-retentionPeriod=30d"; "-usePromCompatibleNaming" } | |
| 11 | ||
| 12 | http { | |
| 13 | containerPort = 8428 | |
| 14 | hostPort = 18428 | |
| 15 | subdomain = "metrics" | |
| 16 | authRole = "infra-admin" | |
| 17 | checkPath = "/health" | |
| 18 | metricsPath = "/metrics" | |
| 19 | } | |
| 20 | ||
| 21 | volumes { ["/data"] {} } | |
| 22 | } |
service/victoria/victoria-metrics/icon-dark.svg deleted-1
| ... | ... | @@ -1 +0,0 @@ |
| 1 | <svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" viewBox="0 0 512 512"><path fill="#fafafa" d="m40.6 65.6 174.1 156.7c23.9 21.5 60.1 21.4 83.9-.1L471.4 65.6C506.7 32.8 408.6.2 256.8 0h-1.6C103.5.2 5.3 32.8 40.6 65.6m-7.2 130 181.3 163.2c23.9 21.5 60.1 21.4 83.9-.1l180.1-163.1v-71.2L289.4 295.8c-18.6 16.9-46.9 16.9-65.5.1L33.4 124.4zm181.3 300.3L43.8 342c-6.7-5.9-10.4-14.5-10.4-23.4v-57.1L224 433.1c18.6 16.8 46.9 16.8 65.5-.1l189.2-171.4v57.2c0 8.8-3.7 17.3-10.3 23.3L298.5 495.9c-23.7 21.4-59.9 21.5-83.8 0"/></svg> | |
| \ No newline at end of file |
service/victoria/victoria-metrics/icon-light.svg deleted-1
| ... | ... | @@ -1 +0,0 @@ |
| 1 | <svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" viewBox="0 0 512 512"><path d="m40.6 65.6 174.1 156.7c23.9 21.5 60.1 21.4 83.9-.1L471.4 65.6C506.7 32.8 408.6.2 256.8 0h-1.6C103.5.2 5.3 32.8 40.6 65.6m-7.2 130 181.3 163.2c23.9 21.5 60.1 21.4 83.9-.1l180.1-163.1v-71.2L289.4 295.8c-18.6 16.9-46.9 16.9-65.5.1L33.4 124.4zm181.3 300.3L43.8 342c-6.7-5.9-10.4-14.5-10.4-23.4v-57.1L224 433.1c18.6 16.8 46.9 16.8 65.5-.1l189.2-171.4v57.2c0 8.8-3.7 17.3-10.3 23.3L298.5 495.9c-23.7 21.4-59.9 21.5-83.8 0"/></svg> | |
| \ No newline at end of file |
service/victoria/victoria-traces.pkl deleted-22
| ... | ... | @@ -1,22 +0,0 @@ |
| 1 | amends "../../config/Service.pkl" | |
| 2 | ||
| 3 | meta { name = "VictoriaTraces"; tagline = "Request traces" } | |
| 4 | rollout = "simple" | |
| 5 | ||
| 6 | container { | |
| 7 | image = "docker.io/victoriametrics/victoria-traces:v0.11.1" | |
| 8 | cpu = 300 | |
| 9 | memory = 512 | |
| 10 | args { "-storageDataPath=/data"; "-retentionPeriod=30d" } | |
| 11 | ||
| 12 | http { | |
| 13 | containerPort = 10428 | |
| 14 | hostPort = 10428 | |
| 15 | subdomain = "traces" | |
| 16 | authRole = "infra-admin" | |
| 17 | checkPath = "/health" | |
| 18 | metricsPath = "/metrics" | |
| 19 | } | |
| 20 | ||
| 21 | volumes { ["/data"] {} } | |
| 22 | } |
service/victoria/victoria-traces/icon.svg deleted-1
| ... | ... | @@ -1 +0,0 @@ |
| 1 | <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64"><rect width="64" height="64" rx="14" fill="#191b2b"/><g fill="none" stroke="#9d8cff" stroke-linecap="round" stroke-linejoin="round" stroke-width="4"><path d="M10 20h15l9 11h20M10 44h15l9-13"/><circle cx="10" cy="20" r="4" fill="#9d8cff"/><circle cx="10" cy="44" r="4" fill="#9d8cff"/><circle cx="54" cy="31" r="4" fill="#9d8cff"/></g></svg> |
service/youtube/yt-feed.pkl deleted-34
| ... | ... | @@ -1,34 +0,0 @@ |
| 1 | amends "../../config/Service.pkl" | |
| 2 | ||
| 3 | import "../../config/site.pkl" as site | |
| 4 | ||
| 5 | local passive = site.preview || site.mediaReadOnly | |
| 6 | ||
| 7 | meta { name = "YouTube Thumbnails" } | |
| 8 | ||
| 9 | requiredSecrets { | |
| 10 | "smtp_host" | |
| 11 | "smtp_user" | |
| 12 | "smtp_pass" | |
| 13 | } | |
| 14 | ||
| 15 | container { | |
| 16 | build = "upscaler" | |
| 17 | cpu = if (passive) 200 else 1000 | |
| 18 | memory = if (passive) 512 else 2048 | |
| 19 | entrypoint = if (passive) "/bin/sh" else null | |
| 20 | args = if (passive) new { | |
| 21 | "-c" | |
| 22 | "python3 /app/upscale.py --check && exec python3 -m http.server 8899 --bind 0.0.0.0 --directory /tmp" | |
| 23 | } else new {} | |
| 24 | http = if (passive) new { containerPort = 8899 } else null | |
| 25 | volumes { | |
| 26 | ["/data"] {} | |
| 27 | ["/media"] { src = site.mediaRoot; readOnly = site.mediaReadOnly } | |
| 28 | } | |
| 29 | env { | |
| 30 | ["STATE_DIR"] = "/data" | |
| 31 | ["INDEP_DIR"] = "/media/\(site.jellyfinFolders["Independent"])" | |
| 32 | ["SR_THREADS"] = "4" | |
| 33 | } | |
| 34 | } |
service/youtube/yt-feed/icon.svg deleted-1
| ... | ... | @@ -1 +0,0 @@ |
| 1 | <svg fill="#ff0033" role="img" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>YouTube</title><path d="M23.498 6.186a3.016 3.016 0 0 0-2.122-2.136C19.505 3.545 12 3.545 12 3.545s-7.505 0-9.377.505A3.017 3.017 0 0 0 .502 6.186C0 8.07 0 12 0 12s0 3.93.502 5.814a3.016 3.016 0 0 0 2.122 2.136c1.871.505 9.376.505 9.376.505s7.505 0 9.377-.505a3.015 3.015 0 0 0 2.122-2.136C24 15.93 24 12 24 12s0-3.93-.502-5.814zM9.545 15.568V8.432L15.818 12l-6.273 3.568z"/></svg> |
service/youtube/yt-feed/upscaler/Dockerfile deleted-5
| ... | ... | @@ -1,5 +0,0 @@ |
| 1 | FROM docker.io/library/python:3.12-slim@sha256:44ff437bba879d4941b710a369a8f19266aea34b29002807f0c487fabc9eec9b | |
| 2 | RUN pip install --no-cache-dir onnxruntime==1.30.0 numpy==2.5.3 Pillow==12.3.0 | |
| 3 | RUN mkdir /app && python3 -c 'import hashlib,urllib.request; u="https://huggingface.co/notaneimu/onnx-image-models/resolve/main/realesr-general-x4v3.onnx"; d=urllib.request.urlopen(u).read(); assert hashlib.sha256(d).hexdigest()=="e8db65652ed421c2f8c92645d8f6fc6b07fd2868a916fdaa1a99c8d28091f097"; open("/app/realesr-general-x4v3.onnx","wb").write(d)' | |
| 4 | COPY upscale.py /app/upscale.py | |
| 5 | CMD ["python3", "-u", "/app/upscale.py"] |
service/youtube/yt-feed/upscaler/upscale.py deleted-216
| ... | ... | @@ -1,216 +0,0 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | # yt-upscaler: keeps the Independent library's thumbnails sharp on a TV. | |
| 3 | # youtube only stores ~720p thumbnails, which jellyfin then stretches across | |
| 4 | # the whole screen as the item backdrop (soft + blocky). this re-fetches the | |
| 5 | # highest-res thumbnail from the image CDN (not bot-walled) and runs it through | |
| 6 | # Real-ESRGAN general-x4v3 (exported to ONNX, so no pickle is ever loaded) at | |
| 7 | # 4x on CPU, writing a crisp <video>.webp that jellyfin uses for card + backdrop. | |
| 8 | import io | |
| 9 | import json | |
| 10 | import os | |
| 11 | import re | |
| 12 | import sys | |
| 13 | import time | |
| 14 | import urllib.request | |
| 15 | ||
| 16 | import numpy as np | |
| 17 | import onnxruntime as ort | |
| 18 | from PIL import Image | |
| 19 | ||
| 20 | INDEP_DIR = os.environ.get("INDEP_DIR", "/media/jellyfin/Independent") | |
| 21 | STATE_DIR = os.environ.get("STATE_DIR", "/state") | |
| 22 | MODEL = os.environ.get("SR_MODEL", "/app/realesr-general-x4v3.onnx") | |
| 23 | MIN_H = int(os.environ.get("SR_MIN_H", "1400")) # >= this tall ⇒ already done | |
| 24 | CAP_H = int(os.environ.get("SR_CAP_H", "2160")) # cap output height (4k-ish) | |
| 25 | TILE = int(os.environ.get("SR_TILE", "256")) # tile size to bound memory | |
| 26 | THREADS = int(os.environ.get("SR_THREADS", "4")) # leave cpu for other services | |
| 27 | SCAN_INTERVAL = int(os.environ.get("SR_SCAN_INTERVAL", "1800")) | |
| 28 | VIDEO_EXTS = (".webm", ".mp4", ".mkv") | |
| 29 | UA = {"User-Agent": "Mozilla/5.0 (yt-upscaler; +https://paperclover.net)"} | |
| 30 | ||
| 31 | status = {"enabled": True, "running": False, "done": 0, "total": 0, | |
| 32 | "current": "", "errors": 0} | |
| 33 | _sess = None | |
| 34 | ||
| 35 | ||
| 36 | def log(m): | |
| 37 | print(m, flush=True) | |
| 38 | ||
| 39 | ||
| 40 | def write_status(): | |
| 41 | try: | |
| 42 | tmp = os.path.join(STATE_DIR, "upscale-status.json.tmp") | |
| 43 | with open(tmp, "w") as f: | |
| 44 | json.dump(status, f) | |
| 45 | os.replace(tmp, os.path.join(STATE_DIR, "upscale-status.json")) | |
| 46 | except OSError: | |
| 47 | pass | |
| 48 | ||
| 49 | ||
| 50 | def enabled(): | |
| 51 | try: | |
| 52 | with open(os.path.join(STATE_DIR, "upscale-enabled.json")) as f: | |
| 53 | return bool(json.load(f)["enabled"]) | |
| 54 | except FileNotFoundError: | |
| 55 | return True | |
| 56 | ||
| 57 | ||
| 58 | def session(): | |
| 59 | global _sess | |
| 60 | if _sess is None: | |
| 61 | opts = ort.SessionOptions() | |
| 62 | opts.intra_op_num_threads = THREADS | |
| 63 | opts.inter_op_num_threads = 1 | |
| 64 | _sess = ort.InferenceSession(MODEL, sess_options=opts, | |
| 65 | providers=["CPUExecutionProvider"]) | |
| 66 | return _sess | |
| 67 | ||
| 68 | ||
| 69 | def upscale(img, scale=4, pad=8): | |
| 70 | # tiled 4x SR; overlap each tile by `pad` px and crop it back to hide seams | |
| 71 | arr = np.asarray(img, dtype=np.float32) / 255.0 | |
| 72 | h, w, _ = arr.shape | |
| 73 | out = np.zeros((h * scale, w * scale, 3), dtype=np.float32) | |
| 74 | sess = session() | |
| 75 | for y in range(0, h, TILE): | |
| 76 | for x in range(0, w, TILE): | |
| 77 | y0, x0 = max(0, y - pad), max(0, x - pad) | |
| 78 | y1, x1 = min(h, y + TILE + pad), min(w, x + TILE + pad) | |
| 79 | patch = arr[y0:y1, x0:x1].transpose(2, 0, 1)[None] | |
| 80 | res = sess.run(None, {sess.get_inputs()[0].name: patch})[0][0].transpose(1, 2, 0) | |
| 81 | th, tw = min(TILE, h - y) * scale, min(TILE, w - x) * scale | |
| 82 | ty, tx = (y - y0) * scale, (x - x0) * scale | |
| 83 | out[y * scale:y * scale + th, x * scale:x * scale + tw] = \ | |
| 84 | res[ty:ty + th, tx:tx + tw] | |
| 85 | return Image.fromarray((out.clip(0, 1) * 255).round().astype("uint8")) | |
| 86 | ||
| 87 | ||
| 88 | def yt_id(base): | |
| 89 | ij = base + ".info.json" | |
| 90 | if os.path.exists(ij): | |
| 91 | try: | |
| 92 | with open(ij) as f: | |
| 93 | vid = json.load(f).get("id") | |
| 94 | if vid: | |
| 95 | return vid | |
| 96 | except (OSError, json.JSONDecodeError): | |
| 97 | pass | |
| 98 | try: | |
| 99 | with open(base + ".nfo") as f: | |
| 100 | m = re.search(r"<plot>(.*?)</plot>", f.read(), re.S) | |
| 101 | if m: | |
| 102 | u = re.search(r"(?:v=|youtu\.be/)([A-Za-z0-9_-]{11})", m.group(1)) | |
| 103 | return u.group(1) if u else None | |
| 104 | except OSError: | |
| 105 | pass | |
| 106 | return None | |
| 107 | ||
| 108 | ||
| 109 | def fetch_cdn_webp(vid): | |
| 110 | for q in ("maxresdefault", "sddefault", "hqdefault"): | |
| 111 | try: | |
| 112 | req = urllib.request.Request( | |
| 113 | f"https://i.ytimg.com/vi_webp/{vid}/{q}.webp", headers=UA) | |
| 114 | with urllib.request.urlopen(req, timeout=30) as r: | |
| 115 | data = r.read() | |
| 116 | if len(data) > 1000: | |
| 117 | return Image.open(io.BytesIO(data)).convert("RGB") | |
| 118 | except Exception: | |
| 119 | continue | |
| 120 | return None | |
| 121 | ||
| 122 | ||
| 123 | def img_height(path): | |
| 124 | try: | |
| 125 | with Image.open(path) as im: | |
| 126 | return im.height | |
| 127 | except Exception: | |
| 128 | return 0 | |
| 129 | ||
| 130 | ||
| 131 | def enhance(video_path): | |
| 132 | base = os.path.splitext(video_path)[0] | |
| 133 | webp, jpg = base + ".webp", base + ".jpg" | |
| 134 | if os.path.exists(webp) and img_height(webp) >= MIN_H: | |
| 135 | return "skip" | |
| 136 | src = None | |
| 137 | vid = yt_id(base) | |
| 138 | if vid: | |
| 139 | src = fetch_cdn_webp(vid) | |
| 140 | if src is None: | |
| 141 | for p in (webp, jpg): | |
| 142 | if os.path.exists(p): | |
| 143 | src = Image.open(p).convert("RGB") | |
| 144 | break | |
| 145 | if src is None: | |
| 146 | return "no-source" | |
| 147 | up = upscale(src) | |
| 148 | if up.height > CAP_H: | |
| 149 | up = up.resize((round(up.width * CAP_H / up.height), CAP_H), Image.LANCZOS) | |
| 150 | tmp = webp + ".tmp" | |
| 151 | up.save(tmp, "WEBP", quality=92, method=6) | |
| 152 | os.replace(tmp, webp) | |
| 153 | if os.path.exists(jpg): | |
| 154 | os.remove(jpg) | |
| 155 | return "done" | |
| 156 | ||
| 157 | ||
| 158 | def independent_videos(): | |
| 159 | out = [] | |
| 160 | for ch in sorted(os.listdir(INDEP_DIR)) if os.path.isdir(INDEP_DIR) else []: | |
| 161 | cdir = os.path.join(INDEP_DIR, ch) | |
| 162 | if not os.path.isdir(cdir): | |
| 163 | continue | |
| 164 | for f in sorted(os.listdir(cdir)): | |
| 165 | if f.lower().endswith(VIDEO_EXTS): | |
| 166 | out.append(os.path.join(cdir, f)) | |
| 167 | return out | |
| 168 | ||
| 169 | ||
| 170 | def main(): | |
| 171 | log(f"yt-upscaler started (model={MODEL}, threads={THREADS})") | |
| 172 | while True: | |
| 173 | status["enabled"] = enabled() | |
| 174 | if not status["enabled"]: | |
| 175 | status.update(running=False, current="") | |
| 176 | write_status() | |
| 177 | time.sleep(5) | |
| 178 | continue | |
| 179 | vids = independent_videos() | |
| 180 | pending = [v for v in vids | |
| 181 | if img_height(os.path.splitext(v)[0] + ".webp") < MIN_H] | |
| 182 | status.update(total=len(vids), done=len(vids) - len(pending), | |
| 183 | running=bool(pending), current="") | |
| 184 | write_status() | |
| 185 | if pending: | |
| 186 | log(f"upscaling {len(pending)} thumbnail(s)…") | |
| 187 | for v in pending: | |
| 188 | if not enabled(): | |
| 189 | break | |
| 190 | status["current"] = os.path.basename(v) | |
| 191 | write_status() | |
| 192 | t = time.time() | |
| 193 | try: | |
| 194 | r = enhance(v) | |
| 195 | if r == "done": | |
| 196 | log(f"upscaled ({time.time()-t:.0f}s): {os.path.basename(v)}") | |
| 197 | elif r != "skip": | |
| 198 | log(f"{r}: {os.path.basename(v)}") | |
| 199 | except Exception as e: | |
| 200 | status["errors"] += 1 | |
| 201 | log(f"failed: {os.path.basename(v)}: {e}") | |
| 202 | status["done"] += 1 | |
| 203 | write_status() | |
| 204 | time.sleep(1) # be polite to the rest of the box | |
| 205 | status.update(running=False, current="") | |
| 206 | write_status() | |
| 207 | if not enabled(): | |
| 208 | continue | |
| 209 | time.sleep(SCAN_INTERVAL) | |
| 210 | ||
| 211 | ||
| 212 | if __name__ == "__main__": | |
| 213 | if sys.argv[1:] == ["--check"]: | |
| 214 | assert upscale(Image.new("RGB", (8, 8), (30, 60, 90))).size == (32, 32) | |
| 215 | else: | |
| 216 | main() |
service/youtube/ytdl-sub.pkl deleted-45
| ... | ... | @@ -1,45 +0,0 @@ |
| 1 | amends "../../config/Service.pkl" | |
| 2 | ||
| 3 | import "../../config/site.pkl" as site | |
| 4 | ||
| 5 | local passive = site.preview || site.mediaReadOnly | |
| 6 | ||
| 7 | meta { name = "YouTube Archiver" } | |
| 8 | ||
| 9 | container { | |
| 10 | image = "ghcr.io/jmbannon/ytdl-sub@sha256:29f27bc2b405b303d9da07f11f53ba39f7575efb94f98fe4c2c65d40a3e83bdd" | |
| 11 | entrypoint = "/bin/sh" | |
| 12 | args { | |
| 13 | "-c" | |
| 14 | if (passive) | |
| 15 | "ytdl-sub --config /config-yt/config.yaml inspect --level 1 /yt-config/subscriptions.yaml >/dev/null && exec python3 -m http.server 8899 --bind 0.0.0.0 --directory /tmp" | |
| 16 | else | |
| 17 | "sh /config-yt/archive-loop.sh" | |
| 18 | } | |
| 19 | cpu = 200 | |
| 20 | memory = if (passive) 256 else 512 | |
| 21 | ||
| 22 | http = if (passive) new { | |
| 23 | containerPort = 8899 | |
| 24 | } else null | |
| 25 | ||
| 26 | volumes { | |
| 27 | ["/config"] {} | |
| 28 | ["/config-yt"] { config = "config" } | |
| 29 | ["/yt-config"] { | |
| 30 | src = "\(site.cloverRoot)/Documents/Config/Youtube Downloader" | |
| 31 | readOnly = true | |
| 32 | } | |
| 33 | ["/media"] {} | |
| 34 | when (!passive) { | |
| 35 | ["/media/jellyfin/Independent"] { | |
| 36 | src = "\(site.mediaRoot)/\(site.jellyfinFolders["Independent"])" | |
| 37 | readOnly = site.mediaReadOnly | |
| 38 | } | |
| 39 | } | |
| 40 | } | |
| 41 | ||
| 42 | env { | |
| 43 | ["HOME"] = "/config" | |
| 44 | } | |
| 45 | } |
service/youtube/ytdl-sub/config/archive-loop.sh deleted-19
| ... | ... | @@ -1,19 +0,0 @@ |
| 1 | #!/bin/sh | |
| 2 | # subscription pass on a 6h cycle — but if youtube has bot-walled the ip | |
| 3 | # ("sign in to confirm you're not a bot"), back off for a whole day instead | |
| 4 | # of hammering it every cycle, which prolongs the wall. | |
| 5 | while true; do | |
| 6 | started=$(date +%s) | |
| 7 | ytdl-sub --config /config-yt/config.yaml sub /yt-config/subscriptions.yaml 2>&1 | tee /tmp/last-pass.log | |
| 8 | finished=$(date +%s) | |
| 9 | if grep -q "confirm you.re not a bot" /tmp/last-pass.log; then | |
| 10 | echo "[archive-loop] bot wall detected; sleeping 24h" | |
| 11 | printf '{"started":%s,"finished":%s,"walled":true,"next":%s}\n' "$started" "$finished" "$((finished + 86400))" > /config/archive-status.json.tmp | |
| 12 | mv /config/archive-status.json.tmp /config/archive-status.json | |
| 13 | sleep 86400 | |
| 14 | else | |
| 15 | printf '{"started":%s,"finished":%s,"walled":false,"next":%s}\n' "$started" "$finished" "$((finished + 21600))" > /config/archive-status.json.tmp | |
| 16 | mv /config/archive-status.json.tmp /config/archive-status.json | |
| 17 | sleep 21600 | |
| 18 | fi | |
| 19 | done |
service/youtube/ytdl-sub/config/config.yaml deleted-42
| ... | ... | @@ -1,42 +0,0 @@ |
| 1 | # ytdl-sub tool configuration — the channel list lives in subscriptions.yaml | |
| 2 | configuration: | |
| 3 | working_directory: "/config/work" | |
| 4 | ||
| 5 | presets: | |
| 6 | # per-channel backlog control: only download uploads on/after download_after | |
| 7 | # (yyyymmdd). subscriptions.yaml sets the default and per-channel values. | |
| 8 | only-after: | |
| 9 | date_range: | |
| 10 | after: "{download_after}" | |
| 11 | overrides: | |
| 12 | download_after: "19700101" | |
| 13 | ||
| 14 | # output for the Independent library, which is a jellyfin "home videos" | |
| 15 | # library: flat files (no Season <year> folder), movie-style nfo (that | |
| 16 | # library type parses nfo with the movie parser, not episodedetails), | |
| 17 | # title without the date prefix, and the thumbnail named exactly like the | |
| 18 | # video so jellyfin uses it as the card image. | |
| 19 | flat-videos: | |
| 20 | nfo_tags: | |
| 21 | nfo_root: "movie" | |
| 22 | tags: | |
| 23 | title: "{title}" | |
| 24 | premiered: "{episode_date_standardized}" | |
| 25 | overrides: | |
| 26 | episode_file_path: "{episode_file_name_sanitized}" | |
| 27 | episode_file_name: "{upload_date_standardized} - {file_title}" | |
| 28 | thumbnail_file_name: "{episode_file_path}.jpg" | |
| 29 | # sponsorblock: cut paid sponsor reads, self-promo (merch/patreon), and | |
| 30 | # like/subscribe reminders out of newly downloaded videos. intros, outros, | |
| 31 | # and all real content are kept. applies to new downloads only (the archive | |
| 32 | # remembers what's already fetched). segment data is crowd-sourced from the | |
| 33 | # sponsorblock api, which isn't affected by youtube bot walls. | |
| 34 | chapters: | |
| 35 | sponsorblock_categories: | |
| 36 | - sponsor | |
| 37 | - selfpromo | |
| 38 | - interaction | |
| 39 | remove_sponsorblock_categories: | |
| 40 | - sponsor | |
| 41 | - selfpromo | |
| 42 | - interaction |
service/youtube/ytdl-sub/icon.svg deleted-1
| ... | ... | @@ -1 +0,0 @@ |
| 1 | <svg fill="#ff0033" role="img" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>YouTube</title><path d="M23.498 6.186a3.016 3.016 0 0 0-2.122-2.136C19.505 3.545 12 3.545 12 3.545s-7.505 0-9.377.505A3.017 3.017 0 0 0 .502 6.186C0 8.07 0 12 0 12s0 3.93.502 5.814a3.016 3.016 0 0 0 2.122 2.136c1.871.505 9.376.505 9.376.505s7.505 0 9.377-.505a3.015 3.015 0 0 0 2.122-2.136C24 15.93 24 12 24 12s0-3.93-.502-5.814zM9.545 15.568V8.432L15.818 12l-6.273 3.568z"/></svg> |
service/ytdl/config/archive-loop.sh created+19
| ... | ... | @@ -0,0 +1,19 @@ |
| 1 | #!/bin/sh | |
| 2 | # subscription pass on a 6h cycle — but if youtube has bot-walled the ip | |
| 3 | # ("sign in to confirm you're not a bot"), back off for a whole day instead | |
| 4 | # of hammering it every cycle, which prolongs the wall. | |
| 5 | while true; do | |
| 6 | started=$(date +%s) | |
| 7 | ytdl-sub --config /config-yt/config.yaml sub /yt-config/subscriptions.yaml 2>&1 | tee /tmp/last-pass.log | |
| 8 | finished=$(date +%s) | |
| 9 | if grep -q "confirm you.re not a bot" /tmp/last-pass.log; then | |
| 10 | echo "[archive-loop] bot wall detected; sleeping 24h" | |
| 11 | printf '{"started":%s,"finished":%s,"walled":true,"next":%s}\n' "$started" "$finished" "$((finished + 86400))" > /config/archive-status.json.tmp | |
| 12 | mv /config/archive-status.json.tmp /config/archive-status.json | |
| 13 | sleep 86400 | |
| 14 | else | |
| 15 | printf '{"started":%s,"finished":%s,"walled":false,"next":%s}\n' "$started" "$finished" "$((finished + 21600))" > /config/archive-status.json.tmp | |
| 16 | mv /config/archive-status.json.tmp /config/archive-status.json | |
| 17 | sleep 21600 | |
| 18 | fi | |
| 19 | done |
service/ytdl/config/config.yaml created+42
| ... | ... | @@ -0,0 +1,42 @@ |
| 1 | # ytdl-sub tool configuration — the channel list lives in subscriptions.yaml | |
| 2 | configuration: | |
| 3 | working_directory: "/config/work" | |
| 4 | ||
| 5 | presets: | |
| 6 | # per-channel backlog control: only download uploads on/after download_after | |
| 7 | # (yyyymmdd). subscriptions.yaml sets the default and per-channel values. | |
| 8 | only-after: | |
| 9 | date_range: | |
| 10 | after: "{download_after}" | |
| 11 | overrides: | |
| 12 | download_after: "19700101" | |
| 13 | ||
| 14 | # output for the Independent library, which is a jellyfin "home videos" | |
| 15 | # library: flat files (no Season <year> folder), movie-style nfo (that | |
| 16 | # library type parses nfo with the movie parser, not episodedetails), | |
| 17 | # title without the date prefix, and the thumbnail named exactly like the | |
| 18 | # video so jellyfin uses it as the card image. | |
| 19 | flat-videos: | |
| 20 | nfo_tags: | |
| 21 | nfo_root: "movie" | |
| 22 | tags: | |
| 23 | title: "{title}" | |
| 24 | premiered: "{episode_date_standardized}" | |
| 25 | overrides: | |
| 26 | episode_file_path: "{episode_file_name_sanitized}" | |
| 27 | episode_file_name: "{upload_date_standardized} - {file_title}" | |
| 28 | thumbnail_file_name: "{episode_file_path}.jpg" | |
| 29 | # sponsorblock: cut paid sponsor reads, self-promo (merch/patreon), and | |
| 30 | # like/subscribe reminders out of newly downloaded videos. intros, outros, | |
| 31 | # and all real content are kept. applies to new downloads only (the archive | |
| 32 | # remembers what's already fetched). segment data is crowd-sourced from the | |
| 33 | # sponsorblock api, which isn't affected by youtube bot walls. | |
| 34 | chapters: | |
| 35 | sponsorblock_categories: | |
| 36 | - sponsor | |
| 37 | - selfpromo | |
| 38 | - interaction | |
| 39 | remove_sponsorblock_categories: | |
| 40 | - sponsor | |
| 41 | - selfpromo | |
| 42 | - interaction |
service/ytdl/icon.svg created+1
| ... | ... | @@ -0,0 +1 @@ |
| 1 | <svg fill="#ff0033" role="img" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>YouTube</title><path d="M23.498 6.186a3.016 3.016 0 0 0-2.122-2.136C19.505 3.545 12 3.545 12 3.545s-7.505 0-9.377.505A3.017 3.017 0 0 0 .502 6.186C0 8.07 0 12 0 12s0 3.93.502 5.814a3.016 3.016 0 0 0 2.122 2.136c1.871.505 9.376.505 9.376.505s7.505 0 9.377-.505a3.015 3.015 0 0 0 2.122-2.136C24 15.93 24 12 24 12s0-3.93-.502-5.814zM9.545 15.568V8.432L15.818 12l-6.273 3.568z"/></svg> |
service/ytdl/service.pkl created+73
| ... | ... | @@ -0,0 +1,73 @@ |
| 1 | amends "../../config/Service.pkl" | |
| 2 | ||
| 3 | import "../../config/site.pkl" as site | |
| 4 | ||
| 5 | local passive = site.preview || site.mediaReadOnly | |
| 6 | ||
| 7 | meta { name = "YTDL" } | |
| 8 | ||
| 9 | requiredSecrets { | |
| 10 | "smtp_host" | |
| 11 | "smtp_user" | |
| 12 | "smtp_pass" | |
| 13 | } | |
| 14 | ||
| 15 | containers { | |
| 16 | ["thumbnails"] { | |
| 17 | build = "upscaler" | |
| 18 | cpu = if (passive) 200 else 1000 | |
| 19 | memory = if (passive) 512 else 2048 | |
| 20 | entrypoint = if (passive) "/bin/sh" else null | |
| 21 | args = if (passive) new { | |
| 22 | "-c" | |
| 23 | "python3 /app/upscale.py --check && exec python3 -m http.server 8899 --bind 0.0.0.0 --directory /tmp" | |
| 24 | } else new {} | |
| 25 | http = if (passive) new { containerPort = 8899 } else null | |
| 26 | volumes { | |
| 27 | ["/data"] {} | |
| 28 | ["/media"] { src = site.mediaRoot; readOnly = site.mediaReadOnly } | |
| 29 | } | |
| 30 | env { | |
| 31 | ["STATE_DIR"] = "/data" | |
| 32 | ["INDEP_DIR"] = "/media/\(site.jellyfinFolders["Independent"])" | |
| 33 | ["SR_THREADS"] = "4" | |
| 34 | } | |
| 35 | } | |
| 36 | ["archiver"] { | |
| 37 | image = "ghcr.io/jmbannon/ytdl-sub@sha256:29f27bc2b405b303d9da07f11f53ba39f7575efb94f98fe4c2c65d40a3e83bdd" | |
| 38 | entrypoint = "/bin/sh" | |
| 39 | args { | |
| 40 | "-c" | |
| 41 | if (passive) | |
| 42 | "ytdl-sub --config /config-yt/config.yaml inspect --level 1 /yt-config/subscriptions.yaml >/dev/null && exec python3 -m http.server 8899 --bind 0.0.0.0 --directory /tmp" | |
| 43 | else | |
| 44 | "sh /config-yt/archive-loop.sh" | |
| 45 | } | |
| 46 | cpu = 200 | |
| 47 | memory = if (passive) 256 else 512 | |
| 48 | ||
| 49 | http = if (passive) new { | |
| 50 | containerPort = 8899 | |
| 51 | } else null | |
| 52 | ||
| 53 | volumes { | |
| 54 | ["/config"] {} | |
| 55 | ["/config-yt"] { config = "config" } | |
| 56 | ["/yt-config"] { | |
| 57 | src = "\(site.cloverRoot)/Documents/Config/Youtube Downloader" | |
| 58 | readOnly = true | |
| 59 | } | |
| 60 | ["/media"] {} | |
| 61 | when (!passive) { | |
| 62 | ["/media/jellyfin/Independent"] { | |
| 63 | src = "\(site.mediaRoot)/\(site.jellyfinFolders["Independent"])" | |
| 64 | readOnly = site.mediaReadOnly | |
| 65 | } | |
| 66 | } | |
| 67 | } | |
| 68 | ||
| 69 | env { | |
| 70 | ["HOME"] = "/config" | |
| 71 | } | |
| 72 | } | |
| 73 | } |
service/ytdl/upscaler/Dockerfile created+5
| ... | ... | @@ -0,0 +1,5 @@ |
| 1 | FROM docker.io/library/python:3.12-slim@sha256:44ff437bba879d4941b710a369a8f19266aea34b29002807f0c487fabc9eec9b | |
| 2 | RUN pip install --no-cache-dir onnxruntime==1.30.0 numpy==2.5.3 Pillow==12.3.0 | |
| 3 | RUN mkdir /app && python3 -c 'import hashlib,urllib.request; u="https://huggingface.co/notaneimu/onnx-image-models/resolve/main/realesr-general-x4v3.onnx"; d=urllib.request.urlopen(u).read(); assert hashlib.sha256(d).hexdigest()=="e8db65652ed421c2f8c92645d8f6fc6b07fd2868a916fdaa1a99c8d28091f097"; open("/app/realesr-general-x4v3.onnx","wb").write(d)' | |
| 4 | COPY upscale.py /app/upscale.py | |
| 5 | CMD ["python3", "-u", "/app/upscale.py"] |
service/ytdl/upscaler/upscale.py created+216
| ... | ... | @@ -0,0 +1,216 @@ |
| 1 | #!/usr/bin/env python3 | |
| 2 | # yt-upscaler: keeps the Independent library's thumbnails sharp on a TV. | |
| 3 | # youtube only stores ~720p thumbnails, which jellyfin then stretches across | |
| 4 | # the whole screen as the item backdrop (soft + blocky). this re-fetches the | |
| 5 | # highest-res thumbnail from the image CDN (not bot-walled) and runs it through | |
| 6 | # Real-ESRGAN general-x4v3 (exported to ONNX, so no pickle is ever loaded) at | |
| 7 | # 4x on CPU, writing a crisp <video>.webp that jellyfin uses for card + backdrop. | |
| 8 | import io | |
| 9 | import json | |
| 10 | import os | |
| 11 | import re | |
| 12 | import sys | |
| 13 | import time | |
| 14 | import urllib.request | |
| 15 | ||
| 16 | import numpy as np | |
| 17 | import onnxruntime as ort | |
| 18 | from PIL import Image | |
| 19 | ||
| 20 | INDEP_DIR = os.environ.get("INDEP_DIR", "/media/jellyfin/Independent") | |
| 21 | STATE_DIR = os.environ.get("STATE_DIR", "/state") | |
| 22 | MODEL = os.environ.get("SR_MODEL", "/app/realesr-general-x4v3.onnx") | |
| 23 | MIN_H = int(os.environ.get("SR_MIN_H", "1400")) # >= this tall ⇒ already done | |
| 24 | CAP_H = int(os.environ.get("SR_CAP_H", "2160")) # cap output height (4k-ish) | |
| 25 | TILE = int(os.environ.get("SR_TILE", "256")) # tile size to bound memory | |
| 26 | THREADS = int(os.environ.get("SR_THREADS", "4")) # leave cpu for other services | |
| 27 | SCAN_INTERVAL = int(os.environ.get("SR_SCAN_INTERVAL", "1800")) | |
| 28 | VIDEO_EXTS = (".webm", ".mp4", ".mkv") | |
| 29 | UA = {"User-Agent": "Mozilla/5.0 (yt-upscaler; +https://paperclover.net)"} | |
| 30 | ||
| 31 | status = {"enabled": True, "running": False, "done": 0, "total": 0, | |
| 32 | "current": "", "errors": 0} | |
| 33 | _sess = None | |
| 34 | ||
| 35 | ||
| 36 | def log(m): | |
| 37 | print(m, flush=True) | |
| 38 | ||
| 39 | ||
| 40 | def write_status(): | |
| 41 | try: | |
| 42 | tmp = os.path.join(STATE_DIR, "upscale-status.json.tmp") | |
| 43 | with open(tmp, "w") as f: | |
| 44 | json.dump(status, f) | |
| 45 | os.replace(tmp, os.path.join(STATE_DIR, "upscale-status.json")) | |
| 46 | except OSError: | |
| 47 | pass | |
| 48 | ||
| 49 | ||
| 50 | def enabled(): | |
| 51 | try: | |
| 52 | with open(os.path.join(STATE_DIR, "upscale-enabled.json")) as f: | |
| 53 | return bool(json.load(f)["enabled"]) | |
| 54 | except FileNotFoundError: | |
| 55 | return True | |
| 56 | ||
| 57 | ||
| 58 | def session(): | |
| 59 | global _sess | |
| 60 | if _sess is None: | |
| 61 | opts = ort.SessionOptions() | |
| 62 | opts.intra_op_num_threads = THREADS | |
| 63 | opts.inter_op_num_threads = 1 | |
| 64 | _sess = ort.InferenceSession(MODEL, sess_options=opts, | |
| 65 | providers=["CPUExecutionProvider"]) | |
| 66 | return _sess | |
| 67 | ||
| 68 | ||
| 69 | def upscale(img, scale=4, pad=8): | |
| 70 | # tiled 4x SR; overlap each tile by `pad` px and crop it back to hide seams | |
| 71 | arr = np.asarray(img, dtype=np.float32) / 255.0 | |
| 72 | h, w, _ = arr.shape | |
| 73 | out = np.zeros((h * scale, w * scale, 3), dtype=np.float32) | |
| 74 | sess = session() | |
| 75 | for y in range(0, h, TILE): | |
| 76 | for x in range(0, w, TILE): | |
| 77 | y0, x0 = max(0, y - pad), max(0, x - pad) | |
| 78 | y1, x1 = min(h, y + TILE + pad), min(w, x + TILE + pad) | |
| 79 | patch = arr[y0:y1, x0:x1].transpose(2, 0, 1)[None] | |
| 80 | res = sess.run(None, {sess.get_inputs()[0].name: patch})[0][0].transpose(1, 2, 0) | |
| 81 | th, tw = min(TILE, h - y) * scale, min(TILE, w - x) * scale | |
| 82 | ty, tx = (y - y0) * scale, (x - x0) * scale | |
| 83 | out[y * scale:y * scale + th, x * scale:x * scale + tw] = \ | |
| 84 | res[ty:ty + th, tx:tx + tw] | |
| 85 | return Image.fromarray((out.clip(0, 1) * 255).round().astype("uint8")) | |
| 86 | ||
| 87 | ||
| 88 | def yt_id(base): | |
| 89 | ij = base + ".info.json" | |
| 90 | if os.path.exists(ij): | |
| 91 | try: | |
| 92 | with open(ij) as f: | |
| 93 | vid = json.load(f).get("id") | |
| 94 | if vid: | |
| 95 | return vid | |
| 96 | except (OSError, json.JSONDecodeError): | |
| 97 | pass | |
| 98 | try: | |
| 99 | with open(base + ".nfo") as f: | |
| 100 | m = re.search(r"<plot>(.*?)</plot>", f.read(), re.S) | |
| 101 | if m: | |
| 102 | u = re.search(r"(?:v=|youtu\.be/)([A-Za-z0-9_-]{11})", m.group(1)) | |
| 103 | return u.group(1) if u else None | |
| 104 | except OSError: | |
| 105 | pass | |
| 106 | return None | |
| 107 | ||
| 108 | ||
| 109 | def fetch_cdn_webp(vid): | |
| 110 | for q in ("maxresdefault", "sddefault", "hqdefault"): | |
| 111 | try: | |
| 112 | req = urllib.request.Request( | |
| 113 | f"https://i.ytimg.com/vi_webp/{vid}/{q}.webp", headers=UA) | |
| 114 | with urllib.request.urlopen(req, timeout=30) as r: | |
| 115 | data = r.read() | |
| 116 | if len(data) > 1000: | |
| 117 | return Image.open(io.BytesIO(data)).convert("RGB") | |
| 118 | except Exception: | |
| 119 | continue | |
| 120 | return None | |
| 121 | ||
| 122 | ||
| 123 | def img_height(path): | |
| 124 | try: | |
| 125 | with Image.open(path) as im: | |
| 126 | return im.height | |
| 127 | except Exception: | |
| 128 | return 0 | |
| 129 | ||
| 130 | ||
| 131 | def enhance(video_path): | |
| 132 | base = os.path.splitext(video_path)[0] | |
| 133 | webp, jpg = base + ".webp", base + ".jpg" | |
| 134 | if os.path.exists(webp) and img_height(webp) >= MIN_H: | |
| 135 | return "skip" | |
| 136 | src = None | |
| 137 | vid = yt_id(base) | |
| 138 | if vid: | |
| 139 | src = fetch_cdn_webp(vid) | |
| 140 | if src is None: | |
| 141 | for p in (webp, jpg): | |
| 142 | if os.path.exists(p): | |
| 143 | src = Image.open(p).convert("RGB") | |
| 144 | break | |
| 145 | if src is None: | |
| 146 | return "no-source" | |
| 147 | up = upscale(src) | |
| 148 | if up.height > CAP_H: | |
| 149 | up = up.resize((round(up.width * CAP_H / up.height), CAP_H), Image.LANCZOS) | |
| 150 | tmp = webp + ".tmp" | |
| 151 | up.save(tmp, "WEBP", quality=92, method=6) | |
| 152 | os.replace(tmp, webp) | |
| 153 | if os.path.exists(jpg): | |
| 154 | os.remove(jpg) | |
| 155 | return "done" | |
| 156 | ||
| 157 | ||
| 158 | def independent_videos(): | |
| 159 | out = [] | |
| 160 | for ch in sorted(os.listdir(INDEP_DIR)) if os.path.isdir(INDEP_DIR) else []: | |
| 161 | cdir = os.path.join(INDEP_DIR, ch) | |
| 162 | if not os.path.isdir(cdir): | |
| 163 | continue | |
| 164 | for f in sorted(os.listdir(cdir)): | |
| 165 | if f.lower().endswith(VIDEO_EXTS): | |
| 166 | out.append(os.path.join(cdir, f)) | |
| 167 | return out | |
| 168 | ||
| 169 | ||
| 170 | def main(): | |
| 171 | log(f"yt-upscaler started (model={MODEL}, threads={THREADS})") | |
| 172 | while True: | |
| 173 | status["enabled"] = enabled() | |
| 174 | if not status["enabled"]: | |
| 175 | status.update(running=False, current="") | |
| 176 | write_status() | |
| 177 | time.sleep(5) | |
| 178 | continue | |
| 179 | vids = independent_videos() | |
| 180 | pending = [v for v in vids | |
| 181 | if img_height(os.path.splitext(v)[0] + ".webp") < MIN_H] | |
| 182 | status.update(total=len(vids), done=len(vids) - len(pending), | |
| 183 | running=bool(pending), current="") | |
| 184 | write_status() | |
| 185 | if pending: | |
| 186 | log(f"upscaling {len(pending)} thumbnail(s)…") | |
| 187 | for v in pending: | |
| 188 | if not enabled(): | |
| 189 | break | |
| 190 | status["current"] = os.path.basename(v) | |
| 191 | write_status() | |
| 192 | t = time.time() | |
| 193 | try: | |
| 194 | r = enhance(v) | |
| 195 | if r == "done": | |
| 196 | log(f"upscaled ({time.time()-t:.0f}s): {os.path.basename(v)}") | |
| 197 | elif r != "skip": | |
| 198 | log(f"{r}: {os.path.basename(v)}") | |
| 199 | except Exception as e: | |
| 200 | status["errors"] += 1 | |
| 201 | log(f"failed: {os.path.basename(v)}: {e}") | |
| 202 | status["done"] += 1 | |
| 203 | write_status() | |
| 204 | time.sleep(1) # be polite to the rest of the box | |
| 205 | status.update(running=False, current="") | |
| 206 | write_status() | |
| 207 | if not enabled(): | |
| 208 | continue | |
| 209 | time.sleep(SCAN_INTERVAL) | |
| 210 | ||
| 211 | ||
| 212 | if __name__ == "__main__": | |
| 213 | if sys.argv[1:] == ["--check"]: | |
| 214 | assert upscale(Image.new("RGB", (8, 8), (30, 60, 90))).size == (32, 32) | |
| 215 | else: | |
| 216 | main() |
tools/dashboard-unit-test.py+9-9
| ... | ... | @@ -42,7 +42,7 @@ def main(): |
| 42 | 42 | for name in ["studio-dashboard", "studio-host"]} |
| 43 | 43 | permissions = [rule for rule in json.loads(shell(*nix, "eval", base + ".systemd.tmpfiles.rules", |
| 44 | 44 | "--offline", "--json", "--option", "eval-cache", "false")) |
| 45 | if rule.startswith("A+ /srv/prod/yt-feed/data ")] | |
| 45 | if rule.startswith("A+ /srv/prod/ytdl/data ")] | |
| 46 | 46 | assert len(permissions) == 1, permissions |
| 47 | 47 | scripts = re.findall(r"^Exec(?:Start|StartPre|StartPost)=(/nix/store/\S+)$", units["studio-dashboard"], re.M) |
| 48 | 48 | assert len(scripts) == 3, scripts |
| ... | ... | @@ -92,7 +92,7 @@ def main(): |
| 92 | 92 | ca.write_bytes(Path("/var/lib/studio/ca-bundle.crt").read_bytes()) |
| 93 | 93 | ca.chmod(0o444) |
| 94 | 94 | acl = root / "permissions.conf" |
| 95 | acl.write_text(permissions[0].replace("/srv/prod/yt-feed/data", str(state)) + "\n") | |
| 95 | acl.write_text(permissions[0].replace("/srv/prod/ytdl/data", str(state)) + "\n") | |
| 96 | 96 | with socket.socket() as reservation: |
| 97 | 97 | reservation.bind(("127.0.0.1", 0)) |
| 98 | 98 | port = reservation.getsockname()[1] |
| ... | ... | @@ -129,7 +129,7 @@ def main(): |
| 129 | 129 | "/var/lib/studio/dashboard.token": str(readonly), |
| 130 | 130 | "/var/lib/studio/ca-bundle.crt": str(ca), |
| 131 | 131 | "/var/lib/studio/dashboard": str(data), |
| 132 | "/srv/prod/yt-feed/data": str(state), | |
| 132 | "/srv/prod/ytdl/data": str(state), | |
| 133 | 133 | "/srv/clover": str(library), |
| 134 | 134 | "--prefix=" + str(state): "--prefix=" + str(state) + " " + str(acl), |
| 135 | 135 | "--publish=127.0.0.1:7072:7072": f"--publish=127.0.0.1:{port}:7072", |
| ... | ... | @@ -140,7 +140,7 @@ def main(): |
| 140 | 140 | content = Path(source).read_text() |
| 141 | 141 | # Only host mount sources move; container paths retain the generated unit's contract. |
| 142 | 142 | for original, replacement in replacements.items(): |
| 143 | if original in ["/srv/prod/yt-feed/data", "/srv/clover"]: | |
| 143 | if original in ["/srv/prod/ytdl/data", "/srv/clover"]: | |
| 144 | 144 | content = content.replace("test -d " + original, "test -d " + replacement) |
| 145 | 145 | content = content.replace("test -d '" + original, "test -d '" + replacement) |
| 146 | 146 | content = content.replace("src=" + original, "src=" + replacement) |
| ... | ... | @@ -188,11 +188,11 @@ def main(): |
| 188 | 188 | sources = {"/data": data, "/run/studio-host": Path("/run/" + host_unit), |
| 189 | 189 | "/run/secrets/dashboard-proxy.token": token, "/run/secrets/dashboard-nomad.token": readonly, |
| 190 | 190 | "/run/secrets/ca-bundle.crt": ca, "/srv/clover": library, |
| 191 | "/srv/clover/Media": library / "Media", "/srv/prod/yt-feed/data": state} | |
| 191 | "/srv/clover/Media": library / "Media", "/srv/prod/ytdl/data": state} | |
| 192 | 192 | assert mounts.keys() == sources.keys(), mounts.keys() |
| 193 | 193 | for target, source in sources.items(): |
| 194 | 194 | assert mounts[target]["Source"] == str(source), mounts[target] |
| 195 | assert mounts[target]["RW"] == (target in ["/data", "/srv/clover", "/srv/prod/yt-feed/data"]), mounts[target] | |
| 195 | assert mounts[target]["RW"] == (target in ["/data", "/srv/clover", "/srv/prod/ytdl/data"]), mounts[target] | |
| 196 | 196 | for namespace in ["net", "pid", "mnt"]: |
| 197 | 197 | assert Path(f"/proc/{info['State']['Pid']}/ns/{namespace}").stat().st_ino != Path(f"/proc/self/ns/{namespace}").stat().st_ino |
| 198 | 198 | status = shell("podman", "exec", container, "/bin/cat", "/proc/1/status") |
| ... | ... | @@ -206,11 +206,11 @@ def main(): |
| 206 | 206 | shell("podman", "exec", container, python, "-c", "import ssl; assert ssl.create_default_context().cert_store_stats()['x509_ca'] > 0") |
| 207 | 207 | refused = "import socket; s=socket.socket(socket.AF_UNIX); s.connect('/run/studio-host/host.sock'); " + "\ntry:\n s.sendall(b'{\"operation\":\"host.sample\"}\\n'); assert s.recv(4) == b''\nexcept (BrokenPipeError, ConnectionResetError):\n pass\n" |
| 208 | 208 | shell("podman", "exec", f"--user=65534:{account.pw_gid}", container, python, "-c", refused) |
| 209 | shell("podman", "exec", container, python, "-c", "from pathlib import Path; p=Path('/srv/prod/yt-feed/data'); (p/'existing.json').write_text('fixture'); (p/'nested/new.json').write_text('fixture')") | |
| 209 | shell("podman", "exec", container, python, "-c", "from pathlib import Path; p=Path('/srv/prod/ytdl/data'); (p/'existing.json').write_text('fixture'); (p/'nested/new.json').write_text('fixture')") | |
| 210 | 210 | assert (state / "existing.json").stat().st_uid == 3118 and state.stat().st_uid == 3118 |
| 211 | 211 | assert (state / "nested/new.json").exists() |
| 212 | shell("podman", "exec", "--user=3118:3000", container, python, "-c", "from pathlib import Path; p=Path('/srv/prod/yt-feed/data'); (p/'nested/new.json').open('a').write('service'); (p/'nested/service.json').write_text('service')") | |
| 213 | shell("podman", "exec", container, python, "-c", "from pathlib import Path; p=Path('/srv/prod/yt-feed/data/nested/service.json'); assert p.read_text() == 'service'; p.open('a').write('dashboard')") | |
| 212 | shell("podman", "exec", "--user=3118:3000", container, python, "-c", "from pathlib import Path; p=Path('/srv/prod/ytdl/data'); (p/'nested/new.json').open('a').write('service'); (p/'nested/service.json').write_text('service')") | |
| 213 | shell("podman", "exec", container, python, "-c", "from pathlib import Path; p=Path('/srv/prod/ytdl/data/nested/service.json'); assert p.read_text() == 'service'; p.open('a').write('dashboard')") | |
| 214 | 214 | assert subprocess.run(["podman", "exec", container, "/bin/touch", "/srv/clover/Media/escaped"], capture_output=True).returncode != 0 |
| 215 | 215 | |
| 216 | 216 | def get(path): |
tools/import-personal-files.py+2-2
| ... | ... | @@ -11,7 +11,7 @@ import urllib.request |
| 11 | 11 | |
| 12 | 12 | |
| 13 | 13 | parser = argparse.ArgumentParser(description="Copy retained app files on the new host before starting the service.") |
| 14 | parser.add_argument("service", choices=["tailscale", "ddns-updater", "copyparty", "ytdl-sub", "pds"]) | |
| 14 | parser.add_argument("service", choices=["tailscale", "ddns-updater", "copyparty", "ytdl", "pds"]) | |
| 15 | 15 | |
| 16 | 16 | |
| 17 | 17 | def main(): |
| ... | ... | @@ -51,7 +51,7 @@ def main(): |
| 51 | 51 | "tailscale": ("tailscale", "var/lib/tailscale"), |
| 52 | 52 | "ddns-updater": ("ddns-updater", "updater/data"), |
| 53 | 53 | "copyparty": ("copyparty/copyparty", "cfg/copyparty"), |
| 54 | "ytdl-sub": ("ytdl-sub", "config"), | |
| 54 | "ytdl": ("ytdl-sub", "config"), | |
| 55 | 55 | "pds": ("pds", "pds"), |
| 56 | 56 | } |
| 57 | 57 | source_name, target_name = directories[service] |
tools/import-yt-feed.sh+17-17
| ... | ... | @@ -1,9 +1,9 @@ |
| 1 | 1 | #!/usr/bin/env bash |
| 2 | 2 | set -euo pipefail |
| 3 | 3 | |
| 4 | instance=${1:-yt-feed} | |
| 5 | [[ $instance == yt-feed || $instance =~ ^yt-feed-preview-[0-9a-f]{8}$ ]] || { | |
| 6 | echo 'Expected yt-feed or its preview ID' >&2 | |
| 4 | instance=${1:-ytdl} | |
| 5 | [[ $instance == ytdl || $instance =~ ^ytdl-preview-[0-9a-f]{8}$ ]] || { | |
| 6 | echo 'Expected ytdl or its preview ID' >&2 | |
| 7 | 7 | exit 1 |
| 8 | 8 | } |
| 9 | 9 | |
| ... | ... | @@ -15,7 +15,7 @@ source_copy_host=$source_host |
| 15 | 15 | source_rsh=ssh |
| 16 | 16 | offline=false |
| 17 | 17 | if [[ -n ${STUDIO_LEGACY_HANDOFF:-} ]]; then |
| 18 | [[ $instance == yt-feed && -n ${STUDIO_DEPLOY_HOST:-} ]] || { | |
| 18 | [[ $instance == ytdl && -n ${STUDIO_DEPLOY_HOST:-} ]] || { | |
| 19 | 19 | echo 'Offline handoff requires production YouTube Triage and a target' >&2; exit 1; |
| 20 | 20 | } |
| 21 | 21 | sh "$(dirname "$0")/check-legacy-handoff.sh" "$STUDIO_LEGACY_HANDOFF" "$target_host" "$target_port" |
| ... | ... | @@ -23,8 +23,8 @@ if [[ -n ${STUDIO_LEGACY_HANDOFF:-} ]]; then |
| 23 | 23 | source_rsh="ssh -p $target_port" |
| 24 | 24 | offline=true |
| 25 | 25 | fi |
| 26 | if [[ $instance == yt-feed ]]; then | |
| 27 | root=/srv/prod/yt-feed | |
| 26 | if [[ $instance == ytdl ]]; then | |
| 27 | root=/srv/prod/ytdl | |
| 28 | 28 | "${remote[@]}" 'test "$(findmnt -n -o FSTYPE --mountpoint /srv/clover/Media)" = zfs; test -d "/srv/clover/Media/Intake - Music"; test -d "/srv/clover/Media/Indie Shows"; test -d /srv/clover/Media/Videos/Independent' || { |
| 29 | 29 | echo 'Mount Media with Intake - Music, Indie Shows, and Videos/Independent before importing YouTube Triage' >&2 |
| 30 | 30 | exit 1 |
| ... | ... | @@ -33,23 +33,23 @@ if [[ $instance == yt-feed ]]; then |
| 33 | 33 | source_running=$(ssh "$source_host" "sudo -n docker inspect -f '{{.State.Running}}' yt-feed") |
| 34 | 34 | [[ $source_running == false ]] || { echo 'Stop Zenith yt-feed before importing production data' >&2; exit 1; } |
| 35 | 35 | fi |
| 36 | response=$("${remote[@]}" "curl -s -w '\n%{http_code}' -H \"X-Nomad-Token: \$(cat /var/lib/studio/nomad.token)\" http://127.0.0.1:4646/v1/job/yt-feed") | |
| 36 | response=$("${remote[@]}" "curl -s -w '\n%{http_code}' -H \"X-Nomad-Token: \$(cat /var/lib/studio/nomad.token)\" http://127.0.0.1:4646/v1/job/ytdl") | |
| 37 | 37 | status=${response##*$'\n'} |
| 38 | 38 | if [[ $status == 200 ]]; then |
| 39 | 39 | stopped=$(python3 -c 'import json,sys; print(str(json.load(sys.stdin)["Stop"]).lower())' <<<"${response%$'\n'*}") |
| 40 | [[ $stopped == true ]] || { echo 'Stop production yt-feed before importing' >&2; exit 1; } | |
| 40 | [[ $stopped == true ]] || { echo 'Stop production YTDL before importing' >&2; exit 1; } | |
| 41 | 41 | elif [[ $status != 404 ]]; then |
| 42 | echo "Could not verify production yt-feed job state: $status" >&2 | |
| 42 | echo "Could not verify production YTDL job state: $status" >&2 | |
| 43 | 43 | exit 1 |
| 44 | 44 | fi |
| 45 | 45 | else |
| 46 | 46 | root=/srv/staging/$instance |
| 47 | 47 | source_id=$("${remote[@]}" "python3 -c 'import json; print(json.load(open(\"/var/lib/studio/stages/$instance.json\"))[\"sourceId\"])'") |
| 48 | [[ $source_id == yt-feed ]] || { echo 'Preview belongs to another service' >&2; exit 1; } | |
| 48 | [[ $source_id == ytdl ]] || { echo 'Preview belongs to another service' >&2; exit 1; } | |
| 49 | 49 | fi |
| 50 | 50 | dataset=$("${remote[@]}" "findmnt -n -o SOURCE --mountpoint $root") |
| 51 | [[ $dataset == */prod/yt-feed || $dataset == */staging/"$instance" ]] || { | |
| 52 | echo "Expected a mounted yt-feed dataset at $root" >&2 | |
| 51 | [[ $dataset == */prod/ytdl || $dataset == */staging/"$instance" ]] || { | |
| 52 | echo "Expected a mounted YTDL dataset at $root" >&2 | |
| 53 | 53 | exit 1 |
| 54 | 54 | } |
| 55 | 55 | |
| ... | ... | @@ -72,7 +72,7 @@ PY |
| 72 | 72 | drift=$(rsync -rnc --delete --out-format='%n' -e "$source_rsh" "$source_copy_host:/mnt/storage1/apps/yt-feed/" "$scratch/") |
| 73 | 73 | [[ -z $drift ]] || { echo 'Source changed during copy; retry the import' >&2; exit 1; } |
| 74 | 74 | |
| 75 | if [[ $instance != yt-feed ]]; then | |
| 75 | if [[ $instance != ytdl ]]; then | |
| 76 | 76 | "${remote[@]}" "NOMAD_TOKEN=\$(cat /var/lib/studio/nomad.token) nomad job stop -yes $instance" |
| 77 | 77 | else |
| 78 | 78 | "${remote[@]}" 'systemctl stop studio-dashboard' |
| ... | ... | @@ -88,12 +88,12 @@ done |
| 88 | 88 | snapshot="$dataset@before-yt-feed-import-$(date +%s)-$$" |
| 89 | 89 | "${remote[@]}" "zfs snapshot $snapshot" |
| 90 | 90 | rsync -a --delete -e "ssh -p $target_port" "$scratch/" "$target_host:$root/data/" |
| 91 | uid=$("${remote[@]}" "python3 -c 'import json; print(json.load(open(\"/var/lib/studio/identities.json\"))[\"yt-feed\"])'") | |
| 91 | uid=$("${remote[@]}" "python3 -c 'import json; print(json.load(open(\"/var/lib/studio/identities.json\"))[\"ytdl\"])'") | |
| 92 | 92 | "${remote[@]}" "chown -R $uid:$uid $root/data" |
| 93 | 93 | drift=$(rsync -rnc --delete --out-format='%n' -e "ssh -p $target_port" "$scratch/" "$target_host:$root/data/") |
| 94 | 94 | [[ -z $drift ]] || { echo "Copied state differs from source; restore $snapshot" >&2; exit 1; } |
| 95 | 95 | |
| 96 | if [[ $instance == yt-feed ]]; then | |
| 96 | if [[ $instance == ytdl ]]; then | |
| 97 | 97 | if [[ $offline == false ]]; then |
| 98 | 98 | [[ $(ssh "$source_host" "sudo -n docker inspect -f '{{.State.Running}}' yt-feed") == false ]] || { |
| 99 | 99 | echo 'Zenith yt-feed restarted during import; leave the home server stopped' >&2 |
| ... | ... | @@ -101,9 +101,9 @@ if [[ $instance == yt-feed ]]; then |
| 101 | 101 | } |
| 102 | 102 | fi |
| 103 | 103 | "${remote[@]}" 'systemctl start studio-dashboard' |
| 104 | echo "Imported production yt-feed from Zenith. Previous dataset state: $snapshot" | |
| 104 | echo "Imported production YTDL from Zenith. Previous dataset state: $snapshot" | |
| 105 | 105 | else |
| 106 | python3 "$(dirname "$0")/deploy.py" stage yt-feed | |
| 106 | python3 "$(dirname "$0")/deploy.py" stage ytdl | |
| 107 | 107 | "${remote[@]}" "zfs destroy $snapshot" |
| 108 | 108 | echo "Imported and tested $instance" |
| 109 | 109 | fi |
tools/personal-cutover.md+2-2
| ... | ... | @@ -19,7 +19,7 @@ Provision managed service datasets and identities before import. Register a stop |
| 19 | 19 | | PDS | `apps/pds/` → `/srv/prod/pds/pds/` | `ssh root@10.0.0.1 'python3 - pds' < tools/import-personal-files.py`; preserve original JWT/admin/PLC/mail secrets first. Checks every SQLite database, including actor stores, and copies actor keys/blocks. Refuses WAL files requiring a consistent SQLite backup. | |
| 20 | 20 | | Sonarr / Radarr | `apps/<app>/` → `/srv/prod/<app>/config/` | `bash tools/import-arr.sh <app>`; consistent SQLite backup, integrity and hash; excludes logs/PIDs. Configure current internal endpoints after startup. | |
| 21 | 21 | | Jackett | `apps/jackett/` → `/srv/prod/jackett/config/Jackett/` | `sh tools/import-jackett.sh jackett`; retains API key and indexer JSON, verifies checksums. | |
| 22 | | YouTube state | `apps/yt-feed/` → `/srv/prod/yt-feed/data/` | `bash tools/import-yt-feed.sh yt-feed`; validates JSON and checksums; stops and restarts dashboard's worker around import. Import mail secrets before activating worker. | | |
| 22 | | YouTube state | `apps/yt-feed/` → `/srv/prod/ytdl/data/` | `bash tools/import-yt-feed.sh ytdl`; validates JSON and checksums; stops and restarts dashboard's worker around import. Import mail secrets before activating worker. | | |
| 23 | 23 | | Clover DB | `/srv/clover/.zfs/snapshot/<snapshot>/Documents/Config/paper clover/` → `/srv/prod/clover-source-of-truth/data/` | `STUDIO_MIGRATION_SNAPSHOT=<snapshot> bash tools/import-source-data.sh`; direct host-local tar stream, tree digest and SQLite integrity. Import original API key. | |
| 24 | 24 | | Tailscale / DDNS / Copyparty / YouTube Archiver / PDS | Sources and mount destinations are defined in [import-personal-files.py](import-personal-files.py). | Run the host-side importer below once per app. No app starts; it checks stopped allocations, snapshots destination, copies, checksum-verifies, and assigns its UID. | |
| 25 | 25 | |
| ... | ... | @@ -27,7 +27,7 @@ Provision managed service datasets and identities before import. Register a stop |
| 27 | 27 | ssh root@10.0.0.1 'python3 - tailscale' < tools/import-personal-files.py |
| 28 | 28 | ssh root@10.0.0.1 'python3 - ddns-updater' < tools/import-personal-files.py |
| 29 | 29 | ssh root@10.0.0.1 'python3 - copyparty' < tools/import-personal-files.py |
| 30 | ssh root@10.0.0.1 'python3 - ytdl-sub' < tools/import-personal-files.py | |
| 30 | ssh root@10.0.0.1 'python3 - ytdl' < tools/import-personal-files.py | |
| 31 | 31 | ``` |
| 32 | 32 | |
| 33 | 33 | Tailscale must retain `tailscaled.state` to preserve its node identity; the copy remains root-owned and mode `0600`. The PDS importer copies every actor store, not only its three root SQLite files; `import-pds.sh` excludes nested SQLite files and is unsuitable for this offline cutover. DDNS retains update history; its provider configuration comes from imported Cloudflare secrets and the service's generated `CONFIG`. Copyparty's active state is the **nested** old `copyparty/copyparty/` directory, including `shares.db`, sessions, IdP database, and salts; it stays nested under the new `/cfg/copyparty`. The [container sets `XDG_CONFIG_HOME=/cfg`](https://github.com/9001/copyparty/blob/hovudstraum/scripts/docker/Dockerfile.ac), and Copyparty appends its app directory. The outer legacy policy config is not copied: `prepare.py` generates the current policy. Its share database contained 17 shares and three selected-file records at the October 5 UTC inspection. Per-volume histories already on Clover/Media stay on those datasets. |
tools/studio.py+6-3
| ... | ... | @@ -238,7 +238,6 @@ def render(data, definitions): |
| 238 | 238 | f" studio_hostname = {q(primary.get('hostname') or '')}", |
| 239 | 239 | f" studio_launcher = {q(str(data['launcher']).lower())}", |
| 240 | 240 | f" studio_auth_role = {q(data.get('access') or primary.get('authRole') or '')}", |
| 241 | f" studio_metrics_path = {q(primary.get('metricsPath') or '')}", | |
| 242 | 241 | f" studio_trace_service = {q(data.get('traceServiceName') or '')}", |
| 243 | 242 | f" studio_metrics_pushed = {q(str(data['metricsPushed']).lower())}", |
| 244 | 243 | f" studio_requires = {q(json.dumps(requirements))}", |
| ... | ... | @@ -331,8 +330,11 @@ def render(data, definitions): |
| 331 | 330 | lines += [" service {", f" name = {q(name if task_name == 'app' else name + '-' + task_name)}", ' provider = "nomad"', f" port = {q(ports[(task_name, kind)])}"] |
| 332 | 331 | if len(data["containers"]) == 1: |
| 333 | 332 | lines.append(f" task = {q(task_name)}") |
| 333 | tags = [] | |
| 334 | if kind == "http" and endpoint.get("metricsPath"): | |
| 335 | tags.append("studio-metrics-path=" + endpoint["metricsPath"]) | |
| 334 | 336 | if kind == "http" and endpoint.get("hostname"): |
| 335 | tags = ["caddy-host=" + host for host in [endpoint["hostname"], *endpoint["plainHostnames"]]] | |
| 337 | tags += ["caddy-host=" + host for host in [endpoint["hostname"], *endpoint["plainHostnames"]]] | |
| 336 | 338 | if endpoint.get("authRole"): |
| 337 | 339 | tags.append("caddy-auth-role=" + endpoint["authRole"]) |
| 338 | 340 | if endpoint.get("userHeader"): |
| ... | ... | @@ -341,6 +343,7 @@ def render(data, definitions): |
| 341 | 343 | tags.append("caddy-real-ip=true") |
| 342 | 344 | if endpoint["tlsInternal"]: |
| 343 | 345 | tags.append("caddy-internal=true") |
| 346 | if tags: | |
| 344 | 347 | lines.append(f" tags = {q(tags)}") |
| 345 | 348 | lines += [" check {", f" type = {q(kind)}"] |
| 346 | 349 | if kind == "http": |
| ... | ... | @@ -494,7 +497,7 @@ def provision(data): |
| 494 | 497 | path.mkdir(parents=True, exist_ok=True) |
| 495 | 498 | os.chown(path, data["uid"], gid) |
| 496 | 499 | path.chmod(0o750) |
| 497 | if data["id"] == "yt-feed" and target == "/data": | |
| 500 | if data["sourceId"] == "ytdl" and target == "/data": | |
| 498 | 501 | command("systemd-tmpfiles", "--create", "--prefix=" + str(path)) |
| 499 | 502 | bundle = STATE / "ca-bundle.crt" |
| 500 | 503 | if any( |
tools/yt-feed-migration.md+2-2
| ... | ... | @@ -2,8 +2,8 @@ |
| 2 | 2 | |
| 3 | 3 | The service keeps six JSON state files in `/mnt/storage1/apps/yt-feed` and editable `feed.yaml` and `subscriptions.yaml` in Clover's `Documents/Config/Youtube Downloader`. Its downloaded videos and upscaler output live in the existing Media dataset, so that dataset moves by ZFS rename as described in [media-cutover.md](media-cutover.md). |
| 4 | 4 | |
| 5 | On 2026-09-26, all six JSON files in `yt-feed-preview-e87b1ca1` matched Zenith by SHA-256 and parsed as JSON. Both Clover configuration files had matching SHA-256 hashes in the VM. The preview Media mount was read-only. Globe now owns the review API and worker; Nomad's `yt-feed` job runs only the thumbnail upscaler. | |
| 5 | On 2026-09-26, all six JSON files in `yt-feed-preview-e87b1ca1` matched Zenith by SHA-256 and parsed as JSON. Both Clover configuration files had matching SHA-256 hashes in the VM. The preview Media mount was read-only. Globe now owns the review API and worker; Nomad's `ytdl` job runs the thumbnail upscaler and archiver. | |
| 6 | 6 | |
| 7 | For production, set `STUDIO_DEPLOY_HOST` and `STUDIO_DEPLOY_PORT` for the new host, then run `bash tools/import-legacy-secrets.sh yt-feed MAILER_ADDRESS MAILER_USERNAME MAILER_PASSWORD`. The order matches `smtp_host`, `smtp_user`, and `smtp_pass` in `service/youtube/yt-feed.pkl`. Stop Zenith's `yt-feed` and the Snow Globe `yt-feed` job before running `bash tools/import-yt-feed.sh yt-feed`; the importer stops Globe's worker, snapshots the Snow Globe dataset, verifies the JSON copy, then restarts Globe. Clover's configuration and the Media dataset must be mounted at their final paths before import. Verify Globe's YouTube queue and one manual review action before cutover. | |
| 7 | For production, set `STUDIO_DEPLOY_HOST` and `STUDIO_DEPLOY_PORT` for the new host, then run `bash tools/import-legacy-secrets.sh ytdl MAILER_ADDRESS MAILER_USERNAME MAILER_PASSWORD`. The order matches `smtp_host`, `smtp_user`, and `smtp_pass` in `service/ytdl/service.pkl`. Stop Zenith's `yt-feed` and the Snow Globe `ytdl` job before running `bash tools/import-yt-feed.sh ytdl`; the importer stops Globe's worker, snapshots the Snow Globe dataset, verifies the JSON copy, then restarts Globe. Clover's configuration and the Media dataset must be mounted at their final paths before import. Verify Globe's YouTube queue and one manual review action before cutover. | |
| 8 | 8 | |
| 9 | 9 | For a same-machine OS replacement, set `STUDIO_LEGACY_HANDOFF` to the [offline handoff](legacy-handoff.md) directory while importing both secrets and JSON state. The production importer then reads the retained files from the mounted old apps dataset on the new host, without old Docker. |