diff --git a/dashboard/server/youtube-worker.py b/dashboard/server/youtube-worker.py index b4ba209fe5cc808eb2f2d2c2ee668f9d16e70b1a..edaf45de929efa026d12359f2b69c152cc36d2f7 100644 --- a/dashboard/server/youtube-worker.py +++ b/dashboard/server/youtube-worker.py @@ -21,7 +21,7 @@ os.umask(0o007) YT_CONFIG_DIR = os.environ.get("STUDIO_YT_CONFIG", "/srv/clover/Documents/Config/Youtube Downloader") FEED_CONFIG = os.path.join(YT_CONFIG_DIR, "feed.yaml") -STATE_DIR = os.environ.get("STUDIO_YT_STATE", "/srv/prod/yt-feed/data") +STATE_DIR = os.environ.get("STUDIO_YT_STATE", "/srv/prod/ytdl/data") INTERVAL = int(os.environ.get("CHECK_INTERVAL", "1800")) SMTP_HOST = os.environ.get("SMTP_HOST", "") SMTP_PORT = int(os.environ.get("SMTP_PORT", "465")) diff --git a/dashboard/src/core.rs b/dashboard/src/core.rs index 535723c74c5915a4e3ba5f12b3229d8d9941fe87..993e5a5fb2849c6de0593867b3d99f002938dea1 100644 --- a/dashboard/src/core.rs +++ b/dashboard/src/core.rs @@ -566,7 +566,7 @@ async fn service(app: Arc, id: &str) -> Result> { false }; let metrics = - if meta["studio_metrics_path"].as_str().is_some_and(|p| !p.is_empty()) || meta["studio_metrics_pushed"] == "true" { + if array(&found["job"]["TaskGroups"]).iter().flat_map(|g| array(&g["Services"])).any(|s| array(&s["Tags"]).iter().any(|tag| string(tag).starts_with("studio-metrics-path="))) || meta["studio_metrics_pushed"] == "true" { telemetry::metric_names(app.clone(), id) .await .unwrap_or(json!([])) @@ -587,7 +587,7 @@ async fn service(app: Arc, id: &str) -> Result> { let rows = host::call(json!({"operation":"storage.datasets"})).await?; Ok(json!(array(&rows).iter().map(|d| json!({"name":d["name"],"mountpoint":d["mountpoint"],"used":d["usedbydataset"],"snapshots":d["usedbysnapshots"]})).collect::>())) }).await?; - let logs = array(&all.value).iter().find(|s| s["id"] == "victoria-logs" && s["url"].is_string()).map(|s| json!({"app":{"id":s["id"],"name":s["name"],"icon":s["icon"]},"url":format!("{}/select/vmui/#/?query={}",string(&s["url"]),encoded(&format!("{{job=\"{id}\"}}")))})); + let logs = array(&all.value).iter().find(|s| s["id"] == "victoria" && s["url"].is_string()).map(|s| json!({"app":{"id":s["id"],"name":s["name"],"icon":s["icon"]},"url":format!("{}/select/vmui/#/?query={}",string(&s["url"]),encoded(&format!("{{job=\"{id}\"}}")))})); let fields = json!({"applicationTraces":application_traces,"applicationMetrics":metrics,"release":release,"deployedAt":number(&found["job"]["SubmitTime"])/1e9,"rollout":if array(&found["job"]["TaskGroups"]).iter().any(|g| number(&g["Update"]["Canary"]) > 0.0) {"overlapped"} else {"simple"},"containers":containers(found),"checks":checks_of(&json!(array(&found["allocs"]).iter().flat_map(|a| array(&a["home_checks"])).collect::>())),"requirements":requirements,"dependents":dependents,"secrets":secrets,"datasets":array(&datasets.value).iter().filter(|d| string(&d["name"]).ends_with(&format!("/prod/{id}"))).collect::>(),"restartsAcknowledged":ack[id],"logs":logs}); summary .as_object_mut() diff --git a/dashboard/src/telemetry.rs b/dashboard/src/telemetry.rs index 3149a14b43ee1e12184d5c43849d4023b15f7157..36977e4eb79125bb58d7b3efdd62a0ceafdf4b1c 100644 --- a/dashboard/src/telemetry.rs +++ b/dashboard/src/telemetry.rs @@ -822,15 +822,19 @@ pub fn start(app: Arc) { .as_object() .unwrap() .iter() - .filter_map(|(id, job)| { - job["job"]["Meta"]["studio_metrics_path"] - .as_str() - .filter(|p| !p.is_empty()) - .map(|p| (id.clone(), p.to_owned())) + .flat_map(|(id, job)| { + array(&job["job"]["TaskGroups"]).iter() + .flat_map(|group| array(&group["Services"])) + .flat_map(move |service| { + array(&service["Tags"]).iter().filter_map(move |tag| { + string(tag).strip_prefix("studio-metrics-path=") + .map(|path| (id.clone(), string(&service["Name"]).to_owned(), path.to_owned())) + }) + }) }) .collect(); stream::iter(scrapes) - .for_each_concurrent(4, |(id, path)| { + .for_each_concurrent(4, |(id, service, path)| { let app = app.clone(); let base = base.clone(); async move { @@ -838,7 +842,7 @@ pub fn start(app: Arc) { let response = app .request( Method::GET, - &format!("{}{path}", endpoint(app.clone(), &id).await?), + &format!("{}{path}", endpoint(app.clone(), &service).await?), )? .timeout(Duration::from_secs(5)) .send() @@ -848,7 +852,7 @@ pub fn start(app: Arc) { Method::POST, &format!( "{base}/api/v1/import/prometheus?{}", - params(&[("extra_label", format!("service={id}"))]) + params(&[("extra_label", format!("service={id}")), ("extra_label", format!("instance={service}"))]) ), )? .body(response.text().await?) diff --git a/dashboard/src/youtube.rs b/dashboard/src/youtube.rs index 65ec0c633ed3c090d50dbeec1e998a5f569187a0..e983300b47accc4dbef01cd26a9a098f6376a71e 100644 --- a/dashboard/src/youtube.rs +++ b/dashboard/src/youtube.rs @@ -42,7 +42,7 @@ impl Worker { ("SMTP_PASS", "smtp_pass"), ] { if let Ok(secret) = host::call( - json!({"operation":"deploy.secret.get","service":"yt-feed","key":name}), + json!({"operation":"deploy.secret.get","service":"ytdl","key":name}), ).await { if let Some(value) = secret.as_str() { command.env(key, value); diff --git a/dashboard/web/styles.css b/dashboard/web/styles.css index b4ce76d891c16cbcc0602281a5e8c1539b1d72fa..d306236776d4f08c2e5e6b00dcb735305cf01534 100644 --- a/dashboard/web/styles.css +++ b/dashboard/web/styles.css @@ -299,7 +299,7 @@ button { font: inherit; color: inherit; } .status.stopped { color: var(--stopped); } .status.working { color: var(--text-2); } .status:is(.deploying, .restarting, .starting) { color: var(--accent); } -.status:is(.deploying, .restarting, .starting, .working) { animation: spin 1.1s cubic-bezier(0.55, 0.15, 0.45, 0.85) infinite; } +.status:is(.deploying, .restarting, .starting, .working) { transform-box: view-box; transform-origin: center; animation: spin 1.1s cubic-bezier(0.55, 0.15, 0.45, 0.85) infinite; } @keyframes spin { to { transform: rotate(360deg); } } .status-label { display: inline-flex; align-items: center; gap: 6px; color: var(--text-2); } diff --git a/nixos/configuration.nix b/nixos/configuration.nix index 42451c6ca0f7e2cf7c683811c6afde1e30b9fb2c..19becc7e52524765eb473fa288b419e77ffe1921 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -183,9 +183,9 @@ in uid=$(id -u studio-dashboard) gid=$(id -g studio-dashboard) optional=() - if test -d /srv/prod/yt-feed/data && test -d '/srv/clover/Documents/Config/Youtube Downloader'; then - optional+=(--volume=/srv/prod/yt-feed/data:/srv/prod/yt-feed/data:rw - --env=STUDIO_YT_STATE=/srv/prod/yt-feed/data + if test -d /srv/prod/ytdl/data && test -d '/srv/clover/Documents/Config/Youtube Downloader'; then + optional+=(--volume=/srv/prod/ytdl/data:/srv/prod/ytdl/data:rw + --env=STUDIO_YT_STATE=/srv/prod/ytdl/data '--env=STUDIO_YT_CONFIG=/srv/clover/Documents/Config/Youtube Downloader' --env=STUDIO_YT_MEDIA=/srv/clover/Media) fi @@ -228,7 +228,7 @@ in chown root:studio-dashboard ${proxyToken} /var/lib/studio/dashboard.token chmod 0640 ${proxyToken} /var/lib/studio/dashboard.token chown -R studio-dashboard:studio-dashboard /var/lib/studio/dashboard - systemd-tmpfiles --create --prefix=/srv/prod/yt-feed/data + systemd-tmpfiles --create --prefix=/srv/prod/ytdl/data if ! podman image exists ${lib.escapeShellArg "${dashboard.image.imageName}:${dashboard.image.imageTag}"}; then podman load --quiet --input ${dashboard.image} fi @@ -249,7 +249,7 @@ in "d /srv/staging 0755 root root -" "d /srv/vm 0755 root root -" "d /srv/logs 0770 clo chloe -" - "A+ /srv/prod/yt-feed/data - - - - g:chloe:rwX,m::rwX,d:g:chloe:rwx,d:m::rwx" + "A+ /srv/prod/ytdl/data - - - - g:chloe:rwX,m::rwX,d:g:chloe:rwx,d:m::rwx" "d /var/lib/studio 0700 root root -" "d /var/lib/studio/dashboard 0700 studio-dashboard studio-dashboard -" "d /var/lib/studio/routes 0700 root root -" diff --git a/service/keycloak/service.pkl b/service/keycloak/service.pkl index 0f2e6ae5f517bf3c02e882a3ad87bcf0584ae666..f181428932eff8fcc22264009835214dd7f48ac8 100644 --- a/service/keycloak/service.pkl +++ b/service/keycloak/service.pkl @@ -17,7 +17,7 @@ local database = new postgres.Database { name = "keycloak_next" } meta { name = "Keycloak"; launcher = false } traceServiceName = module.id -dependsOn { "victoria-traces" } +dependsOn { "victoria" } healthyDeadline = "20m" setup = "configure.py" provide = "provide.py" diff --git a/service/pds/service.pkl b/service/pds/service.pkl index 1e75be090092594faf698e36e5010a0441862efc..9ae290032657c4eb0aaf3e2bc628deb7638a7aa9 100644 --- a/service/pds/service.pkl +++ b/service/pds/service.pkl @@ -9,7 +9,7 @@ traceServiceName = module.id metricsPushed = true rollout = "simple" stageIsolation = "fresh" -dependsOn { "victoria-metrics"; "victoria-traces" } +dependsOn { "victoria" } secrets = if (isolated) new { ["jwt_secret"] {} diff --git a/service/shale/theme.css b/service/shale/theme.css index b20577426dd78ce715c90685b67da0abd3459c9c..84c63161ba0935cf23f64cc093c68aa4f526d418 100644 --- a/service/shale/theme.css +++ b/service/shale/theme.css @@ -57,7 +57,7 @@ --theme-color-primary-darker: #74113a; --theme-body-background-color: #fffafd; --theme-link-color: var(--theme-color-primary); - --theme-link-visited-color: #88436c; + --theme-link-visited-color: var(--theme-link-color); --theme-link-hover-color: var(--theme-color-primary-dark); --theme-link-active-color: var(--theme-color-primary-darker); --theme-focus-color: var(--theme-color-primary-vivid); @@ -166,8 +166,8 @@ border-color: #d9bdca; } - :is(td, span).idleage:is(.seconds, .minutes, .hours) { - color: #216e1f; + :is(td, span).idleage:is(.seconds, .minutes, .hours, .days) { + color: var(--shale-color-cool-dark); } :is(td, span).idleage.weeks, @@ -242,7 +242,7 @@ --theme-text-color: #f7edf2; --theme-text-reverse-color: #21161c; --theme-link-color: #ff8fba; - --theme-link-visited-color: #d9a3c1; + --theme-link-visited-color: var(--theme-link-color); --theme-link-hover-color: #ffc0d8; --theme-link-active-color: #ffd7e6; --theme-focus-color: var(--theme-color-primary-vivid); @@ -356,7 +356,7 @@ } :is(td, span).idleage:is(.seconds, .minutes, .hours, .days) { - color: #90db9b; + color: var(--shale-color-cool-dark); } :is(td, span).idleage.weeks, diff --git a/service/victoria/icon-dark.svg b/service/victoria/icon-dark.svg new file mode 100644 index 0000000000000000000000000000000000000000..7e44bb4085bf6c10b3f39672c8996a1f2feda94c --- /dev/null +++ b/service/victoria/icon-dark.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/service/victoria/icon-light.svg b/service/victoria/icon-light.svg new file mode 100644 index 0000000000000000000000000000000000000000..42cca81748ebf5ac4c8fe1f0fc682096d4b7d2e4 --- /dev/null +++ b/service/victoria/icon-light.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/service/victoria/service.pkl b/service/victoria/service.pkl new file mode 100644 index 0000000000000000000000000000000000000000..019449b6d64973361f470d924be2a2de80fd3833 --- /dev/null +++ b/service/victoria/service.pkl @@ -0,0 +1,56 @@ +amends "../../config/Service.pkl" + +meta { name = "Victoria"; tagline = "Metrics, logs, and traces" } + +containers { + ["logs"] { + image = "docker.io/victoriametrics/victoria-logs:v1.52.0" + cpu = 300 + memory = 512 + args { "-storageDataPath=/logs"; "-retentionPeriod=30d" } + + http { + containerPort = 9428 + subdomain = "logs" + authRole = "infra-admin" + checkPath = "/health" + metricsPath = "/metrics" + } + + volumes { ["/logs"] {} } + } + ["metrics"] { + image = "docker.io/victoriametrics/victoria-metrics:v1.152.0" + cpu = 300 + memory = 512 + args { "-storageDataPath=/metrics"; "-retentionPeriod=30d"; "-usePromCompatibleNaming" } + + http { + containerPort = 8428 + hostPort = 18428 + subdomain = "metrics" + authRole = "infra-admin" + checkPath = "/health" + metricsPath = "/metrics" + } + + volumes { ["/metrics"] {} } + } + ["traces"] { + image = "docker.io/victoriametrics/victoria-traces:v0.11.1" + cpu = 300 + memory = 512 + args { "-storageDataPath=/traces"; "-retentionPeriod=30d" } + + http { + containerPort = 10428 + hostPort = 10428 + subdomain = "traces" + authRole = "infra-admin" + checkPath = "/health" + metricsPath = "/metrics" + } + + volumes { ["/traces"] {} } + } +} diff --git a/service/victoria/victoria-logs.pkl b/service/victoria/victoria-logs.pkl deleted file mode 100644 index f09dfc6686c748b9bb65077164c34aa0fc6ffac6..0000000000000000000000000000000000000000 --- a/service/victoria/victoria-logs.pkl +++ /dev/null @@ -1,21 +0,0 @@ -amends "../../config/Service.pkl" - -meta { name = "VictoriaLogs"; tagline = "Searchable service and system logs" } -rollout = "simple" - -container { - image = "docker.io/victoriametrics/victoria-logs:v1.52.0" - cpu = 300 - memory = 512 - args { "-storageDataPath=/data"; "-retentionPeriod=30d" } - - http { - containerPort = 9428 - subdomain = "logs" - authRole = "infra-admin" - checkPath = "/health" - metricsPath = "/metrics" - } - - volumes { ["/data"] {} } -} diff --git a/service/victoria/victoria-metrics.pkl b/service/victoria/victoria-metrics.pkl deleted file mode 100644 index 91b5a8139198a20b2a1d60335b4bf5bf218a6fcc..0000000000000000000000000000000000000000 --- a/service/victoria/victoria-metrics.pkl +++ /dev/null @@ -1,22 +0,0 @@ -amends "../../config/Service.pkl" - -meta { name = "VictoriaMetrics"; tagline = "Metrics history" } -rollout = "simple" - -container { - image = "docker.io/victoriametrics/victoria-metrics:v1.152.0" - cpu = 300 - memory = 512 - args { "-storageDataPath=/data"; "-retentionPeriod=30d"; "-usePromCompatibleNaming" } - - http { - containerPort = 8428 - hostPort = 18428 - subdomain = "metrics" - authRole = "infra-admin" - checkPath = "/health" - metricsPath = "/metrics" - } - - volumes { ["/data"] {} } -} diff --git a/service/victoria/victoria-metrics/icon-dark.svg b/service/victoria/victoria-metrics/icon-dark.svg deleted file mode 100644 index 7e44bb4085bf6c10b3f39672c8996a1f2feda94c..0000000000000000000000000000000000000000 --- a/service/victoria/victoria-metrics/icon-dark.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/service/victoria/victoria-metrics/icon-light.svg b/service/victoria/victoria-metrics/icon-light.svg deleted file mode 100644 index 42cca81748ebf5ac4c8fe1f0fc682096d4b7d2e4..0000000000000000000000000000000000000000 --- a/service/victoria/victoria-metrics/icon-light.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/service/victoria/victoria-traces.pkl b/service/victoria/victoria-traces.pkl deleted file mode 100644 index f6c15c85d1e1e3e0537ba64cfb3d7060610ae851..0000000000000000000000000000000000000000 --- a/service/victoria/victoria-traces.pkl +++ /dev/null @@ -1,22 +0,0 @@ -amends "../../config/Service.pkl" - -meta { name = "VictoriaTraces"; tagline = "Request traces" } -rollout = "simple" - -container { - image = "docker.io/victoriametrics/victoria-traces:v0.11.1" - cpu = 300 - memory = 512 - args { "-storageDataPath=/data"; "-retentionPeriod=30d" } - - http { - containerPort = 10428 - hostPort = 10428 - subdomain = "traces" - authRole = "infra-admin" - checkPath = "/health" - metricsPath = "/metrics" - } - - volumes { ["/data"] {} } -} diff --git a/service/victoria/victoria-traces/icon.svg b/service/victoria/victoria-traces/icon.svg deleted file mode 100644 index 2d8bd82f456790a26b08434d25dcc1466c073c56..0000000000000000000000000000000000000000 --- a/service/victoria/victoria-traces/icon.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/service/youtube/yt-feed.pkl b/service/youtube/yt-feed.pkl deleted file mode 100644 index 08460b59c39213009908c1a73fc54abe31590ad6..0000000000000000000000000000000000000000 --- a/service/youtube/yt-feed.pkl +++ /dev/null @@ -1,34 +0,0 @@ -amends "../../config/Service.pkl" - -import "../../config/site.pkl" as site - -local passive = site.preview || site.mediaReadOnly - -meta { name = "YouTube Thumbnails" } - -requiredSecrets { - "smtp_host" - "smtp_user" - "smtp_pass" -} - -container { - build = "upscaler" - cpu = if (passive) 200 else 1000 - memory = if (passive) 512 else 2048 - entrypoint = if (passive) "/bin/sh" else null - args = if (passive) new { - "-c" - "python3 /app/upscale.py --check && exec python3 -m http.server 8899 --bind 0.0.0.0 --directory /tmp" - } else new {} - http = if (passive) new { containerPort = 8899 } else null - volumes { - ["/data"] {} - ["/media"] { src = site.mediaRoot; readOnly = site.mediaReadOnly } - } - env { - ["STATE_DIR"] = "/data" - ["INDEP_DIR"] = "/media/\(site.jellyfinFolders["Independent"])" - ["SR_THREADS"] = "4" - } -} diff --git a/service/youtube/yt-feed/icon.svg b/service/youtube/yt-feed/icon.svg deleted file mode 100644 index 42cc80773151addabf812255573a1324952e55c5..0000000000000000000000000000000000000000 --- a/service/youtube/yt-feed/icon.svg +++ /dev/null @@ -1 +0,0 @@ -YouTube diff --git a/service/youtube/yt-feed/upscaler/Dockerfile b/service/youtube/yt-feed/upscaler/Dockerfile deleted file mode 100644 index 7a42cd72914a1302c05c9fdc85ef3cc54437fe3c..0000000000000000000000000000000000000000 --- a/service/youtube/yt-feed/upscaler/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM docker.io/library/python:3.12-slim@sha256:44ff437bba879d4941b710a369a8f19266aea34b29002807f0c487fabc9eec9b -RUN pip install --no-cache-dir onnxruntime==1.30.0 numpy==2.5.3 Pillow==12.3.0 -RUN mkdir /app && python3 -c 'import hashlib,urllib.request; u="https://huggingface.co/notaneimu/onnx-image-models/resolve/main/realesr-general-x4v3.onnx"; d=urllib.request.urlopen(u).read(); assert hashlib.sha256(d).hexdigest()=="e8db65652ed421c2f8c92645d8f6fc6b07fd2868a916fdaa1a99c8d28091f097"; open("/app/realesr-general-x4v3.onnx","wb").write(d)' -COPY upscale.py /app/upscale.py -CMD ["python3", "-u", "/app/upscale.py"] diff --git a/service/youtube/yt-feed/upscaler/upscale.py b/service/youtube/yt-feed/upscaler/upscale.py deleted file mode 100644 index 7fec0250b734659153a3f49906deec76fa83e9b5..0000000000000000000000000000000000000000 --- a/service/youtube/yt-feed/upscaler/upscale.py +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/env python3 -# yt-upscaler: keeps the Independent library's thumbnails sharp on a TV. -# youtube only stores ~720p thumbnails, which jellyfin then stretches across -# the whole screen as the item backdrop (soft + blocky). this re-fetches the -# highest-res thumbnail from the image CDN (not bot-walled) and runs it through -# Real-ESRGAN general-x4v3 (exported to ONNX, so no pickle is ever loaded) at -# 4x on CPU, writing a crisp