diff --git a/.gitignore b/.gitignore index 2ba30b8b70a47d9d0173a99cc0d4210580ab3bb7..593948c9ffa55a3037e9e723cb3a51368282110a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /.apps /certs /sitegen +config/yt-upscaler/realesr-general-x4v3.onnx diff --git a/compose.yaml b/compose.yaml index a096a9bb042129a024e2163e7f52711adc7d6ecb..b3cff9a72eb5b034de83a8974586e92de11ceb8e 100755 --- a/compose.yaml +++ b/compose.yaml @@ -117,6 +117,7 @@ services: - "${STORE_ROOT}:/w" - "${APP_ROOT}/copyparty:/cfg" - ./config/copyparty.conf:/cfg/copyparty.conf:ro + - ./config/copyparty-hooks:/hooks:ro healthcheck: test: ["CMD-SHELL", "wget -qO /dev/null http://127.0.0.1:80/?h"] interval: 30s @@ -152,7 +153,7 @@ services: cpus: "16.0" restart: unless-stopped volumes: - - "$CLOVER_ROOT/Documents/Config/paperclover:/data:rw" + - "$CLOVER_ROOT/Documents/Config/paper clover:/data:rw" # rw: the indexer scrubs exif location data in place - "$CLOVER_ROOT/Published:/published:rw" healthcheck: @@ -246,7 +247,7 @@ services: context: https://tangled.org/tangled.org/3lqs6zdi4nt22.git dockerfile: Dockerfile args: - TAG: "${KNOT_IMAGE_TAG:-v1.14.0-alpha}" + TAG: "${KNOT_IMAGE_TAG:-v1.15.0-alpha}" UID: "${USER_ID:?}" GID: "${GROUP_ID:?}" pull_policy: build @@ -285,7 +286,7 @@ services: context: ./config/spindle dockerfile: Dockerfile args: - TAG: "${SPINDLE_IMAGE_TAG:-v1.14.0-alpha}" + TAG: "${SPINDLE_IMAGE_TAG:-v1.15.0-alpha}" pull_policy: build depends_on: docker-in-docker: @@ -401,7 +402,7 @@ services: - ENABLE_SOCKS=no - SOCKS_USER=admin - SOCKS_PASS=socks - - LAN_NETWORK=192.168.86.0/24 + - LAN_NETWORK=10.0.0.0/24 - NAME_SERVERS=84.200.69.80,37.235.1.174,1.1.1.1,37.235.1.177,84.200.70.40,1.0.0.1 - VPN_INPUT_PORTS=1234 - VPN_OUTPUT_PORTS=5678 @@ -515,6 +516,9 @@ services: HOME: /config volumes: - ./config/yt:/config-yt:ro + # channel list + ytdl-sub config live outside the repo so they can be + # edited from the yt-feed web ui (Clover ▸ Documents/Config/Youtube Downloader) + - "${CLOVER_ROOT}/Documents/Config/Youtube Downloader:/yt-config:ro" # the image hardcodes its lock file at /config, so state lives there - "${APP_ROOT}/ytdl-sub:/config" - "${MEDIA_ROOT}:/media" @@ -540,8 +544,13 @@ services: MAIL_FROM: "yt-feed@${HOME_DOMAIN:?}" MAIL_TO: "${ADMIN_EMAIL:?}" BASE_URL: "https://yt.${HOME_DOMAIN:?}" + # channel lists are edited here and read live; the editor writes to this dir + FEED_CONFIG: "/yt-config/feed.yaml" + YT_CONFIG_DIR: "/yt-config" volumes: - ./config/yt:/config-yt:ro + # editable channel lists (subscriptions.yaml, feed.yaml) — rw for the web editor + - "${CLOVER_ROOT}/Documents/Config/Youtube Downloader:/yt-config:rw" - "${APP_ROOT}/yt-feed:/data" - "${MEDIA_ROOT}:/media" restart: unless-stopped @@ -550,6 +559,32 @@ services: net.paperclover.list.domain: yt net.paperclover.list.priority: 53 net.paperclover.list.access: media-manage + # thumbnail super-resolution (issue #6): upscales Independent thumbnails 4x + # with Real-ESRGAN (ONNX, cpu) so they stay sharp as jellyfin tv backdrops. + # separate from yt-feed because onnxruntime has no wheels for the ytdl-sub + # image's python 3.14; writes upscale-status.json into yt-feed's data dir. + yt-upscaler: + container_name: yt-upscaler + build: + context: config/yt-upscaler + dockerfile: Dockerfile + pull_policy: build + user: "$USER_ID:$GROUP_ID" + environment: + INDEP_DIR: "/media/jellyfin/Independent" + STATE_DIR: "/state" + SR_THREADS: "6" + volumes: + - "${MEDIA_ROOT}:/media" + - "${APP_ROOT}/yt-feed:/state" + deploy: + resources: + limits: + cpus: "8.0" + restart: unless-stopped + labels: + net.paperclover.list.name: YouTube Thumbnail Upscaler + net.paperclover.list.web: "false" # language models opencode: # port 4096 container_name: opencode @@ -684,19 +719,24 @@ services: dawarich-app: # port 30161 image: freikin/dawarich:latest container_name: dawarich-app + networks: + default: + aliases: + - dawarich volumes: + - "${APP_ROOT}/dawarich_tmp:/var/app/tmp" - "${APP_ROOT}/dawarich_public:/var/app/public" - "${APP_ROOT}/dawarich_watched:/var/app/tmp/imports/watched" - "${APP_ROOT}/dawarich_storage:/var/app/storage" - "${APP_ROOT}/dawarich_data:/dawarich_db_data" - entrypoint: web-entrypoint.sh + - "${APP_ROOT}/keycloak:/shared-keys:ro" + # wrap the stock web-entrypoint to inject the OIDC client secret that + # keycloak's init.py wrote to /shared-keys/dawarich (same pattern as forgejo) + entrypoint: ["/bin/sh", "-c"] command: - - bin/rails - - server - - -p - - "30161" - - -b - - "::" + - | + export OIDC_CLIENT_SECRET="$$(cat /shared-keys/dawarich)" + exec web-entrypoint.sh bin/rails server -p 30161 -b :: restart: on-failure user: "$USER_ID:$GROUP_ID" environment: @@ -709,6 +749,9 @@ services: DATABASE_NAME: dawarich MIN_MINUTES_SPENT_IN_CITY: 60 APPLICATION_HOSTS: "localhost,zenith,dawarich.${HOME_DOMAIN}" + OIDC_CLIENT_ID: dawarich + OIDC_ISSUER: "https://auth.${HOME_DOMAIN}/realms/master" + OIDC_REDIRECT_URI: "https://dawarich.${HOME_DOMAIN}/users/auth/openid_connect/callback" TIME_ZONE: America/Los_Angeles APPLICATION_PROTOCOL: http PROMETHEUS_EXPORTER_ENABLED: "false" @@ -898,19 +941,26 @@ services: labels: net.paperclover.list.name: DDNS net.paperclover.list.access: personal - shale: # port probably is 80 - image: astheno/shale + shale: # port 8000 + image: astheno/shale@sha256:1f2144eb7a422871414fdc010f05cf99206f621cd68b71435e50da71ed6dbcde container_name: shale user: "0:0" volumes: - "${APP_ROOT}/shale/repositories_mirrors:/repositories_mirrors" - "${APP_ROOT}/shale/repositories_owned:/repositories_owned" - "${APP_ROOT}/shale/data:/data" + restart: unless-stopped environment: DOMAIN: "shale.${HOME_DOMAIN:?}" - OAUTH2_CLIENT: "snow sign on,https://auth.${HOME_DOMAIN:?}/realms/master|shale|${SHALE_CLIENT_SECRET:?}" + OAUTH2_CLIENT: "oidc,auth.${HOME_DOMAIN:?}/realms/master|shale|${SHALE_CLIENT_SECRET:?}" + SESSION_SECRET: "${SHALE_SESSION_SECRET:?}" SERVER_TITLE: "clover's git" - MIRROR_1: "sitegen,https://git.paperclover.net/clo/sitegen.git,website generator, standard library, and home of paperclover.net" + MIRROR_1: "home-infra,https://git.paperclover.net/clo/home-infra.git,compose files and tasks for my home server" + MIRROR_2: "discord-name-painter,https://git.paperclover.net/clo/discord-name-painter.git,maintainance only. allows any user to set their display name color to any color, by using dynamically created roles." + MIRROR_3: "react-mutation,https://git.paperclover.net/clo/react-mutation.git,create async mutations with trivial optimistic updates and great error handling" + MIRROR_4: "markodown,https://git.paperclover.net/clo/markodown.git,alternate universe where markdown lets you write marko components inline" + MIRROR_5: "toolkit,https://git.paperclover.net/clo/toolkit.git,Clover's Creative Toolkit is collection of macOS software that I use to create." + MIRROR_6: "react-markdown,https://git.paperclover.net/clo/react-markdown.git,memoized markdown renderer for react using the unified plugin ecosystem" # evil inc temporary infrastructure evil-forgejo: # port 3000 container_name: evil-forgejo diff --git a/config/Caddyfile b/config/Caddyfile index 1470159e01f840068dbccfcae6923cedbda73bea..c6082b2e9782cc0e39b48ee9d6d05482387a4390 100644 --- a/config/Caddyfile +++ b/config/Caddyfile @@ -41,6 +41,30 @@ } # services are sorted alphabetically +ai.{$HOME_DOMAIN} { + # inference api (qwen3.8-27b on the 3090). vllm serves both the openai + # routes and anthropic's /v1/messages, so this host answers both protocols. + # + # auth is vllm's own bearer token, NOT reverse_proxy_auth: that snippet does + # an oauth2 browser redirect, which every api client would choke on. + # + # vllm only accepts `Authorization: Bearer `. anthropic-protocol + # clients (claude code) send `x-api-key: ` instead and get a 401. + # translate it so both conventions work against the same key. + @anthropic_auth header X-Api-Key * + handle @anthropic_auth { + reverse_proxy "http://vllm:8000" { + header_up Authorization "Bearer {http.request.header.X-Api-Key}" + # stream tokens as they are generated rather than buffering + flush_interval -1 + } + } + handle { + reverse_proxy "http://vllm:8000" { + flush_interval -1 + } + } +} auth.{$HOME_DOMAIN} { handle / { redir / /apps @@ -96,6 +120,8 @@ file.{$HOME_DOMAIN} { header_up X-Forwarded-Uri {uri} } } + @share_root path /shr /shr/ + respond @share_root 403 @should_auth not path /shr/* reverse_proxy @should_auth "http://forward-auth" { method GET @@ -143,7 +169,7 @@ knot.{$HOME_DOMAIN} { reverse_proxy "http://knot:5555" } shale.{$HOME_DOMAIN} { - reverse_proxy "http://shale" + reverse_proxy "http://shale:8000" } spindle.{$HOME_DOMAIN} { reverse_proxy "http://spindle:6555" @@ -195,6 +221,17 @@ music.{$HOME_DOMAIN} { reverse_proxy "http://navidrome" } } +dav.{$HOME_DOMAIN} { + # plain webdav endpoint for clients that can't do the sso redirect dance + # (onenote 2007, rclone, finder) -- copyparty does its own auth here instead. + # strip the idp headers so a client on this vhost can't claim to be an sso user. + reverse_proxy "http://copyparty" { + header_up -User-Id + header_up -User-Groups + header_up -User-Email + header_up -User-Name + } +} opencode.{$HOME_DOMAIN} { import reverse_proxy_auth "http://opencode:4096" admin } diff --git a/config/copyparty-hooks/reject-apple-cruft.py b/config/copyparty-hooks/reject-apple-cruft.py new file mode 100755 index 0000000000000000000000000000000000000000..171289be45ef9fb1d7c4a33c9bd91f8f0d1593d0 --- /dev/null +++ b/config/copyparty-hooks/reject-apple-cruft.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import os +import sys + +_ = r""" +reject the metadata files macos scatters over network shares + +the samba setup vetoes these server-side; webdav has no equivalent knob, and +DSDontWriteNetworkStores only suppresses .DS_Store -- the ._ AppleDouble files +(resource forks / xattrs) keep coming regardless. finder also retries a failed +write forever, so left alone these generate thousands of PUTs per file. + +enabled globally in copyparty.conf as: + xbu: c,/hooks/reject-apple-cruft.py + + xbu = execute before upload + c = check result, reject upload if error +""" + +BAD_EXACT = { + ".DS_Store", + ".localized", + ".Spotlight-V100", + ".TemporaryItems", + ".Trashes", + ".fseventsd", + ".apdisk", + ".metadata_never_index", + ".metadata_never_index_unless_rootfs", + ".metadata_direct_scope_only", + ".hidden", +} + + +def main(): + name = os.path.basename(sys.argv[1]) + bad = name.startswith("._") or name in BAD_EXACT + sys.exit(1 if bad else 0) + + +if __name__ == "__main__": + main() diff --git a/config/copyparty.conf b/config/copyparty.conf index 025f5e6bcac0d5c8b350b297cf2c26af6d1dfd2b..2aa0292fd2df65c48981a582f48abb6ab5ec3b29 100644 --- a/config/copyparty.conf +++ b/config/copyparty.conf @@ -9,6 +9,9 @@ theme: 2 # monokai name: clover's nas stats, nos-dup # enable the prometheus endpoint, but disable the dupes counter (too slow) + dav-auth # webdav clients must always authenticate (windows gets confused otherwise) + ah-alg: argon2 # passwords in accounts.conf are argon2 hashes, never plaintext + xbu: c,/hooks/reject-apple-cruft.py # veto macos ._ / .DS_Store cruft (samba-style) # keycloak xff-src: lan # accept X-Forwarded-For from `lan` diff --git a/config/keycloak/init.py b/config/keycloak/init.py index 758f5eb1fcd5a973f4b2838273e55da024b1d068..e64415a431cb406be725d7339e84d080d87bdb1a 100755 --- a/config/keycloak/init.py +++ b/config/keycloak/init.py @@ -194,6 +194,39 @@ def configure(): client_data = kc_admin.get_client(client_id) _ = Path("/shared/jellyfin").write_text(client_data["secret"]) + # dawarich + client_id = kc_admin.create_client( + { + "protocol": "openid-connect", + "clientId": "dawarich", + "name": "Dawarich", + "description": "", + "publicClient": False, + "authorizationServicesEnabled": False, + "serviceAccountsEnabled": False, + "implicitFlowEnabled": False, + "directAccessGrantsEnabled": False, + "standardFlowEnabled": True, + "frontchannelLogout": True, + "attributes": { + "saml_idp_initiated_sso_url_name": "", + "standard.token.exchange.enabled": False, + "oauth2.device.authorization.grant.enabled": False, + "oidc.ciba.grant.enabled": False, + "pkce.code.challenge.method": "", + "dpop.bound.access.tokens": "false", + "post.logout.redirect.uris": "*", + }, + "alwaysDisplayInConsole": False, + "rootUrl": "", + "baseUrl": "", + "redirectUris": ["*"], + }, + skip_exists=True, + ) + client_data = kc_admin.get_client(client_id) + _ = Path("/shared/dawarich").write_text(client_data["secret"]) + # forward auth client_id = kc_admin.create_client( { diff --git a/config/spindle/Dockerfile b/config/spindle/Dockerfile index dcf9b56aa70e73e5e8df84947dc4c45309fbaf33..89c319dd124918ebf431bb36a4abe374919472b1 100644 --- a/config/spindle/Dockerfile +++ b/config/spindle/Dockerfile @@ -1,7 +1,7 @@ FROM golang:1.25-alpine AS builder ENV CGO_ENABLED=1 -ARG TAG="v1.14.0-alpha" +ARG TAG="v1.15.0-alpha" WORKDIR /app RUN apk add --no-cache git gcc musl-dev diff --git a/config/yt-feed/app.py b/config/yt-feed/app.py index d366eddbbfbd4a5587c7e82435239d4eaede6ff6..efcb7f9d850e4d4d97b07cdd30c5489d04721f04 100644 --- a/config/yt-feed/app.py +++ b/config/yt-feed/app.py @@ -16,12 +16,15 @@ import urllib.request import xml.etree.ElementTree as ET from email.message import EmailMessage from email.utils import formatdate -from xml.sax.saxutils import escape +from xml.sax.saxutils import escape, unescape import yaml from flask import Flask, Response, redirect, request -FEED_CONFIG = os.environ.get("FEED_CONFIG", "/config-yt/feed.yaml") +FEED_CONFIG = os.environ.get("FEED_CONFIG", "/yt-config/feed.yaml") +# the channel lists live outside the repo (Clover ▸ Documents/Config/Youtube +# Downloader) so they can be edited from the web ui; this dir is mounted rw +YT_CONFIG_DIR = os.environ.get("YT_CONFIG_DIR", "/yt-config") STATE_DIR = os.environ.get("STATE_DIR", "/data") INTERVAL = int(os.environ.get("CHECK_INTERVAL", "1800")) SMTP_HOST = os.environ["SMTP_HOST"] @@ -125,6 +128,59 @@ def safe_name(name): return re.sub(r'[/\\:*?"<>|]', "-", name).strip() or "untitled" +def looks_like_url(s): + return bool(re.match(r"https?://", (s or "").strip(), re.I)) + + +def stable_key(v): + # the card's identity stays put across a stub resolving (id changes, + # stub_id doesn't) so the page can refresh a card in place + return v.get("stub_id") or v["id"] + + +def safe_listdir(p): + try: + return os.listdir(p) + except OSError: + return [] + + +def nfo_fields(path, *tags): + try: + with open(path) as f: + txt = f.read() + except OSError: + return {t: "" for t in tags} + out = {} + for t in tags: + m = re.search(rf"<{t}>(.*?)", txt, re.S) + out[t] = unescape(m.group(1)).strip() if m else "" + return out + + +def under(base, path): + base = os.path.realpath(base) + path = os.path.realpath(path) + return path == base or path.startswith(base + os.sep) + + +def set_nfo_tags(path, updates): + # rewrite individual values in an existing nfo, leaving the rest as-is + try: + with open(path) as f: + txt = f.read() + except OSError: + return + for k, v in updates.items(): + new = f"<{k}>{escape(str(v))}" + if re.search(rf"<{k}>.*?", txt, re.S): + txt = re.sub(rf"<{k}>.*?", new, txt, count=1, flags=re.S) + else: + txt = re.sub(r"(\n\s*)\Z", f"\n {new}\\1", txt) + with open(path, "w") as f: + f.write(txt) + + # ---------------------------------------------------------------- feed poller def channel_id_for(url, cache): @@ -422,7 +478,7 @@ def indie_shows(): PAGE = """ -yt triage +yt downloader -

yt triage · {npending} pending

-{cards} +

yt downloader · {npending} pending + + library ▸ · channels ▸

+{banner} +
{cards}
@@ -474,11 +533,35 @@ function seasonOptions(f, show) {{ sel.onchange = () => f.querySelector("input[name=episode]").value = (seasons[sel.value] || 0) + 1; sel.onchange(); }} -document.querySelectorAll("select[name=dest]").forEach(destChanged); -setInterval(async () => {{ - const r = await fetch("/jobs.html"); - document.getElementById("jobs").innerHTML = await r.text(); -}}, 4000); +function initCard(card) {{ const s = card.querySelector("select[name=dest]"); if (s) destChanged(s); }} +document.querySelectorAll("#cards .card").forEach(initCard); +// any real edit "dirties" a card so the background refresh won't clobber it +const cards = document.getElementById("cards"); +for (const ev of ["input", "change"]) cards.addEventListener(ev, e => {{ + const f = e.target.closest(".card"); if (f) f.dataset.dirty = "1"; +}}, true); +let busy = false; +async function refresh() {{ + try {{ + document.getElementById("jobs").innerHTML = await (await fetch("/jobs.html")).text(); + }} catch (e) {{}} + if (busy) return; busy = true; + try {{ + const tmp = document.createElement("div"); + tmp.innerHTML = await (await fetch("/cards.html")).text(); + const fresh = {{}}; + tmp.querySelectorAll(".card[data-key]").forEach(c => fresh[c.dataset.key] = c); + cards.querySelectorAll(".card[data-key]").forEach(c => {{ + if (!fresh[c.dataset.key] && c.dataset.dirty !== "1") c.remove(); + }}); + for (const key in fresh) {{ + const have = cards.querySelector('.card[data-key="' + key + '"]'); + if (!have) {{ cards.appendChild(fresh[key]); initCard(fresh[key]); }} + else if (have.dataset.dirty !== "1") {{ have.replaceWith(fresh[key]); initCard(fresh[key]); }} + }} + }} catch (e) {{}} finally {{ busy = false; }} +}} +setInterval(refresh, 4000); """ @@ -489,18 +572,19 @@ def card_html(v, shows): for s in shows: opts.append(f'') opts.append('') + key = stable_key(v) img = (f'' if v.get("thumb") else "") meta = ("resolving…" if v.get("unresolved") else f"{escape(v.get('channel', '?'))} · {escape(v.get('published', ''))}") - return f""" + return f""" {img}

{escape(v['title'])}

{meta}

- + - +
@@ -530,20 +614,31 @@ def jobs_html(): return "\n".join(out) +def render_cards(pending, shows): + if not pending: + return "

nothing pending. enjoy the silence.

" + return "\n".join(card_html(v, shows) for v in reversed(list(pending.values()))) + + @app.get("/") def index(): with state_lock: pending = load_json("pending.json", {}) shows = indie_shows() - cards = "\n".join(card_html(v, shows) for v in reversed(list(pending.values()))) - if not pending: - cards = "

nothing pending. enjoy the silence.

" - if wall_active(): - cards = ("
youtube has bot-walled this ip — downloads " - "are paused and will resume automatically once the wall lifts " - "(probed every few hours). queueing still works.
" + cards) - return PAGE.format(npending=len(pending), cards=cards, - jobs=jobs_html(), shows_json=json.dumps(indie_shows())) + banner = ("
youtube has bot-walled this ip — downloads " + "are paused and will resume automatically once the wall lifts " + "(probed every few hours). queueing still works.
" + if wall_active() else "") + return PAGE.format(npending=len(pending), banner=banner, + cards=render_cards(pending, shows), + jobs=jobs_html(), shows_json=json.dumps(shows)) + + +@app.get("/cards.html") +def cards_partial(): + with state_lock: + pending = load_json("pending.json", {}) + return Response(render_cards(pending, indie_shows()), mimetype="text/html") @app.get("/jobs.html") @@ -567,9 +662,14 @@ def retry(): @app.post("/skip") def skip(): + vid = request.form["vid"] with state_lock: pending = load_json("pending.json", {}) - pending.pop(request.form["vid"], None) + # a manually-added stub gets re-keyed to the real video id once it + # resolves, but the card still submits the stub_id — match on either + if vid not in pending: + vid = next((k for k, p in pending.items() if p.get("stub_id") == vid), vid) + pending.pop(vid, None) save_json("pending.json", pending) return redirect("/") @@ -613,10 +713,15 @@ def ingest(): if not show: return Response("missing show name", 400, mimetype="text/plain") # a custom episode title; left equal to the card title means "use the - # video title" (which, for a still-resolving stub, arrives later) + # video title" (which, for a still-resolving stub, arrives later). guard + # against a url leaking in: the field is prefilled with the stub title, + # which for an unresolved paste IS the url — if the stub then resolves + # before submit, that url would otherwise be taken as a real title. ep_title = request.form.get("ep_title", "").strip() - job.update(dest="indie", show=show, - ep_title=ep_title if ep_title and ep_title != v["title"] else "", + if not ep_title or ep_title == v["title"] or ep_title == v.get("link") \ + or looks_like_url(ep_title): + ep_title = "" + job.update(dest="indie", show=show, ep_title=ep_title, season=int(request.form["season"]), episode=int(request.form["episode"])) job["dest_label"] = f"{show} S{job['season']:02d}E{job['episode']:02d}" elif dest == "independent": @@ -633,6 +738,549 @@ def ingest(): return redirect("/") +# ---------------------------------------------------------------- library edit + +def library_entries(): + # every ingested video across the two managed libraries, with its current + # title (from the nfo, falling back to the filename) for search + rename + out = [] + for show in sorted(safe_listdir(INDIE_DIR)): + show_path = os.path.join(INDIE_DIR, show) + if not os.path.isdir(show_path): + continue + for sub in sorted(safe_listdir(show_path)): + sm = re.fullmatch(r"Season (\d+)", sub) + if not sm: + continue + sdir = os.path.join(show_path, sub) + for f in sorted(safe_listdir(sdir)): + if not f.lower().endswith(VIDEO_EXTS): + continue + base, _ = os.path.splitext(f) + em = re.match(r"S(\d+)E(\d+) - (.*)", base) + season, episode = int(sm.group(1)), int(em.group(2)) if em else 1 + nf = nfo_fields(os.path.join(sdir, base + ".nfo"), "title", "plot") + title = nf["title"] or (em.group(3) if em else base) + out.append({ + "type": "indie", "rel": os.path.relpath(os.path.join(sdir, f), "/media"), + "ctx": f"{show} · S{season}E{episode}", "title": title, + "link": nf["plot"] if looks_like_url(nf["plot"]) else "", + "season": season, "episode": episode}) + for ch in sorted(safe_listdir(INDEP_DIR)): + cdir = os.path.join(INDEP_DIR, ch) + if not os.path.isdir(cdir): + continue + for f in sorted(safe_listdir(cdir)): + if not f.lower().endswith(VIDEO_EXTS): + continue + base, _ = os.path.splitext(f) + dm = re.match(r"(\d{4}-\d{2}-\d{2}) - (.*)", base) + nf = nfo_fields(os.path.join(cdir, base + ".nfo"), "title", "plot") + title = nf["title"] or (dm.group(2) if dm else base) + out.append({ + "type": "independent", "rel": os.path.relpath(os.path.join(cdir, f), "/media"), + "ctx": f"{ch} · {dm.group(1) if dm else ''}", "title": title, + "link": nf["plot"] if looks_like_url(nf["plot"]) else ""}) + return out + + +LIB_PAGE = """ + + +yt downloader · library + +

yt library · {n} videos + ◂ home

+ +{rows} + +""" + + +def lib_row(e): + rel = escape(e["rel"], {'"': """}) + search = escape((e["ctx"] + " " + e["title"]).lower(), {'"': """}) + if e["type"] == "indie": + fields = (f'' + f'') + else: + fields = "" + link = e.get("link") + open_link = (f' · ↗ open' if link else "") + return f""" +
{escape(e['ctx'])}{open_link}
+ +
+ + {fields} + +
+""" + + +@app.get("/library") +def library(): + entries = library_entries() + rows = "\n".join(lib_row(e) for e in entries) or "

library is empty.

" + return LIB_PAGE.format(n=len(entries), rows=rows) + + +@app.post("/rename") +def rename(): + rel = request.form.get("rel", "") + new_title = request.form.get("title", "").strip() + full = os.path.realpath(os.path.join("/media", rel)) + indie = under(INDIE_DIR, full) + if not (indie or under(INDEP_DIR, full)): + return Response("path not allowed", 403, mimetype="text/plain") + if not os.path.isfile(full): + return Response("file not found", 404, mimetype="text/plain") + if not new_title: + return Response("title required", 400, mimetype="text/plain") + d, fname = os.path.split(full) + old_base, _ = os.path.splitext(fname) + if indie: + season = int(request.form.get("season", 1)) + episode = int(request.form.get("episode", 1)) + new_base = f"S{season:02d}E{episode:02d} - {safe_name(new_title)}" + nfo_updates = {"title": new_title, "season": season, "episode": episode} + else: + dm = re.match(r"(\d{4}-\d{2}-\d{2}) - ", old_base) + new_base = (f"{dm.group(1)} - " if dm else "") + safe_name(new_title) + nfo_updates = {"title": new_title} + if new_base != old_base: + for f in os.listdir(d): + # rename the video and every sidecar sharing its basename stem + # (.nfo, .jpg, -thumb.jpg, .info.json, …) in lockstep + if f.startswith(old_base): + suffix = f[len(old_base):] + if suffix.startswith(".") or suffix.startswith("-thumb"): + os.rename(os.path.join(d, f), os.path.join(d, new_base + suffix)) + set_nfo_tags(os.path.join(d, new_base + ".nfo"), nfo_updates) + log(f"renamed {old_base!r} → {new_base!r}") + return redirect("/library") + + +# -------------------------------------------------------------- channel config +# /config → friendly gui over the auto-download channel list +# /config/raw → codemirror editor for the raw yaml files (advanced) + +SUBS_FILE = "subscriptions.yaml" +SUB_PRESET_DEFAULT = "Jellyfin TV Show by Date | only-after | flat-videos" +SUB_SECTION_DEFAULT = "= Independent Creators" +# per-channel rule keys the gui understands; order = the "add rule" menu order. +# anything else in a channel entry is left for the raw editor. +RULE_FIELDS = ["download_after", "title_include_keywords", "title_exclude_keywords", + "description_include_keywords", "description_exclude_keywords"] +RAW_LABELS = {"subscriptions.yaml": "auto-download channels", + "feed.yaml": "notify-me channels"} + + +def list_yaml_configs(): + return sorted(f for f in safe_listdir(YT_CONFIG_DIR) + if f.endswith((".yaml", ".yml"))) + + +def write_text_atomic(path, text): + tmp = path + ".tmp" + with open(tmp, "w") as f: + f.write(text) + os.replace(tmp, path) + + +def parse_subscriptions(): + # flatten subscriptions.yaml down to a plain channel list for the gui, + # remembering the wrapping structure so it can be rebuilt verbatim on save + try: + with open(os.path.join(YT_CONFIG_DIR, SUBS_FILE)) as f: + data = yaml.safe_load(f) or {} + except (OSError, yaml.YAMLError): + data = {} + overrides = (data.get("__preset__") or {}).get("overrides") or {} + preset_key = next((k for k in data if k != "__preset__"), SUB_PRESET_DEFAULT) + section = data.get(preset_key) if isinstance(data.get(preset_key), dict) else {} + section_key = next(iter(section), SUB_SECTION_DEFAULT) if section else SUB_SECTION_DEFAULT + chmap = section.get(section_key) if isinstance(section.get(section_key), dict) else {} + channels = [] + for key, val in (chmap or {}).items(): + name = key[1:].strip() if key.startswith("~") else key + if isinstance(val, str): + channels.append({"name": name, "url": val, "rules": {}}) + elif isinstance(val, dict): + rules = {k: val[k] for k in RULE_FIELDS if k in val} + channels.append({"name": name, "url": val.get("url", ""), "rules": rules}) + return {"preset_key": preset_key, "section_key": section_key, + "overrides": overrides, "channels": channels} + + +def build_subscriptions(channels): + # rebuild the file from the gui's channel list, preserving the preset + # wrapper + global overrides; a channel with no rules stays the compact + # "name: url" form, one with rules becomes the "~name:" dict form + cur = parse_subscriptions() + chmap = {} + for ch in channels: + name = (ch.get("name") or "").strip() + url = (ch.get("url") or "").strip() + if not name or not url: + continue + clean = {} + for field in RULE_FIELDS: + v = (ch.get("rules") or {}).get(field) + if isinstance(v, str): + v = v.strip() + if isinstance(v, list): + v = [str(x).strip() for x in v if str(x).strip()] + if v in (None, "", [], {}): + continue + clean[field] = v + if clean: + chmap["~" + name] = {"url": url, **clean} + else: + chmap[name] = url + out = {} + if cur["overrides"]: + out["__preset__"] = {"overrides": cur["overrides"]} + out[cur["preset_key"]] = {cur["section_key"]: chmap} + return yaml.dump(out, sort_keys=False, allow_unicode=True, width=4096, + default_flow_style=False) + + +@app.get("/api/subscriptions") +def api_subscriptions_get(): + return {"channels": parse_subscriptions()["channels"]} + + +@app.post("/api/subscriptions") +def api_subscriptions_save(): + payload = request.get_json(silent=True) or {} + channels = payload.get("channels") + if not isinstance(channels, list): + return {"error": "expected a channels list"}, 400 + text = build_subscriptions(channels) + try: + yaml.safe_load(text) + except yaml.YAMLError as e: + return {"error": f"could not save: {e}"}, 400 + write_text_atomic(os.path.join(YT_CONFIG_DIR, SUBS_FILE), text) + n = sum(1 for c in channels if (c.get("name") or "").strip() + and (c.get("url") or "").strip()) + log(f"subscriptions.yaml saved via gui ({n} channels)") + return {"ok": True, "count": n} + + +@app.get("/api/configs") +def api_configs_get(): + files = [] + for name in list_yaml_configs(): + try: + with open(os.path.join(YT_CONFIG_DIR, name)) as f: + files.append({"name": name, "label": RAW_LABELS.get(name, ""), + "body": f.read()}) + except OSError: + continue + return {"files": files} + + +@app.post("/api/config") +def api_config_save(): + payload = request.get_json(silent=True) or {} + name, body = payload.get("name", ""), payload.get("body", "") + if name not in list_yaml_configs(): + return {"error": "unknown file"}, 400 + try: + yaml.safe_load(body) + except yaml.YAMLError as e: + return {"error": str(e)}, 400 + write_text_atomic(os.path.join(YT_CONFIG_DIR, name), body) + log(f"config saved via raw editor: {name}") + return {"ok": True} + + +GUI_PAGE = """ + + +yt downloader · channels + +

channels ◂ home

+

videos from these channels download automatically.

+
+ +
+ edit raw yaml ▸
+ +""" + + +RAW_PAGE = """ + + +yt downloader · raw yaml + + + +

raw yaml ◂ channels

+
loading…
+ + + +""" + + +@app.get("/config") +def config_page(): + return Response(GUI_PAGE, mimetype="text/html") + + +@app.get("/config/raw") +def config_raw(): + return Response(RAW_PAGE, mimetype="text/html") + + +@app.get("/api/upscale") +def api_upscale_status(): + # progress is written by the separate yt-upscaler service into our data dir + return load_json("upscale-status.json", {"enabled": False}) + + if __name__ == "__main__": # jobs interrupted by a container restart pick up where they left off with state_lock: diff --git a/config/yt-upscaler/Dockerfile b/config/yt-upscaler/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..4b39a6bd0830ad33446b93880b5ff405132dbec3 --- /dev/null +++ b/config/yt-upscaler/Dockerfile @@ -0,0 +1,8 @@ +# standalone thumbnail super-resolution worker. kept off the ytdl-sub base +# image because that image is on python 3.14, which onnxruntime has no wheels +# for yet; this pins a supported python and stays tiny (no yt-dlp/ffmpeg). +FROM python:3.12-slim +RUN pip install --no-cache-dir onnxruntime numpy pillow +COPY realesr-general-x4v3.onnx /app/realesr-general-x4v3.onnx +COPY upscale.py /app/upscale.py +CMD ["python3", "-u", "/app/upscale.py"] diff --git a/config/yt-upscaler/upscale.py b/config/yt-upscaler/upscale.py new file mode 100644 index 0000000000000000000000000000000000000000..5e9abfa5a7d9b52c2517523a938b84cfa0793209 --- /dev/null +++ b/config/yt-upscaler/upscale.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +# yt-upscaler: keeps the Independent library's thumbnails sharp on a TV. +# youtube only stores ~720p thumbnails, which jellyfin then stretches across +# the whole screen as the item backdrop (soft + blocky). this re-fetches the +# highest-res thumbnail from the image CDN (not bot-walled) and runs it through +# Real-ESRGAN general-x4v3 (exported to ONNX, so no pickle is ever loaded) at +# 4x on CPU, writing a crisp