From c6666b14acb9d62bb70b34275156feb553e400f8 Mon Sep 17 00:00:00 2001 From: clover caruso Date: Thu, 11 Jun 2026 20:23:55 -0700 Subject: [PATCH] feat: jellyfin youtube --- compose.yaml | 49 +++ config/Caddyfile | 3 + config/copyparty.conf | 5 - config/static/apps.html | 3 + config/yt-feed/Dockerfile | 9 + config/yt-feed/app.py | 645 +++++++++++++++++++++++++++++++++++ config/yt/archive-loop.sh | 13 + config/yt/config.yaml | 28 ++ config/yt/feed.yaml | 22 ++ config/yt/subscriptions.yaml | 69 ++++ 10 files changed, 841 insertions(+), 5 deletions(-) create mode 100644 config/yt-feed/Dockerfile create mode 100644 config/yt-feed/app.py create mode 100644 config/yt/archive-loop.sh create mode 100644 config/yt/config.yaml create mode 100644 config/yt/feed.yaml create mode 100644 config/yt/subscriptions.yaml diff --git a/compose.yaml b/compose.yaml index 4aa279ca60271b6b4136a2cf192354cd2083abc4..a096a9bb042129a024e2163e7f52711adc7d6ecb 100755 --- a/compose.yaml +++ b/compose.yaml @@ -501,6 +501,55 @@ services: net.paperclover.list.domain: rdr net.paperclover.list.priority: 50 net.paperclover.list.access: media-manage + # youtube archival (issue #6) — channel list in config/yt/subscriptions.yaml + ytdl-sub: + container_name: ytdl-sub + image: ghcr.io/jmbannon/ytdl-sub:latest + user: "$USER_ID:$GROUP_ID" + # bypass the image's s6 init (it only handles PUID/PGID, which user: covers) + # and run the subscription pass on a fixed cycle; the yaml stays read-only + # in this repo instead of living in the container's /config + entrypoint: ["/bin/sh", "-c"] + command: ["sh /config-yt/archive-loop.sh"] + environment: + HOME: /config + volumes: + - ./config/yt:/config-yt:ro + # the image hardcodes its lock file at /config, so state lives there + - "${APP_ROOT}/ytdl-sub:/config" + - "${MEDIA_ROOT}:/media" + restart: unless-stopped + labels: + net.paperclover.list.name: ytdl-sub YouTube Archiver + net.paperclover.list.web: "false" + # new-video triage (issue #6): polls channels in config/yt/feed.yaml, emails + # a notification per upload, and serves the review page on yt.* where each + # video is ingested into indie shows / independent / music — or skipped + yt-feed: # port 80 + container_name: yt-feed + build: + context: config/yt-feed + dockerfile: Dockerfile + pull_policy: build + user: "$USER_ID:$GROUP_ID" + environment: + SMTP_HOST: "${MAILER_ADDRESS:?}" + SMTP_PORT: "465" + SMTP_USER: "${MAILER_USERNAME:?}" + SMTP_PASS: "${MAILER_PASSWORD:?}" + MAIL_FROM: "yt-feed@${HOME_DOMAIN:?}" + MAIL_TO: "${ADMIN_EMAIL:?}" + BASE_URL: "https://yt.${HOME_DOMAIN:?}" + volumes: + - ./config/yt:/config-yt:ro + - "${APP_ROOT}/yt-feed:/data" + - "${MEDIA_ROOT}:/media" + restart: unless-stopped + labels: + net.paperclover.list.name: YouTube Triage + net.paperclover.list.domain: yt + net.paperclover.list.priority: 53 + net.paperclover.list.access: media-manage # language models opencode: # port 4096 container_name: opencode diff --git a/config/Caddyfile b/config/Caddyfile index 895a617f87c02c8daf910a71c94f78fc720e996d..1470159e01f840068dbccfcae6923cedbda73bea 100644 --- a/config/Caddyfile +++ b/config/Caddyfile @@ -223,6 +223,9 @@ speedtest.{$HOME_DOMAIN} { sync.{$HOME_DOMAIN} { import reverse_proxy_auth "http://syncthing:8384" admin } +yt.{$HOME_DOMAIN} { + import reverse_proxy_auth "http://yt-feed" media-manage +} xmpp.{$HOME_DOMAIN} { tls { on_demand diff --git a/config/copyparty.conf b/config/copyparty.conf index 07972d5b554092945b0487a3dc8bda2a01c22042..025f5e6bcac0d5c8b350b297cf2c26af6d1dfd2b 100644 --- a/config/copyparty.conf +++ b/config/copyparty.conf @@ -44,11 +44,6 @@ r.: * rwm: @acct rwmd: snow -[/mirrors] - /w/mirrors - accs: - r.: * - rwmd: snow [/logs] /w/logs accs: diff --git a/config/static/apps.html b/config/static/apps.html index a8299263a5ee98af72c8ee00046b47c240bb073e..06efdf3c6b014386e24ffcefea2ad445d8edf140 100644 --- a/config/static/apps.html +++ b/config/static/apps.html @@ -81,6 +81,9 @@ li::before {
  • opencode opencode.paperclover.net
    • {{ end }} {{ if contains "role:media-manage" $groups }} +
  • youtube triage yt.paperclover.net
    • + {{ end }} + {{ if contains "role:media-manage" $groups }}
  • qbittorrent qbt.paperclover.net
    • {{ end }} {{ if contains "role:media-manage" $groups }} diff --git a/config/yt-feed/Dockerfile b/config/yt-feed/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..20dbf7a9f4d5f19d35f9cb1f9b63830f6d29c861 --- /dev/null +++ b/config/yt-feed/Dockerfile @@ -0,0 +1,9 @@ +# based on the ytdl-sub image so yt-dlp + ffmpeg stay maintained upstream — +# "pull the image" fixes youtube breakage here the same as for the archiver +FROM ghcr.io/jmbannon/ytdl-sub:latest +# always take the newest yt-dlp at build time — the youtube bot-wall +# cat-and-mouse moves faster than the base image +RUN pip install --no-cache-dir -U yt-dlp && pip install --no-cache-dir flask pyyaml +COPY app.py /app/app.py +ENTRYPOINT [] +CMD ["python3", "-u", "/app/app.py"] diff --git a/config/yt-feed/app.py b/config/yt-feed/app.py new file mode 100644 index 0000000000000000000000000000000000000000..d366eddbbfbd4a5587c7e82435239d4eaede6ff6 --- /dev/null +++ b/config/yt-feed/app.py @@ -0,0 +1,645 @@ +#!/usr/bin/env python3 +# yt-triage: polls youtube rss for the channels in feed.yaml, queues new +# uploads for manual review, emails a notification (thumbnail + link back +# here), and serves the review page where each video is ingested into the +# right library — an indie show season (proper S/E naming + episode nfo), +# the flat Independent creators library, or music intake — or skipped. +import json +import os +import queue +import re +import smtplib +import subprocess +import threading +import time +import urllib.request +import xml.etree.ElementTree as ET +from email.message import EmailMessage +from email.utils import formatdate +from xml.sax.saxutils import escape + +import yaml +from flask import Flask, Response, redirect, request + +FEED_CONFIG = os.environ.get("FEED_CONFIG", "/config-yt/feed.yaml") +STATE_DIR = os.environ.get("STATE_DIR", "/data") +INTERVAL = int(os.environ.get("CHECK_INTERVAL", "1800")) +SMTP_HOST = os.environ["SMTP_HOST"] +SMTP_PORT = int(os.environ.get("SMTP_PORT", "465")) +SMTP_USER = os.environ["SMTP_USER"] +SMTP_PASS = os.environ["SMTP_PASS"] +MAIL_FROM = os.environ["MAIL_FROM"] +MAIL_TO = os.environ["MAIL_TO"] +BASE_URL = os.environ.get("BASE_URL", "").rstrip("/") + +INDIE_DIR = "/media/jellyfin/Indie Shows" +INDEP_DIR = "/media/jellyfin/Independent" +MUSIC_DIR = "/media/music_intake" +VIDEO_EXTS = (".webm", ".mp4", ".mkv") + +ATOM = "{http://www.w3.org/2005/Atom}" +YT = "{http://www.youtube.com/xml/schemas/2015}" +MEDIA = "{http://search.yahoo.com/mrss/}" +UA = {"User-Agent": "Mozilla/5.0 (yt-triage; +https://paperclover.net)"} +SEEN_CAP = 300 + +state_lock = threading.Lock() +job_queue = queue.Queue() + +# youtube sometimes bot-walls the whole home ip after heavy traffic. when the +# signature error appears, downloads pause and a cheap probe every few hours +# resumes them once the wall lifts (rss polling is unaffected by walls). +WALL_RE = re.compile(r"confirm you.re not a bot", re.I) +WALL_PROBE_INTERVAL = int(os.environ.get("WALL_PROBE_INTERVAL", "10800")) +PROBE_VIDEO = "https://www.youtube.com/watch?v=jNQXAC9IVRw" + + +class WallError(Exception): + pass + + +def wall_active(): + return load_json("wall.json", {}).get("walled", False) + + +def set_wall(walled): + with state_lock: + save_json("wall.json", {"walled": walled, "since": int(time.time())}) + log(f"bot wall {'detected — downloads paused' if walled else 'lifted — downloads resumed'}") + + +def wall_prober(): + while True: + time.sleep(WALL_PROBE_INTERVAL if wall_active() else 600) + if not wall_active(): + continue + probe = subprocess.run( + ["yt-dlp", "--simulate", "--print", "%(id)s", PROBE_VIDEO], + capture_output=True, text=True, timeout=120) + if probe.returncode == 0: + set_wall(False) + resolve_stragglers() + else: + log("wall probe: still walled") + + +def resolve_stragglers(): + with state_lock: + pending = load_json("pending.json", {}) + for v in pending.values(): + if v.get("unresolved"): + threading.Thread(target=resolve_and_update, args=(v["id"], v["link"]), + daemon=True).start() + + +def log(msg): + print(msg, flush=True) + + +def fetch(url): + req = urllib.request.Request(url, headers=UA) + with urllib.request.urlopen(req, timeout=30) as resp: + return resp.read().decode("utf-8", errors="replace") + + +def state_path(name): + return os.path.join(STATE_DIR, name) + + +def load_json(name, fallback): + try: + with open(state_path(name)) as f: + return json.load(f) + except (FileNotFoundError, json.JSONDecodeError): + return fallback + + +def save_json(name, data): + tmp = state_path(name) + ".tmp" + with open(tmp, "w") as f: + json.dump(data, f, indent=1) + os.replace(tmp, state_path(name)) + + +def safe_name(name): + return re.sub(r'[/\\:*?"<>|]', "-", name).strip() or "untitled" + + +# ---------------------------------------------------------------- feed poller + +def channel_id_for(url, cache): + if url in cache: + return cache[url] + m = re.search(r"/channel/(UC[0-9A-Za-z_-]{22})", url) + if not m: + html = fetch(url) + m = re.search(r"channel_id=(UC[0-9A-Za-z_-]{22})", html) or re.search( + r'"channelId":"(UC[0-9A-Za-z_-]{22})"', html + ) + if not m: + raise ValueError(f"could not resolve channel id for {url}") + cache[url] = m.group(1) + return cache[url] + + +def feed_entries(channel_id): + text = fetch(f"https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}") + entries = [] + for e in ET.fromstring(text).findall(ATOM + "entry"): + vid = e.find(YT + "videoId") + title = e.find(ATOM + "title") + link = e.find(ATOM + "link") + published = e.find(ATOM + "published") + thumb = e.find(f"{MEDIA}group/{MEDIA}thumbnail") + if vid is None or vid.text is None or title is None: + continue + entries.append({ + "id": vid.text, + "title": title.text or "(untitled)", + "link": link.get("href") if link is not None else f"https://youtu.be/{vid.text}", + "published": published.text[:10] if published is not None and published.text else "", + "thumb": thumb.get("url") if thumb is not None else "", + }) + return entries + + +def send_notification(channel, entry): + msg = EmailMessage() + slug = re.sub(r"[^a-z0-9]+", "-", channel.lower()).strip("-") + review = f"{BASE_URL}/#v-{entry['id']}" if BASE_URL else "" + msg["Subject"] = f"[yt] {channel}: {entry['title']}" + msg["From"] = MAIL_FROM + msg["To"] = MAIL_TO + msg["Date"] = formatdate(localtime=True) + msg["Message-ID"] = f"" + msg["References"] = f"" + msg["In-Reply-To"] = f"" + msg.set_content( + f"{channel} uploaded: {entry['title']}\n\n" + f" {entry['link']}\n published {entry['published']}\n\n" + f"review and ingest: {review}\n" + ) + h = escape(entry["title"]) + msg.add_alternative( + f'
    ' + f'

    {escape(channel)} uploaded:

    ' + f'

    ' + f'

    ' + f'

    {h} · {entry["published"]}

    ' + f'

    review & ingest → · watch

    ' + f"
    ", + subtype="html", + ) + with smtplib.SMTP_SSL(SMTP_HOST, SMTP_PORT, timeout=30) as s: + s.login(SMTP_USER, SMTP_PASS) + s.send_message(msg) + + +def poll_once(): + with open(FEED_CONFIG) as f: + channels = (yaml.safe_load(f) or {}).get("channels") or {} + with state_lock: + ids = load_json("channel-ids.json", {}) + seen = load_json("seen.json", {}) + pending = load_json("pending.json", {}) + for name, url in channels.items(): + try: + cid = channel_id_for(url, ids) + entries = feed_entries(cid) + except Exception as e: + log(f"{name}: fetch failed: {e}") + continue + if cid not in seen: + seen[cid] = [e["id"] for e in entries] + log(f"{name}: now tracking ({len(entries)} existing videos skipped)") + continue + known = set(seen[cid]) + for entry in reversed(entries): + if entry["id"] in known: + continue + entry["channel"] = name + pending[entry["id"]] = entry + seen[cid].append(entry["id"]) + known.add(entry["id"]) + log(f"{name}: queued {entry['title']!r}") + try: + send_notification(name, entry) + except Exception as e: + log(f"{name}: notification failed: {e}") + seen[cid] = seen[cid][-SEEN_CAP:] + with state_lock: + save_json("channel-ids.json", ids) + save_json("seen.json", seen) + save_json("pending.json", pending) + + +def poller(): + while True: + try: + poll_once() + except Exception as e: + log(f"poll failed: {e}") + time.sleep(INTERVAL) + + +# ------------------------------------------------------------ download worker + +def update_job(job_id, **fields): + with state_lock: + jobs = load_json("jobs.json", []) + for j in jobs: + if j["id"] == job_id: + j.update(fields) + save_json("jobs.json", jobs) + + +def resolve_url(url): + out = subprocess.run( + ["yt-dlp", "--no-playlist", "--print", + "%(id)s\t%(title)s\t%(channel)s\t%(upload_date>%Y-%m-%d)s\t%(thumbnail)s", url], + capture_output=True, text=True, timeout=90) + if out.returncode != 0: + if WALL_RE.search(out.stderr): + raise WallError(url) + raise ValueError(out.stderr[-300:]) + vid, title, channel, published, thumb = out.stdout.strip().split("\t") + return {"id": vid, "title": title, "channel": channel, "published": published, + "thumb": thumb, "link": f"https://www.youtube.com/watch?v={vid}"} + + +def resolve_and_update(stub_id, url): + try: + entry = resolve_url(url) + except WallError: + set_wall(True) + return + except Exception as e: + log(f"resolve failed for {url}: {e}") + return + with state_lock: + pending = load_json("pending.json", {}) + # if the stub is gone the user already ingested or skipped it + if stub_id in pending: + del pending[stub_id] + entry["stub_id"] = stub_id + pending[entry["id"]] = entry + save_json("pending.json", pending) + + +def write_nfo(filepath, root, tags): + base, _ = os.path.splitext(filepath) + body = "\n".join(f" <{k}>{escape(str(v))}" for k, v in tags.items() if v != "") + with open(base + ".nfo", "w") as f: + f.write(f"\n<{root}>\n{body}\n\n") + + +def run_job(job): + # youtube intermittently bot-walls the home ip after heavy traffic, so + # transient failures get retried with backoff before declaring an error + for delay in (0, 30, 90): + if delay: + update_job(job["id"], status="retrying", progress="") + time.sleep(delay) + try: + if run_job_once(job): + return + except WallError: + raise + except Exception as e: + log(f"job {job['id']} attempt failed: {e}") + update_job(job["id"], status="error", progress="") + + +def run_job_once(job): + if job.get("unresolved"): + update_job(job["id"], status="resolving") + meta = resolve_url(job["url"]) + job.update(title=meta["title"], channel=meta["channel"], + published=meta["published"], url=meta["link"], unresolved=False) + update_job(job["id"], title=meta["title"]) + dest, url = job["dest"], job["url"] + if dest == "indie": + outdir = os.path.join(INDIE_DIR, safe_name(job["show"]), f"Season {job['season']}") + prefix = f"S{job['season']:02d}E{job['episode']:02d}" + ep_name = safe_name(job.get("ep_title") or job["title"]) + out = f"{outdir}/{prefix} - {ep_name}.%(ext)s" + thumb_out = f"thumbnail:{outdir}/{prefix} - {ep_name}-thumb.%(ext)s" + elif dest == "independent": + outdir = os.path.join(INDEP_DIR, safe_name(job["channel"])) + out = f"{outdir}/%(upload_date>%Y-%m-%d)s - %(title)s.%(ext)s" + thumb_out = f"thumbnail:{outdir}/%(upload_date>%Y-%m-%d)s - %(title)s.%(ext)s" + else: + outdir = os.path.join(MUSIC_DIR, safe_name(job["channel"])) + out = f"{outdir}/%(title)s.%(ext)s" + thumb_out = None + os.makedirs(outdir, exist_ok=True) + cmd = ["yt-dlp", "--newline", "--no-playlist", "--embed-chapters", + "--sleep-requests", "0.75", + "--print", "after_move:filepath", "--no-simulate", "-o", out] + if dest == "music": + cmd += ["-x"] + else: + cmd += ["-f", "bv*+ba/b", "--write-thumbnail", "--convert-thumbnails", "jpg", + "-o", thumb_out] + cmd.append(url) + update_job(job["id"], status="downloading") + filepath = None + tail = [] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + for line in proc.stdout: + line = line.rstrip() + tail = (tail + [line])[-30:] + m = re.search(r"\[download\]\s+([\d.]+%)", line) + if m: + update_job(job["id"], progress=m.group(1)) + elif line.startswith("/"): + filepath = line + proc.wait() + if proc.returncode != 0 or (dest != "music" and not filepath): + if WALL_RE.search("\n".join(tail)): + raise WallError(url) + log(f"job {job['id']} attempt failed (exit {proc.returncode})") + return False + if dest == "indie": + write_nfo(filepath, "episodedetails", { + "title": job.get("ep_title") or job["title"], + "season": job["season"], "episode": job["episode"], + "aired": job.get("published", ""), "plot": url, + }) + elif dest == "independent": + write_nfo(filepath, "movie", { + "title": job["title"], "premiered": job.get("published", ""), "plot": url, + }) + update_job(job["id"], status="done", progress="") + log(f"job {job['id']} done: {filepath or job['title']}") + return True + + +def worker(): + while True: + job = job_queue.get() + if wall_active(): + update_job(job["id"], status="waiting", progress="") + while wall_active(): + time.sleep(30) + try: + run_job(job) + except WallError: + set_wall(True) + update_job(job["id"], status="queued", progress="") + job_queue.put(job) + except Exception as e: + update_job(job["id"], status="error") + log(f"job {job['id']} crashed: {e}") + + +# -------------------------------------------------------------------- web app + +app = Flask(__name__) + + +def indie_shows(): + shows = {} + try: + names = sorted(os.listdir(INDIE_DIR)) + except FileNotFoundError: + names = [] + for name in names: + path = os.path.join(INDIE_DIR, name) + if not os.path.isdir(path): + continue + seasons = {} + for sub in os.listdir(path): + m = re.fullmatch(r"Season (\d+)", sub) + if m and os.path.isdir(os.path.join(path, sub)): + count = sum(1 for f in os.listdir(os.path.join(path, sub)) + if f.lower().endswith(VIDEO_EXTS)) + seasons[int(m.group(1))] = count + shows[name] = seasons + return shows + + +PAGE = """ + + +yt triage + +

    yt triage · {npending} pending

    +{cards} +
    + + +
    +
    {jobs}
    + +""" + + +def card_html(v, shows): + opts = ['', + ''] + for s in shows: + opts.append(f'') + opts.append('') + img = (f'' + if v.get("thumb") else "") + meta = ("resolving…" if v.get("unresolved") + else f"{escape(v.get('channel', '?'))} · {escape(v.get('published', ''))}") + return f"""
    + {img} +

    {escape(v['title'])}

    +

    {meta}

    + + + + +
    + + +
    +
    + + +
    +
    """ + + +def jobs_html(): + with state_lock: + jobs = load_json("jobs.json", []) + icon = {"queued": "·", "resolving": "…", "downloading": "↓", "retrying": "↻", + "waiting": "⏸", "done": "✓", "error": "✗"} + out = ["

    recent jobs

    "] if jobs else [] + for j in jobs[:15]: + cls = j["status"] if j["status"] in ("done", "error") else "" + prog = f" {j.get('progress', '')}" if j["status"] == "downloading" else "" + retry = ("" if j["status"] != "error" else + f'
    ' + f'' + f'
    ') + out.append(f"

    {icon.get(j['status'], '·')} " + f"{escape(j['title'])} → {escape(j['dest_label'])}{prog}{retry}

    ") + return "\n".join(out) + + +@app.get("/") +def index(): + with state_lock: + pending = load_json("pending.json", {}) + shows = indie_shows() + cards = "\n".join(card_html(v, shows) for v in reversed(list(pending.values()))) + if not pending: + cards = "

    nothing pending. enjoy the silence.

    " + if wall_active(): + cards = ("
    youtube has bot-walled this ip — downloads " + "are paused and will resume automatically once the wall lifts " + "(probed every few hours). queueing still works.
    " + cards) + return PAGE.format(npending=len(pending), cards=cards, + jobs=jobs_html(), shows_json=json.dumps(indie_shows())) + + +@app.get("/jobs.html") +def jobs_partial(): + return Response(jobs_html(), mimetype="text/html") + + +@app.post("/retry") +def retry(): + jid = request.form["job"] + with state_lock: + jobs = load_json("jobs.json", []) + job = next((j for j in jobs if j["id"] == jid), None) + if job: + job.update(status="queued", progress="") + save_json("jobs.json", jobs) + if job: + job_queue.put(job) + return redirect("/") + + +@app.post("/skip") +def skip(): + with state_lock: + pending = load_json("pending.json", {}) + pending.pop(request.form["vid"], None) + save_json("pending.json", pending) + return redirect("/") + + +@app.post("/add") +def add(): + # accepts one url or a whole batch separated by whitespace/commas. cards + # appear instantly as stubs and resolve in the background — they can be + # ingested before the title/thumbnail has arrived. + urls = [u for u in re.split(r"[\s,]+", request.form["url"].strip()) if u] + with state_lock: + pending = load_json("pending.json", {}) + for i, url in enumerate(urls): + stub_id = f"u{int(time.time() * 1000)}{i}" + pending[stub_id] = {"id": stub_id, "title": url, "channel": "", + "published": "", "thumb": "", "link": url, + "unresolved": True} + threading.Thread(target=resolve_and_update, args=(stub_id, url), + daemon=True).start() + save_json("pending.json", pending) + return redirect("/") + + +@app.post("/ingest") +def ingest(): + vid = request.form["vid"] + with state_lock: + pending = load_json("pending.json", {}) + v = pending.get(vid) + if not v: + # a stub may have resolved (new key) between page load and submit + v = next((p for p in pending.values() if p.get("stub_id") == vid), None) + if not v: + return Response("video not in pending queue", 404, mimetype="text/plain") + dest = request.form["dest"] + job = {"id": f"j{int(time.time() * 1000)}", "url": v["link"], "title": v["title"], + "channel": v.get("channel", "unknown"), "published": v.get("published", ""), + "unresolved": v.get("unresolved", False), "status": "queued", "progress": ""} + if dest.startswith("indie|"): + show = request.form.get("new_show", "").strip() if dest == "indie|__new__" else dest[6:] + if not show: + return Response("missing show name", 400, mimetype="text/plain") + # a custom episode title; left equal to the card title means "use the + # video title" (which, for a still-resolving stub, arrives later) + ep_title = request.form.get("ep_title", "").strip() + job.update(dest="indie", show=show, + ep_title=ep_title if ep_title and ep_title != v["title"] else "", + season=int(request.form["season"]), episode=int(request.form["episode"])) + job["dest_label"] = f"{show} S{job['season']:02d}E{job['episode']:02d}" + elif dest == "independent": + job.update(dest="independent", dest_label="Independent") + else: + job.update(dest="music", dest_label="music_intake") + with state_lock: + pending.pop(v["id"], None) + save_json("pending.json", pending) + jobs = load_json("jobs.json", []) + jobs.insert(0, job) + save_json("jobs.json", jobs[:50]) + job_queue.put(job) + return redirect("/") + + +if __name__ == "__main__": + # jobs interrupted by a container restart pick up where they left off + with state_lock: + for j in reversed(load_json("jobs.json", [])): + if j.get("status") in ("queued", "resolving", "downloading", "retrying"): + job_queue.put(j) + threading.Thread(target=poller, daemon=True).start() + threading.Thread(target=worker, daemon=True).start() + threading.Thread(target=wall_prober, daemon=True).start() + app.run(host="0.0.0.0", port=80, threaded=True) diff --git a/config/yt/archive-loop.sh b/config/yt/archive-loop.sh new file mode 100644 index 0000000000000000000000000000000000000000..cce5ccf2815b829cd77d033d45f91bd75b1fb7fc --- /dev/null +++ b/config/yt/archive-loop.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# subscription pass on a 6h cycle — but if youtube has bot-walled the ip +# ("sign in to confirm you're not a bot"), back off for a whole day instead +# of hammering it every cycle, which prolongs the wall. +while true; do + ytdl-sub --config /config-yt/config.yaml sub /config-yt/subscriptions.yaml 2>&1 | tee /tmp/last-pass.log + if grep -q "confirm you.re not a bot" /tmp/last-pass.log; then + echo "[archive-loop] bot wall detected; sleeping 24h" + sleep 86400 + else + sleep 21600 + fi +done diff --git a/config/yt/config.yaml b/config/yt/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76009bbb1ff6a88756fd558da9c9d898ced7ef58 --- /dev/null +++ b/config/yt/config.yaml @@ -0,0 +1,28 @@ +# ytdl-sub tool configuration — the channel list lives in subscriptions.yaml +configuration: + working_directory: "/config/work" + +presets: + # per-channel backlog control: only download uploads on/after download_after + # (yyyymmdd). subscriptions.yaml sets the default and per-channel values. + only-after: + date_range: + after: "{download_after}" + overrides: + download_after: "19700101" + + # output for the Independent library, which is a jellyfin "home videos" + # library: flat files (no Season folder), movie-style nfo (that + # library type parses nfo with the movie parser, not episodedetails), + # title without the date prefix, and the thumbnail named exactly like the + # video so jellyfin uses it as the card image. + flat-videos: + nfo_tags: + nfo_root: "movie" + tags: + title: "{title}" + premiered: "{episode_date_standardized}" + overrides: + episode_file_path: "{episode_file_name_sanitized}" + episode_file_name: "{upload_date_standardized} - {file_title}" + thumbnail_file_name: "{episode_file_path}.jpg" diff --git a/config/yt/feed.yaml b/config/yt/feed.yaml new file mode 100644 index 0000000000000000000000000000000000000000..360cc5f53e948ddb4286e47c8f3b444be4e57b89 --- /dev/null +++ b/config/yt/feed.yaml @@ -0,0 +1,22 @@ +# channels whose new videos land in the triage queue at yt. for +# manual sorting into shows/music/creators (home-infra issue #6). each also +# sends a notification email with a link to the review page. +# adding a channel is one line; any youtube channel url or @handle url works. +channels: + "ArrowType": "https://www.youtube.com/@ArrowType" + "SethBling": "https://www.youtube.com/@SethBling" + "Voidstar": "https://www.youtube.com/@voidstar-digital" + "MallBat": "https://www.youtube.com/@mallbat" + "Early Eyes": "https://www.youtube.com/@earlyeyes" + "Otaku-Vs": "https://www.youtube.com/@OtakuVs" + "Something Witty Entertainment": "https://www.youtube.com/@SWE" + "Ethan Niser": "https://www.youtube.com/@ethanniser" + "V3rb": "https://www.youtube.com/@VerbDoesStuff" + "dyc3": "https://www.youtube.com/@rollthedyc3" + "Technology Connections": "https://www.youtube.com/@TechnologyConnections" + "jan Misali": "https://www.youtube.com/@HBMmaster" + "awe": https://www.youtube.com/@whyawe + "JJBlair": "https://www.youtube.com/@JJBlairrecording" + "2 Mello": "https://www.youtube.com/@2mello" + "XavierWolf": "https://www.youtube.com/@xavierwolfy" + "chaosyumi": "https://www.youtube.com/@willburtz" diff --git a/config/yt/subscriptions.yaml b/config/yt/subscriptions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d93540d6727765caab50ff3592b8cb1eb67b7aee --- /dev/null +++ b/config/yt/subscriptions.yaml @@ -0,0 +1,69 @@ +# channels that are archived automatically (home-infra issue #6). +# every upload lands in media/jellyfin/Independent// with +# .nfo metadata + thumbnails so jellyfin shows each channel as a series. +# +# to add a channel: one line under the preset, then +# sh sync.sh && sh docker.sh restart ytdl-sub +# (or just sync and wait for the next 6h pass) + +__preset__: + overrides: + tv_show_directory: "/media/jellyfin/Independent" + # default backlog policy: new uploads only. set download_after on a + # channel (the "~name" form) to backfill from a date, or to "19700101" + # for the entire backlog. + download_after: "20260611" + +Jellyfin TV Show by Date | only-after | flat-videos: + = Independent Creators: + # full archive + "~Retro Game Mechanics Explained": + url: "https://www.youtube.com/@RGMechEx" + download_after: "19700101" + # skip videos whose title contains any of these (case-insensitive + # substrings). works on any channel entry. for videos already + # downloaded, just delete the files — the download archive remembers + # them and won't re-fetch. + title_exclude_keywords: + - "q&a session" + - "channel trailer" + - "launching memberships" + - "subscriber milestone" + "~Franco Citera": + url: "https://www.youtube.com/@francocitera" + download_after: "19700101" + # partial backlog + "~bill wurtz": + url: "https://www.youtube.com/@billwurtz" + download_after: "20260401" # 'i'm going off the map' onward + "~Coffeezilla": + url: "https://www.youtube.com/@Coffeezilla" + download_after: "20260609" # 'I Found The $200,000 Missing Lego' onward + "~classic j": + url: "https://www.youtube.com/@classicj7094" + download_after: "20240801" + "~JUNIA": + url: "https://www.youtube.com/@butterflywife" + download_after: "19700101" + "~hbomberguy": + url: https://www.youtube.com/@hbomberguy + download_after: "20190210" + # new uploads only (default policy) + "t3ssel8r": "https://www.youtube.com/@t3ssel8r" + "Nes": "https://www.youtube.com/@nesorion6" + "4096": "https://www.youtube.com/@4096" + "MegaLag": "https://www.youtube.com/@MegaLag" + "Stuff Made Here": "https://www.youtube.com/@StuffMadeHere" + "mali potka": "https://www.youtube.com/@malipotka4294" + "Michael Reeves": "https://www.youtube.com/@MichaelReeves" + "Patrick Foley": "https://www.youtube.com/@patrickfoley489" + "orchard phobia": "https://www.youtube.com/@orchardphobia" + "Captain Disillusion": "https://www.youtube.com/@CaptainDisillusion" + "mnmira": "https://www.youtube.com/@mnmira" + "EmpLemon": "https://www.youtube.com/@EmperorLemon" + "andMo'": "https://www.youtube.com/@andMo" + "~CGP Grey": + url: "https://www.youtube.com/@CGPGrey" + # members-only preview posts can't download and abort the queue + title_exclude_keywords: + - "early preview for bonnie bees" -- 2.54.0