diff --git a/yt_job.py b/yt_job.py index 53c74ec..10ffea1 100644 --- a/yt_job.py +++ b/yt_job.py @@ -51,6 +51,20 @@ async def main(): for e in cyc["errors"]: print(" cyklus-fejl:", e) + # Kanal-plugin'ets indeks: playlister (højst ugentligt pr. kanal) og så indekset. + try: + pl = await ytdl.opdater_playlister() + print(f"yt_job: playlister opdateret for {pl['kanaler']} kanaler, {pl['playlister']} i alt") + for e in pl["fejl"][:10]: + print(" playliste-fejl:", e) + except Exception as e: # noqa: BLE001 + print(f"yt_job: playlister kunne ikke opdateres: {e}") + try: + ix = ytdl.skriv_indeks() + print(f"yt_job: indeks: {ix['channels']} kanaler, {ix['videos']} videoer, {ix['playlists']} playlister") + except Exception as e: # noqa: BLE001 + print(f"yt_job: indekset kunne ikke skrives: {e}") + res = await ytdl.run_cycle(dry_run=False) if res.get("error"): print("yt_job FEJL:", res["error"]) diff --git a/ytdl.py b/ytdl.py index 3515c2f..6802828 100644 --- a/ytdl.py +++ b/ytdl.py @@ -11,12 +11,15 @@ biblioteket — fx det der lå i /slow/shared/yt i forvejen — er urørligt. """ import asyncio import fcntl +import functools import json import os import re import shutil import time import uuid +import xml.etree.ElementTree as ET +from datetime import datetime from xml.sax.saxutils import escape import httpx @@ -395,6 +398,251 @@ def _slet_video(path: str) -> int: return size +# ---------------- indeks til Emby-kanal-plugin'et ---------------- +# Emby kan kun vise serie → sæson → afsnit. Kanal-plugin'et (agw/emby-yt-kanaler) +# viser i stedet Kanal → [Alle videoer | Playlister → Playliste] → videoer, hvor +# en video kan ligge i flere playlister. Det læser dette indeks, som portalen +# skriver ud fra disken (NFO'er og billeder) og playliste-cachen. Plugin'et har +# ingen egen logik for YouTube; alt det ligger her. +INDEKS_NAVN = ".weircon-index.json" +PLAYLISTER_PATH = config.DATA / "yt_playlister.json" +PLAYLISTE_MAKS_ALDER = 7 * 86400 +_VIDEO_ENDELSER = (".mp4", ".mkv", ".webm") +_SRT_RE = re.compile(r"\.([A-Za-z0-9_-]+)\.(srt|vtt)$") + + +def indeks_sti() -> str: + return os.path.join(config.YT_DOWNLOAD_DIR, INDEKS_NAVN) + + +def _laes_nfo(sti: str) -> dict: + """Felter fra en NFO (episodedetails/tvshow): tag -> tekst, uniqueid pr. type.""" + try: + rod = ET.parse(sti).getroot() + except (OSError, ET.ParseError): + return {} + ud = {} + for el in rod: + if el.tag == "uniqueid": + ud[f"uniqueid:{el.get('type') or ''}"] = (el.text or "").strip() + elif el.tag not in ud: + ud[el.tag] = (el.text or "").strip() + return ud + + +@functools.lru_cache(maxsize=1024) +def _varighed_mp4(sti: str, mtime: float): + try: + from mutagen import File as _MFile + m = _MFile(sti) + return int(m.info.length) if m and m.info and m.info.length else None + except Exception: # noqa: BLE001 + return None + + +def _laes_playlister() -> dict: + return _load(PLAYLISTER_PATH, {}) or {} + + +def skriv_indeks() -> dict: + """Skriv indekset (DOWNLOAD_DIR/.weircon-index.json) ud fra disken. Ingen netværk. + Returnerer en lille statistik.""" + rod = config.YT_DOWNLOAD_DIR + kanaler, videoer = [], {} + try: + navne = sorted(os.listdir(rod), key=str.lower) + except OSError: + navne = [] + for navn in navne: + kdir = os.path.join(rod, navn) + if navn.startswith(".") or not os.path.isdir(kdir): + continue + tv = _laes_nfo(os.path.join(kdir, "tvshow.nfo")) + kid = tv.get("uniqueid:youtube") or f"navn:{navn}" + mine = [] + for d, dirs, fs in os.walk(kdir): + dirs[:] = [x for x in dirs if not x.startswith(".")] + for f in fs: + if not f.lower().endswith(_VIDEO_ENDELSER) or f.startswith("."): + continue + sti = os.path.join(d, f) + stem = f.rsplit(".", 1)[0] + nfo = _laes_nfo(os.path.join(d, stem + ".nfo")) + m = re.search(r"\[([A-Za-z0-9_-]{11})\]$", stem) + vid = nfo.get("uniqueid:youtube") or (m.group(1) if m else "") + if not vid: + continue + st_ = os.stat(sti) + thumb = os.path.join(d, stem + "-thumb.jpg") + subs = [] + for side in sorted(fs): + if side.startswith(stem + "."): + sm = _SRT_RE.search(side[len(stem):]) + if sm: + subs.append({"path": os.path.join(d, side), "lang": sm.group(1)}) + videoer[vid] = { + "title": nfo.get("title") or stem, "channel": kid, "path": sti, + "thumb": thumb if os.path.exists(thumb) else None, + "description": nfo.get("plot") or "", + "upload_date": nfo.get("aired") or nfo.get("premiered") or "", + "duration": _varighed_mp4(sti, st_.st_mtime), "added": int(st_.st_mtime), + "subtitles": subs} + mine.append(vid) + if not mine: + continue + mine.sort(key=lambda v: (videoer[v]["upload_date"], videoer[v]["added"]), reverse=True) + poster, fanart = os.path.join(kdir, "poster.jpg"), os.path.join(kdir, "fanart.jpg") + kanaler.append({"id": kid, "name": tv.get("title") or navn, "description": tv.get("plot") or "", + "poster": poster if os.path.exists(poster) else None, + "fanart": fanart if os.path.exists(fanart) else None, + "videos": mine, "playlists": [], "_dir": kdir}) + + # Playlister fra cachen; kun hentede videoer, i YouTubes rækkefølge. + cache = _laes_playlister() + for k in kanaler: + for pl in (cache.get(k["id"]) or {}).get("playlists", []): + vids = [v for v in pl.get("videos", []) if v in videoer] + if not vids: + continue + thumb = os.path.join(k["_dir"], ".playlists", f"{pl['id']}.jpg") + k["playlists"].append({"id": pl["id"], "title": pl.get("title") or pl["id"], + "thumb": thumb if os.path.exists(thumb) else None, "videos": vids}) + k.pop("_dir", None) + + data = {"version": 1, "generated": datetime.now().astimezone().isoformat(timespec="seconds"), + "channels": kanaler, "videos": videoer} + sti = indeks_sti() + tmp = sti + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, indent=1) + os.chmod(tmp, 0o644) + os.replace(tmp, sti) + return {"channels": len(kanaler), "videos": len(videoer), + "playlists": sum(len(k["playlists"]) for k in kanaler)} + + +async def _ytdlp_json(url: str) -> dict: + """yt-dlp --flat-playlist -J med samme nye forsøg som hentning (403/429).""" + err = "" + for pause, _ in FORSOEG: + if pause: + await asyncio.sleep(pause) + proc = await asyncio.create_subprocess_exec( + config.YT_DLP_BIN, "--flat-playlist", "-J", "--no-warnings", "--ignore-errors", url, + stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) + out, e = await proc.communicate() + try: + return json.loads(out.decode("utf-8", "replace") or "{}") + except ValueError: + err = e.decode("utf-8", "replace") + if not midlertidig(err): + break + raise RuntimeError((err.strip().splitlines() or ["ukendt fejl"])[-1][:200]) + + +def _stoerste_thumb(thumbs: list) -> str: + gode = [t for t in thumbs or [] if t.get("url")] + gode.sort(key=lambda t: (t.get("width") or 0) * (t.get("height") or 0), reverse=True) + return gode[0]["url"] if gode else "" + + +async def _hent_playliste(pl_url: str) -> dict: + d = await _ytdlp_json(pl_url) + return {"id": d.get("id") or "", "title": d.get("title") or "", + "channel_id": d.get("channel_id") or d.get("uploader_id") or "", + "thumb_url": _stoerste_thumb(d.get("thumbnails")), + "videos": [e["id"] for e in (d.get("entries") or []) if e and e.get("id")]} + + +async def opdater_playlister(kun_nye: bool = False) -> dict: + """Hent kanalernes playlister (og deres video-id'er) til cachen. + + Hver kanal opdateres højst hver 7. dag; kun_nye=True tager kun kanaler uden + cache. Portalens egne playliste-job og -abonnementer tages altid med og + knyttes til playlistens ejer-kanal. Bagefter skrives indekset. + """ + res = {"kanaler": 0, "playlister": 0, "fejl": []} + cache = _laes_playlister() + rod = config.YT_DOWNLOAD_DIR + kanal_dirs = {} + for navn in sorted(os.listdir(rod)): + kdir = os.path.join(rod, navn) + if navn.startswith(".") or not os.path.isdir(kdir): + continue + kid = _laes_nfo(os.path.join(kdir, "tvshow.nfo")).get("uniqueid:youtube") + if kid: + kanal_dirs[kid] = kdir + nu = time.time() + + for kid in kanal_dirs: + gl = cache.get(kid) or {} + if kun_nye and gl: + continue + if gl and nu - gl.get("updated", 0) < PLAYLISTE_MAKS_ALDER: + continue + try: + d = await _ytdlp_json(f"https://www.youtube.com/channel/{kid}/playlists") + pls = [] + for e in d.get("entries") or []: + if not e or not e.get("id"): + continue + await asyncio.sleep(2) # skån YouTube + try: + pls.append(await _hent_playliste( + e.get("url") or f"https://www.youtube.com/playlist?list={e['id']}")) + except Exception as ex: # noqa: BLE001 + res["fejl"].append(f"{e.get('title') or e['id']}: {ex}") + # Bevar egne playlister (fra job/abonnementer) der ikke står på kanalens fane. + egne = [p for p in gl.get("playlists", []) if p.get("egen")] + ids = {p["id"] for p in pls} + cache[kid] = {"updated": nu, "playlists": pls + [p for p in egne if p["id"] not in ids]} + res["kanaler"] += 1 + except Exception as ex: # noqa: BLE001 + res["fejl"].append(f"{kid}: {ex}") + await asyncio.sleep(2) + + # Portalens egne playlister (job med list= og playliste-abonnementer). + st = get_state() + urls = {j["url"] for j in st["jobs"].values() if "list=" in (j.get("url") or "")} + urls |= {s["url"] for s in st["subs"].values() if s.get("kind") == "playlist"} + kendte = {p["id"] for c in cache.values() for p in c.get("playlists", [])} + for url in sorted(urls): + m = re.search(r"list=([A-Za-z0-9_-]+)", url) + if m and m.group(1) in kendte: + # Allerede i en kanals cache; markér den som egen, så den bevares. + for c in cache.values(): + for p in c.get("playlists", []): + if p["id"] == m.group(1): + p["egen"] = True + continue + try: + await asyncio.sleep(2) + pl = await _hent_playliste( + f"https://www.youtube.com/playlist?list={m.group(1)}" if m else url) + pl["egen"] = True + ejer = pl.get("channel_id") or "" + c = cache.setdefault(ejer, {"updated": 0, "playlists": []}) + c["playlists"] = [p for p in c["playlists"] if p["id"] != pl["id"]] + [pl] + except Exception as ex: # noqa: BLE001 + res["fejl"].append(f"{url}: {ex}") + + # Miniaturer til /.playlists/.jpg (kun når de mangler). + for kid, c in cache.items(): + kdir = kanal_dirs.get(kid) + if not kdir: + continue + for p in c.get("playlists", []): + mappe = os.path.join(kdir, ".playlists") + sti = os.path.join(mappe, f"{p['id']}.jpg") + if p.get("thumb_url") and not os.path.exists(sti): + os.makedirs(mappe, exist_ok=True) + await _hent_billede(p["thumb_url"], sti) + res["playlister"] += 1 + _save(PLAYLISTER_PATH, cache) + skriv_indeks() + return res + + # Fejl der ofte går over af sig selv: YouTube afviser en enkelt forespørgsel # (403/429), eller forbindelsen falder. yt-dlp gentager ikke selv en 403. _MIDLERTIDIG = ("403", "forbidden", "429", "too many requests", "timed out", @@ -841,6 +1089,45 @@ def _marked_played(ud: dict) -> bool: return bool((ud or {}).get("Played")) +async def _kanal_brugerdata(c, users: list, merged: dict) -> None: + """Læg set/favorit fra Emby-kanalen "YouTube" (kanal-plugin'et) oven i + biblioteket for samme fil, så slettereglen og fredningen virker uanset hvor + videoen blev set. Kanalens elementer bærer video-id'et i ProviderIds["YouTube"]; + stien findes via indekset. Mangler kanalen eller indekset, sker der intet.""" + try: + with open(indeks_sti(), encoding="utf-8") as f: + stier = {vid: os.path.realpath(v["path"]) for vid, v in json.load(f)["videos"].items()} + except (OSError, ValueError, KeyError): + return + try: + kanaler = await _emby(c, "/Channels", UserId=users[0]) if users else {} + except Exception: # noqa: BLE001 + return + kanal = next((k for k in (kanaler.get("Items") or []) + if (k.get("Name") or "").lower() == "youtube"), None) + if not kanal: + return + for uid in users: + try: + data = await _emby(c, f"/Users/{uid}/Items", ParentId=kanal["Id"], + Recursive="true", Fields="ProviderIds,UserData") + except Exception: # noqa: BLE001 + continue + for it in data.get("Items", []): + pid = it.get("ProviderIds") or {} + vid = pid.get("YouTube") or pid.get("Youtube") or pid.get("youtube") + sti = stier.get(vid or "") + if not sti: + continue + m = merged.setdefault(sti, {"id": it["Id"], "name": it.get("Name") or os.path.basename(sti), + "path": sti, "played": False, "favorite": False}) + ud = it.get("UserData") or {} + if _marked_played(ud): + m["played"] = True + if ud.get("IsFavorite"): + m["favorite"] = True + + async def emby_yt_items() -> dict: """Alle videoer i YouTube-biblioteket med set/favorit aggregeret over ALLE brugere.""" settings = get_settings() @@ -870,6 +1157,7 @@ async def emby_yt_items() -> dict: m["played"] = True if ud.get("IsFavorite"): m["favorite"] = True + await _kanal_brugerdata(c, users, merged) out["items"] = list(merged.values()) out["ok"] = True except Exception as e: # noqa: BLE001 @@ -878,6 +1166,11 @@ async def emby_yt_items() -> dict: async def refresh_library(): + # Kanal-plugin'ets indeks følger altid disken: efter hentning og sletning. + try: + skriv_indeks() + except Exception: # noqa: BLE001 + pass try: async with httpx.AsyncClient(timeout=30) as c: await c.post(f"{config.EMBY_URL}/emby/Library/Refresh",