From b77a6ad02c2c030aff5b55b9330da1e7351aadca Mon Sep 17 00:00:00 2001 From: Andrew Beveridge Date: Sat, 22 Aug 2026 19:26:27 -0400 Subject: [PATCH 1/6] fix(youtube): add PO token provider + dedicated yt-dlp cache dir (netcup bot-detection) YouTube "Sign in to confirm you're not a bot" downloads started failing after the migration to the netcup datacenter IP. Two root causes, both exposed by the harsher IP reputation of the new box: - No Proof-of-Origin (PO) token provider. YouTube now binds a GVS PO token to downloads (esp. from datacenter IPs); without one, authenticated cookie downloads intermittently hit the bot wall. Add the bgutil PO token provider: a localhost HTTP server (Node, built in provision Stage 5b, confined to loopback via systemd IPAddress rules) plus the yt-dlp plugin in the venv, which auto-detects it. Kept current by update-ytdlp.sh. - Broken yt-dlp cache. The service runs with HOME=/opt/flacfetch where ~/.cache is the Spotify OAuth token *file*, so yt-dlp's default ~/.cache/yt-dlp path died with NotADirectoryError and silently disabled player/sig + PO-token caching (fresh JS-challenge solve every request). Add FLACFETCH_YTDLP_CACHE_DIR override (set to /opt/flacfetch/ytdlp-cache in the flacfetch unit) so yt-dlp gets its own collision-free cache dir. Verified live on flacup: a previously bot-walled video ID now downloads with `PO Token Providers: bgutil:http-1.3.2` active. Note cookies remain required (unauthenticated access is bot-walled on this IP regardless). Co-Authored-By: Claude Opus 4.8 --- deploy/provision.sh | 96 +++++++++++++++++++++++++++++++- flacfetch/downloaders/youtube.py | 40 +++++++++++++ tests/test_youtube_cookies.py | 54 ++++++++++++++++++ 3 files changed, 188 insertions(+), 2 deletions(-) diff --git a/deploy/provision.sh b/deploy/provision.sh index a9c9048..df93a18 100644 --- a/deploy/provision.sh +++ b/deploy/provision.sh @@ -68,6 +68,18 @@ BROWSER_PROFILE_DIR="$DATA_MOUNT/browser-profiles" PLAYWRIGHT_BROWSERS_PATH="$APP_DIR/ms-playwright" # shared, service-user-owned (not /root/.cache) DENO_INSTALL=/opt/deno LIBRESPOT_BIN=/usr/local/bin/librespot +# yt-dlp Proof-of-Origin token provider (bgutil). YouTube increasingly binds a +# GVS PO token to downloads from datacenter IPs; without a provider, authenticated +# (cookie) downloads intermittently fail with "Sign in to confirm you're not a +# bot". The HTTP-server variant runs on localhost and the matching yt-dlp plugin +# (installed into the venv) auto-detects it on the default port. +BGUTIL_POT_DIR=/opt/bgutil-pot +BGUTIL_POT_VERSION="${BGUTIL_POT_VERSION:-1.3.2}" +BGUTIL_POT_PORT="${BGUTIL_POT_PORT:-4416}" +# Dedicated yt-dlp cache dir. HOME/.cache on this box is the Spotify OAuth token +# *file*, so yt-dlp's default ~/.cache/yt-dlp path dies with NotADirectoryError +# and silently disables player/signature + PO-token caching. Give it its own dir. +YTDLP_CACHE_DIR="$APP_DIR/ytdlp-cache" HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" log() { echo -e ">>> $*"; } @@ -377,6 +389,43 @@ FLACFETCH_VERSION="$(python -c 'import flacfetch; print(flacfetch.__version__)' log "flacfetch version: $FLACFETCH_VERSION" python -c 'import yt_dlp_ejs' 2>/dev/null && log "yt-dlp-ejs available" || warn "yt-dlp-ejs not available" +# Dedicated yt-dlp cache dir (see YTDLP_CACHE_DIR note above). Under $APP_DIR so +# Stage 8b's chown to the service user covers it. +mkdir -p "$YTDLP_CACHE_DIR" + +# ============================================================================= +log "Stage 5b — yt-dlp PO Token provider (bgutil, HTTP server)" +# ============================================================================= +# The venv plugin (installed here) talks to a local bgutil HTTP server (built +# below) that mints Proof-of-Origin tokens via YouTube's BotGuard challenge. The +# plugin auto-detects the server on the default port, so no --extractor-args are +# needed. Node.js is the server runtime (deno alone can't populate node_modules). +pip install --upgrade "bgutil-ytdlp-pot-provider" --quiet || warn "bgutil PO-token plugin install failed" +if ! command -v node >/dev/null 2>&1; then + log "installing Node.js (bgutil PO server runtime)" + apt-get install -y nodejs npm >/dev/null 2>&1 \ + || warn "nodejs/npm install failed — PO token provider server will be unavailable" +fi +if command -v node >/dev/null 2>&1; then + if [ ! -d "$BGUTIL_POT_DIR/.git" ]; then + git clone --quiet "https://github.com/Brainicism/bgutil-ytdlp-pot-provider.git" "$BGUTIL_POT_DIR" \ + || warn "bgutil clone failed" + fi + if [ -d "$BGUTIL_POT_DIR/.git" ]; then + git config --global --add safe.directory "$BGUTIL_POT_DIR" + git -C "$BGUTIL_POT_DIR" fetch --tags --quiet || warn "bgutil fetch failed" + git -C "$BGUTIL_POT_DIR" checkout --quiet "$BGUTIL_POT_VERSION" \ + || warn "bgutil checkout $BGUTIL_POT_VERSION failed" + if ( cd "$BGUTIL_POT_DIR/server" && npm ci --silent && npx --yes tsc ) >/dev/null 2>&1; then + log "bgutil PO server built ($BGUTIL_POT_VERSION)" + else + warn "bgutil PO server build failed (PO token provider will be unavailable)" + fi + fi +else + warn "node unavailable — skipping bgutil PO server build" +fi + # ============================================================================= log "Stage 6 — secrets" # ============================================================================= @@ -593,6 +642,7 @@ Environment="FLACFETCH_DOWNLOAD_DIR=${DOWNLOAD_DIR}" Environment="TRANSMISSION_HOST=localhost" Environment="TRANSMISSION_PORT=9091" Environment="YOUTUBE_COOKIES_FILE=${YOUTUBE_COOKIES_FILE}" +Environment="FLACFETCH_YTDLP_CACHE_DIR=${YTDLP_CACHE_DIR}" ExecStart=$VENV/bin/flacfetch serve --host 0.0.0.0 --port 8080 Restart=always RestartSec=10 @@ -617,6 +667,39 @@ RestartSec=5 WantedBy=multi-user.target XVFB_SERVICE +# yt-dlp PO Token provider — bgutil HTTP server (localhost only). +# The server binary has no host-bind flag (binds :: then 0.0.0.0), so we confine +# it to loopback with systemd's IPAddress firewall rather than a host firewall. +if [ -f "$BGUTIL_POT_DIR/server/build/main.js" ]; then +cat > /etc/systemd/system/bgutil-pot.service < "$APP_DIR/update-ytdlp.sh" <<'UPDATE_SCRIPT' #!/bin/bash @@ -625,9 +708,14 @@ exec >> /var/log/ytdlp-update.log 2>&1 echo "yt-dlp update started at $(date)" cd /opt/flacfetch && source venv/bin/activate OLD=$(python -c "import yt_dlp; print(yt_dlp.version.__version__)" 2>/dev/null || echo unknown) -pip install --upgrade yt-dlp yt-dlp-ejs --quiet +POT_OLD=$(pip show bgutil-ytdlp-pot-provider 2>/dev/null | awk '/^Version:/{print $2}') +pip install --upgrade yt-dlp yt-dlp-ejs bgutil-ytdlp-pot-provider --quiet NEW=$(python -c "import yt_dlp; print(yt_dlp.version.__version__)" 2>/dev/null || echo unknown) -if [ "$OLD" != "$NEW" ]; then echo "yt-dlp $OLD -> $NEW"; systemctl restart flacfetch; fi +POT_NEW=$(pip show bgutil-ytdlp-pot-provider 2>/dev/null | awk '/^Version:/{print $2}') +if [ "$OLD" != "$NEW" ] || [ "$POT_OLD" != "$POT_NEW" ]; then + echo "yt-dlp $OLD -> $NEW / pot-plugin $POT_OLD -> $POT_NEW" + systemctl restart flacfetch +fi command -v /opt/deno/bin/deno >/dev/null 2>&1 && /opt/deno/bin/deno upgrade --quiet 2>/dev/null || true echo "Update complete at $(date)" UPDATE_SCRIPT @@ -854,6 +942,10 @@ fi # ---- enable + (re)start ---------------------------------------------------- systemctl daemon-reload systemctl enable --now xvfb >/dev/null 2>&1 || true +if [ -f /etc/systemd/system/bgutil-pot.service ]; then + systemctl enable --now bgutil-pot >/dev/null 2>&1 || true + for _ in $(seq 1 10); do curl -s "http://127.0.0.1:$BGUTIL_POT_PORT/ping" 2>/dev/null | grep -q version && { log "bgutil PO server healthy"; break; }; sleep 1; done +fi systemctl enable --now ytdlp-update.timer >/dev/null 2>&1 || true systemctl enable --now transmission-maintenance.timer >/dev/null 2>&1 || true systemctl enable flacfetch >/dev/null 2>&1 || true diff --git a/flacfetch/downloaders/youtube.py b/flacfetch/downloaders/youtube.py index 201c968..02994ff 100644 --- a/flacfetch/downloaders/youtube.py +++ b/flacfetch/downloaders/youtube.py @@ -62,6 +62,40 @@ def get_cookies_file() -> Optional[str]: return None +def get_ytdlp_cache_dir() -> Optional[str]: + """ + Resolve an explicit yt-dlp cache directory, if one is configured. + + yt-dlp defaults its cache to ``$XDG_CACHE_HOME`` / ``~/.cache/yt-dlp``. On the + flacfetch server the service runs with ``HOME=/opt/flacfetch`` where + ``~/.cache`` is a *file* (the Spotify OAuth token cache), so yt-dlp's default + path resolves to ``/opt/flacfetch/.cache/yt-dlp`` and every cache write dies + with ``NotADirectoryError``. That silently disables player/signature and + PO-token caching, forcing a fresh JS-challenge solve on every request (slower + and more bot-detectable). + + Set ``FLACFETCH_YTDLP_CACHE_DIR`` to a real directory to give yt-dlp its own + cache location that can't collide. When unset (e.g. library/CLI use on a dev + machine with a normal ``~/.cache``), we leave yt-dlp's default untouched. + + Returns: + The cache directory (created if needed) or None to use yt-dlp's default. + """ + cache_dir = os.environ.get("FLACFETCH_YTDLP_CACHE_DIR") + if not cache_dir: + return None + + try: + os.makedirs(cache_dir, exist_ok=True) + except OSError as e: + # Don't let a misconfigured cache path break downloads; yt-dlp treats a + # missing/False cachedir as "caching disabled" and still works. + logger.warning(f"Could not create yt-dlp cache dir {cache_dir!r}: {e}") + return None + + return cache_dir + + def get_ytdlp_base_opts(cookies_file: Optional[str] = None) -> dict: """ Get base yt-dlp options with common settings including cookies if available. @@ -81,6 +115,12 @@ def get_ytdlp_base_opts(cookies_file: Optional[str] = None) -> dict: if cookies_file: opts["cookiefile"] = cookies_file + # Route yt-dlp's cache to a dedicated dir when configured, so it doesn't + # collide with the Spotify token file at HOME/.cache on the server. + cache_dir = get_ytdlp_cache_dir() + if cache_dir: + opts["cachedir"] = cache_dir + return opts diff --git a/tests/test_youtube_cookies.py b/tests/test_youtube_cookies.py index 083b5a5..9f20de0 100644 --- a/tests/test_youtube_cookies.py +++ b/tests/test_youtube_cookies.py @@ -10,6 +10,7 @@ YoutubeDownloader, get_cookies_file, get_ytdlp_base_opts, + get_ytdlp_cache_dir, ) from flacfetch.providers.youtube import YoutubeProvider @@ -75,6 +76,59 @@ def test_uses_provided_cookies_file(self): assert result == {"cookiefile": "/my/custom/cookies.txt"} +class TestGetYtdlpCacheDir: + """Tests for the FLACFETCH_YTDLP_CACHE_DIR override. + + On the server, HOME/.cache is the Spotify token *file*, so yt-dlp's default + ~/.cache/yt-dlp path raises NotADirectoryError and caching is silently lost. + """ + + def test_returns_none_when_env_unset(self): + """No override configured -> use yt-dlp's default (None).""" + with patch.dict(os.environ, {}, clear=True): + assert get_ytdlp_cache_dir() is None + + def test_creates_and_returns_dir_when_env_set(self): + """Env set -> directory is created and returned.""" + with tempfile.TemporaryDirectory() as tmpdir: + cache_dir = os.path.join(tmpdir, "ytdlp-cache") + with patch.dict(os.environ, {"FLACFETCH_YTDLP_CACHE_DIR": cache_dir}): + result = get_ytdlp_cache_dir() + assert result == cache_dir + assert os.path.isdir(cache_dir) + + def test_returns_none_when_dir_uncreatable(self): + """A bad cache path must not break downloads (caching just disabled).""" + with patch.dict(os.environ, {"FLACFETCH_YTDLP_CACHE_DIR": "/some/cache"}): + with patch( + "flacfetch.downloaders.youtube.os.makedirs", + side_effect=OSError("not a directory"), + ): + assert get_ytdlp_cache_dir() is None + + def test_base_opts_include_cachedir_when_set(self): + """get_ytdlp_base_opts wires cachedir through alongside cookies.""" + with tempfile.TemporaryDirectory() as tmpdir: + cache_dir = os.path.join(tmpdir, "ytdlp-cache") + with patch.dict(os.environ, {"FLACFETCH_YTDLP_CACHE_DIR": cache_dir}): + with patch( + "flacfetch.downloaders.youtube.get_cookies_file", + return_value=None, + ): + result = get_ytdlp_base_opts() + assert result == {"cachedir": cache_dir} + + def test_base_opts_no_cachedir_when_unset(self): + """Without the override, base opts carry no cachedir key.""" + with patch.dict(os.environ, {}, clear=True): + with patch( + "flacfetch.downloaders.youtube.get_cookies_file", + return_value=None, + ): + result = get_ytdlp_base_opts() + assert "cachedir" not in result + + class TestYoutubeDownloaderCookies: """Tests for YoutubeDownloader with cookies support.""" From b45729f5f9eb74e2cf2b9bb1b3dae1fde0b284b3 Mon Sep 17 00:00:00 2001 From: Andrew Beveridge Date: Sat, 22 Aug 2026 19:33:58 -0400 Subject: [PATCH 2/6] fix(youtube): isolate cookie file from yt-dlp write-back (root cause of bot wall) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live diagnosis on the netcup box showed this is the dominant cause of the recurring "Sign in to confirm you're not a bot" failures — not PO tokens. yt-dlp saves the (rotated) cookie jar back to `cookiefile` after every run. On the flagged datacenter IP, YouTube *rejects* the rotation ("cookies no longer valid ... rotated in the browser as a security measure"), so the cookies yt-dlp writes back are invalid and poison the shared, keeper-managed file for the next download. Cookies that worked seconds ago start failing within ~20 minutes. On GCE's trusted IP the rotation was accepted, so the write-back was harmless — which is exactly why it worked for months and only broke after the migration. Proven live: with FRESH keeper cookies via an isolated copy, two previously bot-walled videos download instantly; using the shared file, they fail — and the file's mtime changes on every run (even --skip-download), confirming the write-back. Fix: hand each yt-dlp run a throwaway copy of the cookie file (isolated_cookiefile / ytdlp_opts_isolated). The canonical file stays pristine (the credential keeper is its only writer) and concurrent downloads no longer clobber each other. Applied to download, availability check, provider search, and the credential-check validator. Co-Authored-By: Claude Opus 4.8 --- flacfetch/api/services/credential_check.py | 7 +- flacfetch/downloaders/youtube.py | 59 +++++++++++++++- flacfetch/providers/youtube.py | 4 +- tests/test_youtube_cookies.py | 82 +++++++++++++++++++++- 4 files changed, 145 insertions(+), 7 deletions(-) diff --git a/flacfetch/api/services/credential_check.py b/flacfetch/api/services/credential_check.py index c83b293..1da2604 100644 --- a/flacfetch/api/services/credential_check.py +++ b/flacfetch/api/services/credential_check.py @@ -243,7 +243,12 @@ def check_youtube_credentials() -> CredentialCheckResult: 'skip_download': True, } - with yt_dlp.YoutubeDL(ydl_opts) as ydl: + # Use a throwaway cookie copy so this validation check never writes yt-dlp's + # rotated (and, on a flagged IP, rejected) cookies back over the canonical + # keeper-managed file. See downloaders.youtube.isolated_cookiefile. + from ...downloaders.youtube import ytdlp_opts_isolated + + with ytdlp_opts_isolated(ydl_opts) as opts, yt_dlp.YoutubeDL(opts) as ydl: info = ydl.extract_info(test_url, download=False) if info and info.get('title'): diff --git a/flacfetch/downloaders/youtube.py b/flacfetch/downloaders/youtube.py index 02994ff..4ad810b 100644 --- a/flacfetch/downloaders/youtube.py +++ b/flacfetch/downloaders/youtube.py @@ -1,6 +1,9 @@ import logging import os +import shutil +import tempfile import time +from contextlib import contextmanager from dataclasses import dataclass from typing import Optional @@ -124,6 +127,58 @@ def get_ytdlp_base_opts(cookies_file: Optional[str] = None) -> dict: return opts +@contextmanager +def isolated_cookiefile(cookies_file: Optional[str]): + """Yield a throwaway copy of the cookie file for a single yt-dlp run. + + yt-dlp saves the (possibly rotated) cookie jar back to ``cookiefile`` after + every run. On a flagged datacenter IP YouTube *rejects* the rotation + ("The provided YouTube account cookies are no longer valid. They have likely + been rotated in the browser as a security measure."), so the cookies yt-dlp + saves back are invalid and poison the shared file for the next download — + cookies that worked seconds ago start returning "Sign in to confirm you're + not a bot" within minutes. (On a trusted IP the rotation is accepted, so the + write-back is harmless: this only bites since the move to the datacenter box.) + + Handing each run its own copy keeps the canonical, keeper-managed cookie file + pristine (the credential keeper is its only writer) and isolates concurrent + downloads from one another. Yields the temp copy path, or the original value + when there's nothing to copy (no cookies / file missing). + """ + if not cookies_file or not os.path.exists(cookies_file): + yield cookies_file + return + + tmp_path = None + try: + fd, tmp_path = tempfile.mkstemp(prefix="ytdlp-cookies-", suffix=".txt") + os.close(fd) + shutil.copyfile(cookies_file, tmp_path) + yield tmp_path + finally: + if tmp_path and os.path.exists(tmp_path): + try: + os.unlink(tmp_path) + except OSError as e: + logger.warning(f"Could not remove temp cookie copy {tmp_path!r}: {e}") + + +@contextmanager +def ytdlp_opts_isolated(ydl_opts: dict): + """Run yt-dlp with a throwaway copy of the cookie file (see isolated_cookiefile). + + Swaps ``cookiefile`` in a shallow copy of ``ydl_opts`` for a per-run temp copy + so the canonical keeper-managed cookies are never written back to. A no-op when + no ``cookiefile`` is configured. + """ + original = ydl_opts.get("cookiefile") + with isolated_cookiefile(original) as cf: + if original: + yield {**ydl_opts, "cookiefile": cf} + else: + yield ydl_opts + + @dataclass class YoutubeAvailability: """Result of a YouTube video availability check.""" @@ -187,7 +242,7 @@ def check_youtube_availability( }) try: - with yt_dlp.YoutubeDL(ydl_opts) as ydl: + with ytdlp_opts_isolated(ydl_opts) as opts, yt_dlp.YoutubeDL(opts) as ydl: info = ydl.extract_info(url, download=False) if info: return YoutubeAvailability( @@ -393,7 +448,7 @@ def _timeout_hook(d): downloaded_file: str | None = None try: - with yt_dlp.YoutubeDL(ydl_opts) as ydl: + with ytdlp_opts_isolated(ydl_opts) as opts, yt_dlp.YoutubeDL(opts) as ydl: info = ydl.extract_info(release.download_url, download=True) # extract_info returns None when match_filter rejected the media. if info is None: diff --git a/flacfetch/providers/youtube.py b/flacfetch/providers/youtube.py index 277f18c..4511d15 100644 --- a/flacfetch/providers/youtube.py +++ b/flacfetch/providers/youtube.py @@ -4,7 +4,7 @@ from ..core.interfaces import Provider from ..core.models import AudioFormat, MediaSource, Quality, Release, TrackQuery -from ..downloaders.youtube import get_ytdlp_base_opts +from ..downloaders.youtube import get_ytdlp_base_opts, ytdlp_opts_isolated class YoutubeProvider(Provider): @@ -46,7 +46,7 @@ def search(self, query: TrackQuery) -> list[Release]: releases = [] try: - with yt_dlp.YoutubeDL(ydl_opts) as ydl: + with ytdlp_opts_isolated(ydl_opts) as opts, yt_dlp.YoutubeDL(opts) as ydl: info = ydl.extract_info(search_query, download=False) if info and 'entries' in info: for entry in info['entries']: diff --git a/tests/test_youtube_cookies.py b/tests/test_youtube_cookies.py index 9f20de0..52eeefc 100644 --- a/tests/test_youtube_cookies.py +++ b/tests/test_youtube_cookies.py @@ -11,6 +11,8 @@ get_cookies_file, get_ytdlp_base_opts, get_ytdlp_cache_dir, + isolated_cookiefile, + ytdlp_opts_isolated, ) from flacfetch.providers.youtube import YoutubeProvider @@ -129,6 +131,78 @@ def test_base_opts_no_cachedir_when_unset(self): assert "cachedir" not in result +class TestIsolatedCookiefile: + """Tests for the cookie write-back isolation (protects keeper's cookies). + + yt-dlp saves the rotated cookie jar back to the cookiefile on every run; on a + flagged IP those rotations are rejected, poisoning the shared file. Each run + must therefore operate on a throwaway copy. + """ + + def test_none_passes_through(self): + with isolated_cookiefile(None) as cf: + assert cf is None + + def test_missing_file_passes_through(self): + with isolated_cookiefile("/no/such/cookies.txt") as cf: + assert cf == "/no/such/cookies.txt" + + def test_yields_distinct_copy_then_cleans_up(self): + with tempfile.TemporaryDirectory() as tmpdir: + canonical = os.path.join(tmpdir, "youtube_cookies.txt") + with open(canonical, "w") as f: + f.write("# Netscape\noriginal-cookie\n") + + copy_path = None + with isolated_cookiefile(canonical) as cf: + copy_path = cf + assert cf != canonical + assert os.path.exists(cf) + with open(cf) as f: + assert "original-cookie" in f.read() + + # Temp copy is removed after the context exits. + assert not os.path.exists(copy_path) + + def test_writeback_to_copy_does_not_touch_canonical(self): + """The core guarantee: simulated yt-dlp write-back hits only the copy.""" + with tempfile.TemporaryDirectory() as tmpdir: + canonical = os.path.join(tmpdir, "youtube_cookies.txt") + with open(canonical, "w") as f: + f.write("GOOD-KEEPER-COOKIES\n") + + with isolated_cookiefile(canonical) as cf: + # yt-dlp would overwrite the cookiefile with rotated (rejected) data. + with open(cf, "w") as f: + f.write("POISONED-ROTATED-COOKIES\n") + + with open(canonical) as f: + assert f.read() == "GOOD-KEEPER-COOKIES\n" + + +class TestYtdlpOptsIsolated: + """Tests for ytdlp_opts_isolated wrapper.""" + + def test_noop_without_cookiefile(self): + opts = {"quiet": True} + with ytdlp_opts_isolated(opts) as isolated: + assert isolated is opts + + def test_swaps_cookiefile_to_copy_and_preserves_original_dict(self): + with tempfile.TemporaryDirectory() as tmpdir: + canonical = os.path.join(tmpdir, "cookies.txt") + with open(canonical, "w") as f: + f.write("# cookies\n") + + opts = {"cookiefile": canonical, "quiet": True} + with ytdlp_opts_isolated(opts) as isolated: + assert isolated["cookiefile"] != canonical + assert os.path.exists(isolated["cookiefile"]) + assert isolated["quiet"] is True + # Original dict is left untouched (shallow copy semantics). + assert opts["cookiefile"] == canonical + + class TestYoutubeDownloaderCookies: """Tests for YoutubeDownloader with cookies support.""" @@ -171,10 +245,14 @@ def test_download_uses_cookies(self): downloader.download(release, tmpdir) - # Verify yt_dlp was called with cookies + # Verify yt_dlp was called with cookies — but an *isolated copy*, + # not the canonical file, so its cookie write-back can't poison the + # keeper-managed cookies. call_args = mock_yt_dlp.call_args opts = call_args[0][0] - assert opts.get("cookiefile") == cookies_path + cookiefile_used = opts.get("cookiefile") + assert cookiefile_used + assert cookiefile_used != cookies_path def test_download_without_cookies(self): """Test download works without cookies.""" From cd0c02dff8e2d5d89dd061bce60bb2a2503468d0 Mon Sep 17 00:00:00 2001 From: Andrew Beveridge Date: Sat, 22 Aug 2026 19:41:26 -0400 Subject: [PATCH 3/6] refactor(youtube): fall back to canonical cookie file if temp copy fails Harden isolated_cookiefile: a mkstemp/copyfile failure (disk full, perms) now degrades to using the canonical cookie file for that single run instead of raising and failing the download. Losing write-back protection for one run is strictly better than failing outright. Co-Authored-By: Claude Opus 4.8 --- flacfetch/downloaders/youtube.py | 16 +++++++++++++++- tests/test_youtube_cookies.py | 14 ++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/flacfetch/downloaders/youtube.py b/flacfetch/downloaders/youtube.py index 4ad810b..242d36b 100644 --- a/flacfetch/downloaders/youtube.py +++ b/flacfetch/downloaders/youtube.py @@ -154,9 +154,23 @@ def isolated_cookiefile(cookies_file: Optional[str]): fd, tmp_path = tempfile.mkstemp(prefix="ytdlp-cookies-", suffix=".txt") os.close(fd) shutil.copyfile(cookies_file, tmp_path) + except OSError as e: + # Copy failed (disk full, perms, ...). Fall back to the canonical file so + # downloads still work — we lose write-back protection for this one run, + # which is strictly better than failing the download outright. + logger.warning(f"Could not create temp cookie copy, using canonical file: {e}") + if tmp_path and os.path.exists(tmp_path): + try: + os.unlink(tmp_path) + except OSError: + pass + yield cookies_file + return + + try: yield tmp_path finally: - if tmp_path and os.path.exists(tmp_path): + if os.path.exists(tmp_path): try: os.unlink(tmp_path) except OSError as e: diff --git a/tests/test_youtube_cookies.py b/tests/test_youtube_cookies.py index 52eeefc..99bc8c1 100644 --- a/tests/test_youtube_cookies.py +++ b/tests/test_youtube_cookies.py @@ -164,6 +164,20 @@ def test_yields_distinct_copy_then_cleans_up(self): # Temp copy is removed after the context exits. assert not os.path.exists(copy_path) + def test_falls_back_to_canonical_when_copy_fails(self): + """A copy failure must not break the download — fall back to the original.""" + with tempfile.TemporaryDirectory() as tmpdir: + canonical = os.path.join(tmpdir, "cookies.txt") + with open(canonical, "w") as f: + f.write("# cookies\n") + + with patch( + "flacfetch.downloaders.youtube.shutil.copyfile", + side_effect=OSError("disk full"), + ): + with isolated_cookiefile(canonical) as cf: + assert cf == canonical + def test_writeback_to_copy_does_not_touch_canonical(self): """The core guarantee: simulated yt-dlp write-back hits only the copy.""" with tempfile.TemporaryDirectory() as tmpdir: From c15d768917d2ca1bc9633b6cb5c69f63c06b1ce9 Mon Sep 17 00:00:00 2001 From: Andrew Beveridge Date: Sat, 22 Aug 2026 19:50:33 -0400 Subject: [PATCH 4/6] =?UTF-8?q?fix:=20address=20review=20=E2=80=94=20isola?= =?UTF-8?q?te=20health-check=20cookies,=20unblock=20PO=20server=20egress,?= =?UTF-8?q?=20pin=20plugin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - health_check._extract_youtube_info: wrap in ytdlp_opts_isolated so the periodic deep-health YouTube probe no longer writes rotated cookies back over the canonical keeper-managed file (same write-back bug as the download paths). - bgutil-pot.service: drop IPAddressDeny=any / IPAddressAllow=localhost. systemd IPAddress rules filter egress too, which blocked the PO server from reaching Google's BotGuard/WAA endpoints — every mint failed with getaddrinfo EAI_AGAIN. Verified live: minting works once egress is unblocked. Inbound is already restricted by the host nftables firewall (policy drop; no accept for :4416; iif lo accept), so the port stays localhost-only. - Pin bgutil plugin to ==BGUTIL_POT_VERSION (server version) and stop upgrading it in update-ytdlp.sh — mismatched plugin/server versions can break POT minting. Co-Authored-By: Claude Opus 4.8 --- deploy/provision.sh | 31 ++++++++++++++------------ flacfetch/api/services/health_check.py | 7 +++++- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/deploy/provision.sh b/deploy/provision.sh index df93a18..fdc801a 100644 --- a/deploy/provision.sh +++ b/deploy/provision.sh @@ -400,7 +400,11 @@ log "Stage 5b — yt-dlp PO Token provider (bgutil, HTTP server)" # below) that mints Proof-of-Origin tokens via YouTube's BotGuard challenge. The # plugin auto-detects the server on the default port, so no --extractor-args are # needed. Node.js is the server runtime (deno alone can't populate node_modules). -pip install --upgrade "bgutil-ytdlp-pot-provider" --quiet || warn "bgutil PO-token plugin install failed" +# Pin the plugin to the SERVER version — bgutil requires matching plugin/server +# versions; bump BGUTIL_POT_VERSION to upgrade both together (re-provision rebuilds +# the server + re-pins the plugin). update-ytdlp.sh deliberately does NOT bump it. +pip install --upgrade "bgutil-ytdlp-pot-provider==${BGUTIL_POT_VERSION}" --quiet \ + || warn "bgutil PO-token plugin install failed" if ! command -v node >/dev/null 2>&1; then log "installing Node.js (bgutil PO server runtime)" apt-get install -y nodejs npm >/dev/null 2>&1 \ @@ -667,9 +671,13 @@ RestartSec=5 WantedBy=multi-user.target XVFB_SERVICE -# yt-dlp PO Token provider — bgutil HTTP server (localhost only). -# The server binary has no host-bind flag (binds :: then 0.0.0.0), so we confine -# it to loopback with systemd's IPAddress firewall rather than a host firewall. +# yt-dlp PO Token provider — bgutil HTTP server. +# The server binary has no host-bind flag (binds :: then 0.0.0.0), but the host +# nftables firewall already has `policy drop` on input with no accept rule for +# $BGUTIL_POT_PORT, so it's unreachable externally while `iif lo accept` keeps it +# available to local yt-dlp. Do NOT use systemd IPAddressDeny/Allow here: those +# filter EGRESS too, and the server must reach Google's BotGuard/WAA endpoints to +# mint tokens (blocking egress makes every mint fail with getaddrinfo EAI_AGAIN). if [ -f "$BGUTIL_POT_DIR/server/build/main.js" ]; then cat > /etc/systemd/system/bgutil-pot.service <> /var/log/ytdlp-update.log 2>&1 echo "yt-dlp update started at $(date)" cd /opt/flacfetch && source venv/bin/activate OLD=$(python -c "import yt_dlp; print(yt_dlp.version.__version__)" 2>/dev/null || echo unknown) -POT_OLD=$(pip show bgutil-ytdlp-pot-provider 2>/dev/null | awk '/^Version:/{print $2}') -pip install --upgrade yt-dlp yt-dlp-ejs bgutil-ytdlp-pot-provider --quiet +# NOTE: the bgutil PO-token plugin is intentionally NOT upgraded here — it is +# pinned to the bgutil server version by provision.sh (mismatched plugin/server +# versions can break POT minting). Bump BGUTIL_POT_VERSION + re-provision instead. +pip install --upgrade yt-dlp yt-dlp-ejs --quiet NEW=$(python -c "import yt_dlp; print(yt_dlp.version.__version__)" 2>/dev/null || echo unknown) -POT_NEW=$(pip show bgutil-ytdlp-pot-provider 2>/dev/null | awk '/^Version:/{print $2}') -if [ "$OLD" != "$NEW" ] || [ "$POT_OLD" != "$POT_NEW" ]; then - echo "yt-dlp $OLD -> $NEW / pot-plugin $POT_OLD -> $POT_NEW" - systemctl restart flacfetch -fi +if [ "$OLD" != "$NEW" ]; then echo "yt-dlp $OLD -> $NEW"; systemctl restart flacfetch; fi command -v /opt/deno/bin/deno >/dev/null 2>&1 && /opt/deno/bin/deno upgrade --quiet 2>/dev/null || true echo "Update complete at $(date)" UPDATE_SCRIPT diff --git a/flacfetch/api/services/health_check.py b/flacfetch/api/services/health_check.py index 5ab1e34..60a7324 100644 --- a/flacfetch/api/services/health_check.py +++ b/flacfetch/api/services/health_check.py @@ -357,8 +357,13 @@ def _extract_youtube_info(self, ydl_opts: Dict[str, Any]) -> Optional[Dict[str, """Extract info from YouTube test video (runs in thread pool).""" import yt_dlp + # Use a throwaway cookie copy so this periodic health probe never writes + # yt-dlp's rotated (and, on a flagged IP, rejected) cookies back over the + # canonical keeper-managed file. See downloaders.youtube.isolated_cookiefile. + from ...downloaders.youtube import ytdlp_opts_isolated + try: - with yt_dlp.YoutubeDL(ydl_opts) as ydl: + with ytdlp_opts_isolated(ydl_opts) as opts, yt_dlp.YoutubeDL(opts) as ydl: return ydl.extract_info(YOUTUBE_TEST_VIDEO, download=False) except Exception as e: logger.warning(f"YouTube info extraction failed: {e}") From 6d526c65b76cbfe02ba2ea71d7c5c4a4d3bb3c3a Mon Sep 17 00:00:00 2001 From: Andrew Beveridge Date: Sat, 22 Aug 2026 19:52:52 -0400 Subject: [PATCH 5/6] chore: release 0.25.0 (YouTube cookie-isolation + PO token provider) Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 29 +++++++++++++++++++++++++++++ flacfetch/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2342750..37f106b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,35 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.25.0] - 2026-08-22 + +### Fixed +- YouTube downloads intermittently failing with "Sign in to confirm you're not a + bot" after the move to the netcup datacenter box. Root cause: yt-dlp saves the + rotated cookie jar back to the shared `cookiefile` after every run; on the + flagged datacenter IP YouTube *rejects* the rotation ("cookies no longer valid … + rotated in the browser as a security measure"), so the saved-back cookies are + invalid and poison the keeper-managed file for the next download. (On the old + GCE IP the rotation was accepted, so the write-back was harmless — which is why + it worked for months and only broke after migration.) Each yt-dlp run now + operates on a throwaway copy of the cookie file (`isolated_cookiefile` / + `ytdlp_opts_isolated`), so the canonical file — written only by the credential + keeper — stays pristine and concurrent downloads no longer clobber each other. + Applied to the download, availability-check, search, credential-check, and + deep-health paths. +- yt-dlp cache was silently disabled on the server: it runs with + `HOME=/opt/flacfetch` where `~/.cache` is the Spotify OAuth token *file*, so + yt-dlp's default `~/.cache/yt-dlp` path died with `NotADirectoryError` (forcing + a fresh JS-challenge solve every request). A new `FLACFETCH_YTDLP_CACHE_DIR` + override gives yt-dlp its own collision-free cache directory. + +### Added +- Proof-of-Origin (PO) token provider for YouTube. YouTube now binds a GVS PO + token to downloads from datacenter IPs; flacfetch now runs the + `bgutil-ytdlp-pot-provider` HTTP server (Node, localhost) plus the matching + yt-dlp plugin, provisioned and kept current by `deploy/provision.sh`. The host + nftables firewall keeps the token server localhost-only. + ## [0.23.0] - 2026-08-14 ### Fixed diff --git a/flacfetch/__init__.py b/flacfetch/__init__.py index 4f20234..b06eb30 100644 --- a/flacfetch/__init__.py +++ b/flacfetch/__init__.py @@ -1,6 +1,6 @@ """flacfetch - Search and download high-quality audio from multiple sources.""" -__version__ = "0.24.0" +__version__ = "0.25.0" __author__ = "Andrew Beveridge" __email__ = "andrew@beveridge.uk" diff --git a/pyproject.toml b/pyproject.toml index bb29935..bd78953 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "flacfetch" -version = "0.24.0" +version = "0.25.0" description = "Search and download high-quality audio from multiple sources" readme = "README.md" requires-python = ">=3.10,<4.0" From b8d3dabc498124f0bc3b2221d96dadaccc7dcbdd Mon Sep 17 00:00:00 2001 From: Andrew Beveridge Date: Sat, 22 Aug 2026 20:00:08 -0400 Subject: [PATCH 6/6] =?UTF-8?q?fix:=20address=20CodeRabbit=20=E2=80=94=20c?= =?UTF-8?q?achedir=20for=20inline=20callers=20+=20abort=20build=20on=20fai?= =?UTF-8?q?led=20checkout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ytdlp_opts_isolated now also injects the dedicated cachedir (when configured and not already set), so credential_check + health_check — which build ydl_opts inline and bypass get_ytdlp_base_opts — no longer hit HOME/.cache's NotADirectoryError. Centralizes both cookie isolation and cachedir in the single wrapper all yt-dlp calls go through. - provision.sh: only (re)build the bgutil server after a verified fetch+checkout of the pinned tag; on failure keep the existing build+plugin pair instead of compiling an unverified/mismatched revision. Co-Authored-By: Claude Opus 4.8 --- deploy/provision.sh | 17 +++++++---- flacfetch/downloaders/youtube.py | 24 +++++++++++----- tests/test_youtube_cookies.py | 48 ++++++++++++++++++++++---------- 3 files changed, 62 insertions(+), 27 deletions(-) diff --git a/deploy/provision.sh b/deploy/provision.sh index fdc801a..5235c52 100644 --- a/deploy/provision.sh +++ b/deploy/provision.sh @@ -417,13 +417,18 @@ if command -v node >/dev/null 2>&1; then fi if [ -d "$BGUTIL_POT_DIR/.git" ]; then git config --global --add safe.directory "$BGUTIL_POT_DIR" - git -C "$BGUTIL_POT_DIR" fetch --tags --quiet || warn "bgutil fetch failed" - git -C "$BGUTIL_POT_DIR" checkout --quiet "$BGUTIL_POT_VERSION" \ - || warn "bgutil checkout $BGUTIL_POT_VERSION failed" - if ( cd "$BGUTIL_POT_DIR/server" && npm ci --silent && npx --yes tsc ) >/dev/null 2>&1; then - log "bgutil PO server built ($BGUTIL_POT_VERSION)" + # Only (re)build once we've verifiably checked out the pinned version. If the + # fetch/checkout fails (e.g. transient network), keep the existing build+plugin + # pair rather than compiling an unverified/mismatched revision. + if git -C "$BGUTIL_POT_DIR" fetch --tags --quiet \ + && git -C "$BGUTIL_POT_DIR" checkout --quiet "$BGUTIL_POT_VERSION"; then + if ( cd "$BGUTIL_POT_DIR/server" && npm ci --silent && npx --yes tsc ) >/dev/null 2>&1; then + log "bgutil PO server built ($BGUTIL_POT_VERSION)" + else + warn "bgutil PO server build failed (PO token provider will be unavailable)" + fi else - warn "bgutil PO server build failed (PO token provider will be unavailable)" + warn "bgutil fetch/checkout to $BGUTIL_POT_VERSION failed — keeping existing server build" fi fi else diff --git a/flacfetch/downloaders/youtube.py b/flacfetch/downloaders/youtube.py index 242d36b..dfe0599 100644 --- a/flacfetch/downloaders/youtube.py +++ b/flacfetch/downloaders/youtube.py @@ -179,18 +179,28 @@ def isolated_cookiefile(cookies_file: Optional[str]): @contextmanager def ytdlp_opts_isolated(ydl_opts: dict): - """Run yt-dlp with a throwaway copy of the cookie file (see isolated_cookiefile). + """Single chokepoint for every yt-dlp invocation: isolate cookies + set cachedir. - Swaps ``cookiefile`` in a shallow copy of ``ydl_opts`` for a per-run temp copy - so the canonical keeper-managed cookies are never written back to. A no-op when - no ``cookiefile`` is configured. + - Swaps ``cookiefile`` for a per-run throwaway copy so the canonical + keeper-managed cookies are never written back to (see isolated_cookiefile). + - Ensures the dedicated ``cachedir`` is set (see get_ytdlp_cache_dir) so even + callers that build ``ydl_opts`` inline (credential / health checks, which + don't go through get_ytdlp_base_opts) avoid the HOME/.cache collision. + + Both are applied to a shallow copy; the caller's dict is left untouched. A + no-op only when there's no cookiefile and no cache dir override configured. """ + extra: dict = {} + + cache_dir = get_ytdlp_cache_dir() + if cache_dir and not ydl_opts.get("cachedir"): + extra["cachedir"] = cache_dir + original = ydl_opts.get("cookiefile") with isolated_cookiefile(original) as cf: if original: - yield {**ydl_opts, "cookiefile": cf} - else: - yield ydl_opts + extra["cookiefile"] = cf + yield {**ydl_opts, **extra} if extra else ydl_opts @dataclass diff --git a/tests/test_youtube_cookies.py b/tests/test_youtube_cookies.py index 99bc8c1..c9f38a0 100644 --- a/tests/test_youtube_cookies.py +++ b/tests/test_youtube_cookies.py @@ -197,24 +197,44 @@ def test_writeback_to_copy_does_not_touch_canonical(self): class TestYtdlpOptsIsolated: """Tests for ytdlp_opts_isolated wrapper.""" - def test_noop_without_cookiefile(self): - opts = {"quiet": True} - with ytdlp_opts_isolated(opts) as isolated: - assert isolated is opts + def test_noop_without_cookiefile_or_cache(self): + with patch.dict(os.environ, {}, clear=True): + opts = {"quiet": True} + with ytdlp_opts_isolated(opts) as isolated: + assert isolated is opts def test_swaps_cookiefile_to_copy_and_preserves_original_dict(self): + with patch.dict(os.environ, {}, clear=True): + with tempfile.TemporaryDirectory() as tmpdir: + canonical = os.path.join(tmpdir, "cookies.txt") + with open(canonical, "w") as f: + f.write("# cookies\n") + + opts = {"cookiefile": canonical, "quiet": True} + with ytdlp_opts_isolated(opts) as isolated: + assert isolated["cookiefile"] != canonical + assert os.path.exists(isolated["cookiefile"]) + assert isolated["quiet"] is True + # Original dict is left untouched (shallow copy semantics). + assert opts["cookiefile"] == canonical + + def test_injects_cachedir_for_inline_opts(self): + """Inline callers (credential/health checks) get the cache dir too.""" with tempfile.TemporaryDirectory() as tmpdir: - canonical = os.path.join(tmpdir, "cookies.txt") - with open(canonical, "w") as f: - f.write("# cookies\n") + cache_dir = os.path.join(tmpdir, "ytdlp-cache") + with patch.dict(os.environ, {"FLACFETCH_YTDLP_CACHE_DIR": cache_dir}): + opts = {"quiet": True} + with ytdlp_opts_isolated(opts) as isolated: + assert isolated["cachedir"] == cache_dir + assert "cachedir" not in opts # original untouched - opts = {"cookiefile": canonical, "quiet": True} - with ytdlp_opts_isolated(opts) as isolated: - assert isolated["cookiefile"] != canonical - assert os.path.exists(isolated["cookiefile"]) - assert isolated["quiet"] is True - # Original dict is left untouched (shallow copy semantics). - assert opts["cookiefile"] == canonical + def test_does_not_override_existing_cachedir(self): + with tempfile.TemporaryDirectory() as tmpdir: + env_cache = os.path.join(tmpdir, "env-cache") + with patch.dict(os.environ, {"FLACFETCH_YTDLP_CACHE_DIR": env_cache}): + opts = {"cachedir": "/preset/cache"} + with ytdlp_opts_isolated(opts) as isolated: + assert isolated["cachedir"] == "/preset/cache" class TestYoutubeDownloaderCookies: