diff --git a/CHANGELOG.md b/CHANGELOG.md index 583b016..17d153c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,26 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.28.1] - 2026-08-30 + +### Fixed +- **Freeleech tokens were never spent when the `.torrent` was already cached.** + v0.28.0 only sent `usetoken=1` on a `.torrent` cache *miss*, but the local + `.torrent` cache is unrelated to freeleech: a token is registered server-side + by calling `action=download&id=X&usetoken=1`, and the audio *data* is + re-downloaded fresh each time (the cache only stores the tiny `.torrent` + file, not the data). In practice every karaoke-gen download is a `.torrent` + cache hit via `/download-by-id`, so **zero tokens were ever spent**. + - The token is now registered **before** the cache short-circuit, so eligible + downloads spend a token even when the `.torrent` is cached; the cached + `.torrent` is still reused for the actual data fetch. + - **Local 7-day token ledger** (`.tokened` marker files) prevents spending + a second token on a torrent that's already personal-freeleech (matches RED's + 7-day freeleech window), including a lock so concurrent album-batch downloads + of the same torrent can't each spend one. + - Token spends are paced ≥ 1s apart (RED rejects faster spends); pacing only + trails successful spends, so an exhausted-token steady state adds no delay. + ## [0.28.0] - 2026-08-29 ### Added diff --git a/flacfetch/providers/gazelle.py b/flacfetch/providers/gazelle.py index df3c567..27fc894 100644 --- a/flacfetch/providers/gazelle.py +++ b/flacfetch/providers/gazelle.py @@ -1,6 +1,7 @@ """Base provider for Gazelle-based private music trackers (RED, OPS, etc.).""" import html +import threading import time from abc import abstractmethod from pathlib import Path @@ -40,6 +41,11 @@ class GazelleProvider(Provider): # See the RED wiki: a token makes the whole torrent personal-freeleech. FL_TOKEN_MAX_BYTES = 5 * 1024 ** 3 + # Spending a token makes a torrent personal-freeleech for 7 days. Within that + # window a re-download is already free, so we must NOT spend another token on + # the same torrent. We track spends locally with this TTL. + FL_TOKEN_FREELEECH_WINDOW_SECONDS = 7 * 24 * 3600 + def __init__(self, api_key: str, base_url: str, cache_subdir: str, use_fl_token: bool = False): """Initialize the Gazelle provider. @@ -57,6 +63,15 @@ def __init__(self, api_key: str, base_url: str, cache_subdir: str, use_fl_token: self.api_key = api_key self.base_url = base_url.rstrip('/') self.use_fl_token = use_fl_token + # Serialize token spends so concurrent downloads of the same torrent + # (common in album batches) can't each spend a token on it. + self._token_lock = threading.Lock() + # RED requires >= 1s between token spends; pace them (guarded by the lock). + self._last_token_spend_time = 0.0 + # In-memory {torrent_id: spend_time} ledger, a fallback for when the + # on-disk marker can't be written (no cache dir / disk error) so we still + # don't double-spend on the same torrent within the freeleech window. + self._token_spends: dict[str, float] = {} self.session = requests.Session() self.session.headers.update({"Authorization": self.api_key}) self.search_limit = 10 @@ -320,6 +335,78 @@ def _fetch_torrent_from_url(self, url: str, torrent_id: Optional[str] = None, ma return None + def _token_marker_path(self, torrent_id: str) -> Optional[Path]: + """Path of the local marker recording a token spend for a torrent.""" + if not self.cache_dir or not torrent_id: + return None + return self.cache_dir / f"{torrent_id}.tokened" + + def _token_recently_spent(self, torrent_id: str) -> bool: + """True if we spent a token on this torrent within the freeleech window. + + Personal-freeleech lasts 7 days, so a re-download inside that window is + already free -- spending another token would be wasteful. Checks both the + in-memory ledger and the on-disk marker so it survives a missing/unwritable + cache dir (in-memory) and process restarts (on-disk). + """ + if not torrent_id: + return False + now = time.time() + + ts = self._token_spends.get(torrent_id) + if ts is not None and now - ts < self.FL_TOKEN_FREELEECH_WINDOW_SECONDS: + return True + + marker = self._token_marker_path(torrent_id) + if marker and marker.exists(): + try: + if now - marker.stat().st_mtime < self.FL_TOKEN_FREELEECH_WINDOW_SECONDS: + return True + except OSError: + pass + return False + + def _record_token_spend(self, torrent_id: str) -> None: + """Record that a token was spent (or the torrent is already free) for a torrent. + + Records in memory always (double-spend guard even without a cache dir) and + on disk when possible (survives restarts). + """ + if not torrent_id: + return + self._token_spends[torrent_id] = time.time() + marker = self._token_marker_path(torrent_id) + if not marker: + return + try: + marker.touch() + except OSError as e: + logger.warning(f"{self.name}: could not write token marker for {torrent_id}: {e}") + + def _spend_token(self, base_url: str, torrent_id: Optional[str]) -> Optional[bytes]: + """Fetch the .torrent via ``usetoken=1`` to register personal freeleech. + + Returns the torrent bytes on success (token spent, or the torrent was + already free and the server just returned it), or None if the token + could not be spent (too large, none left, error) so the caller can fall + back to a normal download. + """ + # RED rejects tokens spent < 1s apart. Callers hold self._token_lock, so + # pacing here safely serializes spends across concurrent downloads. Only + # paced against the last *successful* spend, so once tokens are exhausted + # (all attempts fail) we don't add a delay to every download. + elapsed = time.time() - self._last_token_spend_time + if elapsed < 1.1: + time.sleep(1.1 - elapsed) + + sep = "&" if "?" in base_url else "?" + token_url = f"{base_url}{sep}usetoken=1" + logger.info(f"{self.name}: attempting Freeleech token download for torrent {torrent_id}") + content = self._fetch_torrent_from_url(token_url, torrent_id) + if content: + self._last_token_spend_time = time.time() + return content + def _fetch_with_optional_token(self, base_url: str, torrent_id: Optional[str], use_token: bool) -> Optional[bytes]: """Fetch a torrent, optionally spending a Freeleech token. @@ -328,6 +415,11 @@ def _fetch_with_optional_token(self, base_url: str, torrent_id: Optional[str], u error), transparently fall back to a normal (ratio-counted) download so a failed token spend never blocks the download. + NOTE: this does not consult the local ``.torrent`` cache -- callers that + want cache reuse (e.g. ``fetch_artifact_by_id``) handle that themselves, + because a cached ``.torrent`` file does NOT mean the download is + freeleech (the token must be registered server-side via ``usetoken=1``). + Args: base_url: Download URL without the ``usetoken`` parameter torrent_id: Torrent ID for caching/logging (may be None) @@ -336,15 +428,18 @@ def _fetch_with_optional_token(self, base_url: str, torrent_id: Optional[str], u Returns: Torrent file contents as bytes, or None on failure """ - if use_token: - sep = "&" if "?" in base_url else "?" - token_url = f"{base_url}{sep}usetoken=1" - logger.info(f"{self.name}: attempting Freeleech token download for torrent {torrent_id}") - content = self._fetch_torrent_from_url(token_url, torrent_id) - if content: - logger.info(f"{self.name}: spent a Freeleech token on torrent {torrent_id}") - return content - logger.warning(f"{self.name}: Freeleech token download failed for torrent {torrent_id}; retrying without token") + if use_token and not (torrent_id and self._token_recently_spent(torrent_id)): + # Serialize so concurrent requests for the same torrent (album + # batches fire many at once) can't each spend a token on it. + with self._token_lock: + if not (torrent_id and self._token_recently_spent(torrent_id)): + content = self._spend_token(base_url, torrent_id) + if content: + if torrent_id: + self._record_token_spend(torrent_id) + logger.info(f"{self.name}: spent a Freeleech token on torrent {torrent_id}") + return content + logger.warning(f"{self.name}: Freeleech token download failed for torrent {torrent_id}; falling back to normal download") return self._fetch_torrent_from_url(base_url, torrent_id) @@ -396,8 +491,25 @@ def fetch_artifact_by_id(self, source_id: str, use_token: Optional[bool] = None) if use_token is None: use_token = self.use_fl_token - # Check Cache first (a cached torrent means we already have it -- never - # spend a token to re-fetch something on disk). + url = f"{self.base_url}/ajax.php?action=download&id={source_id}" + + # If we should spend a token and haven't already tokened this torrent + # within the freeleech window, register the token FIRST -- before the + # cache short-circuit. A cached .torrent file does NOT make a download + # freeleech; the token must be registered server-side via usetoken=1, + # otherwise the (re-)download of the torrent data counts against ratio. + if use_token and not self._token_recently_spent(source_id): + with self._token_lock: + if not self._token_recently_spent(source_id): + content = self._spend_token(url, source_id) + if content: + self._record_token_spend(source_id) + logger.info(f"{self.name}: spent a Freeleech token on torrent {source_id}") + return content + logger.warning(f"{self.name}: Freeleech token could not be spent on torrent {source_id}; using cache/normal download") + + # Serve the cached .torrent if present (token, if any, is already + # registered above; a cached torrent is fine for the actual data fetch). if self.cache_dir: cache_path = self.cache_dir / f"{source_id}.torrent" if cache_path.exists(): @@ -410,9 +522,8 @@ def fetch_artifact_by_id(self, source_id: str, use_token: Optional[bool] = None) except Exception as e: logger.warning(f"Error reading from cache: {e}") - # Construct download URL from torrent ID - url = f"{self.base_url}/ajax.php?action=download&id={source_id}" - return self._fetch_with_optional_token(url, source_id, use_token) + # No token spent and no cache: fetch normally. + return self._fetch_torrent_from_url(url, source_id) def fetch_artifact(self, release: Release) -> Optional[bytes]: """Fetch .torrent file for a release. @@ -439,7 +550,7 @@ def fetch_artifact(self, release: Release) -> Optional[bytes]: except IndexError: pass - # Check Cache first (reuse fetch_artifact_by_id if we have a torrent_id) + # Delegate to the by-id path (handles token spend, cache reuse, fallback). if torrent_id: return self.fetch_artifact_by_id(torrent_id, use_token=use_token) diff --git a/pyproject.toml b/pyproject.toml index 0a6ee8e..2a22f46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "flacfetch" -version = "0.28.0" +version = "0.28.1" description = "Search and download high-quality audio from multiple sources" readme = "README.md" requires-python = ">=3.10,<4.0" diff --git a/tests/test_gazelle.py b/tests/test_gazelle.py index cde389e..420ab6a 100644 --- a/tests/test_gazelle.py +++ b/tests/test_gazelle.py @@ -480,23 +480,28 @@ class TestFreeleechTokens: """Test Freeleech (FL) token spending behavior.""" @pytest.fixture - def provider(self): - """Token-enabled concrete GazelleProvider.""" - return ConcreteGazelleProvider( + def provider(self, tmp_path): + """Token-enabled concrete GazelleProvider with an isolated cache dir.""" + p = ConcreteGazelleProvider( api_key="test", base_url="https://test.example", cache_subdir="test", use_fl_token=True, ) + # Isolate the token ledger / .torrent cache per test. + p.cache_dir = tmp_path + return p @pytest.fixture - def no_token_provider(self): - """Default provider (token spending off).""" - return ConcreteGazelleProvider( + def no_token_provider(self, tmp_path): + """Default provider (token spending off) with an isolated cache dir.""" + p = ConcreteGazelleProvider( api_key="test", base_url="https://test.example", cache_subdir="test", ) + p.cache_dir = tmp_path + return p def _release(self, **kwargs): from flacfetch.core.models import AudioFormat, Quality, Release @@ -621,3 +626,83 @@ def test_valid_bencoded_torrent_accepted(self, provider): provider.session.get = MagicMock(return_value=resp) result = provider._fetch_torrent_from_url("https://test.example/ajax.php?action=download&id=42", "42") assert result == resp.content + + # ---- token spend must NOT be skipped by a cached .torrent ------------- + # (regression: a cached .torrent file does not make a download freeleech) + + def test_token_spent_even_when_torrent_cached(self, provider): + # Pre-seed a cached .torrent for id 42. + (provider.cache_dir / "42.torrent").write_bytes(b"d8:announce...cached") + provider._fetch_torrent_from_url = MagicMock(return_value=b"d8:announce...fresh") + + result = provider.fetch_artifact_by_id("42") + + # Must have hit the download endpoint with usetoken=1 rather than + # short-circuiting on the cache. + assert provider._fetch_torrent_from_url.called + called_url = provider._fetch_torrent_from_url.call_args[0][0] + assert "usetoken=1" in called_url + assert result == b"d8:announce...fresh" + # A ledger marker should now exist so we don't re-spend. + assert provider._token_recently_spent("42") + + def test_no_respend_within_freeleech_window(self, provider): + # A recent token marker means the torrent is already personal-freeleech. + (provider.cache_dir / "77.tokened").touch() + (provider.cache_dir / "77.torrent").write_bytes(b"d8:announce...cached") + provider._fetch_torrent_from_url = MagicMock(return_value=b"should-not-be-called") + + result = provider.fetch_artifact_by_id("77") + + # No usetoken call; served from cache instead. + assert not provider._fetch_torrent_from_url.called + assert result == b"d8:announce...cached" + + def test_token_marker_respects_ttl(self, provider): + import os, time as _t + marker = provider.cache_dir / "88.tokened" + marker.touch() + # Backdate the marker beyond the 7-day window. + old = _t.time() - provider.FL_TOKEN_FREELEECH_WINDOW_SECONDS - 3600 + os.utime(marker, (old, old)) + assert provider._token_recently_spent("88") is False + + def test_failed_token_falls_back_to_cache(self, provider): + (provider.cache_dir / "99.torrent").write_bytes(b"d8:announce...cached") + # Token attempt fails (e.g. >5GB / none left) -> None. + provider._fetch_torrent_from_url = MagicMock(return_value=None) + + result = provider.fetch_artifact_by_id("99") + + # Attempted the token (usetoken=1) once, then fell back to cache. + assert provider._fetch_torrent_from_url.call_count == 1 + assert "usetoken=1" in provider._fetch_torrent_from_url.call_args[0][0] + assert result == b"d8:announce...cached" + # No marker recorded on failure -> a later call may retry. + assert not provider._token_recently_spent("99") + + def test_in_memory_ledger_prevents_respend_without_cache_dir(self, provider): + # No cache dir -> on-disk marker impossible; in-memory ledger must still + # prevent a second token spend on the same torrent. + provider.cache_dir = None + provider._fetch_torrent_from_url = MagicMock(return_value=b"d8:announce...fresh") + + first = provider.fetch_artifact_by_id("55") + assert first == b"d8:announce...fresh" + assert provider._fetch_torrent_from_url.call_count == 1 + assert provider._token_recently_spent("55") is True + + # Second call: no marker on disk, but in-memory ledger blocks re-spend. + # With no cache and token skipped, it does a single normal (no-token) fetch. + provider._fetch_torrent_from_url.reset_mock() + second = provider.fetch_artifact_by_id("55") + assert second == b"d8:announce...fresh" + assert provider._fetch_torrent_from_url.call_count == 1 + assert "usetoken=1" not in provider._fetch_torrent_from_url.call_args[0][0] + + def test_no_token_serves_cache_without_endpoint_call(self, no_token_provider): + (no_token_provider.cache_dir / "42.torrent").write_bytes(b"d8:announce...cached") + no_token_provider._fetch_torrent_from_url = MagicMock(return_value=b"fresh") + result = no_token_provider.fetch_artifact_by_id("42") + assert not no_token_provider._fetch_torrent_from_url.called + assert result == b"d8:announce...cached"