fix: spend FL tokens even when .torrent is cached (v0.28.1)

CHANGELOG.md

@@ -5,6 +5,26 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.28.1] - 2026-08-30
+
+### Fixed
+- **Freeleech tokens were never spent when the `.torrent` was already cached.**
+  v0.28.0 only sent `usetoken=1` on a `.torrent` cache *miss*, but the local
+  `.torrent` cache is unrelated to freeleech: a token is registered server-side
+  by calling `action=download&id=X&usetoken=1`, and the audio *data* is
+  re-downloaded fresh each time (the cache only stores the tiny `.torrent`
+  file, not the data). In practice every karaoke-gen download is a `.torrent`
+  cache hit via `/download-by-id`, so **zero tokens were ever spent**.
+  - The token is now registered **before** the cache short-circuit, so eligible
+    downloads spend a token even when the `.torrent` is cached; the cached
+    `.torrent` is still reused for the actual data fetch.
+  - **Local 7-day token ledger** (`<id>.tokened` marker files) prevents spending
+    a second token on a torrent that's already personal-freeleech (matches RED's
+    7-day freeleech window), including a lock so concurrent album-batch downloads
+    of the same torrent can't each spend one.
+  - Token spends are paced ≥ 1s apart (RED rejects faster spends); pacing only
+    trails successful spends, so an exhausted-token steady state adds no delay.
+
 ## [0.28.0] - 2026-08-29
 
 ### Added

flacfetch/providers/gazelle.py

@@ -1,6 +1,7 @@
 """Base provider for Gazelle-based private music trackers (RED, OPS, etc.)."""
 
 import html
+import threading
 import time
 from abc import abstractmethod
 from pathlib import Path
@@ -40,6 +41,11 @@ class GazelleProvider(Provider):
     # See the RED wiki: a token makes the whole torrent personal-freeleech.
     FL_TOKEN_MAX_BYTES = 5 * 1024 ** 3
 
+    # Spending a token makes a torrent personal-freeleech for 7 days. Within that
+    # window a re-download is already free, so we must NOT spend another token on
+    # the same torrent. We track spends locally with this TTL.
+    FL_TOKEN_FREELEECH_WINDOW_SECONDS = 7 * 24 * 3600
+
     def __init__(self, api_key: str, base_url: str, cache_subdir: str, use_fl_token: bool = False):
         """Initialize the Gazelle provider.
 
@@ -57,6 +63,15 @@ def __init__(self, api_key: str, base_url: str, cache_subdir: str, use_fl_token:
         self.api_key = api_key
         self.base_url = base_url.rstrip('/')
         self.use_fl_token = use_fl_token
+        # Serialize token spends so concurrent downloads of the same torrent
+        # (common in album batches) can't each spend a token on it.
+        self._token_lock = threading.Lock()
+        # RED requires >= 1s between token spends; pace them (guarded by the lock).
+        self._last_token_spend_time = 0.0
+        # In-memory {torrent_id: spend_time} ledger, a fallback for when the
+        # on-disk marker can't be written (no cache dir / disk error) so we still
+        # don't double-spend on the same torrent within the freeleech window.
+        self._token_spends: dict[str, float] = {}
         self.session = requests.Session()
         self.session.headers.update({"Authorization": self.api_key})
         self.search_limit = 10
@@ -320,6 +335,78 @@ def _fetch_torrent_from_url(self, url: str, torrent_id: Optional[str] = None, ma
 
         return None
 
+    def _token_marker_path(self, torrent_id: str) -> Optional[Path]:
+        """Path of the local marker recording a token spend for a torrent."""
+        if not self.cache_dir or not torrent_id:
+            return None
+        return self.cache_dir / f"{torrent_id}.tokened"
+
+    def _token_recently_spent(self, torrent_id: str) -> bool:
+        """True if we spent a token on this torrent within the freeleech window.
+
+        Personal-freeleech lasts 7 days, so a re-download inside that window is
+        already free -- spending another token would be wasteful. Checks both the
+        in-memory ledger and the on-disk marker so it survives a missing/unwritable
+        cache dir (in-memory) and process restarts (on-disk).
+        """
+        if not torrent_id:
+            return False
+        now = time.time()
+
+        ts = self._token_spends.get(torrent_id)
+        if ts is not None and now - ts < self.FL_TOKEN_FREELEECH_WINDOW_SECONDS:
+            return True
+
+        marker = self._token_marker_path(torrent_id)
+        if marker and marker.exists():
+            try:
+                if now - marker.stat().st_mtime < self.FL_TOKEN_FREELEECH_WINDOW_SECONDS:
+                    return True
+            except OSError:
+                pass
+        return False
+
+    def _record_token_spend(self, torrent_id: str) -> None:
+        """Record that a token was spent (or the torrent is already free) for a torrent.
+
+        Records in memory always (double-spend guard even without a cache dir) and
+        on disk when possible (survives restarts).
+        """
+        if not torrent_id:
+            return
+        self._token_spends[torrent_id] = time.time()
+        marker = self._token_marker_path(torrent_id)
+        if not marker:
+            return
+        try:
+            marker.touch()
+        except OSError as e:
+            logger.warning(f"{self.name}: could not write token marker for {torrent_id}: {e}")
+
+    def _spend_token(self, base_url: str, torrent_id: Optional[str]) -> Optional[bytes]:
+        """Fetch the .torrent via ``usetoken=1`` to register personal freeleech.
+
+        Returns the torrent bytes on success (token spent, or the torrent was
+        already free and the server just returned it), or None if the token
+        could not be spent (too large, none left, error) so the caller can fall
+        back to a normal download.
+        """
+        # RED rejects tokens spent < 1s apart. Callers hold self._token_lock, so
+        # pacing here safely serializes spends across concurrent downloads. Only
+        # paced against the last *successful* spend, so once tokens are exhausted
+        # (all attempts fail) we don't add a delay to every download.
+        elapsed = time.time() - self._last_token_spend_time
+        if elapsed < 1.1:
+            time.sleep(1.1 - elapsed)
+
+        sep = "&" if "?" in base_url else "?"
+        token_url = f"{base_url}{sep}usetoken=1"
+        logger.info(f"{self.name}: attempting Freeleech token download for torrent {torrent_id}")
+        content = self._fetch_torrent_from_url(token_url, torrent_id)
+        if content:
+            self._last_token_spend_time = time.time()
+        return content
+
     def _fetch_with_optional_token(self, base_url: str, torrent_id: Optional[str], use_token: bool) -> Optional[bytes]:
         """Fetch a torrent, optionally spending a Freeleech token.
 
@@ -328,6 +415,11 @@ def _fetch_with_optional_token(self, base_url: str, torrent_id: Optional[str], u
         error), transparently fall back to a normal (ratio-counted) download so a
         failed token spend never blocks the download.
 
+        NOTE: this does not consult the local ``.torrent`` cache -- callers that
+        want cache reuse (e.g. ``fetch_artifact_by_id``) handle that themselves,
+        because a cached ``.torrent`` file does NOT mean the download is
+        freeleech (the token must be registered server-side via ``usetoken=1``).
+
         Args:
             base_url: Download URL without the ``usetoken`` parameter
             torrent_id: Torrent ID for caching/logging (may be None)
@@ -336,15 +428,18 @@ def _fetch_with_optional_token(self, base_url: str, torrent_id: Optional[str], u
         Returns:
             Torrent file contents as bytes, or None on failure
         """
-        if use_token:
-            sep = "&" if "?" in base_url else "?"
-            token_url = f"{base_url}{sep}usetoken=1"
-            logger.info(f"{self.name}: attempting Freeleech token download for torrent {torrent_id}")
-            content = self._fetch_torrent_from_url(token_url, torrent_id)
-            if content:
-                logger.info(f"{self.name}: spent a Freeleech token on torrent {torrent_id}")
-                return content
-            logger.warning(f"{self.name}: Freeleech token download failed for torrent {torrent_id}; retrying without token")
+        if use_token and not (torrent_id and self._token_recently_spent(torrent_id)):
+            # Serialize so concurrent requests for the same torrent (album
+            # batches fire many at once) can't each spend a token on it.
+            with self._token_lock:
+                if not (torrent_id and self._token_recently_spent(torrent_id)):
+                    content = self._spend_token(base_url, torrent_id)
+                    if content:
+                        if torrent_id:
+                            self._record_token_spend(torrent_id)
+                        logger.info(f"{self.name}: spent a Freeleech token on torrent {torrent_id}")
+                        return content
+                    logger.warning(f"{self.name}: Freeleech token download failed for torrent {torrent_id}; falling back to normal download")
 
         return self._fetch_torrent_from_url(base_url, torrent_id)
 
@@ -396,8 +491,25 @@ def fetch_artifact_by_id(self, source_id: str, use_token: Optional[bool] = None)
         if use_token is None:
             use_token = self.use_fl_token
 
-        # Check Cache first (a cached torrent means we already have it -- never
-        # spend a token to re-fetch something on disk).
+        url = f"{self.base_url}/ajax.php?action=download&id={source_id}"
+
+        # If we should spend a token and haven't already tokened this torrent
+        # within the freeleech window, register the token FIRST -- before the
+        # cache short-circuit. A cached .torrent file does NOT make a download
+        # freeleech; the token must be registered server-side via usetoken=1,
+        # otherwise the (re-)download of the torrent data counts against ratio.
+        if use_token and not self._token_recently_spent(source_id):
+            with self._token_lock:
+                if not self._token_recently_spent(source_id):
+                    content = self._spend_token(url, source_id)
+                    if content:
+                        self._record_token_spend(source_id)
+                        logger.info(f"{self.name}: spent a Freeleech token on torrent {source_id}")
+                        return content
+                    logger.warning(f"{self.name}: Freeleech token could not be spent on torrent {source_id}; using cache/normal download")
+
+        # Serve the cached .torrent if present (token, if any, is already
+        # registered above; a cached torrent is fine for the actual data fetch).
         if self.cache_dir:
             cache_path = self.cache_dir / f"{source_id}.torrent"
             if cache_path.exists():
@@ -410,9 +522,8 @@ def fetch_artifact_by_id(self, source_id: str, use_token: Optional[bool] = None)
                 except Exception as e:
                     logger.warning(f"Error reading from cache: {e}")
 
-        # Construct download URL from torrent ID
-        url = f"{self.base_url}/ajax.php?action=download&id={source_id}"
-        return self._fetch_with_optional_token(url, source_id, use_token)
+        # No token spent and no cache: fetch normally.
+        return self._fetch_torrent_from_url(url, source_id)
 
     def fetch_artifact(self, release: Release) -> Optional[bytes]:
         """Fetch .torrent file for a release.
@@ -439,7 +550,7 @@ def fetch_artifact(self, release: Release) -> Optional[bytes]:
         except IndexError:
             pass
 
-        # Check Cache first (reuse fetch_artifact_by_id if we have a torrent_id)
+        # Delegate to the by-id path (handles token spend, cache reuse, fallback).
         if torrent_id:
             return self.fetch_artifact_by_id(torrent_id, use_token=use_token)
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "flacfetch"
-version = "0.28.0"
+version = "0.28.1"
 description = "Search and download high-quality audio from multiple sources"
 readme = "README.md"
 requires-python = ">=3.10,<4.0"

tests/test_gazelle.py

@@ -480,23 +480,28 @@ class TestFreeleechTokens:
     """Test Freeleech (FL) token spending behavior."""
 
     @pytest.fixture
-    def provider(self):
-        """Token-enabled concrete GazelleProvider."""
-        return ConcreteGazelleProvider(
+    def provider(self, tmp_path):
+        """Token-enabled concrete GazelleProvider with an isolated cache dir."""
+        p = ConcreteGazelleProvider(
             api_key="test",
             base_url="https://test.example",
             cache_subdir="test",
             use_fl_token=True,
         )
+        # Isolate the token ledger / .torrent cache per test.
+        p.cache_dir = tmp_path
+        return p
 
     @pytest.fixture
-    def no_token_provider(self):
-        """Default provider (token spending off)."""
-        return ConcreteGazelleProvider(
+    def no_token_provider(self, tmp_path):
+        """Default provider (token spending off) with an isolated cache dir."""
+        p = ConcreteGazelleProvider(
             api_key="test",
             base_url="https://test.example",
             cache_subdir="test",
         )
+        p.cache_dir = tmp_path
+        return p
 
     def _release(self, **kwargs):
         from flacfetch.core.models import AudioFormat, Quality, Release
@@ -621,3 +626,83 @@ def test_valid_bencoded_torrent_accepted(self, provider):
         provider.session.get = MagicMock(return_value=resp)
         result = provider._fetch_torrent_from_url("https://test.example/ajax.php?action=download&id=42", "42")
         assert result == resp.content
+
+    # ---- token spend must NOT be skipped by a cached .torrent -------------
+    # (regression: a cached .torrent file does not make a download freeleech)
+
+    def test_token_spent_even_when_torrent_cached(self, provider):
+        # Pre-seed a cached .torrent for id 42.
+        (provider.cache_dir / "42.torrent").write_bytes(b"d8:announce...cached")
+        provider._fetch_torrent_from_url = MagicMock(return_value=b"d8:announce...fresh")
+
+        result = provider.fetch_artifact_by_id("42")
+
+        # Must have hit the download endpoint with usetoken=1 rather than
+        # short-circuiting on the cache.
+        assert provider._fetch_torrent_from_url.called
+        called_url = provider._fetch_torrent_from_url.call_args[0][0]
+        assert "usetoken=1" in called_url
+        assert result == b"d8:announce...fresh"
+        # A ledger marker should now exist so we don't re-spend.
+        assert provider._token_recently_spent("42")
+
+    def test_no_respend_within_freeleech_window(self, provider):
+        # A recent token marker means the torrent is already personal-freeleech.
+        (provider.cache_dir / "77.tokened").touch()
+        (provider.cache_dir / "77.torrent").write_bytes(b"d8:announce...cached")
+        provider._fetch_torrent_from_url = MagicMock(return_value=b"should-not-be-called")
+
+        result = provider.fetch_artifact_by_id("77")
+
+        # No usetoken call; served from cache instead.
+        assert not provider._fetch_torrent_from_url.called
+        assert result == b"d8:announce...cached"
+
+    def test_token_marker_respects_ttl(self, provider):
+        import os, time as _t
+        marker = provider.cache_dir / "88.tokened"
+        marker.touch()
+        # Backdate the marker beyond the 7-day window.
+        old = _t.time() - provider.FL_TOKEN_FREELEECH_WINDOW_SECONDS - 3600
+        os.utime(marker, (old, old))
+        assert provider._token_recently_spent("88") is False
+
+    def test_failed_token_falls_back_to_cache(self, provider):
+        (provider.cache_dir / "99.torrent").write_bytes(b"d8:announce...cached")
+        # Token attempt fails (e.g. >5GB / none left) -> None.
+        provider._fetch_torrent_from_url = MagicMock(return_value=None)
+
+        result = provider.fetch_artifact_by_id("99")
+
+        # Attempted the token (usetoken=1) once, then fell back to cache.
+        assert provider._fetch_torrent_from_url.call_count == 1
+        assert "usetoken=1" in provider._fetch_torrent_from_url.call_args[0][0]
+        assert result == b"d8:announce...cached"
+        # No marker recorded on failure -> a later call may retry.
+        assert not provider._token_recently_spent("99")
+
+    def test_in_memory_ledger_prevents_respend_without_cache_dir(self, provider):
+        # No cache dir -> on-disk marker impossible; in-memory ledger must still
+        # prevent a second token spend on the same torrent.
+        provider.cache_dir = None
+        provider._fetch_torrent_from_url = MagicMock(return_value=b"d8:announce...fresh")
+
+        first = provider.fetch_artifact_by_id("55")
+        assert first == b"d8:announce...fresh"
+        assert provider._fetch_torrent_from_url.call_count == 1
+        assert provider._token_recently_spent("55") is True
+
+        # Second call: no marker on disk, but in-memory ledger blocks re-spend.
+        # With no cache and token skipped, it does a single normal (no-token) fetch.
+        provider._fetch_torrent_from_url.reset_mock()
+        second = provider.fetch_artifact_by_id("55")
+        assert second == b"d8:announce...fresh"
+        assert provider._fetch_torrent_from_url.call_count == 1
+        assert "usetoken=1" not in provider._fetch_torrent_from_url.call_args[0][0]
+
+    def test_no_token_serves_cache_without_endpoint_call(self, no_token_provider):
+        (no_token_provider.cache_dir / "42.torrent").write_bytes(b"d8:announce...cached")
+        no_token_provider._fetch_torrent_from_url = MagicMock(return_value=b"fresh")
+        result = no_token_provider.fetch_artifact_by_id("42")
+        assert not no_token_provider._fetch_torrent_from_url.called
+        assert result == b"d8:announce...cached"