From f0432f5156d2cd1c036c19f216ec57e0858418f6 Mon Sep 17 00:00:00 2001 From: Sharma Date: Tue, 14 Jul 2026 01:46:07 +0530 Subject: [PATCH 1/2] feat: add model cache and runtime footprint report Closes #345 --- backend/scripts/model_footprint_report.py | 107 ++++ backend/src/find_api/core/model_footprint.py | 546 +++++++++++++++++++ backend/src/find_api/routers/status.py | 16 + backend/tests/test_model_footprint.py | 340 ++++++++++++ backend/tests/test_status.py | 50 ++ docs/guides/model-footprint.md | 111 ++++ docs/overhaul/inventory/lane-f-ml.md | 8 + 7 files changed, 1178 insertions(+) create mode 100644 backend/scripts/model_footprint_report.py create mode 100644 backend/src/find_api/core/model_footprint.py create mode 100644 backend/tests/test_model_footprint.py create mode 100644 docs/guides/model-footprint.md diff --git a/backend/scripts/model_footprint_report.py b/backend/scripts/model_footprint_report.py new file mode 100644 index 00000000..76cfa86f --- /dev/null +++ b/backend/scripts/model_footprint_report.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +""" +Model footprint report — what each ML model downloads and loads. + +Prints, per model: configured identifier, on-disk cache size, loaded/ +unloaded state (this process), execution device, and last-use time. +Also prints light/full pack totals and notes the proposed CPU pack is +not implemented yet (see docs/overhaul/inventory/lane-f-ml.md, issue #45). + +This is a local admin tool: unlike GET /status/models/footprint, it +prints filesystem cache paths. Run it on the machine whose cache you +want to inspect. + +Never downloads model weights — every lookup is a local filesystem read +or a local cache-index read (e.g. huggingface_hub.scan_cache_dir()). + +Usage (from the backend directory): + uv run python scripts/model_footprint_report.py + uv run python scripts/model_footprint_report.py --json + uv run python scripts/model_footprint_report.py --no-paths +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +def _human_bytes(n: int) -> str: + value = float(n) + for unit in ("B", "KB", "MB", "GB", "TB"): + if value < 1024 or unit == "TB": + return f"{value:.1f} {unit}" if unit != "B" else f"{int(value)} B" + value /= 1024 + return f"{value:.1f} TB" + + +def _print_human(report: dict, show_paths: bool) -> None: + print(f"Model footprint report — generated {report['generated_at']}") + print() + + header = f"{'MODEL':<12} {'PACKS':<12} {'CACHED':<7} {'SIZE':>10} {'LOADED':<7} {'DEVICE':<22} LAST USED" + print(header) + print("-" * len(header)) + for m in report["models"]: + cache = m["cache"] + size = _human_bytes(cache["bytes_on_disk"]) if cache["cached"] else "—" + print( + f"{m['key']:<12} {','.join(m['packs']):<12} " + f"{'yes' if cache['cached'] else 'no':<7} {size:>10} " + f"{'yes' if m['loaded'] else 'no':<7} {m['device']:<22} " + f"{m['last_used'] or '—'}" + ) + print(f" identifier: {m['identifier']}") + if cache.get("note"): + print(f" note: {cache['note']}") + if show_paths and cache.get("path"): + print(f" path: {cache['path']}") + for note in m.get("notes", []): + print(f" note: {note}") + print() + + print("Pack totals:") + for pack_name in ("light", "full"): + totals = report["packs"][pack_name] + print( + f" {pack_name:<7} {totals['cached_count']}/{totals['total_count']} " + f"cached, {_human_bytes(totals['bytes_on_disk'])} on disk" + ) + proposed = report["packs"]["proposed_cpu"] + print(f" proposed_cpu status: {proposed['status']}") + print(f" {proposed['note']}") + for model_label in proposed["models"]: + print(f" - {model_label}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--json", action="store_true", help="Print JSON instead of a table." + ) + parser.add_argument( + "--no-paths", + action="store_true", + help="Omit filesystem cache paths (they are included by default " + "since this script is meant to run locally).", + ) + args = parser.parse_args() + + # Add src to path so this runs standalone (matches other backend/scripts). + sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + from find_api.core.model_footprint import build_report + + report = build_report(include_paths=not args.no_paths) + + if args.json: + print(json.dumps(report, indent=2)) + else: + _print_human(report, show_paths=not args.no_paths) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/src/find_api/core/model_footprint.py b/backend/src/find_api/core/model_footprint.py new file mode 100644 index 00000000..b491a399 --- /dev/null +++ b/backend/src/find_api/core/model_footprint.py @@ -0,0 +1,546 @@ +"""Model footprint reporting. + +Builds a read-only inventory of the ML models Find can load: configured +identifier, on-disk cache footprint, loaded/unloaded state, execution +device, and last-use time. Consumed by: + +- ``GET /status/models/footprint`` (admin-only; never returns filesystem + paths — see ``build_report(include_paths=False)``). +- ``backend/scripts/model_footprint_report.py`` (local CLI; may show paths + since it runs with the operator's own filesystem access). + +Design constraints (see issue tracker + docs/overhaul/inventory/lane-f-ml.md): + +- Never downloads model weights. Every lookup here is a local filesystem + read, or a read of a local cache *index* (``huggingface_hub.scan_cache_dir`` + reads on-disk metadata only, no network). +- Never raises. Any failure to resolve a cache location degrades to + "not cached" rather than crashing the report. +- Never exposes credentials or private media metadata. The only thing this + module reads is ML model cache directories, not user data. +""" + +from __future__ import annotations + +import os +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Callable, Optional + +from find_api.core.config import settings +from find_api.core.hardware import ( + current_torch_device, + detect_capabilities, + resolve_execution, +) +from find_api.core.model_manager import get_model_manager + +# --- Pack identifiers -------------------------------------------------------- +PACK_LIGHT = "light" +PACK_FULL = "full" +PACK_PROPOSED_CPU = "proposed_cpu" # not implemented — tracked by issue #45 + + +# --- Cache lookup ------------------------------------------------------------- +@dataclass(frozen=True) +class CacheInfo: + """Best-effort description of a model's on-disk footprint.""" + + resolver: str + path: Optional[str] + exists: bool + bytes_on_disk: int + file_count: int + last_modified: Optional[float] # epoch seconds + note: Optional[str] = None + + def to_dict(self, include_path: bool) -> dict: + payload = { + "cached": self.exists, + "bytes_on_disk": self.bytes_on_disk, + "file_count": self.file_count, + "last_modified": _iso(self.last_modified), + "resolver": self.resolver, + } + if self.note: + payload["note"] = self.note + if include_path: + payload["path"] = self.path + return payload + + +_EMPTY_CACHE = CacheInfo( + resolver="none", + path=None, + exists=False, + bytes_on_disk=0, + file_count=0, + last_modified=None, +) + + +def _dir_size(path: Path) -> tuple[int, int, Optional[float]]: + """Sum file sizes/mtimes under ``path``. Never raises.""" + total = 0 + count = 0 + latest: Optional[float] = None + try: + for root, _dirs, files in os.walk(path): + for fname in files: + fp = Path(root) / fname + try: + st = fp.stat() + except OSError: + continue + total += st.st_size + count += 1 + if latest is None or st.st_mtime > latest: + latest = st.st_mtime + except OSError: + pass + return total, count, latest + + +def _current_hf_hub_cache_dir() -> str: + """Resolve the HF Hub cache directory from *current* env vars. + + ``huggingface_hub`` bakes ``HF_HUB_CACHE`` into a module-level constant + at import time, so it can miss ``HF_HOME``/``HUGGINGFACE_HUB_CACHE`` + changes made afterwards (e.g. by our own settings loading, or by tests). + Recomputing it here keeps this report accurate for the environment as + it is right now. + """ + explicit = os.getenv("HUGGINGFACE_HUB_CACHE") or os.getenv("HF_HUB_CACHE") + if explicit: + return explicit + hf_home = os.getenv("HF_HOME") or os.path.expanduser("~/.cache/huggingface") + return os.path.join(hf_home, "hub") + + +def _hf_hub_cache_matches(*needles: str) -> Optional[CacheInfo]: + """Best-effort lookup of a Hugging Face Hub cached repo matching any needle. + + Reads the local Hub cache *index* only (``scan_cache_dir``); this never + triggers a download or network call. + """ + try: + from huggingface_hub import scan_cache_dir + except Exception: + return None + + try: + cache_info = scan_cache_dir(cache_dir=_current_hf_hub_cache_dir()) + except Exception: + return None + + lowered = [n.lower() for n in needles if n] + if not lowered: + return None + + matches = [ + repo + for repo in cache_info.repos + if any(needle in repo.repo_id.lower() for needle in lowered) + ] + if not matches: + return None + + total_bytes = sum(int(repo.size_on_disk) for repo in matches) + total_files = sum(int(repo.nb_files) for repo in matches) + last_modified = max((float(repo.last_modified) for repo in matches), default=None) + paths = ", ".join(str(repo.repo_path) for repo in matches) + return CacheInfo( + resolver="hf_hub_cache", + path=paths, + exists=True, + bytes_on_disk=total_bytes, + file_count=total_files, + last_modified=last_modified, + note=None + if len(matches) == 1 + else f"{len(matches)} matching cached repos summed", + ) + + +def _open_clip_legacy_cache(*needles: str) -> Optional[CacheInfo]: + """Fallback for open_clip checkpoints stored outside the HF Hub cache.""" + cache_dir = os.getenv("OPEN_CLIP_CACHE_DIR") or os.path.expanduser( + "~/.cache/clip" + ) + root = Path(cache_dir) + if not root.exists(): + return None + + lowered = [n.lower() for n in needles if n] + try: + matched_files = [ + p + for p in root.rglob("*") + if p.is_file() and any(n in p.name.lower() for n in lowered) + ] + except OSError: + return None + if not matched_files: + return None + + total = 0 + latest: Optional[float] = None + for p in matched_files: + try: + st = p.stat() + except OSError: + continue + total += st.st_size + if latest is None or st.st_mtime > latest: + latest = st.st_mtime + + return CacheInfo( + resolver="open_clip_cache_dir", + path=str(root), + exists=True, + bytes_on_disk=total, + file_count=len(matched_files), + last_modified=latest, + ) + + +def resolve_siglip_cache() -> CacheInfo: + needles = (settings.CLIP_MODEL, settings.CLIP_PRETRAINED, "siglip") + return ( + _hf_hub_cache_matches(*needles) + or _open_clip_legacy_cache(*needles) + or CacheInfo( + resolver="none", + path=None, + exists=False, + bytes_on_disk=0, + file_count=0, + last_modified=None, + note="Checked the Hugging Face Hub cache and $OPEN_CLIP_CACHE_DIR " + "(default ~/.cache/clip); neither contained a matching checkpoint.", + ) + ) + + +def resolve_florence_cache() -> CacheInfo: + repo_id = settings.BLIP_MODEL # e.g. "microsoft/Florence-2-base" + short_name = repo_id.split("/")[-1] if repo_id else "" + return _hf_hub_cache_matches(repo_id, short_name) or CacheInfo( + resolver="none", + path=None, + exists=False, + bytes_on_disk=0, + file_count=0, + last_modified=None, + note="Not found in the Hugging Face Hub cache.", + ) + + +def resolve_yolo_cache() -> CacheInfo: + """Best-effort — Ultralytics does not guarantee a single cache location. + + Checks the process working directory (the default auto-download target), + the Ultralytics-configured ``weights_dir`` when importable, and the + conventional ``~/.cache/ultralytics`` fallback. + """ + weight_name = settings.YOLO_MODEL + candidates: list[Path] = [Path.cwd() / weight_name] + + try: + from ultralytics.utils import SETTINGS as _ultra_settings # type: ignore + + weights_dir = _ultra_settings.get("weights_dir") + if weights_dir: + candidates.append(Path(weights_dir) / weight_name) + except Exception: + pass + + candidates.append(Path.home() / ".cache" / "ultralytics" / weight_name) + + for candidate in candidates: + try: + if candidate.is_file(): + st = candidate.stat() + return CacheInfo( + resolver="ultralytics_weights_file", + path=str(candidate), + exists=True, + bytes_on_disk=st.st_size, + file_count=1, + last_modified=st.st_mtime, + note="Ultralytics does not expose one guaranteed cache " + "path; only the common download locations were checked.", + ) + except OSError: + continue + + return CacheInfo( + resolver="ultralytics_weights_file", + path=None, + exists=False, + bytes_on_disk=0, + file_count=0, + last_modified=None, + note="Not found in the working directory, the configured " + "weights_dir, or ~/.cache/ultralytics.", + ) + + +def resolve_insightface_cache() -> CacheInfo: + home = Path(os.getenv("INSIGHTFACE_HOME", os.path.expanduser("~/.insightface"))) + for sub in ("models/antelopev2", "models/antelopev2/antelopev2"): + candidate = home / sub + if candidate.is_dir(): + total, count, latest = _dir_size(candidate) + if count: + return CacheInfo( + resolver="insightface_home", + path=str(candidate), + exists=True, + bytes_on_disk=total, + file_count=count, + last_modified=latest, + ) + return CacheInfo( + resolver="insightface_home", + path=str(home / "models/antelopev2"), + exists=False, + bytes_on_disk=0, + file_count=0, + last_modified=None, + note="Checked $INSIGHTFACE_HOME (default ~/.insightface).", + ) + + +def resolve_paddleocr_cache() -> CacheInfo: + """PaddleOCR 3.x resolves weights through PaddleX; older installs use + ``~/.paddleocr``. Both are checked.""" + candidates = [ + os.getenv("PADDLE_PDX_CACHE_HOME"), + os.path.expanduser("~/.paddlex/official_models"), + os.path.expanduser("~/.paddleocr"), + ] + for candidate in candidates: + if not candidate: + continue + path = Path(candidate) + if path.is_dir(): + total, count, latest = _dir_size(path) + if count: + return CacheInfo( + resolver="paddlex_cache_home", + path=str(path), + exists=True, + bytes_on_disk=total, + file_count=count, + last_modified=latest, + ) + return CacheInfo( + resolver="paddlex_cache_home", + path=None, + exists=False, + bytes_on_disk=0, + file_count=0, + last_modified=None, + note="Checked $PADDLE_PDX_CACHE_HOME, ~/.paddlex/official_models, " + "and ~/.paddleocr.", + ) + + +# --- Model registry ----------------------------------------------------------- +def _insightface_device() -> str: + plan = resolve_execution(settings.ACCEL_MODE, detect_capabilities()) + return plan.providers[0] if plan.providers else "cpu" + + +@dataclass(frozen=True) +class ModelSpec: + key: str # ModelManager registration key (matches use_model()/get_model() names) + label: str + kind: str + packs: tuple[str, ...] + identifier: Callable[[], str] + cache_resolver: Callable[[], CacheInfo] + device_resolver: Callable[[], str] + + +MODEL_SPECS: tuple[ModelSpec, ...] = ( + ModelSpec( + key="siglip", + label="SigLIP image/text embedding", + kind="open_clip", + packs=(PACK_LIGHT, PACK_FULL), + identifier=lambda: f"{settings.CLIP_MODEL}/{settings.CLIP_PRETRAINED}", + cache_resolver=resolve_siglip_cache, + device_resolver=current_torch_device, + ), + ModelSpec( + key="florence-2", + label="Florence-2 captioning", + kind="transformers", + packs=(PACK_FULL,), + identifier=lambda: settings.BLIP_MODEL, + cache_resolver=resolve_florence_cache, + device_resolver=current_torch_device, + ), + ModelSpec( + key="yolo", + label="YOLO object detection", + kind="ultralytics", + packs=(PACK_FULL,), + identifier=lambda: settings.YOLO_MODEL, + cache_resolver=resolve_yolo_cache, + device_resolver=current_torch_device, + ), + ModelSpec( + key="insightface", + label="InsightFace face detection/recognition", + kind="insightface", + packs=(PACK_FULL,), + identifier=lambda: "antelopev2", + cache_resolver=resolve_insightface_cache, + device_resolver=_insightface_device, + ), + ModelSpec( + key="paddleocr", + label="PaddleOCR text extraction", + kind="paddleocr", + packs=(PACK_FULL,), + identifier=lambda: "PP-OCR (en)", + cache_resolver=resolve_paddleocr_cache, + device_resolver=lambda: "cpu", + ), +) + +# Proposed CPU-optimized ONNX pack (docs/overhaul/inventory/lane-f-ml.md, +# issue #45). Not implemented yet, so there is nothing on disk to measure — +# listed so pack totals/documentation have a stable place to grow into once +# it ships. +PROPOSED_CPU_MODELS: tuple[dict, ...] = ( + {"label": "CLIP ViT-B-32 (ONNX, openai)", "replaces": "siglip"}, + {"label": "InsightFace buffalo_s (ONNX)", "replaces": "insightface"}, + {"label": "PP-OCRv5 mobile (ONNX)", "replaces": "paddleocr"}, +) + + +# --- Report assembly ----------------------------------------------------------- +def _iso(epoch: Optional[float]) -> Optional[str]: + if epoch is None: + return None + return datetime.fromtimestamp(epoch, tz=timezone.utc).isoformat() + + +def _aggregate_manager_status() -> set[str]: + """Model keys reported loaded by this process or any process that has + published status to Redis. Best-effort; degrades to local-only.""" + manager = get_model_manager() + local_status = manager.get_status() + loaded: set[str] = set(local_status.get("loaded_models", [])) + + try: + import json + + from find_api.core.queue import get_redis_connection + + redis_conn = get_redis_connection() + for key in redis_conn.scan_iter("find:model_status:*"): + try: + raw = redis_conn.get(key) + if not raw: + continue + status = json.loads(raw) + loaded.update(status.get("loaded_models", [])) + except Exception: + continue + except Exception: + pass + + return loaded + + +def build_report(include_paths: bool = False) -> dict: + """Build the full model footprint report. + + ``include_paths=False`` (the default, used by the admin API) omits + filesystem paths from the cache entries — only sizes, counts, and + timestamps are returned. ``include_paths=True`` is for the local CLI + script only; never wire it up to a network-reachable endpoint. + """ + manager = get_model_manager() + loaded_keys = _aggregate_manager_status() + + models: list[dict] = [] + pack_totals: dict[str, dict] = { + PACK_LIGHT: {"bytes_on_disk": 0, "cached_count": 0, "total_count": 0}, + PACK_FULL: {"bytes_on_disk": 0, "cached_count": 0, "total_count": 0}, + } + + for spec in MODEL_SPECS: + try: + identifier = spec.identifier() + except Exception as exc: # noqa: BLE001 + identifier = f"" + + try: + cache_info = spec.cache_resolver() + except Exception: # noqa: BLE001 + cache_info = _EMPTY_CACHE + + try: + device = spec.device_resolver() + except Exception: # noqa: BLE001 + device = "unknown" + + last_used_epoch = manager.last_used.get(spec.key) + + notes: list[str] = [] + if last_used_epoch is None: + notes.append( + "last_used reflects this process only and no in-process use " + "has been recorded yet" + ) + + models.append( + { + "key": spec.key, + "label": spec.label, + "kind": spec.kind, + "packs": list(spec.packs), + "identifier": identifier, + "loaded": spec.key in loaded_keys, + "device": device, + "last_used": _iso(last_used_epoch), + "cache": cache_info.to_dict(include_paths), + "notes": notes, + } + ) + + for pack in spec.packs: + totals = pack_totals.get(pack) + if totals is None: + continue + totals["total_count"] += 1 + if cache_info.exists: + totals["cached_count"] += 1 + totals["bytes_on_disk"] += cache_info.bytes_on_disk + + return { + "generated_at": _iso(time.time()), + "models": models, + "packs": { + PACK_LIGHT: pack_totals[PACK_LIGHT], + PACK_FULL: pack_totals[PACK_FULL], + PACK_PROPOSED_CPU: { + "status": "not_implemented", + "note": ( + "CPU-optimized ONNX pack proposed in " + "docs/overhaul/inventory/lane-f-ml.md; tracked by issue #45. " + "Nothing is downloaded yet, so there is no footprint to " + "measure." + ), + "models": [m["label"] for m in PROPOSED_CPU_MODELS], + }, + }, + } diff --git a/backend/src/find_api/routers/status.py b/backend/src/find_api/routers/status.py index ed3063f2..22813d4b 100644 --- a/backend/src/find_api/routers/status.py +++ b/backend/src/find_api/routers/status.py @@ -9,6 +9,7 @@ from find_api.core.config import settings from find_api.core.dependencies import get_admin_user, get_required_user +from find_api.core.model_footprint import build_report from find_api.core.model_manager import get_model_manager from find_api.core.queue import get_job, get_redis_connection from find_api.models.user import User @@ -16,6 +17,21 @@ router = APIRouter() +@router.get("/status/models/footprint") +def get_model_footprint(_admin: Optional[User] = Depends(get_admin_user)): + """ + Report what each ML model downloads and loads: configured identifier, + on-disk cache size, loaded/unloaded state, execution device, and + last-use time. + + Admin-only, and never returns filesystem paths (see + ``find_api.core.model_footprint`` for the full-path local CLI + equivalent). Use this as the measurement source before changing ML + model defaults or installer download sizes. + """ + return build_report(include_paths=False) + + @router.get("/status/models") def get_loaded_models(_admin: Optional[User] = Depends(get_admin_user)): """ diff --git a/backend/tests/test_model_footprint.py b/backend/tests/test_model_footprint.py new file mode 100644 index 00000000..1a58b1ae --- /dev/null +++ b/backend/tests/test_model_footprint.py @@ -0,0 +1,340 @@ +"""Tests for the model footprint report (find_api.core.model_footprint). + +All caches used here are temporary directories built by the tests +themselves — nothing is downloaded, and no network access is required or +expected. Where a lookup would normally hit huggingface_hub, we patch the +download-capable functions to raise so an accidental network call fails +the test loudly instead of silently succeeding. +""" + +from __future__ import annotations + +import json +import os + +import pytest + +from find_api.core import model_footprint as mf +from find_api.core.config import settings + + +def _make_hf_cache(tmp_path, repo_id: str, blob_bytes: bytes) -> tuple[str, bool]: + """Build a minimal huggingface_hub cache layout under tmp_path. + + Mirrors the on-disk shape `huggingface_hub.scan_cache_dir()` expects: + hub/models----/{blobs,snapshots,refs}. + """ + hf_home = tmp_path / "hf_home" + hub = hf_home / "hub" + repo_dir = hub / f"models--{repo_id.replace('/', '--')}" + blobs_dir = repo_dir / "blobs" + snap_dir = repo_dir / "snapshots" / "abc123" + refs_dir = repo_dir / "refs" + blobs_dir.mkdir(parents=True) + snap_dir.mkdir(parents=True) + refs_dir.mkdir(parents=True) + + blob_path = blobs_dir / "blob1" + blob_path.write_bytes(blob_bytes) + snapshot_file = snap_dir / "model.safetensors" + symlink_used = True + try: + snapshot_file.symlink_to(blob_path) + except (OSError, NotImplementedError): + import shutil + + shutil.copyfile(blob_path, snapshot_file) + symlink_used = False + (refs_dir / "main").write_text("abc123") + + return str(hf_home), symlink_used + + +_NO_SYMLINK_REASON = ( + "requires real filesystem symlinks to build a valid huggingface_hub " + "cache (scan_cache_dir needs them); not available without Windows " + "Developer Mode/admin rights or an equivalent POSIX permission" +) + + +@pytest.fixture(autouse=True) +def _no_network(monkeypatch): + """Fail loudly if any resolver tries to actually download something.""" + + def _forbidden(*args, **kwargs): + raise AssertionError( + "model_footprint must never trigger a download; " + "this should have been a local-only cache lookup" + ) + + try: + import huggingface_hub + + monkeypatch.setattr(huggingface_hub, "hf_hub_download", _forbidden) + monkeypatch.setattr(huggingface_hub, "snapshot_download", _forbidden) + except ImportError: + pass + + +class TestHFHubCacheResolution: + def test_finds_cached_repo_by_full_id(self, tmp_path, monkeypatch): + hf_home, symlink_used = _make_hf_cache(tmp_path, "microsoft/Florence-2-base", b"0" * 2048) + if not symlink_used: + pytest.skip(_NO_SYMLINK_REASON) + monkeypatch.setenv("HF_HOME", hf_home) + monkeypatch.setattr(settings, "BLIP_MODEL", "microsoft/Florence-2-base") + + info = mf.resolve_florence_cache() + + assert info.exists is True + assert info.bytes_on_disk == 2048 + assert info.file_count == 1 + assert info.resolver == "hf_hub_cache" + assert info.last_modified is not None + + def test_no_matching_repo_reports_not_cached(self, tmp_path, monkeypatch): + hf_home,_ = _make_hf_cache(tmp_path, "someone/unrelated-model", b"0" * 10) + monkeypatch.setenv("HF_HOME", hf_home) + monkeypatch.setattr(settings, "BLIP_MODEL", "microsoft/Florence-2-base") + + info = mf.resolve_florence_cache() + + assert info.exists is False + assert info.bytes_on_disk == 0 + assert info.note + + def test_missing_huggingface_hub_degrades_gracefully(self, monkeypatch): + """If huggingface_hub can't be imported, resolution must not raise.""" + import builtins + + real_import = builtins.__import__ + + def _blocked_import(name, *args, **kwargs): + if name == "huggingface_hub": + raise ImportError("simulated missing dependency") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", _blocked_import) + + result = mf._hf_hub_cache_matches("anything") + assert result is None + + +class TestOpenClipLegacyCache: + def test_finds_checkpoint_in_legacy_cache_dir(self, tmp_path, monkeypatch): + cache_dir = tmp_path / "open_clip_cache" + cache_dir.mkdir() + (cache_dir / "ViT-B-16-SigLIP_webli.bin").write_bytes(b"x" * 4096) + + monkeypatch.setenv("OPEN_CLIP_CACHE_DIR", str(cache_dir)) + # Force the HF Hub path to miss so we exercise the fallback. + monkeypatch.setattr(mf, "_hf_hub_cache_matches", lambda *needles: None) + + info = mf.resolve_siglip_cache() + + assert info.exists is True + assert info.bytes_on_disk == 4096 + assert info.resolver == "open_clip_cache_dir" + + def test_nothing_cached_anywhere(self, tmp_path, monkeypatch): + monkeypatch.setenv("OPEN_CLIP_CACHE_DIR", str(tmp_path / "does-not-exist")) + monkeypatch.setattr(mf, "_hf_hub_cache_matches", lambda *needles: None) + + info = mf.resolve_siglip_cache() + + assert info.exists is False + assert info.bytes_on_disk == 0 + + +class TestYoloCache: + def test_finds_weights_in_working_directory(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(settings, "YOLO_MODEL", "yolo26n.pt") + (tmp_path / "yolo26n.pt").write_bytes(b"y" * 512) + + info = mf.resolve_yolo_cache() + + assert info.exists is True + assert info.bytes_on_disk == 512 + assert info.resolver == "ultralytics_weights_file" + + def test_not_found_reports_checked_locations(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(settings, "YOLO_MODEL", "does-not-exist.pt") + + info = mf.resolve_yolo_cache() + + assert info.exists is False + assert info.bytes_on_disk == 0 + assert "not found" not in info.note.lower() or info.note # note is present + + +class TestInsightFaceCache: + def test_finds_model_pack(self, tmp_path, monkeypatch): + home = tmp_path / "insightface_home" + pack_dir = home / "models" / "antelopev2" + pack_dir.mkdir(parents=True) + (pack_dir / "glintr100.onnx").write_bytes(b"a" * 1000) + (pack_dir / "scrfd_10g_bnkps.onnx").write_bytes(b"b" * 500) + + monkeypatch.setenv("INSIGHTFACE_HOME", str(home)) + + info = mf.resolve_insightface_cache() + + assert info.exists is True + assert info.bytes_on_disk == 1500 + assert info.file_count == 2 + + def test_falls_back_to_nested_layout(self, tmp_path, monkeypatch): + home = tmp_path / "insightface_home" + pack_dir = home / "models" / "antelopev2" / "antelopev2" + pack_dir.mkdir(parents=True) + (pack_dir / "glintr100.onnx").write_bytes(b"a" * 42) + + monkeypatch.setenv("INSIGHTFACE_HOME", str(home)) + + info = mf.resolve_insightface_cache() + + assert info.exists is True + assert info.bytes_on_disk == 42 + + def test_missing_pack_reports_not_cached(self, tmp_path, monkeypatch): + monkeypatch.setenv("INSIGHTFACE_HOME", str(tmp_path / "empty")) + + info = mf.resolve_insightface_cache() + + assert info.exists is False + assert info.bytes_on_disk == 0 + + +class TestPaddleOCRCache: + def test_finds_models_via_pdx_cache_home(self, tmp_path, monkeypatch): + cache_home = tmp_path / "paddlex_models" + cache_home.mkdir() + (cache_home / "det.onnx").write_bytes(b"d" * 300) + + monkeypatch.setenv("PADDLE_PDX_CACHE_HOME", str(cache_home)) + + info = mf.resolve_paddleocr_cache() + + assert info.exists is True + assert info.bytes_on_disk == 300 + + def test_missing_reports_checked_locations(self, monkeypatch): + monkeypatch.delenv("PADDLE_PDX_CACHE_HOME", raising=False) + monkeypatch.setattr( + os.path, "expanduser", lambda p: p.replace("~", "/nonexistent-home") + ) + + info = mf.resolve_paddleocr_cache() + + assert info.exists is False + assert info.note + + +class TestBuildReport: + def _wire_all_caches(self, tmp_path, monkeypatch): + """Point every model at a small, fully cached, temporary footprint.""" + hf_home, symlink_used = _make_hf_cache(tmp_path, "microsoft/Florence-2-base", b"f" * 100) + monkeypatch.setenv("HF_HOME", hf_home) + monkeypatch.setattr(settings, "BLIP_MODEL", "microsoft/Florence-2-base") + + clip_cache = tmp_path / "open_clip_cache" + clip_cache.mkdir() + (clip_cache / "ViT-B-16-SigLIP_webli.bin").write_bytes(b"c" * 50) + monkeypatch.setenv("OPEN_CLIP_CACHE_DIR", str(clip_cache)) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(settings, "YOLO_MODEL", "yolo26n.pt") + (tmp_path / "yolo26n.pt").write_bytes(b"y" * 30) + + insight_home = tmp_path / "insightface_home" + (insight_home / "models" / "antelopev2").mkdir(parents=True) + (insight_home / "models" / "antelopev2" / "glintr100.onnx").write_bytes( + b"i" * 20 + ) + monkeypatch.setenv("INSIGHTFACE_HOME", str(insight_home)) + + paddle_home = tmp_path / "paddlex_models" + paddle_home.mkdir() + (paddle_home / "det.onnx").write_bytes(b"p" * 10) + monkeypatch.setenv("PADDLE_PDX_CACHE_HOME", str(paddle_home)) + return symlink_used + + def test_pack_totals_sum_correctly(self, tmp_path, monkeypatch): + symlink_used = self._wire_all_caches(tmp_path, monkeypatch) + if not symlink_used: + pytest.skip(_NO_SYMLINK_REASON) + report = mf.build_report(include_paths=True) + + # light pack = siglip only + assert report["packs"]["light"]["total_count"] == 1 + assert report["packs"]["light"]["cached_count"] == 1 + assert report["packs"]["light"]["bytes_on_disk"] == 50 + + # full pack = all five models + assert report["packs"]["full"]["total_count"] == 5 + assert report["packs"]["full"]["cached_count"] == 5 + assert report["packs"]["full"]["bytes_on_disk"] == 100 + 50 + 30 + 20 + 10 + + proposed = report["packs"]["proposed_cpu"] + assert proposed["status"] == "not_implemented" + assert len(proposed["models"]) == 3 + + def test_report_has_five_model_entries_with_required_fields( + self, tmp_path, monkeypatch + ): + self._wire_all_caches(tmp_path, monkeypatch) + + report = mf.build_report(include_paths=True) + + assert {m["key"] for m in report["models"]} == { + "siglip", + "florence-2", + "yolo", + "insightface", + "paddleocr", + } + for entry in report["models"]: + assert entry["identifier"] + assert "loaded" in entry + assert "device" in entry + assert "last_used" in entry + assert "cache" in entry + assert "bytes_on_disk" in entry["cache"] + + def test_include_paths_false_never_leaks_filesystem_paths( + self, tmp_path, monkeypatch + ): + self._wire_all_caches(tmp_path, monkeypatch) + + report = mf.build_report(include_paths=False) + serialized = json.dumps(report) + + assert str(tmp_path) not in serialized + for entry in report["models"]: + assert "path" not in entry["cache"] + + def test_include_paths_true_exposes_paths_for_local_cli( + self, tmp_path, monkeypatch + ): + self._wire_all_caches(tmp_path, monkeypatch) + + report = mf.build_report(include_paths=True) + + yolo_entry = next(m for m in report["models"] if m["key"] == "yolo") + assert yolo_entry["cache"]["path"] == str(tmp_path / "yolo26n.pt") + + def test_never_raises_when_everything_is_missing(self, tmp_path, monkeypatch): + empty = tmp_path / "nothing-here" + monkeypatch.setenv("HF_HOME", str(empty / "hf")) + monkeypatch.setenv("OPEN_CLIP_CACHE_DIR", str(empty / "clip")) + monkeypatch.setenv("INSIGHTFACE_HOME", str(empty / "insightface")) + monkeypatch.setenv("PADDLE_PDX_CACHE_HOME", str(empty / "paddle")) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(settings, "YOLO_MODEL", "does-not-exist.pt") + + report = mf.build_report(include_paths=False) + + assert report["packs"]["full"]["cached_count"] == 0 + assert all(not m["cache"]["cached"] for m in report["models"]) diff --git a/backend/tests/test_status.py b/backend/tests/test_status.py index 280b0a10..3ebda52f 100644 --- a/backend/tests/test_status.py +++ b/backend/tests/test_status.py @@ -170,3 +170,53 @@ def test_loaded_models_endpoint_includes_worker_snapshot(client): body["processes"]["worker"]["failed_models"]["florence-2"]["error"] == "load failed" ) + + +def test_model_footprint_endpoint_returns_report_without_paths(client, tmp_path): + """GET /api/status/models/footprint reports cache/loaded/device info and + never leaks filesystem paths, even though the endpoint is admin-gated.""" + fake_redis = MagicMock() + fake_redis.scan_iter.return_value = [] + fake_redis.get.return_value = None + + with ( + patch("find_api.core.model_footprint.get_model_manager") as mock_get_manager, + patch( + "find_api.core.queue.get_redis_connection", + return_value=fake_redis, + ), + ): + mock_manager = mock_get_manager.return_value + mock_manager.get_status.return_value = { + "process": "api", + "loaded_models": ["siglip"], + "in_flight": {}, + "failed_models": {}, + "max_loaded_models": 5, + "updated_at": 0, + } + mock_manager.last_used = {} + + response = client.get("/api/status/models/footprint") + + assert response.status_code == 200 + body = response.json() + + assert {m["key"] for m in body["models"]} == { + "siglip", + "florence-2", + "yolo", + "insightface", + "paddleocr", + } + + siglip_entry = next(m for m in body["models"] if m["key"] == "siglip") + assert siglip_entry["loaded"] is True + + for entry in body["models"]: + assert "path" not in entry["cache"] + + assert set(body["packs"].keys()) == {"light", "full", "proposed_cpu"} + assert body["packs"]["light"]["total_count"] == 1 + assert body["packs"]["full"]["total_count"] == 5 + assert body["packs"]["proposed_cpu"]["status"] == "not_implemented" diff --git a/docs/guides/model-footprint.md b/docs/guides/model-footprint.md new file mode 100644 index 00000000..ee55c744 --- /dev/null +++ b/docs/guides/model-footprint.md @@ -0,0 +1,111 @@ +# Model Footprint Report + +Before changing ML model defaults, adding an installer, or writing a +benchmark, you need a straight answer to "what does this actually download +and load?" This report gives you that answer without guessing at cache +paths or hand-measuring directories. + +It's the measurement source for +[#45 — design installer model downloads and cache management](https://github.com/Abhash-Chakraborty/Find/issues/45) +and for model benchmark issues: don't propose new defaults or pack sizes +without running this first. + +## What it reports, per model + +- **Identifier** — the exact model/checkpoint configured via settings + (e.g. `microsoft/Florence-2-base`, `yolo26n.pt`). +- **Cache footprint** — whether it's on disk, total bytes, file count, and + when it was last modified. +- **Loaded state** — whether it's currently loaded in this process (or any + process that has published status to Redis). +- **Device / provider** — CPU, CUDA, MPS, or the resolved ONNX execution + provider, from the same detection `find_api.core.hardware` uses (see + [hardware-acceleration.md](./hardware-acceleration.md)). +- **Last-use time** — from the in-process `ModelManager`; this is + local-process-only, not aggregated across workers. + +Nothing here downloads a model. Cache lookups are local filesystem reads or +local cache-*index* reads (`huggingface_hub.scan_cache_dir()` reads on-disk +metadata only). Every resolver degrades to "not cached" on failure instead +of raising — a missing library or unset cache dir never breaks the report. + +## Running it + +```bash +cd backend +uv run python scripts/model_footprint_report.py # human-readable table, with paths +uv run python scripts/model_footprint_report.py --json # JSON, with paths +uv run python scripts/model_footprint_report.py --no-paths # either mode, no filesystem paths +``` + +This is a **local CLI tool** — it prints cache paths by default because it +runs with your own filesystem access. The equivalent API endpoint does not: + +``` +GET /api/status/models/footprint (admin-only) +``` + +The API response never includes filesystem paths, arbitrary host +information, or anything about media/user data — only sizes, counts, +identifiers, and timestamps. If you need paths from a deployed instance, +use the CLI script on that machine directly; don't add paths to the API +response. + +## Runtime packs + +Find's models fall into three groups, used to reason about install size: + +| Pack | Contents | Status | +|---|---|---| +| **light** | SigLIP embedding model only (text/image search — the core feature) | Implemented; measured by this report | +| **full** | SigLIP + Florence-2 captioning + YOLO object detection + InsightFace + PaddleOCR | Implemented; measured by this report | +| **proposed_cpu** | CPU-optimized ONNX replacements: CLIP ViT-B-32 (ONNX), InsightFace `buffalo_s` (ONNX), PP-OCRv5 mobile (ONNX) | **Not implemented.** Proposed in [lane-f-ml.md](../overhaul/inventory/lane-f-ml.md); tracked by #45. The report lists these models by name with `status: "not_implemented"` and no size, since there's nothing on disk yet to measure. | + +The report's `packs` section gives you `cached_count` / `total_count` and +summed `bytes_on_disk` for `light` and `full` directly from your machine's +actual cache — always trust that over any number in this doc, including the +ones below. + +### Approximate current sizes (for planning only — re-run the script for ground truth) + +These are public, approximate download sizes for the currently-configured +checkpoints, gathered for rough installer-size planning. They are **not** a +substitute for running the report on a real machine, and will drift as +models/versions change: + +| Model | Pack | Approx. size | +|---|---|---| +| SigLIP (`ViT-B-16-SigLIP`/`webli`) | light, full | ~0.4–0.8 GB depending on precision | +| Florence-2-base | full | ~0.46 GB (safetensors) | +| YOLO (`yolo26n.pt`, nano) | full | a few MB | +| InsightFace `antelopev2` | full | ~0.4 GB | +| PaddleOCR (en, det+rec+cls) | full | tens of MB | + +Rough full-pack total: **on the order of 1.5–2 GB**. This is exactly the +kind of number the installer work in #45 needs pinned down precisely — use +`model_footprint_report.py` against a real, fully-loaded cache rather than +this table when it matters. + +### Proposed CPU pack + +Not implemented. `buffalo_s` (InsightFace's smaller ONNX face pack) is +publicly documented at roughly 0.16 GB versus `antelopev2`'s ~0.4 GB — a +meaningful reduction, which is the whole motivation for the proposed pack. +Once ViT-B-32 (ONNX) and PP-OCRv5 mobile are actually wired up, add real +cache-resolver entries to +`backend/src/find_api/core/model_footprint.py::PROPOSED_CPU_MODELS` and this +row moves from "proposed" to "measured" like the other two packs. + +## Extending the report + +Model definitions live in `backend/src/find_api/core/model_footprint.py` as +a tuple of `ModelSpec` entries: a manager key, display label, pack +membership, an identifier function, a cache resolver, and a device +resolver. To add a model: + +1. Add a `resolve__cache()` function that finds the on-disk footprint + for that library (best-effort, wrapped so it can never raise). +2. Add a `ModelSpec` entry to `MODEL_SPECS` referencing it. +3. Add tests in `backend/tests/test_model_footprint.py` using a temporary + fake cache directory — never a real download — following the existing + `Test*Cache` classes as a template. diff --git a/docs/overhaul/inventory/lane-f-ml.md b/docs/overhaul/inventory/lane-f-ml.md index f4aefc0b..495ee647 100644 --- a/docs/overhaul/inventory/lane-f-ml.md +++ b/docs/overhaul/inventory/lane-f-ml.md @@ -106,3 +106,11 @@ Find runs PyTorch/library models in-process, leased through a singleton `ModelMa | Keep YOLO / Florence-2 / ModelManager | **S** (no-op) | No change. | **Cross-cutting risk:** any embedding model swap forces a re-embed/re-index of existing photos and faces; sequence behind a migration plan. + +## 7. Measuring current footprint + +Actual on-disk sizes, loaded state, and device for each model currently in +Find are measured by +[`model_footprint_report.py`](../../guides/model-footprint.md), not +estimated by hand — run it before proposing pack sizes or installer +download budgets. From b0c703a04ad62f662f579ad9eb2fd4253913c08d Mon Sep 17 00:00:00 2001 From: Abhash Chakraborty <80592559+Abhash-Chakraborty@users.noreply.github.com> Date: Mon, 3 Aug 2026 17:57:09 +0530 Subject: [PATCH 2/2] fix(model-footprint): cover the HF cache path in CI and make the CLI ASCII MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit backend-check was failing on two of this PR's own tests. Root cause: the HF cache tests build a real on-disk Hub cache and call the real huggingface_hub, but the dev dependency group deliberately excludes the ML extras, so huggingface_hub is not installed in CI. _hf_hub_cache_matches bails out at the import and reports "not cached" — which is also what an empty cache looks like, so test_finds_cached_repo_by_full_id failed and test_pack_totals_sum_correctly saw 4 of 5 models cached instead of 5. It only ever passed locally on Windows because both tests skip there for a different reason (scan_cache_dir needs real symlinks, which need Developer Mode). Two more tests in that class were passing for the wrong reason — "not cached" is the expected result either way — so the whole HF path had no real coverage anywhere. Stub scan_cache_dir instead of building a real cache. That drops both the huggingface_hub and the symlink requirement, so the needle-matching and aggregation logic — the part that is ours — is now genuinely covered on every platform. The real-library check is kept as a separate integration test guarded by importorskip, and new cases cover case-insensitive matching, multi-repo summing with its note text, and a corrupted cache degrading to "not cached" rather than raising. Separately, the CLI used an em dash as its "no value" placeholder. The default Windows console codepage is cp1252, where that renders as mojibake, and the project ships a Windows desktop build. The script is now ASCII-only, with tests asserting the rendered output encodes as cp1252 and that the source stays ASCII. Verified: report contains no filesystem paths or host identifiers with include_paths=False, --json and --no-paths both behave, and all three CLI modes render. Full backend suite 688 passed, 7 skipped; ruff check and format clean. Not changed: the shared Redis client has no socket timeout, so the cross-process loaded-model scan in _aggregate_manager_status can hang on an unreachable-but-accepting Redis. That client is shared with the rq workers, which rely on long blocking reads, so it needs its own change rather than one buried in this PR. --- backend/scripts/model_footprint_report.py | 16 +- backend/tests/test_model_footprint.py | 193 ++++++++++++++++++++-- 2 files changed, 188 insertions(+), 21 deletions(-) diff --git a/backend/scripts/model_footprint_report.py b/backend/scripts/model_footprint_report.py index 0615a3bc..d1b99c69 100644 --- a/backend/scripts/model_footprint_report.py +++ b/backend/scripts/model_footprint_report.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ -Model footprint report — what each ML model downloads and loads. +Model footprint report - what each ML model downloads and loads. Prints, per model: configured identifier, on-disk cache size, loaded/ unloaded state (this process), execution device, and last-use time. @@ -11,7 +11,7 @@ prints filesystem cache paths. Run it on the machine whose cache you want to inspect. -Never downloads model weights — every lookup is a local filesystem read +Never downloads model weights - every lookup is a local filesystem read or a local cache-index read (e.g. huggingface_hub.scan_cache_dir()). Usage (from the backend directory): @@ -37,8 +37,14 @@ def _human_bytes(n: int) -> str: return f"{value:.1f} TB" +# Placeholder for "no value". Deliberately ASCII: the default Windows console +# codepage is cp1252, where an em dash renders as mojibake, and this is a +# console tool on a platform the project ships a desktop build for. +_NONE = "-" + + def _print_human(report: dict, show_paths: bool) -> None: - print(f"Model footprint report — generated {report['generated_at']}") + print(f"Model footprint report - generated {report['generated_at']}") print() header = f"{'MODEL':<12} {'PACKS':<12} {'CACHED':<7} {'SIZE':>10} {'LOADED':<7} {'DEVICE':<22} LAST USED" @@ -46,12 +52,12 @@ def _print_human(report: dict, show_paths: bool) -> None: print("-" * len(header)) for m in report["models"]: cache = m["cache"] - size = _human_bytes(cache["bytes_on_disk"]) if cache["cached"] else "—" + size = _human_bytes(cache["bytes_on_disk"]) if cache["cached"] else _NONE print( f"{m['key']:<12} {','.join(m['packs']):<12} " f"{'yes' if cache['cached'] else 'no':<7} {size:>10} " f"{'yes' if m['loaded'] else 'no':<7} {m['device']:<22} " - f"{m['last_used'] or '—'}" + f"{m['last_used'] or _NONE}" ) print(f" identifier: {m['identifier']}") if cache.get("note"): diff --git a/backend/tests/test_model_footprint.py b/backend/tests/test_model_footprint.py index 133b621a..36af72cf 100644 --- a/backend/tests/test_model_footprint.py +++ b/backend/tests/test_model_footprint.py @@ -57,6 +57,41 @@ def _make_hf_cache(tmp_path, repo_id: str, blob_bytes: bytes) -> tuple[str, bool ) +class _FakeCachedRepo: + """The subset of huggingface_hub's CachedRepoInfo that we read.""" + + def __init__(self, repo_id, size_on_disk, nb_files, last_modified, repo_path): + self.repo_id = repo_id + self.size_on_disk = size_on_disk + self.nb_files = nb_files + self.last_modified = last_modified + self.repo_path = repo_path + + +def _install_fake_hf_hub(monkeypatch, repos): + """Inject a stub ``huggingface_hub`` exposing only ``scan_cache_dir``. + + The backend's dev dependency group deliberately excludes the ML extras, so + the real ``huggingface_hub`` is absent in CI. Building a real on-disk Hub + cache therefore tested nothing there: ``_hf_hub_cache_matches`` bailed out + at the import and reported "not cached", which is also what a genuinely + empty cache looks like. Stubbing the single function this module calls + keeps the needle-matching and aggregation logic — the part that is ours — + covered on every platform, with no symlink privileges required. + """ + import sys + import types + + module = types.ModuleType("huggingface_hub") + + def _scan_cache_dir(cache_dir=None): + return types.SimpleNamespace(repos=list(repos)) + + module.scan_cache_dir = _scan_cache_dir + monkeypatch.setitem(sys.modules, "huggingface_hub", module) + return module + + @pytest.fixture(autouse=True) def _no_network(monkeypatch): """Fail loudly if any resolver tries to actually download something.""" @@ -77,13 +112,15 @@ def _forbidden(*args, **kwargs): class TestHFHubCacheResolution: - def test_finds_cached_repo_by_full_id(self, tmp_path, monkeypatch): - hf_home, symlink_used = _make_hf_cache( - tmp_path, "microsoft/Florence-2-base", b"0" * 2048 + def test_finds_cached_repo_by_full_id(self, monkeypatch): + _install_fake_hf_hub( + monkeypatch, + [ + _FakeCachedRepo( + "microsoft/Florence-2-base", 2048, 1, 1_700_000_000.0, "/cache/f" + ) + ], ) - if not symlink_used: - pytest.skip(_NO_SYMLINK_REASON) - monkeypatch.setenv("HF_HOME", hf_home) monkeypatch.setattr(settings, "BLIP_MODEL", "microsoft/Florence-2-base") info = mf.resolve_florence_cache() @@ -94,9 +131,47 @@ def test_finds_cached_repo_by_full_id(self, tmp_path, monkeypatch): assert info.resolver == "hf_hub_cache" assert info.last_modified is not None - def test_no_matching_repo_reports_not_cached(self, tmp_path, monkeypatch): - hf_home, _ = _make_hf_cache(tmp_path, "someone/unrelated-model", b"0" * 10) - monkeypatch.setenv("HF_HOME", hf_home) + def test_matching_is_case_insensitive_and_substring(self, monkeypatch): + _install_fake_hf_hub( + monkeypatch, + [ + _FakeCachedRepo( + "MICROSOFT/Florence-2-BASE", 64, 1, 1_700_000_000.0, "/cache/f" + ) + ], + ) + monkeypatch.setattr(settings, "BLIP_MODEL", "microsoft/Florence-2-base") + + assert mf.resolve_florence_cache().exists is True + + def test_multiple_matching_repos_are_summed_and_noted(self, monkeypatch): + _install_fake_hf_hub( + monkeypatch, + [ + _FakeCachedRepo( + "microsoft/Florence-2-base", 100, 2, 1_700_000_000.0, "/cache/a" + ), + _FakeCachedRepo( + "microsoft/Florence-2-base-ft", 40, 3, 1_800_000_000.0, "/cache/b" + ), + _FakeCachedRepo("someone/unrelated", 999, 9, 1.0, "/cache/c"), + ], + ) + monkeypatch.setattr(settings, "BLIP_MODEL", "microsoft/Florence-2-base") + + info = mf.resolve_florence_cache() + + assert info.bytes_on_disk == 140 + assert info.file_count == 5 + # Newest of the matches, and the unrelated repo must not drag it back. + assert info.last_modified is not None + assert "2 matching cached repos summed" in (info.note or "") + + def test_no_matching_repo_reports_not_cached(self, monkeypatch): + _install_fake_hf_hub( + monkeypatch, + [_FakeCachedRepo("someone/unrelated-model", 10, 1, 1.0, "/cache/u")], + ) monkeypatch.setattr(settings, "BLIP_MODEL", "microsoft/Florence-2-base") info = mf.resolve_florence_cache() @@ -105,6 +180,45 @@ def test_no_matching_repo_reports_not_cached(self, tmp_path, monkeypatch): assert info.bytes_on_disk == 0 assert info.note + def test_scan_failure_degrades_to_not_cached(self, monkeypatch): + """A corrupted or unreadable cache must not blow up the report.""" + import sys + import types + + module = types.ModuleType("huggingface_hub") + + def _boom(cache_dir=None): + raise OSError("corrupted cache") + + module.scan_cache_dir = _boom + monkeypatch.setitem(sys.modules, "huggingface_hub", module) + + assert mf._hf_hub_cache_matches("anything") is None + + def test_real_library_reads_an_on_disk_cache(self, tmp_path, monkeypatch): + """Integration check against the genuine huggingface_hub, when present. + + Skipped in CI (ML extras are not installed there) and on filesystems + without symlink privileges, which scan_cache_dir requires. The stubbed + tests above are what actually gate CI. + """ + pytest.importorskip("huggingface_hub") + hf_home, symlink_used = _make_hf_cache( + tmp_path, "microsoft/Florence-2-base", b"0" * 2048 + ) + if not symlink_used: + pytest.skip(_NO_SYMLINK_REASON) + monkeypatch.setenv("HF_HOME", hf_home) + monkeypatch.delenv("HF_HUB_CACHE", raising=False) + monkeypatch.delenv("HUGGINGFACE_HUB_CACHE", raising=False) + monkeypatch.setattr(settings, "BLIP_MODEL", "microsoft/Florence-2-base") + + info = mf.resolve_florence_cache() + + assert info.exists is True + assert info.bytes_on_disk == 2048 + assert info.resolver == "hf_hub_cache" + def test_missing_huggingface_hub_degrades_gracefully(self, monkeypatch): """If huggingface_hub can't be imported, resolution must not raise.""" import builtins @@ -237,10 +351,17 @@ def test_missing_reports_checked_locations(self, monkeypatch): class TestBuildReport: def _wire_all_caches(self, tmp_path, monkeypatch): """Point every model at a small, fully cached, temporary footprint.""" - hf_home, symlink_used = _make_hf_cache( - tmp_path, "microsoft/Florence-2-base", b"f" * 100 + # Florence resolves through huggingface_hub, which is not installed in + # the dev/CI environment, so stub it rather than building a real Hub + # cache — that also drops the symlink privilege requirement. + _install_fake_hf_hub( + monkeypatch, + [ + _FakeCachedRepo( + "microsoft/Florence-2-base", 100, 1, 1_700_000_000.0, "/cache/f" + ) + ], ) - monkeypatch.setenv("HF_HOME", hf_home) monkeypatch.setattr(settings, "BLIP_MODEL", "microsoft/Florence-2-base") clip_cache = tmp_path / "open_clip_cache" @@ -263,12 +384,9 @@ def _wire_all_caches(self, tmp_path, monkeypatch): paddle_home.mkdir() (paddle_home / "det.onnx").write_bytes(b"p" * 10) monkeypatch.setenv("PADDLE_PDX_CACHE_HOME", str(paddle_home)) - return symlink_used def test_pack_totals_sum_correctly(self, tmp_path, monkeypatch): - symlink_used = self._wire_all_caches(tmp_path, monkeypatch) - if not symlink_used: - pytest.skip(_NO_SYMLINK_REASON) + self._wire_all_caches(tmp_path, monkeypatch) report = mf.build_report(include_paths=True) # light pack = siglip only @@ -342,3 +460,46 @@ def test_never_raises_when_everything_is_missing(self, tmp_path, monkeypatch): assert report["packs"]["full"]["cached_count"] == 0 assert all(not m["cache"]["cached"] for m in report["models"]) + + +class TestCliRendering: + """The local CLI must render on a default Windows console. + + Windows defaults to the cp1252 codepage, where the em dash the report + previously used as its "no value" placeholder comes out as mojibake. The + project ships a Windows desktop build, so operators do hit this. + """ + + @staticmethod + def _load_cli(): + import importlib.util + from pathlib import Path + + script = ( + Path(__file__).resolve().parents[1] + / "scripts" + / "model_footprint_report.py" + ) + spec = importlib.util.spec_from_file_location("_mf_cli", script) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module, script + + def test_human_output_is_cp1252_safe(self, capsys): + cli, _ = self._load_cli() + report = mf.build_report(include_paths=False) + + cli._print_human(report, show_paths=False) + + out = capsys.readouterr().out + assert out.strip() + # Would raise UnicodeEncodeError on a strict legacy console. + out.encode("cp1252") + # Placeholders for "not cached" / "never used" must still be present. + assert " - " in out or out.rstrip().endswith("-") + + def test_script_source_is_ascii_only(self): + _, script = self._load_cli() + source = script.read_text(encoding="utf-8") + non_ascii = sorted({ch for ch in source if ord(ch) > 127}) + assert not non_ascii, f"non-ASCII characters in CLI script: {non_ascii}"