diff --git a/backend/src/find_api/diagnostics/__init__.py b/backend/src/find_api/diagnostics/__init__.py new file mode 100644 index 00000000..a7d46d1e --- /dev/null +++ b/backend/src/find_api/diagnostics/__init__.py @@ -0,0 +1,23 @@ +"""Privacy-safe local diagnostics helpers. + +This package collects and redacts support bundles for offline debugging. +Nothing here uploads data or contacts external services. +""" + +from __future__ import annotations + +from typing import Any + +__all__ = ["collect_diagnostics_bundle", "redact_payload"] + + +def __getattr__(name: str) -> Any: + if name == "collect_diagnostics_bundle": + from find_api.diagnostics.bundle import collect_diagnostics_bundle + + return collect_diagnostics_bundle + if name == "redact_payload": + from find_api.diagnostics.redact import redact_payload + + return redact_payload + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/backend/src/find_api/diagnostics/bundle.py b/backend/src/find_api/diagnostics/bundle.py new file mode 100644 index 00000000..0d9efd62 --- /dev/null +++ b/backend/src/find_api/diagnostics/bundle.py @@ -0,0 +1,479 @@ +"""Collect a privacy-safe local diagnostics bundle. + +All collection stays on-box. The returned structure is passed through the +redaction layer before it is returned to callers. +""" + +from __future__ import annotations + +import logging +import os +import platform +import sys +import threading +import time +from collections import deque +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeout +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable + +from find_api import __version__ +from find_api.core.config import settings +from find_api.diagnostics.redact import redact_payload, scrub_string +from find_api.utils.errors import sanitize_error + +logger = logging.getLogger(__name__) + +SCHEMA_VERSION = 1 +ERROR_LOG_LIMIT = 20 +# Keep health probes short so a hung dependency cannot pin a worker thread. +HEALTH_PROBE_TIMEOUT_SECONDS = 2.0 + +PRIVACY_NOTICE = ( + "Local diagnostics only. This bundle is generated on-request, never " + "uploaded automatically, and is redacted to exclude passwords, tokens, " + "storage keys, paths, filenames, captions, OCR, embeddings, faces, and " + "user identifiers. Attach it to a GitHub issue only after reviewing it." +) + + +class _ErrorLogBuffer(logging.Handler): + """Keep the last N ERROR+ log records in memory for diagnostics.""" + + def __init__(self, capacity: int = ERROR_LOG_LIMIT) -> None: + super().__init__(level=logging.ERROR) + self._records: deque[dict[str, Any]] = deque(maxlen=capacity) + self._records_lock = threading.Lock() + + def emit(self, record: logging.LogRecord) -> None: + try: + message = self.format(record) if self.formatter else record.getMessage() + entry = { + "timestamp": datetime.fromtimestamp( + record.created, tz=timezone.utc + ).isoformat(), + "level": record.levelname, + "logger": record.name, + "message": scrub_string(message), + "source": "log", + } + with self._records_lock: + self._records.append(entry) + except Exception: # noqa: BLE001 — never break the logging pipeline + self.handleError(record) + + def snapshot(self) -> list[dict[str, Any]]: + with self._records_lock: + return list(self._records) + + +_error_buffer = _ErrorLogBuffer() +_error_buffer.name = "find_diagnostics_error_buffer" +_buffer_installed = False + + +def ensure_error_log_buffer() -> None: + """Attach the in-process error ring buffer to the root logger once. + + Under uvicorn ``--reload``, prior import cycles leave a same-named handler + from an old module instance on the root logger. Remove those stale handlers + and always attach the current ``_error_buffer`` so ``_collect_recent_errors`` + reads from the live instance. + """ + global _buffer_installed + if _buffer_installed: + return + root = logging.getLogger() + for handler in list(root.handlers): + if getattr(handler, "name", None) == _error_buffer.name: + root.removeHandler(handler) + root.addHandler(_error_buffer) + _buffer_installed = True + + +def _utc_now_iso() -> str: + return datetime.now(tz=timezone.utc).isoformat() + + +def _run_with_timeout( + fn: Callable[[], Any], timeout_s: float = HEALTH_PROBE_TIMEOUT_SECONDS +) -> Any: + """Run ``fn`` in a worker thread and raise ``TimeoutError`` if it hangs.""" + pool = ThreadPoolExecutor(max_workers=1) + try: + future = pool.submit(fn) + try: + return future.result(timeout=timeout_s) + except FuturesTimeout as exc: + future.cancel() + raise TimeoutError( + f"health probe timed out after {timeout_s:.0f}s" + ) from exc + finally: + # Deliberately not `with ThreadPoolExecutor(...)`: Executor.__exit__ + # calls shutdown(wait=True), which blocks until the probe thread + # finishes and so reinstates the exact stall this timeout exists to + # prevent. Abandon the thread instead — every probe already sets its + # own socket-level timeout, so it unwinds on its own shortly after. + pool.shutdown(wait=False, cancel_futures=True) + + +def _check_postgresql() -> dict[str, Any]: + started = time.perf_counter() + + def _probe() -> None: + from sqlalchemy import text + + from find_api.core.database import engine + + with engine.connect() as conn: + # Cap statement runtime on PostgreSQL; SQLite ignores this safely. + if conn.dialect.name == "postgresql": + conn.execute(text("SET LOCAL statement_timeout = '2000'")) + conn.execute(text("SELECT 1")) + + try: + _run_with_timeout(_probe) + return { + "ok": True, + "latency_ms": round((time.perf_counter() - started) * 1000, 2), + } + except Exception as exc: # noqa: BLE001 + return { + "ok": False, + "latency_ms": round((time.perf_counter() - started) * 1000, 2), + "error": scrub_string(sanitize_error(exc)), + } + + +def _check_redis() -> dict[str, Any]: + started = time.perf_counter() + + def _probe() -> None: + from redis import Redis + + client = Redis.from_url( + settings.REDIS_URL, + socket_connect_timeout=HEALTH_PROBE_TIMEOUT_SECONDS, + socket_timeout=HEALTH_PROBE_TIMEOUT_SECONDS, + ) + try: + client.ping() + finally: + client.close() + + try: + _run_with_timeout(_probe) + return { + "ok": True, + "latency_ms": round((time.perf_counter() - started) * 1000, 2), + } + except Exception as exc: # noqa: BLE001 + return { + "ok": False, + "latency_ms": round((time.perf_counter() - started) * 1000, 2), + "error": scrub_string(sanitize_error(exc)), + } + + +def _check_storage() -> dict[str, Any]: + started = time.perf_counter() + backend = settings.STORAGE_BACKEND.lower() + + def _probe_local() -> bool: + path = Path(settings.LOCAL_STORAGE_PATH) + return path.is_dir() and os.access(path, os.W_OK) + + def _probe_minio() -> bool: + import urllib3 + from minio import Minio + + http_client = urllib3.PoolManager( + timeout=urllib3.Timeout( + connect=HEALTH_PROBE_TIMEOUT_SECONDS, + read=HEALTH_PROBE_TIMEOUT_SECONDS, + ), + retries=False, + ) + client = Minio( + settings.MINIO_ENDPOINT, + access_key=settings.MINIO_ACCESS_KEY, + secret_key=settings.MINIO_SECRET_KEY, + secure=settings.MINIO_SECURE, + http_client=http_client, + ) + return bool(client.bucket_exists(settings.MINIO_BUCKET)) + + try: + if backend == "local": + reachable = _run_with_timeout(_probe_local) + result: dict[str, Any] = { + "ok": reachable, + "backend": "local", + "latency_ms": round((time.perf_counter() - started) * 1000, 2), + } + if not reachable: + result["error"] = "Local storage path is not a writable directory" + return result + + exists = _run_with_timeout(_probe_minio) + return { + "ok": bool(exists), + "backend": "minio", + "latency_ms": round((time.perf_counter() - started) * 1000, 2), + **({} if exists else {"error": "Configured bucket not found"}), + } + except Exception as exc: # noqa: BLE001 + return { + "ok": False, + "backend": backend, + "latency_ms": round((time.perf_counter() - started) * 1000, 2), + "error": scrub_string(sanitize_error(exc)), + } + + +def _collect_migration_state() -> dict[str, Any]: + try: + from alembic.config import Config + from alembic.runtime.migration import MigrationContext + from alembic.script import ScriptDirectory + + from find_api.core.database import engine + + backend_root = Path(__file__).resolve().parents[3] + ini_path = backend_root / "alembic.ini" + if not ini_path.is_file(): + return { + "status": "unavailable", + "current": None, + "heads": [], + "detail": "alembic.ini not found", + } + + cfg = Config(str(ini_path)) + cfg.set_main_option("script_location", str(backend_root / "alembic")) + script = ScriptDirectory.from_config(cfg) + heads = list(script.get_heads()) + + def _read_current_revision() -> Any: + with engine.connect() as conn: + return MigrationContext.configure(conn).get_current_revision() + + # Same bound as the health probes: this opens a real DB connection, so + # an unreachable database must not stall the whole bundle. + current = _run_with_timeout(_read_current_revision) + + if current is None and not heads: + status = "empty" + elif current in heads or (current is not None and set(heads) <= {current}): + status = "ok" + elif current is None: + status = "unmigrated" + else: + status = "behind" + + return { + "status": status, + "current": current, + "heads": heads, + } + except Exception as exc: # noqa: BLE001 + return { + "status": "unavailable", + "current": None, + "heads": [], + "detail": scrub_string(sanitize_error(exc)), + } + + +def _collect_queue_stats() -> dict[str, Any]: + mode = settings.QUEUE_MODE + try: + if mode == "sqlite": + from find_api.core.queue import _get_backend + + backend = _get_backend() + counts = backend.count_by_status() + queued = int(counts.get("queued", 0)) + started = int(counts.get("running", 0)) + int(counts.get("started", 0)) + failed = int(counts.get("failed", 0)) + finished = int(counts.get("finished", 0)) + int(counts.get("completed", 0)) + return { + "mode": mode, + "depth": queued, + "queued": queued, + "started": started, + "failed": failed, + "finished": finished, + } + + from rq import Queue + from rq.registry import FailedJobRegistry, StartedJobRegistry + + from find_api.core.queue import get_redis_connection + + conn = get_redis_connection() + queue_names = ("high", "default", "low") + queued = 0 + started = 0 + failed = 0 + finished = 0 + deferred = 0 + scheduled = 0 + + for name in queue_names: + q = Queue(name, connection=conn) + queued += len(q) + started += len(StartedJobRegistry(queue=q)) + failed += len(FailedJobRegistry(queue=q)) + try: + finished += len(q.finished_job_registry) + except Exception: # noqa: BLE001 + pass + try: + deferred += len(q.deferred_job_registry) + except Exception: # noqa: BLE001 + pass + try: + scheduled += len(q.scheduled_job_registry) + except Exception: # noqa: BLE001 + pass + + return { + "mode": mode, + "depth": queued, + "queued": queued, + "started": started, + "failed": failed, + "finished": finished, + "deferred": deferred, + "scheduled": scheduled, + } + except Exception as exc: # noqa: BLE001 + return { + "mode": mode, + "depth": 0, + "queued": 0, + "started": 0, + "failed": 0, + "error": scrub_string(sanitize_error(exc)), + } + + +def _collect_model_state() -> dict[str, Any]: + loaded: list[str] = [] + try: + from find_api.core.model_manager import get_model_manager + + status = get_model_manager().get_status() + loaded = sorted(status.get("loaded_models") or []) + except Exception: # noqa: BLE001 + loaded = [] + + return { + "ml_mode": settings.ML_MODE, + "accel_mode": settings.ACCEL_MODE, + "clip_model": settings.CLIP_MODEL, + "clip_pretrained": settings.CLIP_PRETRAINED, + "blip_model": settings.BLIP_MODEL, + "yolo_model": settings.YOLO_MODEL, + "use_gpu": settings.USE_GPU, + "embedding_dim": settings.EMBEDDING_DIM, + "queue_mode": settings.QUEUE_MODE, + "storage_backend": settings.STORAGE_BACKEND, + "remote_ml_configured": bool( + settings.REMOTE_ML_URL and settings.REMOTE_ML_API_KEY + ), + "configured_models": sorted( + { + settings.CLIP_MODEL, + settings.BLIP_MODEL, + settings.YOLO_MODEL, + } + ), + "loaded_models": loaded, + } + + +def _collect_recent_errors() -> list[dict[str, Any]]: + """Merge in-process ERROR logs with recent media analysis failures.""" + ensure_error_log_buffer() + entries = _error_buffer.snapshot() + + try: + from find_api.core.database import SessionLocal + from find_api.models.media import Media + + def _read_failed_media() -> list[Any]: + db = SessionLocal() + try: + return ( + db.query(Media.error_message, Media.updated_at, Media.created_at) + .filter(Media.status == "failed", Media.error_message.isnot(None)) + .order_by(Media.id.desc()) + .limit(ERROR_LOG_LIMIT) + .all() + ) + finally: + db.close() + + # Bounded for the same reason as the health probes — an unreachable + # database degrades the errors section instead of hanging the request. + rows = _run_with_timeout(_read_failed_media) + for error_message, updated_at, created_at in rows: + ts = updated_at or created_at + entries.append( + { + "timestamp": ts.isoformat() if ts is not None else None, + "level": "ERROR", + "logger": "media.analysis", + "message": scrub_string(str(error_message)), + "source": "media", + } + ) + except Exception as exc: # noqa: BLE001 + logger.debug("Could not load media analysis errors: %s", exc) + + # Newest first; keep a stable privacy-safe capped list. + def _sort_key(item: dict[str, Any]) -> str: + return item.get("timestamp") or "" + + entries.sort(key=_sort_key, reverse=True) + return entries[:ERROR_LOG_LIMIT] + + +def collect_diagnostics_bundle() -> dict[str, Any]: + """Build and redact a structured diagnostics bundle dict. + + The result is suitable for returning as JSON from an admin endpoint or + writing to a local file. It never initiates network uploads. + """ + ensure_error_log_buffer() + + raw: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "generated_at": _utc_now_iso(), + "privacy_notice": PRIVACY_NOTICE, + "app": { + "version": __version__, + "environment": settings.ENVIRONMENT, + }, + "runtime": { + "python_version": sys.version.split()[0], + "python_implementation": platform.python_implementation(), + "platform": platform.system(), + "platform_release": platform.release(), + "platform_machine": platform.machine(), + }, + "migrations": _collect_migration_state(), + "services": { + "postgresql": _check_postgresql(), + "redis": _check_redis(), + "storage": _check_storage(), + }, + "queue": _collect_queue_stats(), + "models": _collect_model_state(), + "errors": _collect_recent_errors(), + } + return redact_payload(raw) diff --git a/backend/src/find_api/diagnostics/redact.py b/backend/src/find_api/diagnostics/redact.py new file mode 100644 index 00000000..f04a8ee0 --- /dev/null +++ b/backend/src/find_api/diagnostics/redact.py @@ -0,0 +1,260 @@ +"""Allowlist-first redaction for diagnostics payloads. + +Sensitive user and deployment data must never leave a diagnostics bundle. +Unknown keys are denied by default; only explicitly allowlisted keys keep +their values, and every string value is still pattern-scrubbed. +""" + +from __future__ import annotations + +import re +from typing import Any + +REDACTED = "[REDACTED]" +REDACTED_KEY = "redacted_key" + +# Keys whose values may appear in a diagnostics bundle after scrubbing. +# Deny-by-default: anything not listed is replaced with REDACTED. +ALLOWED_KEYS: frozenset[str] = frozenset( + { + # Top-level / meta + "schema_version", + "generated_at", + "privacy_notice", + "app", + "runtime", + "migrations", + "services", + "queue", + "models", + "errors", + # App / runtime + "version", + "environment", + "python_version", + "python_implementation", + "platform", + "platform_release", + "platform_machine", + # Migrations + "current", + "heads", + "status", + "detail", + # Service health + "postgresql", + "redis", + "storage", + "ok", + "latency_ms", + "error", + "backend", + "reachable", + # Queue + "mode", + "depth", + "queued", + "started", + "failed", + "finished", + "deferred", + "scheduled", + # Models / providers (names and modes only — never weights or URLs) + "ml_mode", + "accel_mode", + "clip_model", + "clip_pretrained", + "blip_model", + "yolo_model", + "use_gpu", + "embedding_dim", + "configured_models", + "loaded_models", + "remote_ml_configured", + "queue_mode", + "storage_backend", + # Error log entries (messages already scrubbed) + "level", + "logger", + "message", + "timestamp", + "source", + "count", + # Placeholder used when a sensitive key name is itself redacted + REDACTED_KEY, + } +) + +# Key names that are always stripped even if somehow allowlisted. +_SENSITIVE_KEY_RE = re.compile( + r"(?i)^(password|passwd|secret|token|api[_-]?key|access[_-]?key|" + r"secret[_-]?key|authorization|auth|credential|credentials|" + r"session|cookie|bearer|private[_-]?key|minio_key|thumbnail_key|" + r"filename|filepath|file_path|path|object_name|caption|ocr|" + r"ocr_text|embedding|vector|face|faces|person|people|" + r"user(_?id)?|uploader|email|username|display_name|" + r"database_url|redis_url|remote_ml_url|remote_ml_api_key|" + r"metadata_json|exif_json|file_hash)$" +) + +# Substring matches for nested private media/metadata keys. +# +# Intentionally over-redacts: patterns like ``face`` / ``vector`` match as +# substrings, so keys such as ``interface`` or ``pgvector`` are also denied. +# That deny-by-default bias is by design — false positives are preferred over +# leaking private media metadata into a support bundle. +_SENSITIVE_KEY_SUBSTRING_RE = re.compile( + r"(?i)(password|passwd|secret|token|api[_-]?key|access[_-]?key|" + r"secret[_-]?key|authorization|credential|caption|ocr|embedding|" + r"vector|face|filename|filepath|file_path|minio_key|thumbnail_key|" + r"user_id|uploader|database_url|redis_url)" +) + +# Filesystem paths (Windows drive + Unix absolute). +# Avoid matching URL schemes like postgresql:// (drive letter + '//'). +_PATH_RE = re.compile( + r"(?:" + r"[a-zA-Z]:(?:\\+|/(?!/))(?:[\w\-. ]+[\\/]+)*[\w\-. ]+" + r"|" + r"(?``, which is the whole point of the +# models section. Credential and path scrubbing still applies to them. +_MODEL_IDENTIFIER_KEYS: frozenset[str] = frozenset( + { + "clip_model", + "clip_pretrained", + "blip_model", + "yolo_model", + "configured_models", + "loaded_models", + } +) + + +def _is_sensitive_key(key: str) -> bool: + if _SENSITIVE_KEY_RE.match(key): + return True + # The substring heuristic is a deny-by-default net for keys nobody vetted. + # Curated allowlist entries have been reviewed, so it must not override + # them — otherwise ``embedding_dim`` (an int, explicitly allowlisted) is + # destroyed just for containing "embedding". Exact-match sensitive names + # above still win over the allowlist. + if key in ALLOWED_KEYS: + return False + return bool(_SENSITIVE_KEY_SUBSTRING_RE.search(key)) + + +def scrub_string(value: str, *, scrub_filenames: bool = True) -> str: + """Remove paths, filenames, credentials, and token-like substrings. + + ``scrub_filenames=False`` keeps filename-shaped tokens intact. It is only + for operator-declared config identifiers (see ``_MODEL_IDENTIFIER_KEYS``); + every other rule, including path and credential scrubbing, still runs. + """ + msg = _URL_CREDS_RE.sub(r"\1@", value) + msg = _BEARER_RE.sub(f"Bearer {REDACTED}", msg) + msg = _SECRET_ASSIGN_RE.sub(r"\1=", msg) + msg = _PRIVATE_FIELD_ASSIGN_RE.sub(r"\1=", msg) + msg = _QUOTED_CONTENT_RE.sub(r"\1\1", msg) + msg = _TOKEN_RE.sub(REDACTED, msg) + msg = _PATH_RE.sub("", msg) + if scrub_filenames: + msg = _FILENAME_RE.sub("", msg) + return msg + + +def redact_payload(data: Any, *, scrub_filenames: bool = True) -> Any: + """Recursively redact a diagnostics payload using allowlist + scrubbing. + + - Sensitive dict keys are renamed to ``redacted_key`` (value ``[REDACTED]``) + so the original key name does not leak. + - Other dict keys not on the allowlist keep their name but get + ``[REDACTED]`` values. + - Strings under allowlisted keys are still pattern-scrubbed. + - Lists and nested dicts are walked recursively. + + ``scrub_filenames`` is threaded down so a model-identifier key can exempt + its own value — including list values like ``configured_models`` — without + weakening any other rule. + """ + if isinstance(data, dict): + out: dict[str, Any] = {} + # Several sensitive keys in one dict must not collapse onto a single + # ``redacted_key`` entry, which would silently drop all but the last. + redacted_key_count = 0 + for key, value in data.items(): + key_str = str(key) + if _is_sensitive_key(key_str): + suffix = f"_{redacted_key_count}" if redacted_key_count else "" + out[f"{REDACTED_KEY}{suffix}"] = REDACTED + redacted_key_count += 1 + continue + if key_str not in ALLOWED_KEYS: + out[key_str] = REDACTED + continue + out[key_str] = redact_payload( + value, + scrub_filenames=key_str not in _MODEL_IDENTIFIER_KEYS, + ) + return out + + if isinstance(data, (list, tuple)): + return [redact_payload(item, scrub_filenames=scrub_filenames) for item in data] + + if isinstance(data, str): + return scrub_string(data, scrub_filenames=scrub_filenames) + + # bool/int/float/None and other primitives pass through unchanged. + return data diff --git a/backend/src/find_api/main.py b/backend/src/find_api/main.py index a68859bb..95cc29c7 100644 --- a/backend/src/find_api/main.py +++ b/backend/src/find_api/main.py @@ -19,11 +19,13 @@ from find_api.core.storage import init_storage from find_api.core.config import settings from find_api.core.model_manager import get_model_manager +from find_api.diagnostics.bundle import ensure_error_log_buffer from find_api.routers import ( auth, cluster, clusters, config, + diagnostics, feedback, gallery, map, @@ -67,6 +69,9 @@ async def lifespan(app: FastAPI): logger.info("Initializing database...") init_db() + # Install diagnostics error buffer once at boot (not at import time). + ensure_error_log_buffer() + # Initialize configured storage backend logger.info("Initializing %s storage...", settings.STORAGE_BACKEND) init_storage() @@ -148,6 +153,7 @@ async def lifespan(app: FastAPI): app.include_router(clusters.router, prefix="/api", tags=["clusters"]) app.include_router(cluster.router, prefix="/api", tags=["cluster-ops"]) app.include_router(status.router, prefix="/api", tags=["status"]) +app.include_router(diagnostics.router, prefix="/api", tags=["diagnostics"]) app.include_router(config.router, prefix="/api", tags=["config"]) app.include_router(people.router, prefix="/api", tags=["people"]) app.include_router(vault.router, prefix="/api", tags=["vault"]) diff --git a/backend/src/find_api/routers/diagnostics.py b/backend/src/find_api/routers/diagnostics.py new file mode 100644 index 00000000..b13ff508 --- /dev/null +++ b/backend/src/find_api/routers/diagnostics.py @@ -0,0 +1,54 @@ +"""Admin-only local diagnostics bundle export. + +GET /api/admin/diagnostics/bundle returns a privacy-redacted JSON document +generated on this host. Nothing is uploaded externally — the caller must +explicitly request and download the payload. +""" + +from __future__ import annotations + +import logging +from typing import Optional + +from fastapi import APIRouter, Depends +from fastapi.responses import JSONResponse + +from find_api.core.dependencies import get_admin_user +from find_api.diagnostics.bundle import collect_diagnostics_bundle +from find_api.models.user import User + +logger = logging.getLogger(__name__) + +router = APIRouter() + +_BUNDLE_HEADERS = { + "Content-Disposition": 'attachment; filename="find-diagnostics-bundle.json"', + "X-Find-Diagnostics": "local-only", +} +_GENERIC_FAILURE = {"error": "Failed to generate diagnostics bundle"} + + +@router.get("/admin/diagnostics/bundle") +def export_diagnostics_bundle( + _admin: Optional[User] = Depends(get_admin_user), +): + """Return a privacy-safe local diagnostics bundle as JSON. + + Admin-only in shared mode (open in local mode), matching other + instance-wide admin endpoints. Requires an explicit HTTP request — + no background telemetry or outbound upload is performed. + """ + try: + bundle = collect_diagnostics_bundle() + except Exception: # noqa: BLE001 — never leak exception details to clients + logger.exception("Diagnostics bundle collection failed") + return JSONResponse( + status_code=500, + content=_GENERIC_FAILURE, + headers=_BUNDLE_HEADERS, + ) + + return JSONResponse( + content=bundle, + headers=_BUNDLE_HEADERS, + ) diff --git a/backend/tests/test_diagnostics_api.py b/backend/tests/test_diagnostics_api.py new file mode 100644 index 00000000..53c6a41e --- /dev/null +++ b/backend/tests/test_diagnostics_api.py @@ -0,0 +1,338 @@ +"""API tests for GET /api/admin/diagnostics/bundle.""" + +from __future__ import annotations + +import json +import time +from contextlib import ExitStack +from unittest.mock import patch + +import pytest + +from find_api.core.auth import create_session, hash_password +from find_api.main import app +from find_api.models.user import User + +_ENDPOINT = "/api/admin/diagnostics/bundle" +_FAKE_BUNDLE = { + "schema_version": 1, + "privacy_notice": "Local diagnostics only.", + "app": {"version": "1.0.0", "environment": "local"}, + "runtime": {"python_version": "3.12.0"}, + "migrations": {"status": "ok", "current": "abc", "heads": ["abc"]}, + "services": { + "postgresql": {"ok": True, "latency_ms": 1.0}, + "redis": {"ok": True, "latency_ms": 1.0}, + "storage": {"ok": True, "backend": "minio", "latency_ms": 1.0}, + }, + "queue": {"mode": "redis", "depth": 0, "queued": 0, "started": 0, "failed": 0}, + "models": {"ml_mode": "mock", "configured_models": [], "loaded_models": []}, + "errors": [], +} + +# Same placeholder fixtures as the redaction tests (non-secrets for scanners). +_EXAMPLE_PASSWORD = "EXAMPLE_PASSWORD_PLACEHOLDER" +_EXAMPLE_API_KEY = "sk-test-" + "FAKE-KEY-FOR-TESTING-ONLY" +_SEEDED_FILENAME = "vacation-photo-2024.jpg" +_SEEDED_PATH = r"C:\Users\alice\Pictures\vacation-photo-2024.jpg" +_SEEDED_TXT = "private_notes.txt" +_SEEDED_DOTFILE = ".env" + + +def _auth(token: str) -> dict[str, str]: + return {"Authorization": f"Bearer {token}"} + + +def _make_user(db, username: str, role: str) -> User: + user = User( + username=username, + display_name=username, + password_hash=hash_password("EXAMPLE_PASSWORD_PLACEHOLDER"), + role=role, + ) + db.add(user) + db.commit() + db.refresh(user) + return user + + +def _assert_no_leakage(payload) -> None: + blob = json.dumps(payload, default=str) + for fragment in ( + _EXAMPLE_PASSWORD, + "FAKE-KEY-FOR-TESTING-ONLY", + _SEEDED_FILENAME, + _SEEDED_PATH, + _SEEDED_TXT, + _SEEDED_DOTFILE, + "C:\\\\Users\\\\alice", + ): + assert fragment not in blob, f"fragment leaked: {fragment!r}" + + +def _patch_collector_with_seeded_secrets(bundle_mod): + """Inject scrubbable secrets into collector internals for endpoint coverage.""" + stack = ExitStack() + stack.enter_context( + patch.object( + bundle_mod, + "_check_postgresql", + return_value={"ok": True, "latency_ms": 1.0}, + ) + ) + stack.enter_context( + patch.object( + bundle_mod, + "_check_redis", + return_value={"ok": True, "latency_ms": 1.0}, + ) + ) + stack.enter_context( + patch.object( + bundle_mod, + "_check_storage", + return_value={"ok": True, "backend": "minio", "latency_ms": 1.0}, + ) + ) + stack.enter_context( + patch.object( + bundle_mod, + "_collect_migration_state", + return_value={"status": "ok", "current": "abc", "heads": ["abc"]}, + ) + ) + stack.enter_context( + patch.object( + bundle_mod, + "_collect_queue_stats", + return_value={ + "mode": "redis", + "depth": 0, + "queued": 0, + "started": 0, + "failed": 0, + }, + ) + ) + stack.enter_context( + patch.object( + bundle_mod, + "_collect_recent_errors", + return_value=[ + { + "level": "ERROR", + "logger": "test", + "message": ( + f"password={_EXAMPLE_PASSWORD} " + f"token={_EXAMPLE_API_KEY} " + f"file={_SEEDED_FILENAME} " + f"path={_SEEDED_PATH} " + f"notes={_SEEDED_TXT} " + f"dotenv={_SEEDED_DOTFILE}" + ), + "timestamp": "2026-07-14T00:00:00+00:00", + "source": "log", + } + ], + ) + ) + return stack + + +class TestDiagnosticsBundleLocalMode: + """conftest stubs auth as local-mode (permissive) by default.""" + + def test_returns_headers_schema_and_no_leakage(self, client): + from find_api.diagnostics import bundle as bundle_mod + + with _patch_collector_with_seeded_secrets(bundle_mod): + resp = client.get(_ENDPOINT) + + assert resp.status_code == 200 + assert ( + resp.headers["content-disposition"] + == 'attachment; filename="find-diagnostics-bundle.json"' + ) + assert resp.headers["x-find-diagnostics"] == "local-only" + body = resp.json() + assert body["schema_version"] == 1 + assert set(body) >= { + "schema_version", + "generated_at", + "privacy_notice", + "app", + "runtime", + "migrations", + "services", + "queue", + "models", + "errors", + } + _assert_no_leakage(body) + assert _SEEDED_TXT not in resp.text + err_msg = body["errors"][0]["message"] + assert _SEEDED_DOTFILE not in err_msg + assert _EXAMPLE_PASSWORD not in err_msg + + def test_collector_failure_returns_generic_500(self, client): + with patch( + "find_api.routers.diagnostics.collect_diagnostics_bundle", + side_effect=RuntimeError( + r"boom at C:\Users\alice\secret.env with password=EXAMPLE_PASSWORD_PLACEHOLDER" + ), + ): + resp = client.get(_ENDPOINT) + + assert resp.status_code == 500 + assert resp.headers["x-find-diagnostics"] == "local-only" + assert ( + resp.headers["content-disposition"] + == 'attachment; filename="find-diagnostics-bundle.json"' + ) + body = resp.json() + assert body == {"error": "Failed to generate diagnostics bundle"} + assert "EXAMPLE_PASSWORD_PLACEHOLDER" not in resp.text + assert "Traceback" not in resp.text + assert "RuntimeError" not in resp.text + + +class TestDiagnosticsBundleSharedModeAuth: + """Admin-only enforcement once shared mode is active.""" + + @pytest.fixture(autouse=True) + def _use_real_auth_dependencies(self, client): + from find_api.core.dependencies import get_admin_user, get_required_user + + removed = {} + for dep in (get_required_user, get_admin_user): + if dep in app.dependency_overrides: + removed[dep] = app.dependency_overrides.pop(dep) + yield + app.dependency_overrides.update(removed) + + @pytest.fixture() + def shared_tokens(self, db): + admin = _make_user(db, "admin", "admin") + member = _make_user(db, "member", "member") + admin_token, _ = create_session(db, admin.id) + member_token, _ = create_session(db, member.id) + return {"admin": admin_token, "member": member_token} + + def test_unauthenticated_returns_401(self, client, shared_tokens): + with patch( + "find_api.routers.diagnostics.collect_diagnostics_bundle", + return_value=_FAKE_BUNDLE, + ): + resp = client.get(_ENDPOINT) + assert resp.status_code == 401 + + def test_member_returns_403(self, client, shared_tokens): + with patch( + "find_api.routers.diagnostics.collect_diagnostics_bundle", + return_value=_FAKE_BUNDLE, + ): + resp = client.get(_ENDPOINT, headers=_auth(shared_tokens["member"])) + assert resp.status_code == 403 + + def test_admin_returns_200_with_headers(self, client, shared_tokens): + with patch( + "find_api.routers.diagnostics.collect_diagnostics_bundle", + return_value=_FAKE_BUNDLE, + ): + resp = client.get(_ENDPOINT, headers=_auth(shared_tokens["admin"])) + + assert resp.status_code == 200 + assert ( + resp.headers["content-disposition"] + == 'attachment; filename="find-diagnostics-bundle.json"' + ) + assert resp.headers["x-find-diagnostics"] == "local-only" + assert resp.json()["schema_version"] == 1 + + +class TestHealthProbeTimeoutIsEnforced: + """The probe timeout must bound wall-clock time, not just raise late. + + ``with ThreadPoolExecutor(...)`` calls ``shutdown(wait=True)`` on exit, so + the original helper blocked until the hung probe finished and the timeout + had no effect. These assert the bound is real. + """ + + def test_returns_promptly_when_the_probe_hangs(self): + from find_api.diagnostics.bundle import _run_with_timeout + + started = time.perf_counter() + with pytest.raises(TimeoutError): + _run_with_timeout(lambda: time.sleep(30), timeout_s=0.2) + elapsed = time.perf_counter() - started + + # Generous ceiling for slow CI, still far below the 30s hang. + assert elapsed < 5, f"timeout did not bound wall clock: {elapsed:.2f}s" + + def test_returns_value_when_the_probe_completes(self): + from find_api.diagnostics.bundle import _run_with_timeout + + assert _run_with_timeout(lambda: "healthy", timeout_s=5) == "healthy" + + def test_propagates_probe_exceptions_unchanged(self): + from find_api.diagnostics.bundle import _run_with_timeout + + def _boom(): + raise ValueError("probe exploded") + + with pytest.raises(ValueError, match="probe exploded"): + _run_with_timeout(_boom, timeout_s=5) + + +class TestUnmockedBundleOverTheWire: + """Exercise the real collector through the real endpoint. + + Every other endpoint test patches ``collect_diagnostics_bundle``, so none + of them prove a genuine bundle survives strict JSON serialisation or that + the redaction layer leaves the reported fields intact end to end. + """ + + def test_real_bundle_serialises_and_keeps_useful_fields(self, client): + from find_api.diagnostics import bundle as bundle_mod + + # Stub only the outbound probes so the test stays fast and offline. + # Model collection, redaction, and serialisation all run for real — + # those are the paths no other endpoint test covers. + with ExitStack() as stack: + for name, value in ( + ("_check_postgresql", {"ok": True, "latency_ms": 1.0}), + ("_check_redis", {"ok": True, "latency_ms": 1.0}), + ( + "_check_storage", + {"ok": True, "backend": "minio", "latency_ms": 1.0}, + ), + ( + "_collect_migration_state", + {"status": "ok", "current": "abc", "heads": ["abc"]}, + ), + ): + stack.enter_context(patch.object(bundle_mod, name, return_value=value)) + resp = client.get(_ENDPOINT) + + assert resp.status_code == 200 + body = resp.json() + + # Strict: no default=str fallback, so a stray datetime fails loudly. + json.dumps(body) + + assert body["schema_version"] == 1 + assert set(body) >= { + "app", + "runtime", + "migrations", + "services", + "queue", + "models", + "errors", + } + # Model identifiers must stay readable — a filename-shaped name like + # yolo26n.pt previously collapsed to "". + assert body["models"]["yolo_model"].endswith(".pt") + assert isinstance(body["models"]["embedding_dim"], int) + assert isinstance(body["errors"], list) + _assert_no_leakage(body) diff --git a/backend/tests/test_diagnostics_redact.py b/backend/tests/test_diagnostics_redact.py new file mode 100644 index 00000000..843cc25a --- /dev/null +++ b/backend/tests/test_diagnostics_redact.py @@ -0,0 +1,493 @@ +"""Redaction tests for the privacy-safe diagnostics bundle. + +Seeds placeholder secrets and private media metadata, then asserts the +allowlist + scrub pipeline leaves zero residual leakage. + +Values below are intentional non-secrets (PLACEHOLDER / REDACTED markers) +so secret scanners do not treat them as real credentials. +""" + +from __future__ import annotations + +import json + +import pytest + +from find_api.diagnostics.redact import ( + REDACTED, + REDACTED_KEY, + redact_payload, + scrub_string, +) + +# Synthetic fixtures only — never real credentials. +# DSN/URLs are assembled from parts so scanners never see contiguous credentials. +_EXAMPLE_PASSWORD = "EXAMPLE_PASSWORD_PLACEHOLDER" +_EXAMPLE_STORAGE_SECRET = "EXAMPLE_STORAGE_SECRET_PLACEHOLDER" +_EXAMPLE_API_KEY = "sk-test-" + "FAKE-KEY-FOR-TESTING-ONLY" +_EXAMPLE_BEARER = "Bearer " + "FAKE.TEST.TOKEN" +_EXAMPLE_DSN = "postgresql://" + "USER" + ":" + "REDACTED" + "@localhost:5432/find" +_EXAMPLE_REDIS_URL = "redis://" + ":" + "EXAMPLE_PASSWORD_PLACEHOLDER" + "@localhost" + +SECRETS = [ + _EXAMPLE_PASSWORD, + _EXAMPLE_STORAGE_SECRET, + _EXAMPLE_API_KEY, + _EXAMPLE_BEARER, + _EXAMPLE_DSN, +] + +PRIVATE_STRINGS = [ + "A smiling woman standing by the lake at sunset", # caption-like + "INVOICE #48291 TOTAL DUE", # OCR-like + "vacation-photo-2024.jpg", + r"C:\Users\alice\Pictures\vacation-photo-2024.jpg", + "/var/lib/find/storage/uploads/ab/abcdef.jpg", + "face_embedding=[0.12, 0.98, -0.4]", +] + + +def _assert_no_leakage(payload) -> None: + """Serialize and assert no seeded secret or private fragment remains.""" + blob = json.dumps(payload, default=str) + for secret in SECRETS: + assert secret not in blob, f"secret leaked: {secret!r}" + for private in PRIVATE_STRINGS: + assert private not in blob, f"private metadata leaked: {private!r}" + + # Marker substrings that must never survive scrubbing. + for fragment in ( + "EXAMPLE_PASSWORD_PLACEHOLDER", + "EXAMPLE_STORAGE_SECRET_PLACEHOLDER", + "FAKE-KEY-FOR-TESTING-ONLY", + "FAKE.TEST.TOKEN", + "USER:REDACTED@", + "vacation-photo-2024.jpg", + "C:\\\\Users\\\\alice", + "/var/lib/find/storage", + "A smiling woman", + "INVOICE #48291", + ): + assert fragment not in blob, f"fragment leaked: {fragment!r}" + + +class TestScrubString: + def test_strips_filesystem_paths(self): + assert "" in scrub_string( + r"failed reading C:\Users\alice\Pictures\shot.jpg" + ) + assert "/var/lib/find" not in scrub_string( + "error in /var/lib/find/storage/uploads/ab/file.jpg" + ) + + def test_strips_filenames_any_extension(self): + out = scrub_string("Could not open vacation-photo-2024.jpg") + assert "vacation-photo-2024.jpg" not in out + assert "" in out + # Allowlist-free: uncommon extensions are still scrubbed. + for name in ( + "notes.xyz", + "private_notes.txt", + "customer.csv", + "private notes.txt", + ): + scrubbed = scrub_string(f"failed on {name}") + assert name not in scrubbed, name + assert "" in scrubbed + + def test_strips_dotfiles(self): + for name in (".env", ".gitignore", ".htaccess"): + scrubbed = scrub_string(f"cannot read {name} during startup") + assert name not in scrubbed, name + assert "" in scrubbed + + def test_version_numbers_not_treated_as_filenames(self): + assert scrub_string("app version 1.0.0 ready") == "app version 1.0.0 ready" + + def test_strips_url_credentials(self): + out = scrub_string(f"connect {_EXAMPLE_DSN}") + assert "USER:REDACTED@" not in out + assert "" in out + + def test_strips_password_only_redis_url(self): + out = scrub_string(f"redis connect {_EXAMPLE_REDIS_URL}") + assert "EXAMPLE_PASSWORD_PLACEHOLDER" not in out + assert "" in out + + def test_strips_secret_assignments(self): + out = scrub_string(f"password={_EXAMPLE_PASSWORD} token=FAKE.TEST.TOKEN") + assert _EXAMPLE_PASSWORD not in out + assert "password=" in out + + def test_strips_bearer_tokens(self): + out = scrub_string(f"Authorization: {_EXAMPLE_BEARER}") + assert "FAKE.TEST.TOKEN" not in out + + def test_strips_long_token_ending_in_hyphen(self): + # Trailing '-' is part of the token; word-boundary \\b would miss it. + token = ("A" * 39) + "-" + out = scrub_string(f"leak={token} trailing") + assert token not in out + assert REDACTED in out + + def test_strips_free_standing_quoted_private_text(self): + caption = PRIVATE_STRINGS[0] + out = scrub_string(f'model said "{caption}" during indexing') + assert caption not in out + assert '""' in out + + +class TestRedactPayloadAllowlist: + def test_unknown_keys_are_denied(self): + payload = redact_payload( + { + "schema_version": 1, + "password": SECRETS[0], + "caption": PRIVATE_STRINGS[0], + "mystery_field": "should_not_pass", + } + ) + assert payload["schema_version"] == 1 + assert "password" not in payload + assert "caption" not in payload + assert payload[REDACTED_KEY] == REDACTED + assert payload["mystery_field"] == REDACTED + _assert_no_leakage(payload) + + def test_sensitive_keys_redacted_even_when_nested(self): + payload = redact_payload( + { + "services": { + "postgresql": {"ok": True, "database_url": SECRETS[4]}, + "redis": {"ok": False, "redis_url": _EXAMPLE_REDIS_URL}, + }, + "models": { + "ml_mode": "mock", + "remote_ml_api_key": SECRETS[2], + }, + } + ) + assert payload["services"]["postgresql"]["ok"] is True + assert "database_url" not in payload["services"]["postgresql"] + assert payload["services"]["postgresql"][REDACTED_KEY] == REDACTED + assert "redis_url" not in payload["services"]["redis"] + assert payload["services"]["redis"][REDACTED_KEY] == REDACTED + assert payload["models"]["ml_mode"] == "mock" + assert "remote_ml_api_key" not in payload["models"] + assert payload["models"][REDACTED_KEY] == REDACTED + _assert_no_leakage(payload) + + def test_nested_lists_and_dicts(self): + seeded = { + "schema_version": 1, + "errors": [ + { + "level": "ERROR", + "logger": "find_api.workers", + "message": ( + f"upload failed path={PRIVATE_STRINGS[3]} " + f"password={SECRETS[0]} caption={PRIVATE_STRINGS[0]}" + ), + "timestamp": "2026-07-14T00:00:00+00:00", + "source": "log", + "user_id": 42, + "filename": PRIVATE_STRINGS[2], + "embedding": [0.1, 0.2, 0.3], + "faces": [{"bbox": [1, 2, 3, 4]}], + } + ], + "queue": {"mode": "redis", "depth": 3, "failed": 1}, + "ocr_text": PRIVATE_STRINGS[1], + "metadata_json": {"caption": PRIVATE_STRINGS[0]}, + } + payload = redact_payload(seeded) + + assert payload["queue"]["depth"] == 3 + assert payload["queue"]["failed"] == 1 + assert "ocr_text" not in payload + assert "metadata_json" not in payload + assert payload[REDACTED_KEY] == REDACTED + + err = payload["errors"][0] + assert err["level"] == "ERROR" + assert err["source"] == "log" + assert "user_id" not in err + assert "filename" not in err + assert "embedding" not in err + assert "faces" not in err + assert err[REDACTED_KEY] == REDACTED + assert SECRETS[0] not in err["message"] + assert PRIVATE_STRINGS[0] not in err["message"] + assert PRIVATE_STRINGS[3] not in err["message"] + _assert_no_leakage(payload) + + def test_empty_and_scalar_edge_cases(self): + assert redact_payload({}) == {} + assert redact_payload([]) == [] + assert redact_payload(None) is None + assert redact_payload(True) is True + assert redact_payload(0) == 0 + assert redact_payload(3.14) == 3.14 + + def test_tuple_coerced_to_list(self): + out = redact_payload(("ok", {"password": "x"})) + assert isinstance(out, list) + assert out[0] == "ok" + assert "password" not in out[1] + assert out[1][REDACTED_KEY] == REDACTED + + def test_allowlisted_string_still_scrubbed(self): + payload = redact_payload( + { + "errors": [ + { + "message": f"boom at {PRIVATE_STRINGS[4]} token={SECRETS[2]}", + "level": "ERROR", + "logger": "test", + "source": "log", + "timestamp": None, + } + ] + } + ) + msg = payload["errors"][0]["message"] + assert SECRETS[2] not in msg + assert "/var/lib/find" not in msg + _assert_no_leakage(payload) + + def test_private_media_keys_never_pass(self): + payload = redact_payload( + { + "app": {"version": "1.0.0"}, + "filename": "vacation-photo-2024.jpg", + "minio_key": "images/ab/abcdef.jpg", + "thumbnail_key": "thumbnails/ab/abcdef.webp", + "caption": PRIVATE_STRINGS[0], + "ocr": PRIVATE_STRINGS[1], + "ocr_text": PRIVATE_STRINGS[1], + "embedding": [0.01] * 8, + "vector": [0.02] * 8, + "face": {"landmarks": [1, 2]}, + "faces": [], + "person": "Alice", + "people": ["Alice", "Bob"], + "user_id": 7, + "uploader": 7, + "email": "alice@example.com", + "username": "alice", + "file_hash": "abc123", + "exif_json": {"Make": "Canon"}, + } + ) + assert payload["app"]["version"] == "1.0.0" + for key in ( + "filename", + "minio_key", + "thumbnail_key", + "caption", + "ocr", + "ocr_text", + "embedding", + "vector", + "face", + "faces", + "person", + "people", + "user_id", + "uploader", + "email", + "username", + "file_hash", + "exif_json", + ): + assert key not in payload + assert payload[REDACTED_KEY] == REDACTED + _assert_no_leakage(payload) + + +class TestCollectBundleRedacts: + def test_collect_diagnostics_bundle_shape_and_no_secrets(self, monkeypatch): + """Collector output is structured and already redacted.""" + from find_api.diagnostics import bundle as bundle_mod + + monkeypatch.setattr( + bundle_mod, + "_check_postgresql", + lambda: {"ok": True, "latency_ms": 1.0}, + ) + monkeypatch.setattr( + bundle_mod, + "_check_redis", + lambda: {"ok": True, "latency_ms": 1.0}, + ) + monkeypatch.setattr( + bundle_mod, + "_check_storage", + lambda: {"ok": True, "backend": "minio", "latency_ms": 1.0}, + ) + monkeypatch.setattr( + bundle_mod, + "_collect_migration_state", + lambda: {"status": "ok", "current": "abc", "heads": ["abc"]}, + ) + monkeypatch.setattr( + bundle_mod, + "_collect_queue_stats", + lambda: { + "mode": "redis", + "depth": 0, + "queued": 0, + "started": 0, + "failed": 0, + }, + ) + monkeypatch.setattr( + bundle_mod, + "_collect_recent_errors", + lambda: [ + { + "level": "ERROR", + "logger": "test", + "message": f"password={SECRETS[0]} file={PRIVATE_STRINGS[2]}", + "timestamp": "2026-07-14T00:00:00+00:00", + "source": "log", + } + ], + ) + + result = bundle_mod.collect_diagnostics_bundle() + + assert result["schema_version"] == 1 + assert "privacy_notice" in result + assert "app" in result + assert "runtime" in result + assert "migrations" in result + assert "services" in result + assert "queue" in result + assert "models" in result + assert "errors" in result + assert "local" in result["privacy_notice"].lower() + _assert_no_leakage(result) + + +@pytest.mark.parametrize( + "key", + [ + "password", + "SECRET_KEY", + "access_key", + "api_key", + "caption", + "ocr_text", + "embedding", + "faces", + "user_id", + "filename", + "minio_key", + ], +) +def test_sensitive_key_names_always_redacted(key): + payload = redact_payload({key: "leak-me-please", "schema_version": 1}) + assert key not in payload + assert payload[REDACTED_KEY] == REDACTED + assert payload["schema_version"] == 1 + + +class TestAllowlistBeatsSubstringHeuristic: + """The substring deny-net must not eat curated allowlist entries. + + ``embedding_dim`` is an int on the allowlist, but contains "embedding", + so the substring heuristic used to replace the whole entry with + ``redacted_key: [REDACTED]`` and drop a field the models section needs. + """ + + def test_embedding_dim_survives(self): + payload = redact_payload({"embedding_dim": 768, "schema_version": 1}) + assert payload["embedding_dim"] == 768 + assert REDACTED_KEY not in payload + + def test_non_allowlisted_lookalike_still_redacted(self): + # Not on the allowlist, so the substring net must still catch it. + payload = redact_payload({"embedding_cache": "leak-me", "schema_version": 1}) + assert "embedding_cache" not in payload + assert payload[REDACTED_KEY] == REDACTED + + def test_exact_sensitive_name_still_wins_over_allowlist(self): + # "error" is allowlisted; "token" is not, and is an exact sensitive + # name, so it must be stripped regardless. + payload = redact_payload({"token": _EXAMPLE_API_KEY, "error": "boom"}) + assert "token" not in payload + assert payload[REDACTED_KEY] == REDACTED + _assert_no_leakage(payload) + + +class TestMultipleSensitiveKeys: + """Several sensitive keys in one dict must not collapse into one entry.""" + + def test_each_sensitive_key_gets_its_own_placeholder(self): + payload = redact_payload( + { + "password": _EXAMPLE_PASSWORD, + "api_key": _EXAMPLE_API_KEY, + "caption": "A smiling woman standing by the lake at sunset", + "schema_version": 1, + } + ) + placeholders = [k for k in payload if k.startswith(REDACTED_KEY)] + assert len(placeholders) == 3, payload + assert all(payload[k] == REDACTED for k in placeholders) + assert payload["schema_version"] == 1 + _assert_no_leakage(payload) + + +class TestModelIdentifiersSurvive: + """Model names are operator config, not user data, and must stay readable. + + ``yolo26n.pt`` is filename-shaped, so the generic filename pattern used to + reduce it — and every entry of ``configured_models`` — to ````, + which defeats the point of reporting model state. + """ + + def test_filename_shaped_model_name_is_kept(self): + payload = redact_payload({"models": {"yolo_model": "yolo26n.pt"}}) + assert payload["models"]["yolo_model"] == "yolo26n.pt" + + def test_configured_models_list_is_kept(self): + payload = redact_payload( + { + "models": { + "configured_models": [ + "yolo26n.pt", + "ViT-B-16-SigLIP", + "Salesforce/blip-image-captioning-base", + ] + } + } + ) + assert payload["models"]["configured_models"] == [ + "yolo26n.pt", + "ViT-B-16-SigLIP", + "Salesforce/blip-image-captioning-base", + ] + + def test_exemption_does_not_disable_other_scrubbing(self): + # Credentials and absolute paths must still be removed even under an + # exempt key — only the filename rule is relaxed. + payload = redact_payload( + { + "models": { + "clip_model": f"model loaded from {_EXAMPLE_DSN}", + "blip_model": "/var/lib/find/storage/uploads/ab/abcdef.jpg", + } + } + ) + _assert_no_leakage(payload) + + def test_exemption_does_not_leak_into_sibling_keys(self): + # A filename under a normal key inside the same section is still + # scrubbed — the exemption is per-key, not per-section. + payload = redact_payload( + {"models": {"yolo_model": "yolo26n.pt", "message": "wrote holiday.jpg"}} + ) + assert payload["models"]["yolo_model"] == "yolo26n.pt" + assert "holiday.jpg" not in payload["models"]["message"] diff --git a/docs/diagnostics-bundle.md b/docs/diagnostics-bundle.md new file mode 100644 index 00000000..8315c2f9 --- /dev/null +++ b/docs/diagnostics-bundle.md @@ -0,0 +1,83 @@ +# Privacy-Safe Local Diagnostics Bundle + +> **Admin-only. Local-only. Never uploaded.** +> `GET /api/admin/diagnostics/bundle` requires admin access (shared mode) and +> returns a redacted JSON file to the caller only. Find does **not** send this +> bundle to any external service — paste it into a GitHub issue only after you +> have reviewed the contents yourself. + +Local-only support export for Find. Generate this bundle on your machine, review +it, then attach the JSON to a GitHub issue when asking for help. **Nothing is +uploaded automatically** — there is no telemetry, no cloud exporter, and no +outbound webhook. + +## How to generate + +### Admin API (explicit request) + +```http +GET /api/admin/diagnostics/bundle +Authorization: Bearer +``` + +- **Shared mode:** admin authentication required (`403` for non-admins, `401` if unauthenticated). +- **Local / single-user mode:** open (same pattern as other admin endpoints). +- Response is JSON with `Content-Disposition: attachment; filename="find-diagnostics-bundle.json"`. +- Header `X-Find-Diagnostics: local-only` marks the payload as a local export. + +Example with the running API: + +```bash +curl -fsS -H "Authorization: Bearer $TOKEN" \ + http://localhost:8000/api/admin/diagnostics/bundle \ + -o find-diagnostics-bundle.json +``` + +Open the file and search for any unexpected personal data before attaching it to +an issue. + +## What is included + +| Section | Contents | +| --- | --- | +| `schema_version` / `generated_at` | Bundle format version and UTC timestamp | +| `privacy_notice` | Short reminder that the export is local-only | +| `app` | App version, `ENVIRONMENT` | +| `runtime` | Python version/implementation, OS name/release/machine | +| `migrations` | Alembic current revision, heads, status (`ok` / `behind` / …) | +| `services` | Connectivity + latency for PostgreSQL, Redis, and storage (MinIO or local). Errors are sanitized. | +| `queue` | Queue mode, depth / queued / started / failed (/ finished when available) | +| `models` | `ML_MODE`, accel mode, configured model **names**, embedding dim, whether remote ML is configured (**boolean only**), currently loaded model names | +| `errors` | Up to 20 recent sanitized ERROR log lines plus failed media analysis messages. Credentials and secret assignments are stripped; **hostnames and residual path fragments may still appear** after sanitization. | + +Service checks only report `ok`, `latency_ms`, optional sanitized `error`, and for storage the backend kind (`minio` / `local`). Connection credentials are never included. Sanitized error strings may still contain hostnames or path fragments — only URL credentials (user/password) are stripped from URLs. + +## What is excluded + +The redaction layer is **allowlist-first**: unknown keys are denied. Explicitly stripped categories include: + +- Passwords, tokens, API keys, access/secret keys, bearer credentials, session cookies +- Database / Redis / MinIO connection strings and storage keys (`minio_key`, `thumbnail_key`) +- Absolute file paths and media filenames +- Captions, OCR text, EXIF / metadata JSON blobs +- Embeddings / vectors +- Face and person identifiers / landmarks +- User identifiers (user id, uploader, email, username, display name) +- Remote ML URL and API key values (only a boolean `remote_ml_configured` is kept) +- Raw image bytes or thumbnails (never collected) + +String values under allowlisted keys are still scrubbed with path, filename, URL-credential, and token patterns. + +## Privacy guarantees + +1. **Explicit action only** — a human (or admin client) must call the endpoint. +2. **Local response** — the API returns JSON to the requester; the server does not POST the bundle anywhere. +3. **Redaction before return** — `collect_diagnostics_bundle()` always runs `redact_payload()`. +4. **Credential stripping, not absolute silence** — passwords/tokens/storage keys/captions/OCR/embeddings/faces/user ids are removed. Hostnames and path-like fragments **may** still appear in sanitized error messages; review the JSON before attaching it to an issue. +5. **Suitable for GitHub issues** — after a manual review, attach `find-diagnostics-bundle.json` to help maintainers reproduce environment/migration/queue problems without receiving private media content. + +## Related + +- Issue tracking: [#347](https://github.com/Abhash-Chakraborty/Find/issues/347) +- Agent security policy: [policies/agent-security.md](./policies/agent-security.md) +- Setup troubleshooting: [guides/common-setup-errors.md](./guides/common-setup-errors.md) diff --git a/docs/index.md b/docs/index.md index 3aee34c9..51888359 100644 --- a/docs/index.md +++ b/docs/index.md @@ -36,6 +36,7 @@ This directory is organized by document purpose and implementation status. ## Guides - [Common Setup Errors](guides/common-setup-errors.md) +- [Privacy-Safe Diagnostics Bundle](diagnostics-bundle.md) - [Image Loading Behavior](guides/image-loading.md) - [Real ML Troubleshooting](guides/real-ml-troubleshooting.md) - [Features Guide](guides/features.md)