feat(tier3): enforce structured JSON schemas and rate-limit retries in LLM judges

CHANGELOG.md

@@ -4,6 +4,24 @@ All notable changes to SkillEvaluator are documented in this file.
 
 ## Unreleased
 
+### Added
+
+- Transparent HTTP 429 (rate-limiting), transient 5xx, and timeout recovery for
+  LLM judges in both the Harbor container verifier (`eval.py`) and host runtime
+  (`LLMClient`). Features zero-dependency full jitter exponential backoff,
+  RFC-7231 `Retry-After` header parsing, defensive environment overrides
+  (`SKILL_EVAL_LLM_MAX_RETRIES`, `SKILL_EVAL_LLM_RETRY_BASE_DELAY`, and
+  `SKILL_EVAL_LLM_RETRY_MAX_DELAY`), and
+  automatic container forwarding via Harbor `task.toml` without altering
+  benchmark metrics or scoring formulas.
+- Provider-aware structured JSON schema enforcement (`response_format` for
+  OpenAI-compatible / Gemini Vertex / NVIDIA NIM endpoints and `output_config`
+  for Anthropic `/v1/messages`) across `judge_accuracy`, `judge_goal_accuracy`,
+  and `judge_behavior_check`, with automatic `HTTP 400`/`422` downgrade and
+  per-target memoization (`_SCHEMA_UNSUPPORTED_TARGETS`), boolean prompt
+  alignment, and a guard for missing `message` fields on reasoning token
+  exhaustion.
+
 ## 0.3.0 - 2026-09-17
 
 ### Added

src/skillevaluator/inference/client.py

@@ -25,6 +25,7 @@ class supports two usage patterns:
 from urllib.parse import urlsplit
 
 from skillevaluator.constants import LLM_VERIFY_MODEL, LLM_VERIFY_TEMPERATURE
+from skillevaluator.inference.retry import resolve_retry_config, retry_call_with_backoff
 from skillevaluator.inference.types import EmptyLLMResponseError, LLMClientError
 from skillevaluator.logging_config import get_logger
 from skillevaluator.provider_config import (
@@ -104,6 +105,74 @@ def _temperature_kwargs(model: str, temperature: float | None) -> dict[str, floa
     return {"temperature": temperature}
 
 
+_SCHEMA_UNSUPPORTED_TARGETS: set[tuple[str, str, str]] = set()
+
+
+def _build_openai_response_format(schema: dict[str, Any], schema_name: str = "judge_response") -> dict[str, Any]:
+    """Build OpenAI-compatible JSON schema response_format payload."""
+    return {
+        "type": "json_schema",
+        "json_schema": {
+            "name": schema_name,
+            "strict": True,
+            "schema": schema,
+        },
+    }
+
+
+def _build_anthropic_output_config(schema: dict[str, Any]) -> dict[str, Any]:
+    """Build Anthropic Messages API output_config payload."""
+    return {
+        "format": {
+            "type": "json_schema",
+            "schema": schema,
+        }
+    }
+
+
+def _is_schema_unsupported_error(exc: Exception) -> bool:
+    status_code = getattr(exc, "status_code", None)
+    if status_code is None:
+        response = getattr(exc, "response", None)
+        status_code = getattr(response, "status_code", None)
+    return status_code in {400, 422} or isinstance(exc, TypeError)
+
+
+def _call_with_schema_fallback(
+    call_fn: Any,
+    call_kwargs: dict[str, Any],
+    *,
+    schema_key: str,
+    target_key: tuple[str, str, str],
+    use_schema: bool,
+) -> Any:
+    """Invoke call_fn and downgrade to prompt-only on HTTP 400/422 schema errors."""
+    try:
+        return call_fn(**call_kwargs)
+    except Exception as exc:
+        if use_schema and _is_schema_unsupported_error(exc):
+            _SCHEMA_UNSUPPORTED_TARGETS.add(target_key)
+            logger.warning(
+                "Structured output schema unsupported by provider=%s model=%s; "
+                "downgrading to prompt-only JSON and memoizing target.",
+                target_key[0],
+                target_key[2],
+            )
+            call_kwargs.pop(schema_key, None)
+            return call_fn(**call_kwargs)
+        raise
+
+
+def _extract_choice_content(response: Any) -> str:
+    choices = getattr(response, "choices", None) or []
+    first_choice = choices[0] if choices else None
+    message = getattr(first_choice, "message", None) if first_choice is not None else None
+    content = getattr(message, "content", None) if message is not None else ""
+    if not content:
+        return ""
+    return content.strip()
+
+
 class LLMClient:
     """Public-provider client for chat completions.
 
@@ -132,12 +201,19 @@ def __init__(
         *,
         max_tokens: int | None = None,
         temperature: float | None = None,
+        max_retries: int | None = None,
+        retry_base_delay: float | None = None,
+        retry_max_delay: float | None = None,
     ) -> None:
         self._model = model
         self._base_url = base_url
         self._api_key = api_key
         self._max_tokens = max_tokens if max_tokens is not None else self.default_max_tokens
         self._temperature = temperature if temperature is not None else self.default_temperature
+        retry_cfg = resolve_retry_config()
+        self._max_retries = max_retries if max_retries is not None else retry_cfg.max_retries
+        self._retry_base_delay = retry_base_delay if retry_base_delay is not None else retry_cfg.base_delay
+        self._retry_max_delay = retry_max_delay if retry_max_delay is not None else retry_cfg.max_delay
         self._client: Any = None
         self._provider_config: ProviderConfig | None = None
 
@@ -159,6 +235,21 @@ def api_key(self) -> str | None:
     def temperature(self) -> float | None:
         return self._temperature
 
+    @property
+    def max_retries(self) -> int:
+        """Return the maximum number of retry attempts for transient errors."""
+        return self._max_retries
+
+    @property
+    def retry_base_delay(self) -> float:
+        """Return the initial base backoff delay in seconds."""
+        return self._retry_base_delay
+
+    @property
+    def retry_max_delay(self) -> float:
+        """Return the maximum delay ceiling in seconds for a retry backoff."""
+        return self._retry_max_delay
+
     # -- client management ------------------------------------------------
 
     def _resolved_config(self) -> ProviderConfig:
@@ -236,62 +327,98 @@ def _get_client(self) -> Any:
 
     # -- direct-use methods -----------------------------------------------
 
-    def completions(self, system_prompt: str, user_prompt: str) -> str:
+    def completions(
+        self,
+        system_prompt: str,
+        user_prompt: str,
+        *,
+        response_schema: dict[str, Any] | None = None,
+        schema_name: str = "judge_response",
+    ) -> str:
         """Send a chat completion request and return the response text.
 
         Raises :class:`LLMClientError` when the response is empty.
         """
         config = self._resolved_config()
         client = self._get_client()
-        if config.provider == "anthropic":
+        target_key = (config.provider, config.base_url or "", config.model)
+
+        def _invoke_provider() -> str:
+            use_schema = response_schema is not None and target_key not in _SCHEMA_UNSUPPORTED_TARGETS
+            if config.provider == "anthropic":
+                call_kwargs: dict[str, Any] = {
+                    "model": config.model,
+                    "max_tokens": self._max_tokens or 4096,
+                    "system": system_prompt,
+                    "messages": [{"role": "user", "content": user_prompt}],
+                    **_temperature_kwargs(config.model, self._temperature),
+                }
+                if use_schema and response_schema is not None:
+                    call_kwargs["output_config"] = _build_anthropic_output_config(response_schema)
+                response = _call_with_schema_fallback(
+                    client.messages.create,
+                    call_kwargs,
+                    schema_key="output_config",
+                    target_key=target_key,
+                    use_schema=use_schema,
+                )
+                content = "".join(
+                    str(block.text) for block in response.content if getattr(block, "type", None) == "text"
+                )
+                if not content:
+                    raise EmptyLLMResponseError("LLM returned empty response content")
+                return content.strip()
+            if config.provider == "bedrock":
+                try:
+                    from litellm import completion
+                except ImportError as exc:
+                    raise LLMClientError(
+                        "The 'litellm' package is required for Bedrock LLM operations. Install with: pip install 'skillevaluator[llm]'"
+                    ) from exc
+                response = completion(
+                    model=config.litellm_model,
+                    messages=[
+                        {"role": "system", "content": system_prompt},
+                        {"role": "user", "content": user_prompt},
+                    ],
+                    aws_region_name=config.region,
+                    **_temperature_kwargs(config.model, self._temperature),
+                    **({"max_tokens": self._max_tokens} if self._max_tokens is not None else {}),
+                )
+                content = _extract_choice_content(response)
+                if not content:
+                    raise EmptyLLMResponseError("LLM returned empty response content")
+                return content
             call_kwargs: dict[str, Any] = {
                 "model": config.model,
-                "max_tokens": self._max_tokens or 4096,
-                "system": system_prompt,
-                "messages": [{"role": "user", "content": user_prompt}],
-                **_temperature_kwargs(config.model, self._temperature),
-            }
-            response = client.messages.create(**call_kwargs)
-            content = "".join(str(block.text) for block in response.content if getattr(block, "type", None) == "text")
-            if not content:
-                raise EmptyLLMResponseError("LLM returned empty response content")
-            return content.strip()
-        if config.provider == "bedrock":
-            try:
-                from litellm import completion
-            except ImportError as exc:
-                raise LLMClientError(
-                    "The 'litellm' package is required for Bedrock LLM operations. Install with: pip install 'skillevaluator[llm]'"
-                ) from exc
-            response = completion(
-                model=config.litellm_model,
-                messages=[
+                "messages": [
                     {"role": "system", "content": system_prompt},
                     {"role": "user", "content": user_prompt},
                 ],
-                aws_region_name=config.region,
                 **_temperature_kwargs(config.model, self._temperature),
-                **({"max_tokens": self._max_tokens} if self._max_tokens is not None else {}),
+                **_token_limit_kwargs(config, self._max_tokens),
+            }
+            if use_schema and response_schema is not None:
+                call_kwargs["response_format"] = _build_openai_response_format(response_schema, schema_name)
+
+            response = _call_with_schema_fallback(
+                client.chat.completions.create,
+                call_kwargs,
+                schema_key="response_format",
+                target_key=target_key,
+                use_schema=use_schema,
             )
-            content = response.choices[0].message.content
+            content = _extract_choice_content(response)
             if not content:
                 raise EmptyLLMResponseError("LLM returned empty response content")
-            return str(content).strip()
-        call_kwargs: dict[str, Any] = {
-            "model": config.model,
-            "messages": [
-                {"role": "system", "content": system_prompt},
-                {"role": "user", "content": user_prompt},
-            ],
-            **_temperature_kwargs(config.model, self._temperature),
-            **_token_limit_kwargs(config, self._max_tokens),
-        }
-
-        response = client.chat.completions.create(**call_kwargs)
-        content = response.choices[0].message.content
-        if not content:
-            raise EmptyLLMResponseError("LLM returned empty response content")
-        return content.strip()
+            return content
+
+        return retry_call_with_backoff(
+            _invoke_provider,
+            max_retries=self._max_retries,
+            base_delay=self._retry_base_delay,
+            max_delay=self._retry_max_delay,
+        )
 
     def extract_json_from_response(self, system_prompt: str, user_prompt: str) -> dict:
         """Send a completion and parse JSON from the response."""