fix(metrics): parse LLM judge scores from structured JSON responses

openagent_eval/metrics/generation/llm_judge.py

@@ -6,6 +6,8 @@
 
 from __future__ import annotations
 
+import json
+import re
 from dataclasses import dataclass
 from typing import Any
 
@@ -14,6 +16,13 @@
 from openagent_eval.metrics.base import BaseMetric, MetricResult
 from openagent_eval.providers.base.llm import LLMProvider
 
+_JSON_SCORE_INSTRUCTION = (
+    'Respond with ONLY a JSON object on a single line: {{"score": <number from 0.0 to 1.0>}}'
+)
+_JSON_FENCE_RE = re.compile(r"```(?:json)?\s*(\{.*?\})\s*```", re.DOTALL | re.IGNORECASE)
+_SCORE_LABEL_RE = re.compile(r"score[\"'\s:]*(\d+\.?\d*)", re.IGNORECASE)
+_BARE_DECIMAL_RE = re.compile(r"^\s*(\d+\.?\d*)\s*$")
+
 
 @dataclass(frozen=True)
 class JudgeCriteria:
@@ -41,7 +50,7 @@ class JudgeCriteria:
         "Score from 0.0 (not supported at all) to 1.0 (fully supported).\n\n"
         "Context: {premise}\n\n"
         "Answer: {hypothesis}\n\n"
-        "Score:"
+        f"{_JSON_SCORE_INSTRUCTION}"
     ),
 )
 
@@ -53,7 +62,7 @@ class JudgeCriteria:
         "Score from 0.0 (completely irrelevant) to 1.0 (fully addresses the question).\n\n"
         "Question: {premise}\n\n"
         "Answer: {hypothesis}\n\n"
-        "Score:"
+        f"{_JSON_SCORE_INSTRUCTION}"
     ),
 )
 
@@ -66,7 +75,7 @@ class JudgeCriteria:
         "Score from 0.0 (very incomplete) to 1.0 (fully comprehensive).\n\n"
         "Topic: {premise}\n\n"
         "Answer: {hypothesis}\n\n"
-        "Score:"
+        f"{_JSON_SCORE_INSTRUCTION}"
     ),
 )
 
@@ -164,29 +173,70 @@ def evaluate(self, **kwargs: Any) -> MetricResult:
 
     def _parse_score(self, response: str) -> float:
         """Parse numeric score from LLM response text."""
-        import re
-
-        # Try to find a decimal number
-        match = re.search(r'(\d+\.?\d*)', response.strip())
-        if match:
-            score = float(match.group(1))
-            # Normalize to 0-1 range if needed
-            if score > 1.0:
-                # Could be percentage (85 -> 0.85) or scale (8.5 -> 0.85)
-                if score <= 10.0:
-                    score = score / 10.0
-                else:
-                    score = min(score / 100.0, 1.0)
-            return max(0.0, min(1.0, score))
-
-        # Fallback: check for keywords
-        response_lower = response.lower()
-        if any(word in response_lower for word in ["yes", "true", "fully", "completely", "excellent"]):
-            return 1.0
-        if any(word in response_lower for word in ["no", "false", "not at all", "poor"]):
-            return 0.0
-
-        return 0.5  # Default neutral score
+        text = response.strip()
+        if not text:
+            logger.warning("LLM judge: empty response, using default score 0.5")
+            return 0.5
+
+        for candidate in self._json_score_candidates(text):
+            try:
+                payload = json.loads(candidate)
+            except json.JSONDecodeError:
+                continue
+            if isinstance(payload, dict):
+                normalized = {str(key).lower(): value for key, value in payload.items()}
+                if "score" in normalized:
+                    try:
+                        score = float(normalized["score"])
+                    except (TypeError, ValueError):
+                        logger.warning(
+                            "LLM judge: invalid JSON score value in response: {}",
+                            response[:200],
+                        )
+                        continue
+                    return self._clamp_normalized_score(score)
+
+        label_match = _SCORE_LABEL_RE.search(text)
+        if label_match:
+            logger.warning(
+                "LLM judge: parsed score from non-JSON labeled response: {}",
+                response[:200],
+            )
+            return self._clamp_normalized_score(float(label_match.group(1)))
+
+        bare_match = _BARE_DECIMAL_RE.match(text)
+        if bare_match:
+            logger.warning(
+                "LLM judge: parsed score from bare numeric response: {}",
+                response[:200],
+            )
+            return self._clamp_normalized_score(float(bare_match.group(1)))
+
+        logger.warning(
+            "LLM judge: failed to parse score from response, using default 0.5: {}",
+            response[:200],
+        )
+        return 0.5
+
+    @staticmethod
+    def _json_score_candidates(text: str) -> list[str]:
+        candidates: list[str] = []
+        fence_match = _JSON_FENCE_RE.search(text)
+        if fence_match:
+            candidates.append(fence_match.group(1))
+        candidates.append(text)
+        return candidates
+
+    @staticmethod
+    def _clamp_normalized_score(score: float) -> float:
+        if score > 1.0:
+            if score <= 10.0:
+                score = score / 10.0
+            elif score <= 100.0:
+                score = score / 100.0
+            else:
+                score = min(score / 100.0, 1.0)
+        return max(0.0, min(1.0, score))
 
 
 class AsyncLLMJudgeMetric(LLMJudgeMetric):

tests/unit/test_metrics/test_nli.py

@@ -316,31 +316,76 @@ def test_llm_judge_missing_inputs(self):
 
     def test_llm_judge_parse_score_decimal(self):
         metric = LLMJudgeMetric(provider=self.mock_provider)
-        score = metric._parse_score("The score is 0.85")
+        score = metric._parse_score('{"score": 0.85}')
         assert score == 0.85
 
+    def test_llm_judge_parse_score_plain_decimal_backward_compat(self):
+        metric = LLMJudgeMetric(provider=self.mock_provider)
+        score = metric._parse_score("0.92")
+        assert score == 0.92
+
+    def test_llm_judge_parse_score_labeled_fallback(self):
+        metric = LLMJudgeMetric(provider=self.mock_provider)
+        score = metric._parse_score("Score: 0.85")
+        assert score == 0.85
+
+    def test_llm_judge_parse_score_case_insensitive_json_key(self):
+        metric = LLMJudgeMetric(provider=self.mock_provider)
+        score = metric._parse_score('{"Score": 0.77}')
+        assert score == 0.77
+
+    def test_llm_judge_parse_score_empty_response(self):
+        metric = LLMJudgeMetric(provider=self.mock_provider)
+        score = metric._parse_score("   ")
+        assert score == 0.5
+
+    def test_llm_judge_parse_score_invalid_json_value(self):
+        metric = LLMJudgeMetric(provider=self.mock_provider)
+        score = metric._parse_score('{"score": "high"}')
+        assert score == 0.5
+
+    def test_llm_judge_parse_score_ten_point_scale(self):
+        metric = LLMJudgeMetric(provider=self.mock_provider)
+        score = metric._parse_score('{"score": 8.5}')
+        assert score == 0.85
+
+    def test_llm_judge_parse_score_json_with_fence(self):
+        metric = LLMJudgeMetric(provider=self.mock_provider)
+        score = metric._parse_score('```json\n{"score": 0.72}\n```')
+        assert score == 0.72
+
     def test_llm_judge_parse_score_percentage(self):
         metric = LLMJudgeMetric(provider=self.mock_provider)
-        score = metric._parse_score("85%")
+        score = metric._parse_score('{"score": 85}')
         assert score == 0.85  # 85 / 100
 
     def test_llm_judge_parse_score_keyword_yes(self):
         metric = LLMJudgeMetric(provider=self.mock_provider)
         score = metric._parse_score("Yes, the answer is excellent")
-        assert score == 1.0
+        assert score == 0.5
 
     def test_llm_judge_parse_score_keyword_no(self):
         metric = LLMJudgeMetric(provider=self.mock_provider)
         score = metric._parse_score("No, not at all")
-        assert score == 0.0
+        assert score == 0.5
+
+    def test_llm_judge_parse_score_avoids_false_keyword_match(self):
+        metric = LLMJudgeMetric(provider=self.mock_provider)
+        score = metric._parse_score("This is not excellent but acceptable")
+        assert score == 0.5
+
+    def test_llm_judge_parse_score_prefers_json_over_ambiguous_text(self):
+        metric = LLMJudgeMetric(provider=self.mock_provider)
+        score = metric._parse_score('There are 5 reasons why this is a score of 9 {"score": 0.9}')
+        assert score == 0.9
 
     def test_llm_judge_parse_score_neutral(self):
         metric = LLMJudgeMetric(provider=self.mock_provider)
         score = metric._parse_score("some random text")
         assert score == 0.5
 
     def test_llm_judge_evaluate_success(self):
-        self.mock_provider.generate.return_value = "0.92"
+        self.mock_provider.generate.return_value = '{"score": 0.92}'
         metric = LLMJudgeMetric(provider=self.mock_provider)
         result = metric.evaluate(premise="context", hypothesis="answer")
         assert result.score == 0.92