From f47b640700ff9dd7beadc613f6f72c34fd76aa49 Mon Sep 17 00:00:00 2001 From: syf2211 Date: Thu, 23 Jul 2026 00:03:21 +0000 Subject: [PATCH 1/3] fix(metrics): parse LLM judge scores from JSON responses Fixes #74 - Request structured JSON scores in built-in judge prompts - Parse JSON (including fenced blocks) before regex fallbacks - Remove keyword heuristics that produced false positives - Log warnings when falling back to regex or default score --- .../metrics/generation/llm_judge.py | 102 +++++++++++++----- tests/unit/test_metrics/test_nli.py | 55 +++++++++- 2 files changed, 126 insertions(+), 31 deletions(-) diff --git a/openagent_eval/metrics/generation/llm_judge.py b/openagent_eval/metrics/generation/llm_judge.py index ddb7300..39aa92d 100644 --- a/openagent_eval/metrics/generation/llm_judge.py +++ b/openagent_eval/metrics/generation/llm_judge.py @@ -6,6 +6,8 @@ from __future__ import annotations +import json +import re from dataclasses import dataclass from typing import Any @@ -14,6 +16,13 @@ from openagent_eval.metrics.base import BaseMetric, MetricResult from openagent_eval.providers.base.llm import LLMProvider +_JSON_SCORE_INSTRUCTION = ( + 'Respond with ONLY a JSON object on a single line: {{"score": }}' +) +_JSON_FENCE_RE = re.compile(r"```(?:json)?\s*(\{.*?\})\s*```", re.DOTALL | re.IGNORECASE) +_SCORE_LABEL_RE = re.compile(r"score[\"'\s:]*(\d+\.?\d*)", re.IGNORECASE) +_BARE_DECIMAL_RE = re.compile(r"^\s*(\d+\.?\d*)\s*$") + @dataclass(frozen=True) class JudgeCriteria: @@ -41,7 +50,7 @@ class JudgeCriteria: "Score from 0.0 (not supported at all) to 1.0 (fully supported).\n\n" "Context: {premise}\n\n" "Answer: {hypothesis}\n\n" - "Score:" + f"{_JSON_SCORE_INSTRUCTION}" ), ) @@ -53,7 +62,7 @@ class JudgeCriteria: "Score from 0.0 (completely irrelevant) to 1.0 (fully addresses the question).\n\n" "Question: {premise}\n\n" "Answer: {hypothesis}\n\n" - "Score:" + f"{_JSON_SCORE_INSTRUCTION}" ), ) @@ -66,7 +75,7 @@ class JudgeCriteria: "Score from 0.0 (very incomplete) to 1.0 (fully comprehensive).\n\n" "Topic: {premise}\n\n" "Answer: {hypothesis}\n\n" - "Score:" + f"{_JSON_SCORE_INSTRUCTION}" ), ) @@ -164,29 +173,70 @@ def evaluate(self, **kwargs: Any) -> MetricResult: def _parse_score(self, response: str) -> float: """Parse numeric score from LLM response text.""" - import re - - # Try to find a decimal number - match = re.search(r'(\d+\.?\d*)', response.strip()) - if match: - score = float(match.group(1)) - # Normalize to 0-1 range if needed - if score > 1.0: - # Could be percentage (85 -> 0.85) or scale (8.5 -> 0.85) - if score <= 10.0: - score = score / 10.0 - else: - score = min(score / 100.0, 1.0) - return max(0.0, min(1.0, score)) - - # Fallback: check for keywords - response_lower = response.lower() - if any(word in response_lower for word in ["yes", "true", "fully", "completely", "excellent"]): - return 1.0 - if any(word in response_lower for word in ["no", "false", "not at all", "poor"]): - return 0.0 - - return 0.5 # Default neutral score + text = response.strip() + if not text: + logger.warning("LLM judge: empty response, using default score 0.5") + return 0.5 + + for candidate in self._json_score_candidates(text): + try: + payload = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(payload, dict): + normalized = {str(key).lower(): value for key, value in payload.items()} + if "score" in normalized: + try: + score = float(normalized["score"]) + except (TypeError, ValueError): + logger.warning( + "LLM judge: invalid JSON score value in response: {}", + response[:200], + ) + continue + return self._clamp_normalized_score(score) + + label_match = _SCORE_LABEL_RE.search(text) + if label_match: + logger.warning( + "LLM judge: parsed score from non-JSON labeled response: {}", + response[:200], + ) + return self._clamp_normalized_score(float(label_match.group(1))) + + bare_match = _BARE_DECIMAL_RE.match(text) + if bare_match: + logger.warning( + "LLM judge: parsed score from bare numeric response: {}", + response[:200], + ) + return self._clamp_normalized_score(float(bare_match.group(1))) + + logger.warning( + "LLM judge: failed to parse score from response, using default 0.5: {}", + response[:200], + ) + return 0.5 + + @staticmethod + def _json_score_candidates(text: str) -> list[str]: + candidates: list[str] = [] + fence_match = _JSON_FENCE_RE.search(text) + if fence_match: + candidates.append(fence_match.group(1)) + candidates.append(text) + return candidates + + @staticmethod + def _clamp_normalized_score(score: float) -> float: + if score > 1.0: + if score <= 10.0: + score = score / 10.0 + elif score <= 100.0: + score = score / 100.0 + else: + score = min(score / 100.0, 1.0) + return max(0.0, min(1.0, score)) class AsyncLLMJudgeMetric(LLMJudgeMetric): diff --git a/tests/unit/test_metrics/test_nli.py b/tests/unit/test_metrics/test_nli.py index f62597a..01bb6d6 100644 --- a/tests/unit/test_metrics/test_nli.py +++ b/tests/unit/test_metrics/test_nli.py @@ -311,23 +311,68 @@ def test_llm_judge_missing_inputs(self): def test_llm_judge_parse_score_decimal(self): metric = LLMJudgeMetric(provider=self.mock_provider) - score = metric._parse_score("The score is 0.85") + score = metric._parse_score('{"score": 0.85}') assert score == 0.85 + def test_llm_judge_parse_score_plain_decimal_backward_compat(self): + metric = LLMJudgeMetric(provider=self.mock_provider) + score = metric._parse_score("0.92") + assert score == 0.92 + + def test_llm_judge_parse_score_labeled_fallback(self): + metric = LLMJudgeMetric(provider=self.mock_provider) + score = metric._parse_score("Score: 0.85") + assert score == 0.85 + + def test_llm_judge_parse_score_case_insensitive_json_key(self): + metric = LLMJudgeMetric(provider=self.mock_provider) + score = metric._parse_score('{"Score": 0.77}') + assert score == 0.77 + + def test_llm_judge_parse_score_empty_response(self): + metric = LLMJudgeMetric(provider=self.mock_provider) + score = metric._parse_score(" ") + assert score == 0.5 + + def test_llm_judge_parse_score_invalid_json_value(self): + metric = LLMJudgeMetric(provider=self.mock_provider) + score = metric._parse_score('{"score": "high"}') + assert score == 0.5 + + def test_llm_judge_parse_score_ten_point_scale(self): + metric = LLMJudgeMetric(provider=self.mock_provider) + score = metric._parse_score('{"score": 8.5}') + assert score == 0.85 + + def test_llm_judge_parse_score_json_with_fence(self): + metric = LLMJudgeMetric(provider=self.mock_provider) + score = metric._parse_score('```json\n{"score": 0.72}\n```') + assert score == 0.72 + def test_llm_judge_parse_score_percentage(self): metric = LLMJudgeMetric(provider=self.mock_provider) - score = metric._parse_score("85%") + score = metric._parse_score('{"score": 85}') assert score == 0.85 # 85 / 100 def test_llm_judge_parse_score_keyword_yes(self): metric = LLMJudgeMetric(provider=self.mock_provider) score = metric._parse_score("Yes, the answer is excellent") - assert score == 1.0 + assert score == 0.5 def test_llm_judge_parse_score_keyword_no(self): metric = LLMJudgeMetric(provider=self.mock_provider) score = metric._parse_score("No, not at all") - assert score == 0.0 + assert score == 0.5 + + def test_llm_judge_parse_score_avoids_false_keyword_match(self): + metric = LLMJudgeMetric(provider=self.mock_provider) + score = metric._parse_score("This is not excellent but acceptable") + assert score == 0.5 + + def test_llm_judge_parse_score_prefers_json_over_ambiguous_text(self): + metric = LLMJudgeMetric(provider=self.mock_provider) + score = metric._parse_score('There are 5 reasons why this is a score of 9 {"score": 0.9}') + assert score == 0.9 def test_llm_judge_parse_score_neutral(self): metric = LLMJudgeMetric(provider=self.mock_provider) @@ -335,7 +380,7 @@ def test_llm_judge_parse_score_neutral(self): assert score == 0.5 def test_llm_judge_evaluate_success(self): - self.mock_provider.generate.return_value = "0.92" + self.mock_provider.generate.return_value = '{"score": 0.92}' metric = LLMJudgeMetric(provider=self.mock_provider) result = metric.evaluate(premise="context", hypothesis="answer") assert result.score == 0.92 From eabb9f9ff2984d3b1c62a2110bdbc98dd99a6db1 Mon Sep 17 00:00:00 2001 From: syf2211 Date: Thu, 23 Jul 2026 06:04:54 +0000 Subject: [PATCH 2/3] test(metrics): cover malformed JSON score fallback in LLM judge Add explicit cases for invalid JSON payloads so the labeled-score fallback chain stays pinned as suggested in review. --- tests/unit/test_metrics/test_nli.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/unit/test_metrics/test_nli.py b/tests/unit/test_metrics/test_nli.py index 7a1d5d4..165668b 100644 --- a/tests/unit/test_metrics/test_nli.py +++ b/tests/unit/test_metrics/test_nli.py @@ -344,6 +344,16 @@ def test_llm_judge_parse_score_invalid_json_value(self): score = metric._parse_score('{"score": "high"}') assert score == 0.5 + def test_llm_judge_parse_score_malformed_json_falls_back(self): + metric = LLMJudgeMetric(provider=self.mock_provider) + score = metric._parse_score('{"score" 0.85}') + assert score == 0.5 + + def test_llm_judge_parse_score_malformed_json_falls_back_to_labeled(self): + metric = LLMJudgeMetric(provider=self.mock_provider) + score = metric._parse_score('{"score" 0.85} Score: 0.85') + assert score == 0.85 + def test_llm_judge_parse_score_ten_point_scale(self): metric = LLMJudgeMetric(provider=self.mock_provider) score = metric._parse_score('{"score": 8.5}') From 844ae8fb4e72064ef546e3d744aa1728572b86c2 Mon Sep 17 00:00:00 2001 From: "opencode-agent[bot]" Date: Thu, 23 Jul 2026 09:15:29 +0000 Subject: [PATCH 3/3] Fixed 2 regex bugs, all 48 tests pass. Co-authored-by: himanshu231204 --- openagent_eval/metrics/generation/llm_judge.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openagent_eval/metrics/generation/llm_judge.py b/openagent_eval/metrics/generation/llm_judge.py index 39aa92d..3314603 100644 --- a/openagent_eval/metrics/generation/llm_judge.py +++ b/openagent_eval/metrics/generation/llm_judge.py @@ -20,7 +20,8 @@ 'Respond with ONLY a JSON object on a single line: {{"score": }}' ) _JSON_FENCE_RE = re.compile(r"```(?:json)?\s*(\{.*?\})\s*```", re.DOTALL | re.IGNORECASE) -_SCORE_LABEL_RE = re.compile(r"score[\"'\s:]*(\d+\.?\d*)", re.IGNORECASE) +_JSON_OBJECT_RE = re.compile(r"\{[^{}]*\}") +_SCORE_LABEL_RE = re.compile(r"score\s*[:=]\s*(\d+\.?\d*)", re.IGNORECASE) _BARE_DECIMAL_RE = re.compile(r"^\s*(\d+\.?\d*)\s*$") @@ -224,6 +225,9 @@ def _json_score_candidates(text: str) -> list[str]: fence_match = _JSON_FENCE_RE.search(text) if fence_match: candidates.append(fence_match.group(1)) + obj_match = _JSON_OBJECT_RE.search(text) + if obj_match: + candidates.append(obj_match.group()) candidates.append(text) return candidates