From 2125f0098fa36bb09559906a0879bc6a79ae7177 Mon Sep 17 00:00:00 2001 From: syf2211 Date: Thu, 23 Jul 2026 12:03:21 +0000 Subject: [PATCH] fix(diagnosis): return BlameTarget.NONE for healthy items instead of UNKNOWN When no failures are detected, blame attribution now returns NONE with confidence 1.0 instead of UNKNOWN with confidence 0.0. This separates 'analysis succeeded, no issues' from 'could not determine blame'. Fixes #67 --- openagent_eval/cli/commands/diagnose.py | 1 + openagent_eval/diagnosis/analyzer.py | 2 +- openagent_eval/diagnosis/blame.py | 4 ++-- openagent_eval/diagnosis/models.py | 7 ++++++- tests/unit/test_diagnosis/test_blame.py | 11 ++++++----- tests/unit/test_diagnosis/test_integration.py | 2 +- tests/unit/test_diagnosis/test_models.py | 7 ++++--- 7 files changed, 21 insertions(+), 13 deletions(-) diff --git a/openagent_eval/cli/commands/diagnose.py b/openagent_eval/cli/commands/diagnose.py index 4685a7d..7c4651d 100644 --- a/openagent_eval/cli/commands/diagnose.py +++ b/openagent_eval/cli/commands/diagnose.py @@ -32,6 +32,7 @@ BlameTarget.GENERATION.value: "[yellow]Generation[/yellow]", BlameTarget.CHUNKING.value: "[cyan]Chunking[/cyan]", BlameTarget.DATASET.value: "[magenta]Dataset[/magenta]", + BlameTarget.NONE.value: "[green]Healthy[/green]", BlameTarget.UNKNOWN.value: "[dim]Unknown[/dim]", } diff --git a/openagent_eval/diagnosis/analyzer.py b/openagent_eval/diagnosis/analyzer.py index 2378297..bedbcca 100644 --- a/openagent_eval/diagnosis/analyzer.py +++ b/openagent_eval/diagnosis/analyzer.py @@ -129,7 +129,7 @@ def analyze( all_chunking_issues.extend(chunking_issues) # Check if item is healthy - if blame_result.target == BlameTarget.UNKNOWN: + if blame_result.target == BlameTarget.NONE: healthy_count += 1 # Build recommendations diff --git a/openagent_eval/diagnosis/blame.py b/openagent_eval/diagnosis/blame.py index 53d8212..4e13906 100644 --- a/openagent_eval/diagnosis/blame.py +++ b/openagent_eval/diagnosis/blame.py @@ -225,8 +225,8 @@ def _determine_blame( """Determine the primary blame target from all detected failures.""" if not failures: return BlameResult( - target=BlameTarget.UNKNOWN, - confidence=0.0, + target=BlameTarget.NONE, + confidence=1.0, reason="No failures detected.", failure_modes=[], ) diff --git a/openagent_eval/diagnosis/models.py b/openagent_eval/diagnosis/models.py index 0c2b6e3..7e7c157 100644 --- a/openagent_eval/diagnosis/models.py +++ b/openagent_eval/diagnosis/models.py @@ -31,12 +31,17 @@ class FailureMode(str, Enum): class BlameTarget(str, Enum): - """Which component is blamed for a failure.""" + """Which component is blamed for a failure. + + NONE indicates analysis completed with no failures detected. + UNKNOWN is reserved for cases where blame could not be determined. + """ RETRIEVAL = "retrieval" GENERATION = "generation" CHUNKING = "chunking" DATASET = "dataset" + NONE = "none" UNKNOWN = "unknown" diff --git a/tests/unit/test_diagnosis/test_blame.py b/tests/unit/test_diagnosis/test_blame.py index c5901f5..3f46bbe 100644 --- a/tests/unit/test_diagnosis/test_blame.py +++ b/tests/unit/test_diagnosis/test_blame.py @@ -27,8 +27,8 @@ def setup_method(self) -> None: # Healthy cases # ------------------------------------------------------------------ - def test_healthy_item_returns_unknown(self) -> None: - """An item with good scores should return UNKNOWN blame.""" + def test_healthy_item_returns_none(self) -> None: + """An item with good scores should return NONE blame (no failures).""" scores = ComponentScores( question="What is Python?", retrieval_scores={"context_precision": 0.9, "context_recall": 0.85}, @@ -38,7 +38,8 @@ def test_healthy_item_returns_unknown(self) -> None: answer_length=500, ) result = self.blamer.analyze(scores) - assert result.target == BlameTarget.UNKNOWN + assert result.target == BlameTarget.NONE + assert result.confidence == 1.0 assert len(result.failure_modes) == 0 def test_empty_scores_returns_empty_retrieval(self) -> None: @@ -162,8 +163,8 @@ def test_single_context_with_decent_precision(self) -> None: answer_length=300, ) result = self.blamer.analyze(scores) - # May blame chunking or unknown depending on thresholds - assert result.target in (BlameTarget.CHUNKING, BlameTarget.UNKNOWN) + # May blame chunking or none depending on thresholds + assert result.target in (BlameTarget.CHUNKING, BlameTarget.NONE) def test_uneven_context_lengths(self) -> None: """Highly uneven context lengths should detect chunking issue.""" diff --git a/tests/unit/test_diagnosis/test_integration.py b/tests/unit/test_diagnosis/test_integration.py index a24bdc6..6869205 100644 --- a/tests/unit/test_diagnosis/test_integration.py +++ b/tests/unit/test_diagnosis/test_integration.py @@ -167,7 +167,7 @@ def test_blame_attribution_standalone(self) -> None: answer_length=200, ) result = blamer.analyze(healthy) - assert result.target == BlameTarget.UNKNOWN + assert result.target == BlameTarget.NONE # Test empty retrieval empty = ComponentScores( diff --git a/tests/unit/test_diagnosis/test_models.py b/tests/unit/test_diagnosis/test_models.py index 483f806..e22a655 100644 --- a/tests/unit/test_diagnosis/test_models.py +++ b/tests/unit/test_diagnosis/test_models.py @@ -38,9 +38,9 @@ def test_is_string_enum(self) -> None: class TestBlameTarget: """Tests for BlameTarget enum.""" - def test_has_five_targets(self) -> None: - """BlameTarget should have exactly 5 targets.""" - assert len(BlameTarget) == 5 + def test_has_six_targets(self) -> None: + """BlameTarget should have exactly 6 targets.""" + assert len(BlameTarget) == 6 def test_values(self) -> None: """BlameTarget values should be descriptive strings.""" @@ -48,6 +48,7 @@ def test_values(self) -> None: assert BlameTarget.GENERATION.value == "generation" assert BlameTarget.CHUNKING.value == "chunking" assert BlameTarget.DATASET.value == "dataset" + assert BlameTarget.NONE.value == "none" assert BlameTarget.UNKNOWN.value == "unknown"