diff --git a/openagent_eval/cli/commands/diagnose.py b/openagent_eval/cli/commands/diagnose.py index 4cadfc2..8ff48ed 100644 --- a/openagent_eval/cli/commands/diagnose.py +++ b/openagent_eval/cli/commands/diagnose.py @@ -32,6 +32,7 @@ BlameTarget.GENERATION.value: "[yellow]Generation[/yellow]", BlameTarget.CHUNKING.value: "[cyan]Chunking[/cyan]", BlameTarget.DATASET.value: "[magenta]Dataset[/magenta]", + BlameTarget.NONE.value: "[green]Healthy[/green]", BlameTarget.UNKNOWN.value: "[dim]Unknown[/dim]", } diff --git a/openagent_eval/diagnosis/analyzer.py b/openagent_eval/diagnosis/analyzer.py index cb85828..ccb7141 100644 --- a/openagent_eval/diagnosis/analyzer.py +++ b/openagent_eval/diagnosis/analyzer.py @@ -135,7 +135,7 @@ def analyze( all_chunking_issues.extend(chunking_issues) # Check if item is healthy - if blame_result.target == BlameTarget.UNKNOWN: + if blame_result.target == BlameTarget.NONE: healthy_count += 1 # Build recommendations diff --git a/openagent_eval/diagnosis/blame.py b/openagent_eval/diagnosis/blame.py index 53d8212..4e13906 100644 --- a/openagent_eval/diagnosis/blame.py +++ b/openagent_eval/diagnosis/blame.py @@ -225,8 +225,8 @@ def _determine_blame( """Determine the primary blame target from all detected failures.""" if not failures: return BlameResult( - target=BlameTarget.UNKNOWN, - confidence=0.0, + target=BlameTarget.NONE, + confidence=1.0, reason="No failures detected.", failure_modes=[], ) diff --git a/openagent_eval/diagnosis/models.py b/openagent_eval/diagnosis/models.py index 0c2b6e3..7e7c157 100644 --- a/openagent_eval/diagnosis/models.py +++ b/openagent_eval/diagnosis/models.py @@ -31,12 +31,17 @@ class FailureMode(str, Enum): class BlameTarget(str, Enum): - """Which component is blamed for a failure.""" + """Which component is blamed for a failure. + + NONE indicates analysis completed with no failures detected. + UNKNOWN is reserved for cases where blame could not be determined. + """ RETRIEVAL = "retrieval" GENERATION = "generation" CHUNKING = "chunking" DATASET = "dataset" + NONE = "none" UNKNOWN = "unknown" diff --git a/tests/unit/test_diagnosis/test_blame.py b/tests/unit/test_diagnosis/test_blame.py index c5901f5..3f46bbe 100644 --- a/tests/unit/test_diagnosis/test_blame.py +++ b/tests/unit/test_diagnosis/test_blame.py @@ -27,8 +27,8 @@ def setup_method(self) -> None: # Healthy cases # ------------------------------------------------------------------ - def test_healthy_item_returns_unknown(self) -> None: - """An item with good scores should return UNKNOWN blame.""" + def test_healthy_item_returns_none(self) -> None: + """An item with good scores should return NONE blame (no failures).""" scores = ComponentScores( question="What is Python?", retrieval_scores={"context_precision": 0.9, "context_recall": 0.85}, @@ -38,7 +38,8 @@ def test_healthy_item_returns_unknown(self) -> None: answer_length=500, ) result = self.blamer.analyze(scores) - assert result.target == BlameTarget.UNKNOWN + assert result.target == BlameTarget.NONE + assert result.confidence == 1.0 assert len(result.failure_modes) == 0 def test_empty_scores_returns_empty_retrieval(self) -> None: @@ -162,8 +163,8 @@ def test_single_context_with_decent_precision(self) -> None: answer_length=300, ) result = self.blamer.analyze(scores) - # May blame chunking or unknown depending on thresholds - assert result.target in (BlameTarget.CHUNKING, BlameTarget.UNKNOWN) + # May blame chunking or none depending on thresholds + assert result.target in (BlameTarget.CHUNKING, BlameTarget.NONE) def test_uneven_context_lengths(self) -> None: """Highly uneven context lengths should detect chunking issue.""" diff --git a/tests/unit/test_diagnosis/test_integration.py b/tests/unit/test_diagnosis/test_integration.py index 533f52a..9d34a2d 100644 --- a/tests/unit/test_diagnosis/test_integration.py +++ b/tests/unit/test_diagnosis/test_integration.py @@ -169,7 +169,7 @@ def test_blame_attribution_standalone(self) -> None: answer_length=200, ) result = blamer.analyze(healthy) - assert result.target == BlameTarget.UNKNOWN + assert result.target == BlameTarget.NONE # Test empty retrieval empty = ComponentScores( diff --git a/tests/unit/test_diagnosis/test_models.py b/tests/unit/test_diagnosis/test_models.py index 483f806..e22a655 100644 --- a/tests/unit/test_diagnosis/test_models.py +++ b/tests/unit/test_diagnosis/test_models.py @@ -38,9 +38,9 @@ def test_is_string_enum(self) -> None: class TestBlameTarget: """Tests for BlameTarget enum.""" - def test_has_five_targets(self) -> None: - """BlameTarget should have exactly 5 targets.""" - assert len(BlameTarget) == 5 + def test_has_six_targets(self) -> None: + """BlameTarget should have exactly 6 targets.""" + assert len(BlameTarget) == 6 def test_values(self) -> None: """BlameTarget values should be descriptive strings.""" @@ -48,6 +48,7 @@ def test_values(self) -> None: assert BlameTarget.GENERATION.value == "generation" assert BlameTarget.CHUNKING.value == "chunking" assert BlameTarget.DATASET.value == "dataset" + assert BlameTarget.NONE.value == "none" assert BlameTarget.UNKNOWN.value == "unknown"