Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions openagent_eval/cli/commands/diagnose.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
BlameTarget.GENERATION.value: "[yellow]Generation[/yellow]",
BlameTarget.CHUNKING.value: "[cyan]Chunking[/cyan]",
BlameTarget.DATASET.value: "[magenta]Dataset[/magenta]",
BlameTarget.NONE.value: "[green]Healthy[/green]",
BlameTarget.UNKNOWN.value: "[dim]Unknown[/dim]",
}

Expand Down
2 changes: 1 addition & 1 deletion openagent_eval/diagnosis/analyzer.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ def analyze(
all_chunking_issues.extend(chunking_issues)

# Check if item is healthy
if blame_result.target == BlameTarget.UNKNOWN:
if blame_result.target == BlameTarget.NONE:
healthy_count += 1

# Build recommendations
Expand Down
4 changes: 2 additions & 2 deletions openagent_eval/diagnosis/blame.py
Original file line number Diff line number Diff line change
Expand Up @@ -225,8 +225,8 @@ def _determine_blame(
"""Determine the primary blame target from all detected failures."""
if not failures:
return BlameResult(
target=BlameTarget.UNKNOWN,
confidence=0.0,
target=BlameTarget.NONE,
confidence=1.0,
reason="No failures detected.",
failure_modes=[],
)
Expand Down
7 changes: 6 additions & 1 deletion openagent_eval/diagnosis/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,17 @@ class FailureMode(str, Enum):


class BlameTarget(str, Enum):
"""Which component is blamed for a failure."""
"""Which component is blamed for a failure.

NONE indicates analysis completed with no failures detected.
UNKNOWN is reserved for cases where blame could not be determined.
"""

RETRIEVAL = "retrieval"
GENERATION = "generation"
CHUNKING = "chunking"
DATASET = "dataset"
NONE = "none"
UNKNOWN = "unknown"


Expand Down
11 changes: 6 additions & 5 deletions tests/unit/test_diagnosis/test_blame.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ def setup_method(self) -> None:
# Healthy cases
# ------------------------------------------------------------------

def test_healthy_item_returns_unknown(self) -> None:
"""An item with good scores should return UNKNOWN blame."""
def test_healthy_item_returns_none(self) -> None:
"""An item with good scores should return NONE blame (no failures)."""
scores = ComponentScores(
question="What is Python?",
retrieval_scores={"context_precision": 0.9, "context_recall": 0.85},
Expand All @@ -38,7 +38,8 @@ def test_healthy_item_returns_unknown(self) -> None:
answer_length=500,
)
result = self.blamer.analyze(scores)
assert result.target == BlameTarget.UNKNOWN
assert result.target == BlameTarget.NONE
assert result.confidence == 1.0
assert len(result.failure_modes) == 0

def test_empty_scores_returns_empty_retrieval(self) -> None:
Expand Down Expand Up @@ -162,8 +163,8 @@ def test_single_context_with_decent_precision(self) -> None:
answer_length=300,
)
result = self.blamer.analyze(scores)
# May blame chunking or unknown depending on thresholds
assert result.target in (BlameTarget.CHUNKING, BlameTarget.UNKNOWN)
# May blame chunking or none depending on thresholds
assert result.target in (BlameTarget.CHUNKING, BlameTarget.NONE)

def test_uneven_context_lengths(self) -> None:
"""Highly uneven context lengths should detect chunking issue."""
Expand Down
2 changes: 1 addition & 1 deletion tests/unit/test_diagnosis/test_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ def test_blame_attribution_standalone(self) -> None:
answer_length=200,
)
result = blamer.analyze(healthy)
assert result.target == BlameTarget.UNKNOWN
assert result.target == BlameTarget.NONE

# Test empty retrieval
empty = ComponentScores(
Expand Down
7 changes: 4 additions & 3 deletions tests/unit/test_diagnosis/test_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,16 +38,17 @@ def test_is_string_enum(self) -> None:
class TestBlameTarget:
"""Tests for BlameTarget enum."""

def test_has_five_targets(self) -> None:
"""BlameTarget should have exactly 5 targets."""
assert len(BlameTarget) == 5
def test_has_six_targets(self) -> None:
"""BlameTarget should have exactly 6 targets."""
assert len(BlameTarget) == 6

def test_values(self) -> None:
"""BlameTarget values should be descriptive strings."""
assert BlameTarget.RETRIEVAL.value == "retrieval"
assert BlameTarget.GENERATION.value == "generation"
assert BlameTarget.CHUNKING.value == "chunking"
assert BlameTarget.DATASET.value == "dataset"
assert BlameTarget.NONE.value == "none"
assert BlameTarget.UNKNOWN.value == "unknown"


Expand Down
Loading