fix(diagnosis): validate contexts before chunking analysis

openagent_eval/diagnosis/analyzer.py

@@ -105,6 +105,7 @@ def analyze(
 
         for item in results:
             scores = self._extract_scores(item)
+            contexts = self._extract_validated_contexts(item)
             question = scores.question
 
             # Run blame attribution
@@ -119,12 +120,17 @@ def analyze(
                     blame_key = failure.blame.value
                     blame_counts[blame_key] = blame_counts.get(blame_key, 0) + 1
 
+            metadata = item.get("metadata")
+
+            if not isinstance(metadata, dict):
+                metadata = None
+
             # Run chunking analysis
             if scores.context_count > 0:
                 chunking_issues = self._chunking_analyzer.analyze(
-                    question,
-                    item.get("contexts", []) or [],
-                    item.get("metadata"),
+                        question,
+                        contexts,
+                        metadata,
                 )
                 all_chunking_issues.extend(chunking_issues)
 
@@ -135,11 +141,11 @@ def analyze(
         # Build recommendations
         recommendations = self._build_recommendations(blame_counts, failure_counts)
 
-        # Compute overall health
-        if not results:
-            overall_health = 1.0
-        else:
-            overall_health = healthy_count / len(results)
+        overall_health = (
+            1.0
+            if not results
+            else healthy_count / len(results)
+        )
 
         return DiagnosisReport(
             total_items=len(results),
@@ -189,20 +195,45 @@ def _extract_scores(self, item: dict[str, object]) -> ComponentScores:
             else:
                 generation_scores[key] = val
 
-        contexts = item.get("contexts", []) or []
-        if not isinstance(contexts, list):
-            contexts = []
+        raw_contexts = item.get("contexts", []) or []
+
+        if not isinstance(raw_contexts, list):
+            contexts: list[str] = []
+        else:
+            contexts = [
+                context
+                for context in raw_contexts
+                if isinstance(context, str)
+            ]
 
         return ComponentScores(
             question=question,
             retrieval_scores=retrieval_scores,
             generation_scores=generation_scores,
             context_count=len(contexts),
-            context_lengths=[len(str(c)) for c in contexts],
+            context_lengths=[len(c) for c in contexts],
             answer_length=len(str(item.get("answer", ""))),
             latency_ms=metadata.get("latency_ms"),
         )
 
+
+    def _extract_validated_contexts(
+        self,
+        item: dict[str, object],
+    ) -> list[str]:
+        """Extract validated context strings from an evaluation item."""
+
+        raw_contexts = item.get("contexts", []) or []
+
+        if not isinstance(raw_contexts, list):
+            return []
+
+        return [
+            context
+            for context in raw_contexts
+            if isinstance(context, str)
+        ]
+
     def _build_recommendations(
         self,
         blame_counts: dict[str, int],

tests/unit/test_diagnosis/test_integration.py

@@ -7,8 +7,10 @@
 from __future__ import annotations
 
 import json
-from pathlib import Path
+from typing import TYPE_CHECKING
 
+if TYPE_CHECKING:
+    from pathlib import Path
 
 from openagent_eval.diagnosis import DiagnosisAnalyzer
 from openagent_eval.diagnosis.blame import BlameAttribution
@@ -218,6 +220,51 @@ def test_report_serialization_roundtrip(self) -> None:
         assert len(deserialized["recommendations"]) == len(report.recommendations)
 
 
+    def test_chunking_receives_validated_contexts(self) -> None:
+        """Chunking analyzer should receive only validated string contexts."""
+
+        analyzer = DiagnosisAnalyzer()
+
+        captured: dict[str, object] = {}
+
+        def fake_chunking(
+            question: str,
+            contexts: list[str],
+            metadata: dict[str, float] | None,
+        ) -> list:
+            captured["question"] = question
+            captured["contexts"] = contexts
+            captured["metadata"] = metadata
+            return []
+
+        analyzer._chunking_analyzer.analyze = fake_chunking  # type: ignore[method-assign]
+
+        results = [
+            {
+                "question": "What is Python?",
+                "answer": "Python is a programming language.",
+                "contexts": [
+                    "Python is a programming language.",
+                    {"content": "invalid"},
+                    123,
+                    None,
+                ],
+                "metrics": {},
+                "metadata": {},
+            }
+        ]
+
+        analyzer.analyze(results)
+
+        assert captured["question"] == "What is Python?"
+        assert captured["metadata"] == {}
+        assert captured["contexts"] == [
+            "Python is a programming language."
+        ]
+        assert len(captured["contexts"]) == 1
+
+
+
 class TestDiagnosisReportFileOutput:
     """Test saving diagnosis report to file."""
 
@@ -252,3 +299,4 @@ def test_save_report_to_json(self, tmp_path: Path) -> None:
 
         assert loaded["total_items"] == 1
         assert loaded["overall_health"] == 1.0
+