fix: map Harbor trial folders to eval case ids

CHANGELOG.md

@@ -12,6 +12,8 @@ All notable changes to SkillEvaluator are documented in this file.
 
 ### Fixed
 
+- `create-eval-dataset --refine` now maps Harbor trial folders
+  (`case-id__suffix`) back to eval case ids so trajectories attach.
 - License detection no longer treats a frontmatter `license` identifier as
   authoritative when a LICENSE file declares a different license. Claiming
   MIT while shipping GPL-3.0 now fails closed. Every LICENSE/COPYING file is

src/skillevaluator/tier3/generate_dataset.py

@@ -496,6 +496,11 @@ def _ensure_project_imports():
         sys.path.insert(0, src_dir)
 
 
+def _case_id_from_trial_dir(trial_dir_name: str) -> str:
+    """Map Harbor trial folders like ``case-001__Lmi47iy`` back to the eval case id."""
+    return trial_dir_name.split("__", 1)[0]
+
+
 def _discover_trajectories(
     skill_path: Path,
     from_results: str | None = None,
@@ -534,7 +539,9 @@ def _discover_trajectories(
         for trial_dir in sorted(trials_dir.iterdir()):
             if not trial_dir.is_dir():
                 continue
-            case_id = trial_dir.name
+            case_id = _case_id_from_trial_dir(trial_dir.name)
+            if not case_id:
+                continue
             traj_path = trial_dir / "trajectory.json"
             traj, meta = load_trajectory_with_fallback(traj_path, logs_dir=trial_dir)
             if traj and traj.get("steps"):

tests/tier3/test_generate_dataset_results.py

@@ -37,6 +37,29 @@ def test_discover_trajectories_uses_env_results_root(tmp_path, monkeypatch):
     assert _discover_trajectories(skill) == {"case-001": trajectory}
 
 
+def test_discover_trajectories_maps_harbor_trial_folder_to_case_id(tmp_path, monkeypatch):
+    """Harbor persists trials as ``{case_id}__{suffix}``; refine looks up by case id."""
+    skill = tmp_path / "demo"
+    skill.mkdir()
+    results_root = tmp_path / "results"
+    run_id = "20260709_120000"
+    run_dir = results_root / "demo" / run_id
+    trial = run_dir / "claude-code" / "with-skill" / "trials" / "demo-001__Lmi47iy"
+    trial.mkdir(parents=True)
+    trajectory = {"steps": [{"tool_calls": [{"tool": "Read"}]}]}
+    trial.joinpath("trajectory.json").write_text(json.dumps(trajectory), encoding="utf-8")
+    (run_dir / "run_config.json").write_text("{}", encoding="utf-8")
+    (run_dir / "result.json").write_text(json.dumps({"run_id": run_id}), encoding="utf-8")
+    (results_root / "demo" / "latest").symlink_to(run_id)
+
+    monkeypatch.setenv("SKILLEVALUATOR_RESULTS_DIR", str(results_root))
+
+    found = _discover_trajectories(skill)
+    assert "demo-001" in found
+    assert "demo-001__Lmi47iy" not in found
+    assert found["demo-001"] == trajectory
+
+
 def test_discover_trajectories_results_dir_overrides_env(tmp_path, monkeypatch):
     skill = tmp_path / "my-skill"
     skill.mkdir()