fix: map Harbor trial folders to eval case ids

CHANGELOG.md

@@ -19,6 +19,9 @@ All notable changes to SkillEvaluator are documented in this file.
 
 ### Fixed
 
+- `create-eval-dataset --refine` resolves Harbor trial case ids from persisted
+  `reward.json` `entry_id` metadata, using folder-name parsing only as an
+  unambiguous legacy fallback.
 - Tier 3 local mode now drops evaluator-managed empty process-loader resets
   while continuing to reject non-empty loader overrides, allowing generated
   tasks to reach agent execution

src/skillevaluator/tier3/generate_dataset.py

@@ -503,6 +503,46 @@ def _ensure_project_imports():
         sys.path.insert(0, src_dir)
 
 
+def _read_reward_entry_id(trial_dir: Path) -> str:
+    for reward_path in (trial_dir / "reward.json", trial_dir / "verifier" / "reward.json"):
+        if not reward_path.is_file():
+            continue
+        try:
+            payload = json.loads(reward_path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError):
+            continue
+        if isinstance(payload, dict):
+            entry_id = str(payload.get("entry_id") or "").strip()
+            if entry_id:
+                return entry_id
+    return ""
+
+
+def _read_result_entry_id(trial_dir: Path) -> str:
+    """Resolve case id from Harbor ``result.json`` when reward metadata is absent."""
+    result_path = trial_dir / "result.json"
+    if not result_path.is_file():
+        return ""
+    try:
+        payload = json.loads(result_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return ""
+    if not isinstance(payload, dict):
+        return ""
+    from skillevaluator.tier3.harbor.collector import _entry_id_from_harbor_result
+
+    return _entry_id_from_harbor_result(payload)
+
+
+def _case_id_from_trial_dir(trial_dir: Path) -> str:
+    """Resolve eval case id from persisted Harbor metadata, else the folder name."""
+    if entry_id := _read_reward_entry_id(trial_dir):
+        return entry_id
+    if entry_id := _read_result_entry_id(trial_dir):
+        return entry_id
+    return trial_dir.name
+
+
 def _discover_trajectories(
     skill_path: Path,
     from_results: str | None = None,
@@ -541,7 +581,9 @@ def _discover_trajectories(
         for trial_dir in sorted(trials_dir.iterdir()):
             if not trial_dir.is_dir():
                 continue
-            case_id = trial_dir.name
+            case_id = _case_id_from_trial_dir(trial_dir)
+            if not case_id:
+                continue
             traj_path = trial_dir / "trajectory.json"
             traj, meta = load_trajectory_with_fallback(traj_path, logs_dir=trial_dir)
             if traj and traj.get("steps"):

tests/tier3/test_generate_dataset_results.py

@@ -6,6 +6,7 @@
 import stat
 import sys
 from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
 
 import pytest
 
@@ -37,6 +38,168 @@ def test_discover_trajectories_uses_env_results_root(tmp_path, monkeypatch):
     assert _discover_trajectories(skill) == {"case-001": trajectory}
 
 
+def test_discover_trajectories_maps_harbor_trial_folder_to_case_id(tmp_path, monkeypatch):
+    """Harbor persists trials as ``{case_id}__{suffix}``; refine looks up by case id."""
+    skill = tmp_path / "demo"
+    skill.mkdir()
+    results_root = tmp_path / "results"
+    run_id = "20260709_120000"
+    run_dir = results_root / "demo" / run_id
+    trial = run_dir / "claude-code" / "with-skill" / "trials" / "demo-001__Lmi47iy"
+    trial.mkdir(parents=True)
+    trajectory = {"steps": [{"tool_calls": [{"tool": "Read"}]}]}
+    trial.joinpath("trajectory.json").write_text(json.dumps(trajectory), encoding="utf-8")
+    trial.joinpath("result.json").write_text(
+        json.dumps(
+            {
+                "trial_name": "demo-001__Lmi47iy",
+                "config": {"task": {"path": "tasks/demo-001"}},
+            }
+        ),
+        encoding="utf-8",
+    )
+    (run_dir / "run_config.json").write_text("{}", encoding="utf-8")
+    (run_dir / "result.json").write_text(json.dumps({"run_id": run_id}), encoding="utf-8")
+    (results_root / "demo" / "latest").symlink_to(run_id)
+
+    monkeypatch.setenv("SKILLEVALUATOR_RESULTS_DIR", str(results_root))
+
+    found = _discover_trajectories(skill)
+    assert "demo-001" in found
+    assert "demo-001__Lmi47iy" not in found
+    assert found["demo-001"] == trajectory
+
+
+def _write_results_trial(
+    tmp_path: Path,
+    *,
+    skill_name: str,
+    trial_folder: str,
+    trajectory: dict[str, object],
+    reward: dict[str, object] | None = None,
+) -> Path:
+    skill = tmp_path / skill_name
+    skill.mkdir(exist_ok=True)
+    results_root = tmp_path / "results"
+    run_id = "20260709_120000"
+    run_dir = results_root / skill_name / run_id
+    trial = run_dir / "claude-code" / "with-skill" / "trials" / trial_folder
+    trial.mkdir(parents=True)
+    trial.joinpath("trajectory.json").write_text(json.dumps(trajectory), encoding="utf-8")
+    if reward is not None:
+        trial.joinpath("reward.json").write_text(json.dumps(reward), encoding="utf-8")
+    (run_dir / "run_config.json").write_text("{}", encoding="utf-8")
+    (run_dir / "result.json").write_text(json.dumps({"run_id": run_id}), encoding="utf-8")
+    (results_root / skill_name / "latest").symlink_to(run_id)
+    return skill
+
+
+def test_discover_trajectories_prefers_reward_entry_id_over_folder_prefix(tmp_path, monkeypatch):
+    """Stop-on-pass folders must not collapse to the job prefix when entry_id is present."""
+    trajectory = {"steps": [{"tool_calls": [{"tool": "Read"}]}]}
+    skill = _write_results_trial(
+        tmp_path,
+        skill_name="demo",
+        trial_folder="harbor-job__demo-001__Lmi47iy",
+        trajectory=trajectory,
+        reward={"entry_id": "demo-001", "overall": 1.0},
+    )
+    monkeypatch.setenv("SKILLEVALUATOR_RESULTS_DIR", str(tmp_path / "results"))
+
+    found = _discover_trajectories(skill)
+    assert list(found) == ["demo-001"]
+    assert found["demo-001"] == trajectory
+
+
+def test_discover_trajectories_keeps_distinct_case_ids_with_double_underscore(tmp_path, monkeypatch):
+    """Case ids containing ``__`` must not collapse to a shared prefix without metadata."""
+    trajectory_a = {"steps": [{"message": "a"}]}
+    trajectory_b = {"steps": [{"message": "b"}]}
+    skill = tmp_path / "demo"
+    skill.mkdir()
+    results_root = tmp_path / "results"
+    run_id = "20260709_120000"
+    run_dir = results_root / "demo" / run_id
+    trials_dir = run_dir / "claude-code" / "with-skill" / "trials"
+    for folder, traj, entry_id in (
+        ("case__one", trajectory_a, "case__one"),
+        ("case__two", trajectory_b, "case__two"),
+    ):
+        trial = trials_dir / folder
+        trial.mkdir(parents=True)
+        trial.joinpath("trajectory.json").write_text(json.dumps(traj), encoding="utf-8")
+        trial.joinpath("reward.json").write_text(json.dumps({"entry_id": entry_id}), encoding="utf-8")
+    (run_dir / "run_config.json").write_text("{}", encoding="utf-8")
+    (run_dir / "result.json").write_text(json.dumps({"run_id": run_id}), encoding="utf-8")
+    (results_root / "demo" / "latest").symlink_to(run_id)
+    monkeypatch.setenv("SKILLEVALUATOR_RESULTS_DIR", str(results_root))
+
+    found = _discover_trajectories(skill)
+    assert set(found) == {"case__one", "case__two"}
+    assert found["case__one"] == trajectory_a
+    assert found["case__two"] == trajectory_b
+
+
+def test_discover_trajectories_ambiguous_folder_without_reward_uses_full_name(tmp_path, monkeypatch):
+    """Without reward metadata, ambiguous ``__`` folders keep the full directory name."""
+    trajectory = {"steps": [{"tool_calls": []}]}
+    skill = _write_results_trial(
+        tmp_path,
+        skill_name="demo",
+        trial_folder="case__one",
+        trajectory=trajectory,
+    )
+    monkeypatch.setenv("SKILLEVALUATOR_RESULTS_DIR", str(tmp_path / "results"))
+
+    found = _discover_trajectories(skill)
+    assert list(found) == ["case__one"]
+
+
+def test_discover_trajectories_preserves_versioned_case_id_without_reward(tmp_path, monkeypatch):
+    """IDs like ``case__v2`` must not be truncated when only the folder name is present."""
+    trajectory = {"steps": [{"tool_calls": []}]}
+    skill = _write_results_trial(
+        tmp_path,
+        skill_name="demo",
+        trial_folder="case__v2",
+        trajectory=trajectory,
+    )
+    monkeypatch.setenv("SKILLEVALUATOR_RESULTS_DIR", str(tmp_path / "results"))
+
+    found = _discover_trajectories(skill)
+    assert list(found) == ["case__v2"]
+
+
+def test_discover_trajectories_resolves_shortuuid_folder_from_result_json(tmp_path, monkeypatch):
+    """All-letter Harbor tails resolve via result.json task metadata, not suffix guessing."""
+    trajectory = {"steps": [{"tool_calls": []}]}
+    skill = tmp_path / "demo"
+    skill.mkdir()
+    results_root = tmp_path / "results"
+    run_id = "20260709_120000"
+    run_dir = results_root / "demo" / run_id
+    trial_folder = "case-001__LRZctSP"
+    trial = run_dir / "claude-code" / "with-skill" / "trials" / trial_folder
+    trial.mkdir(parents=True)
+    trial.joinpath("trajectory.json").write_text(json.dumps(trajectory), encoding="utf-8")
+    trial.joinpath("result.json").write_text(
+        json.dumps(
+            {
+                "trial_name": trial_folder,
+                "config": {"task": {"path": "tasks/case-001"}},
+            }
+        ),
+        encoding="utf-8",
+    )
+    (run_dir / "run_config.json").write_text("{}", encoding="utf-8")
+    (run_dir / "result.json").write_text(json.dumps({"run_id": run_id}), encoding="utf-8")
+    (results_root / "demo" / "latest").symlink_to(run_id)
+    monkeypatch.setenv("SKILLEVALUATOR_RESULTS_DIR", str(results_root))
+
+    found = _discover_trajectories(skill)
+    assert list(found) == ["case-001"]
+
+
 def test_discover_trajectories_results_dir_overrides_env(tmp_path, monkeypatch):
     skill = tmp_path / "my-skill"
     skill.mkdir()