From 1240fbe4792423c98eba0512323ea68f947ed174 Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Thu, 27 Aug 2026 01:45:33 +0530 Subject: [PATCH 1/2] fix(tier3): map Harbor trial folders to eval case ids --refine keyed trajectories by the raw trial directory name, but Harbor writes folders like case-001__Lmi47iy. Split off the suffix so trajectories.get(case id) actually finds them. Fixes #93 Signed-off-by: mimran-khan --- CHANGELOG.md | 2 ++ src/skillevaluator/tier3/generate_dataset.py | 9 +++++++- tests/tier3/test_generate_dataset_results.py | 23 ++++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2ad6515..77bd5899 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ All notable changes to SkillEvaluator are documented in this file. ### Fixed +- `create-eval-dataset --refine` now maps Harbor trial folders + (`case-id__suffix`) back to eval case ids so trajectories attach. - Tier 3 paired pass@k evidence now respects Python's active integer-string conversion limit, preserves nonzero Wilson interval widths and paired-effect directions at large case counts, and documents exact-rational omission diff --git a/src/skillevaluator/tier3/generate_dataset.py b/src/skillevaluator/tier3/generate_dataset.py index ca21d2dd..36cb215e 100644 --- a/src/skillevaluator/tier3/generate_dataset.py +++ b/src/skillevaluator/tier3/generate_dataset.py @@ -496,6 +496,11 @@ def _ensure_project_imports(): sys.path.insert(0, src_dir) +def _case_id_from_trial_dir(trial_dir_name: str) -> str: + """Map Harbor trial folders like ``case-001__Lmi47iy`` back to the eval case id.""" + return trial_dir_name.split("__", 1)[0] + + def _discover_trajectories( skill_path: Path, from_results: str | None = None, @@ -534,7 +539,9 @@ def _discover_trajectories( for trial_dir in sorted(trials_dir.iterdir()): if not trial_dir.is_dir(): continue - case_id = trial_dir.name + case_id = _case_id_from_trial_dir(trial_dir.name) + if not case_id: + continue traj_path = trial_dir / "trajectory.json" traj, meta = load_trajectory_with_fallback(traj_path, logs_dir=trial_dir) if traj and traj.get("steps"): diff --git a/tests/tier3/test_generate_dataset_results.py b/tests/tier3/test_generate_dataset_results.py index 45d1175d..52f6c97f 100644 --- a/tests/tier3/test_generate_dataset_results.py +++ b/tests/tier3/test_generate_dataset_results.py @@ -37,6 +37,29 @@ def test_discover_trajectories_uses_env_results_root(tmp_path, monkeypatch): assert _discover_trajectories(skill) == {"case-001": trajectory} +def test_discover_trajectories_maps_harbor_trial_folder_to_case_id(tmp_path, monkeypatch): + """Harbor persists trials as ``{case_id}__{suffix}``; refine looks up by case id.""" + skill = tmp_path / "demo" + skill.mkdir() + results_root = tmp_path / "results" + run_id = "20260709_120000" + run_dir = results_root / "demo" / run_id + trial = run_dir / "claude-code" / "with-skill" / "trials" / "demo-001__Lmi47iy" + trial.mkdir(parents=True) + trajectory = {"steps": [{"tool_calls": [{"tool": "Read"}]}]} + trial.joinpath("trajectory.json").write_text(json.dumps(trajectory), encoding="utf-8") + (run_dir / "run_config.json").write_text("{}", encoding="utf-8") + (run_dir / "result.json").write_text(json.dumps({"run_id": run_id}), encoding="utf-8") + (results_root / "demo" / "latest").symlink_to(run_id) + + monkeypatch.setenv("SKILLEVALUATOR_RESULTS_DIR", str(results_root)) + + found = _discover_trajectories(skill) + assert "demo-001" in found + assert "demo-001__Lmi47iy" not in found + assert found["demo-001"] == trajectory + + def test_discover_trajectories_results_dir_overrides_env(tmp_path, monkeypatch): skill = tmp_path / "my-skill" skill.mkdir() From cddc820297e888e1b84ed587dadaac3965053e09 Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Thu, 3 Sep 2026 23:27:52 +0530 Subject: [PATCH 2/2] fix(tier3): resolve trial case ids from persisted Harbor metadata Stop guessing Harbor suffix shape from folder names. Prefer reward entry_id, then result.json task metadata, and keep the full directory name when neither is present. Signed-off-by: mimran-khan --- src/skillevaluator/tier3/generate_dataset.py | 39 +++++++------- tests/tier3/test_generate_dataset_results.py | 54 ++++++++++++++++++++ 2 files changed, 74 insertions(+), 19 deletions(-) diff --git a/src/skillevaluator/tier3/generate_dataset.py b/src/skillevaluator/tier3/generate_dataset.py index a85280b1..c504dd96 100644 --- a/src/skillevaluator/tier3/generate_dataset.py +++ b/src/skillevaluator/tier3/generate_dataset.py @@ -503,16 +503,6 @@ def _ensure_project_imports(): sys.path.insert(0, src_dir) -def _looks_like_harbor_suffix(suffix: str) -> bool: - """True when a ``__`` tail looks like Harbor's random or attempt suffix.""" - if not suffix: - return False - lowered = suffix.lower() - if lowered.startswith("attempt"): - return True - return len(suffix) <= 12 and suffix.isalnum() and any(ch.isdigit() for ch in suffix) - - def _read_reward_entry_id(trial_dir: Path) -> str: for reward_path in (trial_dir / "reward.json", trial_dir / "verifier" / "reward.json"): if not reward_path.is_file(): @@ -528,18 +518,29 @@ def _read_reward_entry_id(trial_dir: Path) -> str: return "" +def _read_result_entry_id(trial_dir: Path) -> str: + """Resolve case id from Harbor ``result.json`` when reward metadata is absent.""" + result_path = trial_dir / "result.json" + if not result_path.is_file(): + return "" + try: + payload = json.loads(result_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return "" + if not isinstance(payload, dict): + return "" + from skillevaluator.tier3.harbor.collector import _entry_id_from_harbor_result + + return _entry_id_from_harbor_result(payload) + + def _case_id_from_trial_dir(trial_dir: Path) -> str: - """Resolve eval case id from reward metadata, with conservative folder fallback.""" + """Resolve eval case id from persisted Harbor metadata, else the folder name.""" if entry_id := _read_reward_entry_id(trial_dir): return entry_id - - name = trial_dir.name - if "__" not in name: - return name - prefix, suffix = name.split("__", 1) - if prefix and _looks_like_harbor_suffix(suffix): - return prefix - return name + if entry_id := _read_result_entry_id(trial_dir): + return entry_id + return trial_dir.name def _discover_trajectories( diff --git a/tests/tier3/test_generate_dataset_results.py b/tests/tier3/test_generate_dataset_results.py index 5cce8e8b..d61ba69c 100644 --- a/tests/tier3/test_generate_dataset_results.py +++ b/tests/tier3/test_generate_dataset_results.py @@ -49,6 +49,15 @@ def test_discover_trajectories_maps_harbor_trial_folder_to_case_id(tmp_path, mon trial.mkdir(parents=True) trajectory = {"steps": [{"tool_calls": [{"tool": "Read"}]}]} trial.joinpath("trajectory.json").write_text(json.dumps(trajectory), encoding="utf-8") + trial.joinpath("result.json").write_text( + json.dumps( + { + "trial_name": "demo-001__Lmi47iy", + "config": {"task": {"path": "tasks/demo-001"}}, + } + ), + encoding="utf-8", + ) (run_dir / "run_config.json").write_text("{}", encoding="utf-8") (run_dir / "result.json").write_text(json.dumps({"run_id": run_id}), encoding="utf-8") (results_root / "demo" / "latest").symlink_to(run_id) @@ -146,6 +155,51 @@ def test_discover_trajectories_ambiguous_folder_without_reward_uses_full_name(tm assert list(found) == ["case__one"] +def test_discover_trajectories_preserves_versioned_case_id_without_reward(tmp_path, monkeypatch): + """IDs like ``case__v2`` must not be truncated when only the folder name is present.""" + trajectory = {"steps": [{"tool_calls": []}]} + skill = _write_results_trial( + tmp_path, + skill_name="demo", + trial_folder="case__v2", + trajectory=trajectory, + ) + monkeypatch.setenv("SKILLEVALUATOR_RESULTS_DIR", str(tmp_path / "results")) + + found = _discover_trajectories(skill) + assert list(found) == ["case__v2"] + + +def test_discover_trajectories_resolves_shortuuid_folder_from_result_json(tmp_path, monkeypatch): + """All-letter Harbor tails resolve via result.json task metadata, not suffix guessing.""" + trajectory = {"steps": [{"tool_calls": []}]} + skill = tmp_path / "demo" + skill.mkdir() + results_root = tmp_path / "results" + run_id = "20260709_120000" + run_dir = results_root / "demo" / run_id + trial_folder = "case-001__LRZctSP" + trial = run_dir / "claude-code" / "with-skill" / "trials" / trial_folder + trial.mkdir(parents=True) + trial.joinpath("trajectory.json").write_text(json.dumps(trajectory), encoding="utf-8") + trial.joinpath("result.json").write_text( + json.dumps( + { + "trial_name": trial_folder, + "config": {"task": {"path": "tasks/case-001"}}, + } + ), + encoding="utf-8", + ) + (run_dir / "run_config.json").write_text("{}", encoding="utf-8") + (run_dir / "result.json").write_text(json.dumps({"run_id": run_id}), encoding="utf-8") + (results_root / "demo" / "latest").symlink_to(run_id) + monkeypatch.setenv("SKILLEVALUATOR_RESULTS_DIR", str(results_root)) + + found = _discover_trajectories(skill) + assert list(found) == ["case-001"] + + def test_discover_trajectories_results_dir_overrides_env(tmp_path, monkeypatch): skill = tmp_path / "my-skill" skill.mkdir()