From 5b87f8c6975a647ee1e8759bedc3bbba9addb300 Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Sat, 29 Aug 2026 03:38:04 +0530 Subject: [PATCH 01/12] feat(cli): write catalog-summary.json after catalog validate Emit a machine-readable fleet rollup at the reports root with per-skill status, optional severity totals from child JSON reports, and report paths. Create the output directory when needed so summary writes survive early skill failures. Fixes #120 Signed-off-by: mimran-khan Signed-off-by: mimran-khan --- CHANGELOG.md | 6 +++ src/skillevaluator/cli.py | 96 +++++++++++++++++++++++++++++++++++++++ tests/test_commands.py | 32 +++++++++---- 3 files changed, 125 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f54c4e84..8e7f6aa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to SkillEvaluator are documented in this file. ## Unreleased +### Added + +- Catalog validation now writes `catalog-summary.json` at the reports root with + per-skill pass/fail status, optional severity rollups from child JSON reports, + and paths to per-skill report directories. + ### Fixed - Tier 3 accuracy and custom goal judges now retry one malformed (including diff --git a/src/skillevaluator/cli.py b/src/skillevaluator/cli.py index 0ed29141..68de9590 100644 --- a/src/skillevaluator/cli.py +++ b/src/skillevaluator/cli.py @@ -6,8 +6,10 @@ from __future__ import annotations import copy +import json import logging import math +from datetime import UTC, datetime from pathlib import Path import click @@ -691,6 +693,89 @@ def _print_catalog_summary(total: int, failures: list[tuple[str, str]], reports_ console_.print(Text.assemble((" reports ", MUTED), (f"{reports_root}//", MUTED))) +CATALOG_SUMMARY_FILENAME = "catalog-summary.json" + + +def _latest_skill_json_report(skill_report_dir: Path) -> Path | None: + """Return the newest per-skill machine-readable report when present.""" + if not skill_report_dir.is_dir(): + return None + candidates = sorted(skill_report_dir.glob("skillevaluator-output-*.json"), reverse=True) + if candidates: + return candidates[0] + return None + + +def _catalog_skill_entry( + skill_name: str, + skill_report_dir: Path, + *, + passed: bool, + reason: str, +) -> dict[str, object]: + entry: dict[str, object] = { + "name": skill_name, + "passed": passed, + "report_dir": skill_name, + } + if not passed: + entry["reason"] = reason + + json_report = _latest_skill_json_report(skill_report_dir) + if json_report is None: + return entry + + entry["json_report"] = json_report.name + try: + payload = json.loads(json_report.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return entry + if not isinstance(payload, dict): + return entry + + for key in ("overall_passed", "overall_status", "incomplete_scans", "severity_counts"): + if key in payload: + entry[key] = payload[key] + return entry + + +def _write_catalog_summary(output_dir: Path, skills: list[dict[str, object]]) -> Path: + """Write a machine-readable fleet rollup for catalog validation.""" + total = len(skills) + passed = sum(1 for skill in skills if skill.get("passed")) + failed = total - passed + severity_totals = { + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + } + for skill in skills: + counts = skill.get("severity_counts") + if not isinstance(counts, dict): + continue + for key in severity_totals: + value = counts.get(key) + if isinstance(value, int): + severity_totals[key] += value + + summary: dict[str, object] = { + "total": total, + "passed": passed, + "failed": failed, + "overall_passed": failed == 0, + "reports_root": output_dir.name, + "summary_path": CATALOG_SUMMARY_FILENAME, + "severity_totals": severity_totals, + "skills": skills, + "generated_at": datetime.now(tz=UTC).isoformat(), + } + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / CATALOG_SUMMARY_FILENAME + output_path.write_text(json.dumps(summary, indent=2, default=str, allow_nan=False), encoding="utf-8") + return output_path + + def _validate_catalog( ctx: click.Context, *, @@ -725,6 +810,17 @@ def _validate_catalog( failures.append((skill_dir.name, str(getattr(exc, "message", exc)))) except Exception as exc: # unexpected: keep the catalog running, report it on the scoreboard failures.append((skill_dir.name, f"unexpected error: {exc}")) + failure_map = dict(failures) + skill_entries = [ + _catalog_skill_entry( + skill_dir.name, + output_dir / skill_dir.name, + passed=skill_dir.name not in failure_map, + reason=failure_map.get(skill_dir.name, ""), + ) + for skill_dir in skill_dirs + ] + _write_catalog_summary(output_dir, skill_entries) _print_catalog_summary(len(skill_dirs), failures, output_dir) if failures: raise click.ClickException( diff --git a/tests/test_commands.py b/tests/test_commands.py index b9e28b50..0b727bbe 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -326,6 +326,13 @@ def test_validate_catalog_runs_each_skill_as_separate_job() -> None: assert result.exit_code == 0, result.output assert any(Path("out/simple").glob("*.html")) assert any(Path("out/simple2").glob("*.html")) + summary_path = Path("out/catalog-summary.json") + assert summary_path.is_file() + summary = json.loads(summary_path.read_text(encoding="utf-8")) + assert summary["total"] == 2 + assert summary["passed"] == 2 + assert summary["overall_passed"] is True + assert len(summary["skills"]) == 2 def test_validate_catalog_rejects_one_previous_version_for_every_skill() -> None: @@ -726,11 +733,15 @@ def _failing_tier1(*_args, **_kwargs) -> list[ValidationResult]: cli, ["validate", str(catalog.resolve()), "--no-llm", "--no-dedup", "--checks", "schema", "-o", "out"] ) - out = _plain_text(result.output) - assert "skill 1/2" in out and "skill 2/2" in out - assert "Catalog Result" in out - assert "0/2 skills passed" in out - assert result.exit_code != 0 + out = _plain_text(result.output) + assert "skill 1/2" in out and "skill 2/2" in out + assert "Catalog Result" in out + assert "0/2 skills passed" in out + assert result.exit_code != 0 + summary = json.loads(Path("out/catalog-summary.json").read_text(encoding="utf-8")) + assert summary["failed"] == 2 + assert summary["overall_passed"] is False + assert all(not skill["passed"] for skill in summary["skills"]) def test_render_evaluation_result_invokes_findings_report(monkeypatch) -> None: @@ -870,10 +881,13 @@ def _boom(*_args, **_kwargs): cli, ["validate", str(catalog.resolve()), "--no-llm", "--no-dedup", "--checks", "schema", "-o", "out"] ) - out = _plain_text(result.output) - assert "Catalog Result" in out - assert "unexpected error: validator exploded" in out - assert result.exit_code != 0 + out = _plain_text(result.output) + assert "Catalog Result" in out + assert "unexpected error: validator exploded" in out + assert result.exit_code != 0 + summary = json.loads(Path("out/catalog-summary.json").read_text(encoding="utf-8")) + assert summary["failed"] == 2 + assert summary["skills"][0]["reason"] == "unexpected error: validator exploded" def test_summarize_tier2_empty_results_name_a_reason() -> None: From 416112bdaf3c78165760ac5d6d2a88bd9d84a882 Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Sat, 29 Aug 2026 03:45:05 +0530 Subject: [PATCH 02/12] feat(cli): add parallel catalog validation with --workers Catalog validate accepts --workers N to run skills in isolated child processes. Values above 1 skip the per-skill pipeline view and rebuild per-skill argv from the parent Click context or sys.argv. Fixes #122 Signed-off-by: mimran-khan Signed-off-by: mimran-khan --- CHANGELOG.md | 2 + src/skillevaluator/cli.py | 245 +++++++++++++++++++++++++++++++++- tests/golden/cli_surface.json | 18 +++ tests/test_commands.py | 38 ++++++ 4 files changed, 300 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e7f6aa8..ba71bd25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ All notable changes to SkillEvaluator are documented in this file. - Catalog validation now writes `catalog-summary.json` at the reports root with per-skill pass/fail status, optional severity rollups from child JSON reports, and paths to per-skill report directories. +- Catalog `validate` accepts `--workers N` to validate skills in parallel child + processes (default 1 preserves the serial per-skill pipeline view). ### Fixed diff --git a/src/skillevaluator/cli.py b/src/skillevaluator/cli.py index 68de9590..e99ce3fb 100644 --- a/src/skillevaluator/cli.py +++ b/src/skillevaluator/cli.py @@ -9,8 +9,10 @@ import json import logging import math +from concurrent.futures import ProcessPoolExecutor, as_completed from datetime import UTC, datetime from pathlib import Path +from typing import Any import click @@ -696,6 +698,164 @@ def _print_catalog_summary(total: int, failures: list[tuple[str, str]], reports_ CATALOG_SUMMARY_FILENAME = "catalog-summary.json" +def _catalog_child_argv_from_sys(skill_dir: Path, output_dir: Path, parent_argv: list[str]) -> list[str]: + """Rebuild ``validate`` argv for one catalog skill from ``sys.argv``.""" + argv = list(parent_argv) + try: + validate_idx = next(i for i, arg in enumerate(argv) if arg == "validate") + except StopIteration: + return ["validate", str(skill_dir), "-o", str(output_dir)] + + tail = argv[validate_idx + 1 :] + if tail and not tail[0].startswith("-"): + tail = tail[1:] + + child_tail: list[str] = [] + skip_next = False + for arg in tail: + if skip_next: + skip_next = False + continue + if arg in {"--workers", "-o", "--output-dir"}: + skip_next = True + continue + if arg.startswith("--workers=") or arg.startswith("--output-dir=") or arg.startswith("-o="): + continue + child_tail.append(arg) + + return ["validate", str(skill_dir), *child_tail, "-o", str(output_dir)] + + +def _catalog_child_argv_from_ctx(ctx: click.Context, skill_dir: Path, output_dir: Path) -> list[str]: + """Rebuild ``validate`` argv from the active Click context (pytest-safe).""" + params = ctx.params + argv: list[str] = ["validate", str(skill_dir)] + + if params.get("verbose"): + argv.append("--verbose") + if params.get("full"): + argv.append("--full") + if params.get("tiers"): + argv.extend(["--tiers", str(params["tiers"])]) + if params.get("checks"): + argv.extend(["--checks", str(params["checks"])]) + if params.get("previous_version"): + argv.extend(["--previous-version", str(params["previous_version"])]) + if params.get("fail_fast"): + argv.append("--fail-fast") + if params.get("continue_on_failure"): + argv.append("-c") + if params.get("llm"): + argv.append("--llm") + else: + argv.append("--no-llm") + if params.get("llm_verify"): + argv.append("--llm-verify") + if not params.get("dedup", True): + argv.append("--no-dedup") + block_on_dedup = params.get("block_on_dedup") + if block_on_dedup is True: + argv.append("--block-on-dedup") + elif block_on_dedup is False: + argv.append("--no-block-on-dedup") + min_score = params.get("min_score", 70) + if min_score != 70: + argv.extend(["--min-score", str(min_score)]) + if params.get("external"): + argv.append("--external") + if params.get("policy_path"): + argv.extend(["--policy", str(params["policy_path"])]) + if params.get("profile"): + argv.extend(["--profile", str(params["profile"])]) + if params.get("agent_eval"): + argv.append("--tier3") + block_on_agent_eval = params.get("block_on_agent_eval") + if block_on_agent_eval is True: + argv.append("--block-on-agent-eval") + elif block_on_agent_eval is False: + argv.append("--no-block-on-agent-eval") + if params.get("autopilot"): + argv.append("--autopilot") + agents = params.get("agents", "codex") + if agents != "codex": + argv.extend(["--agents", str(agents)]) + env_mode = params.get("env_mode", "docker") + if env_mode != "docker": + argv.extend(["--env-mode", str(env_mode)]) + if params.get("skip_baseline"): + argv.append("--skip-baseline") + if params.get("n_concurrent") is not None: + argv.extend(["--n-concurrent", str(params["n_concurrent"])]) + if params.get("max_agents") is not None: + argv.extend(["--max-agents", str(params["max_agents"])]) + if params.get("n_attempts") is not None: + argv.extend(["--n-attempts", str(params["n_attempts"])]) + if params.get("pass_threshold") is not None: + argv.extend(["--pass-threshold", str(params["pass_threshold"])]) + stop_on_pass = params.get("stop_on_pass") + if stop_on_pass is True: + argv.append("--stop-on-pass") + elif stop_on_pass is False: + argv.append("--no-stop-on-pass") + if params.get("model"): + argv.extend(["--model", str(params["model"])]) + for override in params.get("agent_model") or (): + argv.extend(["--agent-model", str(override)]) + if params.get("grading_mode"): + argv.extend(["--grading-mode", str(params["grading_mode"])]) + if params.get("results_dir"): + argv.extend(["--results-dir", str(params["results_dir"])]) + for skill in params.get("include_skills") or (): + argv.extend(["--include-skill", str(skill)]) + if params.get("copy_repo"): + argv.append("--copy-repo") + if params.get("timeout_multiplier") is not None: + argv.extend(["--timeout-multiplier", str(params["timeout_multiplier"])]) + if params.get("harbor_keep_jobs"): + argv.append("--harbor-keep-jobs") + for fmt in params.get("report_formats") or ("cli",): + if fmt == "cli": + continue + argv.extend(["-r", fmt]) + argv.extend(["-o", str(output_dir)]) + return argv + + +def _catalog_child_argv( + ctx: click.Context, + skill_dir: Path, + output_dir: Path, + parent_argv: list[str], +) -> list[str]: + if "validate" in parent_argv: + return _catalog_child_argv_from_sys(skill_dir, output_dir, parent_argv) + return _catalog_child_argv_from_ctx(ctx, skill_dir, output_dir) + + +def _run_catalog_skill_worker(job: dict[str, Any]) -> tuple[str, bool, str]: + """Run one catalog skill validation in a child process.""" + import os + + from click.testing import CliRunner + + workdir = job.get("cwd") + if workdir: + os.chdir(workdir) + + skill_name = str(job["skill_name"]) + result = CliRunner().invoke(cli, job["argv"]) + if result.exit_code == 0: + return skill_name, True, "" + exc = result.exception + if isinstance(exc, SystemExit): + return skill_name, False, "validation failed" + if exc is not None: + if isinstance(exc, click.ClickException): + return skill_name, False, str(getattr(exc, "message", exc)) + return skill_name, False, f"unexpected error: {exc}" + return skill_name, False, "validation failed" + + def _latest_skill_json_report(skill_report_dir: Path) -> Path | None: """Return the newest per-skill machine-readable report when present.""" if not skill_report_dir.is_dir(): @@ -781,12 +941,14 @@ def _validate_catalog( *, resolved_target: Path, output_dir: Path, + workers: int = 1, ) -> None: - """Run the full validate pipeline once per skill in the catalog, serially. + """Run the full validate pipeline once per skill in the catalog. Each skill is an independent job with its own pipeline view, reports (under ``//``), and verdict; the catalog exits nonzero - when any skill failed. + when any skill failed. With ``workers`` above 1, skills validate in + parallel child processes and the per-skill pipeline view is skipped. """ if ctx.params.get("previous_version"): raise click.ClickException( @@ -795,6 +957,10 @@ def _validate_catalog( ) skill_dirs = sorted(marker.parent for marker in resolved_target.glob("*/SKILL.md")) + if workers > 1: + _validate_catalog_parallel(ctx, skill_dirs=skill_dirs, output_dir=output_dir, workers=workers) + return + failures: list[tuple[str, str]] = [] for index, skill_dir in enumerate(skill_dirs, start=1): _print_catalog_divider(index, len(skill_dirs), skill_dir.name) @@ -829,6 +995,67 @@ def _validate_catalog( ) +def _validate_catalog_parallel( + ctx: click.Context, + *, + skill_dirs: list[Path], + output_dir: Path, + workers: int, +) -> None: + """Validate catalog skills concurrently in isolated child processes.""" + from skillevaluator.reporting.console_ui import make_view_console + + if not skill_dirs: + _write_catalog_summary(output_dir, []) + _print_catalog_summary(0, [], output_dir) + return + + import sys + + parent_argv = list(sys.argv) + workdir = str(Path.cwd()) + output_dir.mkdir(parents=True, exist_ok=True) + make_view_console().print( + f"[dim]Validating {len(skill_dirs)} skills with {workers} worker" + f"{'s' if workers != 1 else ''} (parallel catalog mode; per-skill pipeline view disabled)[/dim]" + ) + + jobs = [ + { + "skill_name": skill_dir.name, + "argv": _catalog_child_argv(ctx, skill_dir, output_dir / skill_dir.name, parent_argv), + "cwd": workdir, + } + for skill_dir in skill_dirs + ] + failures: list[tuple[str, str]] = [] + with ProcessPoolExecutor(max_workers=workers) as executor: + futures = [executor.submit(_run_catalog_skill_worker, job) for job in jobs] + for future in as_completed(futures): + skill_name, passed, reason = future.result() + if not passed: + failures.append((skill_name, reason)) + + failures.sort(key=lambda item: item[0]) + failure_map = dict(failures) + skill_entries = [ + _catalog_skill_entry( + skill_dir.name, + output_dir / skill_dir.name, + passed=skill_dir.name not in failure_map, + reason=failure_map.get(skill_dir.name, ""), + ) + for skill_dir in skill_dirs + ] + _write_catalog_summary(output_dir, skill_entries) + _print_catalog_summary(len(skill_dirs), failures, output_dir) + if failures: + raise click.ClickException( + f"{len(failures)}/{len(skill_dirs)} skills failed validation: " + + ", ".join(name for name, _reason in failures) + ) + + def _rerun_hint(target_path: Path, agent_eval: bool) -> str: """Reconstruct the user's actual command for the FAIL panel's rerun line. @@ -1042,6 +1269,16 @@ def _print_run_banner(target_path: Path, content_type: str, profile: str | None) help_group=_RUN_GROUP, help="Custom policy YAML overlaid on top of --profile.", ) +@click.option( + "--workers", + type=click.IntRange(1), + default=1, + show_default=True, + cls=GroupedOption, + help_group=_RUN_GROUP, + help="Concurrent catalog skill jobs when validating a folder of skills. " + "Values above 1 run skills in parallel processes and disable the per-skill pipeline view.", +) @click.option( "--dedup/--no-dedup", "--tier2/--no-tier2", @@ -1245,6 +1482,7 @@ def validate( copy_repo: bool, timeout_multiplier: float | None, harbor_keep_jobs: bool, + workers: int, report_formats: tuple[str, ...], output_dir: Path, ) -> None: @@ -1315,7 +1553,7 @@ def validate( from skillevaluator.constants import CONTENT_TYPE_UNKNOWN # A directory of skills (no root SKILL.md) is a catalog: run the pipeline - # once per skill, serially, each as its own job with its own reports. + # once per skill, each as its own job with its own reports. if ( resolved_type in (CONTENT_TYPE_SKILL, CONTENT_TYPE_UNKNOWN) and target_path.is_dir() @@ -1326,6 +1564,7 @@ def validate( click.get_current_context(), resolved_target=target_path, output_dir=output_dir, + workers=workers, ) return diff --git a/tests/golden/cli_surface.json b/tests/golden/cli_surface.json index bc1aa820..4b770fb4 100644 --- a/tests/golden/cli_surface.json +++ b/tests/golden/cli_surface.json @@ -1512,6 +1512,15 @@ "param_type": "option", "type": "file" }, + { + "default": "1", + "name": "workers", + "opts": [ + "--workers" + ], + "param_type": "option", + "type": "integer range" + }, { "default": "True", "is_flag": true, @@ -2796,6 +2805,15 @@ "param_type": "option", "type": "file" }, + { + "default": "1", + "name": "workers", + "opts": [ + "--workers" + ], + "param_type": "option", + "type": "integer range" + }, { "default": "True", "is_flag": true, diff --git a/tests/test_commands.py b/tests/test_commands.py index 0b727bbe..6e38cdad 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -335,6 +335,44 @@ def test_validate_catalog_runs_each_skill_as_separate_job() -> None: assert len(summary["skills"]) == 2 +def test_validate_catalog_workers_runs_skills_in_parallel() -> None: + runner = CliRunner() + with runner.isolated_filesystem(): + catalog = Path("catalog") + for name in ("simple", "simple2"): + shutil.copytree(FIXTURE, catalog / name) + second = catalog / "simple2" / "SKILL.md" + second.write_text( + second.read_text(encoding="utf-8").replace("name: simple", "name: simple2"), + encoding="utf-8", + ) + result = runner.invoke( + cli, + [ + "validate", + str(catalog.resolve()), + "--workers", + "2", + "--no-llm", + "--no-dedup", + "--checks", + "quality", + "-o", + "out", + ], + ) + + out = _plain_text(result.output) + assert "parallel catalog mode" in out + assert "Catalog Result" in out + assert result.exit_code == 0, result.output + summary = json.loads(Path("out/catalog-summary.json").read_text(encoding="utf-8")) + assert summary["total"] == 2 + assert summary["passed"] == 2 + assert any(Path("out/simple").glob("*.html")) + assert any(Path("out/simple2").glob("*.html")) + + def test_validate_catalog_rejects_one_previous_version_for_every_skill() -> None: runner = CliRunner() with runner.isolated_filesystem(): From 8c51e1b741b751d2af0f1c08197aa054ffe4bf0f Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Sat, 29 Aug 2026 17:20:30 +0530 Subject: [PATCH 03/12] fix(cli): stabilize parallel catalog workers and summary writes Rebuild child argv without dropping positional catalog paths, track fresh per-skill JSON reports instead of stale files, write catalog-summary.json atomically, and fix --include-skills forwarding for context fallback. Signed-off-by: mimran-khan Signed-off-by: mimran-khan --- src/skillevaluator/cli.py | 52 +++++++++++++++++++++++++++++---------- tests/test_commands.py | 27 ++++++++++++++++++++ 2 files changed, 66 insertions(+), 13 deletions(-) diff --git a/src/skillevaluator/cli.py b/src/skillevaluator/cli.py index e99ce3fb..2d03e5d3 100644 --- a/src/skillevaluator/cli.py +++ b/src/skillevaluator/cli.py @@ -707,8 +707,6 @@ def _catalog_child_argv_from_sys(skill_dir: Path, output_dir: Path, parent_argv: return ["validate", str(skill_dir), "-o", str(output_dir)] tail = argv[validate_idx + 1 :] - if tail and not tail[0].startswith("-"): - tail = tail[1:] child_tail: list[str] = [] skip_next = False @@ -721,6 +719,8 @@ def _catalog_child_argv_from_sys(skill_dir: Path, output_dir: Path, parent_argv: continue if arg.startswith("--workers=") or arg.startswith("--output-dir=") or arg.startswith("-o="): continue + if not arg.startswith("-"): + continue child_tail.append(arg) return ["validate", str(skill_dir), *child_tail, "-o", str(output_dir)] @@ -806,7 +806,7 @@ def _catalog_child_argv_from_ctx(ctx: click.Context, skill_dir: Path, output_dir if params.get("results_dir"): argv.extend(["--results-dir", str(params["results_dir"])]) for skill in params.get("include_skills") or (): - argv.extend(["--include-skill", str(skill)]) + argv.extend(["--include-skills", str(skill)]) if params.get("copy_repo"): argv.append("--copy-repo") if params.get("timeout_multiplier") is not None: @@ -832,7 +832,7 @@ def _catalog_child_argv( return _catalog_child_argv_from_ctx(ctx, skill_dir, output_dir) -def _run_catalog_skill_worker(job: dict[str, Any]) -> tuple[str, bool, str]: +def _run_catalog_skill_worker(job: dict[str, Any]) -> tuple[str, bool, str, str | None]: """Run one catalog skill validation in a child process.""" import os @@ -843,17 +843,32 @@ def _run_catalog_skill_worker(job: dict[str, Any]) -> tuple[str, bool, str]: os.chdir(workdir) skill_name = str(job["skill_name"]) + output_dir = Path(job["output_dir"]) + existing_reports = ( + set(output_dir.glob("skillevaluator-output-*.json")) if output_dir.is_dir() else set() + ) result = CliRunner().invoke(cli, job["argv"]) + json_report_name = _new_skill_json_report_name(output_dir, existing_reports) if result.exit_code == 0: - return skill_name, True, "" + return skill_name, True, "", json_report_name exc = result.exception if isinstance(exc, SystemExit): - return skill_name, False, "validation failed" + return skill_name, False, "validation failed", json_report_name if exc is not None: if isinstance(exc, click.ClickException): - return skill_name, False, str(getattr(exc, "message", exc)) - return skill_name, False, f"unexpected error: {exc}" - return skill_name, False, "validation failed" + return skill_name, False, str(getattr(exc, "message", exc)), json_report_name + return skill_name, False, f"unexpected error: {exc}", json_report_name + return skill_name, False, "validation failed", json_report_name + + +def _new_skill_json_report_name(output_dir: Path, existing_reports: set[Path]) -> str | None: + """Return the JSON report filename produced during this worker run.""" + if not output_dir.is_dir(): + return None + new_reports = set(output_dir.glob("skillevaluator-output-*.json")) - existing_reports + if not new_reports: + return None + return sorted(new_reports, reverse=True)[0].name def _latest_skill_json_report(skill_report_dir: Path) -> Path | None: @@ -872,6 +887,7 @@ def _catalog_skill_entry( *, passed: bool, reason: str, + json_report_name: str | None = None, ) -> dict[str, object]: entry: dict[str, object] = { "name": skill_name, @@ -881,8 +897,11 @@ def _catalog_skill_entry( if not passed: entry["reason"] = reason - json_report = _latest_skill_json_report(skill_report_dir) - if json_report is None: + if json_report_name: + json_report = skill_report_dir / json_report_name + else: + json_report = _latest_skill_json_report(skill_report_dir) + if json_report is None or not json_report.is_file(): return entry entry["json_report"] = json_report.name @@ -901,6 +920,8 @@ def _catalog_skill_entry( def _write_catalog_summary(output_dir: Path, skills: list[dict[str, object]]) -> Path: """Write a machine-readable fleet rollup for catalog validation.""" + from skillevaluator.reporting.base import _write_report_atomically + total = len(skills) passed = sum(1 for skill in skills if skill.get("passed")) failed = total - passed @@ -932,7 +953,8 @@ def _write_catalog_summary(output_dir: Path, skills: list[dict[str, object]]) -> } output_dir.mkdir(parents=True, exist_ok=True) output_path = output_dir / CATALOG_SUMMARY_FILENAME - output_path.write_text(json.dumps(summary, indent=2, default=str, allow_nan=False), encoding="utf-8") + payload = json.dumps(summary, indent=2, default=str, allow_nan=False).encode("utf-8") + _write_report_atomically(output_path, payload) return output_path @@ -1025,14 +1047,17 @@ def _validate_catalog_parallel( "skill_name": skill_dir.name, "argv": _catalog_child_argv(ctx, skill_dir, output_dir / skill_dir.name, parent_argv), "cwd": workdir, + "output_dir": str(output_dir / skill_dir.name), } for skill_dir in skill_dirs ] failures: list[tuple[str, str]] = [] + worker_reports: dict[str, str | None] = {} with ProcessPoolExecutor(max_workers=workers) as executor: futures = [executor.submit(_run_catalog_skill_worker, job) for job in jobs] for future in as_completed(futures): - skill_name, passed, reason = future.result() + skill_name, passed, reason, json_report_name = future.result() + worker_reports[skill_name] = json_report_name if not passed: failures.append((skill_name, reason)) @@ -1044,6 +1069,7 @@ def _validate_catalog_parallel( output_dir / skill_dir.name, passed=skill_dir.name not in failure_map, reason=failure_map.get(skill_dir.name, ""), + json_report_name=worker_reports.get(skill_dir.name), ) for skill_dir in skill_dirs ] diff --git a/tests/test_commands.py b/tests/test_commands.py index 6e38cdad..8fd517a0 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -373,6 +373,33 @@ def test_validate_catalog_workers_runs_skills_in_parallel() -> None: assert any(Path("out/simple2").glob("*.html")) +def test_validate_catalog_workers_accepts_options_before_target_path() -> None: + runner = CliRunner() + with runner.isolated_filesystem(): + catalog = Path("catalog") + shutil.copytree(FIXTURE, catalog / "simple") + result = runner.invoke( + cli, + [ + "validate", + "--workers", + "2", + str(catalog.resolve()), + "--no-llm", + "--no-dedup", + "--checks", + "quality", + "-o", + "out", + ], + ) + + assert result.exit_code == 0, result.output + summary = json.loads(Path("out/catalog-summary.json").read_text(encoding="utf-8")) + assert summary["total"] == 1 + assert summary["passed"] == 1 + + def test_validate_catalog_rejects_one_previous_version_for_every_skill() -> None: runner = CliRunner() with runner.isolated_filesystem(): From 9b66899671ee4c867c66988d5718ea811b151244 Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Mon, 31 Aug 2026 22:24:30 +0530 Subject: [PATCH 04/12] fix(cli): address rng1995 parallel catalog worker review Rebuild child argv from Click params, preserve -r cli when selected, track per-run JSON in serial catalog mode, and stop attaching stale reports when no new JSON was produced this run. Signed-off-by: mimran-khan Signed-off-by: mimran-khan --- src/skillevaluator/cli.py | 41 +++++++++++---------- tests/test_commands.py | 75 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 21 deletions(-) diff --git a/src/skillevaluator/cli.py b/src/skillevaluator/cli.py index 2d03e5d3..cdf5df92 100644 --- a/src/skillevaluator/cli.py +++ b/src/skillevaluator/cli.py @@ -814,8 +814,6 @@ def _catalog_child_argv_from_ctx(ctx: click.Context, skill_dir: Path, output_dir if params.get("harbor_keep_jobs"): argv.append("--harbor-keep-jobs") for fmt in params.get("report_formats") or ("cli",): - if fmt == "cli": - continue argv.extend(["-r", fmt]) argv.extend(["-o", str(output_dir)]) return argv @@ -871,16 +869,6 @@ def _new_skill_json_report_name(output_dir: Path, existing_reports: set[Path]) - return sorted(new_reports, reverse=True)[0].name -def _latest_skill_json_report(skill_report_dir: Path) -> Path | None: - """Return the newest per-skill machine-readable report when present.""" - if not skill_report_dir.is_dir(): - return None - candidates = sorted(skill_report_dir.glob("skillevaluator-output-*.json"), reverse=True) - if candidates: - return candidates[0] - return None - - def _catalog_skill_entry( skill_name: str, skill_report_dir: Path, @@ -900,8 +888,9 @@ def _catalog_skill_entry( if json_report_name: json_report = skill_report_dir / json_report_name else: - json_report = _latest_skill_json_report(skill_report_dir) - if json_report is None or not json_report.is_file(): + return entry + + if not json_report.is_file(): return entry entry["json_report"] = json_report.name @@ -984,20 +973,32 @@ def _validate_catalog( return failures: list[tuple[str, str]] = [] + skill_reports: dict[str, str | None] = {} for index, skill_dir in enumerate(skill_dirs, start=1): _print_catalog_divider(index, len(skill_dirs), skill_dir.name) + skill_output = output_dir / skill_dir.name + existing_reports = ( + set(skill_output.glob("skillevaluator-output-*.json")) if skill_output.is_dir() else set() + ) overrides = { **ctx.params, "target_path": skill_dir, "content_type": "skill", - "output_dir": output_dir / skill_dir.name, + "output_dir": skill_output, } + passed = True + reason = "" try: ctx.invoke(validate, **overrides) except click.ClickException as exc: - failures.append((skill_dir.name, str(getattr(exc, "message", exc)))) + passed = False + reason = str(getattr(exc, "message", exc)) + failures.append((skill_dir.name, reason)) except Exception as exc: # unexpected: keep the catalog running, report it on the scoreboard - failures.append((skill_dir.name, f"unexpected error: {exc}")) + passed = False + reason = f"unexpected error: {exc}" + failures.append((skill_dir.name, reason)) + skill_reports[skill_dir.name] = _new_skill_json_report_name(skill_output, existing_reports) failure_map = dict(failures) skill_entries = [ _catalog_skill_entry( @@ -1005,6 +1006,7 @@ def _validate_catalog( output_dir / skill_dir.name, passed=skill_dir.name not in failure_map, reason=failure_map.get(skill_dir.name, ""), + json_report_name=skill_reports.get(skill_dir.name), ) for skill_dir in skill_dirs ] @@ -1032,9 +1034,6 @@ def _validate_catalog_parallel( _print_catalog_summary(0, [], output_dir) return - import sys - - parent_argv = list(sys.argv) workdir = str(Path.cwd()) output_dir.mkdir(parents=True, exist_ok=True) make_view_console().print( @@ -1045,7 +1044,7 @@ def _validate_catalog_parallel( jobs = [ { "skill_name": skill_dir.name, - "argv": _catalog_child_argv(ctx, skill_dir, output_dir / skill_dir.name, parent_argv), + "argv": _catalog_child_argv_from_ctx(ctx, skill_dir, output_dir / skill_dir.name), "cwd": workdir, "output_dir": str(output_dir / skill_dir.name), } diff --git a/tests/test_commands.py b/tests/test_commands.py index 8fd517a0..8635237c 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -10,6 +10,7 @@ import pytest from click.testing import CliRunner +from skillevaluator import cli as cli_module from skillevaluator.cli import cli from skillevaluator.tier3.commands import parse_agent_model_overrides, parse_agents @@ -357,6 +358,8 @@ def test_validate_catalog_workers_runs_skills_in_parallel() -> None: "--no-dedup", "--checks", "quality", + "-r", + "html", "-o", "out", ], @@ -400,6 +403,78 @@ def test_validate_catalog_workers_accepts_options_before_target_path() -> None: assert summary["passed"] == 1 +def test_validate_catalog_workers_preserves_min_score_and_json_format() -> None: + runner = CliRunner() + with runner.isolated_filesystem(): + catalog = Path("catalog") + shutil.copytree(FIXTURE, catalog / "simple") + result = runner.invoke( + cli, + [ + "validate", + str(catalog.resolve()), + "--workers", + "2", + "--no-llm", + "--no-dedup", + "--checks", + "quality", + "-r", + "json", + "-o", + "out", + ], + ) + assert result.exit_code == 0, result.output + assert any(Path("out/simple").glob("skillevaluator-output-*.json")) + + +def test_validate_catalog_workers_cli_only_report_format() -> None: + runner = CliRunner() + with runner.isolated_filesystem(): + catalog = Path("catalog") + shutil.copytree(FIXTURE, catalog / "simple") + result = runner.invoke( + cli, + [ + "validate", + str(catalog.resolve()), + "--workers", + "2", + "--no-llm", + "--no-dedup", + "--checks", + "quality", + "-r", + "cli", + "-o", + "out", + ], + ) + assert result.exit_code == 0, result.output + assert not any(Path("out/simple").glob("*.html")) + assert not any(Path("out/simple").glob("skillevaluator-output-*.json")) + + +def test_catalog_skill_entry_skips_stale_json_without_report_name(tmp_path: Path) -> None: + skill_dir = tmp_path / "simple" + skill_dir.mkdir() + stale = skill_dir / "skillevaluator-output-19990101T000000.json" + stale.write_text( + json.dumps({"overall_passed": True, "severity_counts": {"high": 7}}), + encoding="utf-8", + ) + entry = cli_module._catalog_skill_entry( + "simple", + skill_dir, + passed=False, + reason="validation failed", + json_report_name=None, + ) + assert "overall_passed" not in entry + assert "severity_counts" not in entry + + def test_validate_catalog_rejects_one_previous_version_for_every_skill() -> None: runner = CliRunner() with runner.isolated_filesystem(): From 249682192fc7a5a46df8cc5478639023e40d4144 Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Thu, 3 Sep 2026 15:56:03 +0530 Subject: [PATCH 05/12] fix(cli): drop unused passed assignments in catalog loop Signed-off-by: mimran-khan Signed-off-by: mimran-khan --- src/skillevaluator/cli.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/skillevaluator/cli.py b/src/skillevaluator/cli.py index 39ba4650..9ec3f562 100644 --- a/src/skillevaluator/cli.py +++ b/src/skillevaluator/cli.py @@ -987,16 +987,13 @@ def _validate_catalog( "content_type": "skill", "output_dir": skill_output, } - passed = True reason = "" try: ctx.invoke(validate, **overrides) except click.ClickException as exc: - passed = False reason = str(getattr(exc, "message", exc)) failures.append((skill_dir.name, reason)) except Exception as exc: # unexpected: keep the catalog running, report it on the scoreboard - passed = False reason = f"unexpected error: {exc}" failures.append((skill_dir.name, reason)) skill_reports[skill_dir.name] = _new_skill_json_report_name(skill_output, existing_reports) From c8ece196ab51739822d61adc6a319bd2c86ceeb0 Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Thu, 3 Sep 2026 23:33:43 +0530 Subject: [PATCH 06/12] fix(cli): honor implicit catalog report defaults and JSON selection Do not forward the implicit cli default to catalog workers, and ignore SARIF sidecars when picking the standard JSON report for fleet rollup. Signed-off-by: mimran-khan Signed-off-by: mimran-khan --- src/skillevaluator/cli.py | 28 +++++++++++++++----------- tests/test_commands.py | 41 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 11 deletions(-) diff --git a/src/skillevaluator/cli.py b/src/skillevaluator/cli.py index 9ec3f562..1fa48fbf 100644 --- a/src/skillevaluator/cli.py +++ b/src/skillevaluator/cli.py @@ -699,6 +699,17 @@ def _print_catalog_summary(total: int, failures: list[tuple[str, str]], reports_ CATALOG_SUMMARY_FILENAME = "catalog-summary.json" +def _standard_json_report_paths(output_dir: Path) -> set[Path]: + """Return standard JSON reports, excluding SARIF sidecars that share the glob.""" + if not output_dir.is_dir(): + return set() + return { + path + for path in output_dir.glob("skillevaluator-output-*.json") + if path.is_file() and not path.name.endswith(".sarif.json") + } + + def _catalog_child_argv_from_sys(skill_dir: Path, output_dir: Path, parent_argv: list[str]) -> list[str]: """Rebuild ``validate`` argv for one catalog skill from ``sys.argv``.""" argv = list(parent_argv) @@ -814,8 +825,9 @@ def _catalog_child_argv_from_ctx(ctx: click.Context, skill_dir: Path, output_dir argv.extend(["--timeout-multiplier", str(params["timeout_multiplier"])]) if params.get("harbor_keep_jobs"): argv.append("--harbor-keep-jobs") - for fmt in params.get("report_formats") or ("cli",): - argv.extend(["-r", fmt]) + if _report_formats_explicit(): + for fmt in params.get("report_formats") or (): + argv.extend(["-r", fmt]) argv.extend(["-o", str(output_dir)]) return argv @@ -843,9 +855,7 @@ def _run_catalog_skill_worker(job: dict[str, Any]) -> tuple[str, bool, str, str skill_name = str(job["skill_name"]) output_dir = Path(job["output_dir"]) - existing_reports = ( - set(output_dir.glob("skillevaluator-output-*.json")) if output_dir.is_dir() else set() - ) + existing_reports = _standard_json_report_paths(output_dir) result = CliRunner().invoke(cli, job["argv"]) json_report_name = _new_skill_json_report_name(output_dir, existing_reports) if result.exit_code == 0: @@ -862,9 +872,7 @@ def _run_catalog_skill_worker(job: dict[str, Any]) -> tuple[str, bool, str, str def _new_skill_json_report_name(output_dir: Path, existing_reports: set[Path]) -> str | None: """Return the JSON report filename produced during this worker run.""" - if not output_dir.is_dir(): - return None - new_reports = set(output_dir.glob("skillevaluator-output-*.json")) - existing_reports + new_reports = _standard_json_report_paths(output_dir) - existing_reports if not new_reports: return None return sorted(new_reports, reverse=True)[0].name @@ -978,9 +986,7 @@ def _validate_catalog( for index, skill_dir in enumerate(skill_dirs, start=1): _print_catalog_divider(index, len(skill_dirs), skill_dir.name) skill_output = output_dir / skill_dir.name - existing_reports = ( - set(skill_output.glob("skillevaluator-output-*.json")) if skill_output.is_dir() else set() - ) + existing_reports = _standard_json_report_paths(skill_output) overrides = { **ctx.params, "target_path": skill_dir, diff --git a/tests/test_commands.py b/tests/test_commands.py index 8635237c..4d8d9448 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -456,6 +456,47 @@ def test_validate_catalog_workers_cli_only_report_format() -> None: assert not any(Path("out/simple").glob("skillevaluator-output-*.json")) +def test_validate_catalog_workers_implicit_default_reports() -> None: + runner = CliRunner() + with runner.isolated_filesystem(): + catalog = Path("catalog") + shutil.copytree(FIXTURE, catalog / "simple") + result = runner.invoke( + cli, + [ + "validate", + str(catalog.resolve()), + "--workers", + "2", + "--no-llm", + "--no-dedup", + "--checks", + "quality", + "-o", + "out", + ], + ) + assert result.exit_code == 0, result.output + assert any(Path("out/simple").glob("*.html")) + assert any(Path("out/simple").glob("skillevaluator-output-*.json")) + + +def test_new_skill_json_report_name_ignores_sarif_sidecar(tmp_path: Path) -> None: + skill_dir = tmp_path / "simple" + skill_dir.mkdir() + existing = { + skill_dir / "skillevaluator-output-20260101T000000.sarif.json", + } + for path in existing: + path.write_text("{}", encoding="utf-8") + standard = skill_dir / "skillevaluator-output-20260101T000001.json" + standard.write_text( + json.dumps({"overall_passed": True, "severity_counts": {"high": 1}}), + encoding="utf-8", + ) + assert cli_module._new_skill_json_report_name(skill_dir, existing) == standard.name + + def test_catalog_skill_entry_skips_stale_json_without_report_name(tmp_path: Path) -> None: skill_dir = tmp_path / "simple" skill_dir.mkdir() From 89b3f1ef2ffb5c98ac5fae2c6cc9ae8ca380626b Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Thu, 10 Sep 2026 04:06:22 +0530 Subject: [PATCH 07/12] fix(cli): bind catalog workers to exact validate JSON reports Record the JSON artifact emitted by each validate invocation instead of rediscovering reports via directory diffs, which missed same-second overwrites in parallel catalog mode. Also stop pre-creating catalog summary parents through symlinked paths. Signed-off-by: Narendran Raghavan Signed-off-by: mimran-khan --- src/skillevaluator/cli.py | 59 ++++++++------- tests/test_commands.py | 150 ++++++++++++++++++++++++++++++++++---- 2 files changed, 166 insertions(+), 43 deletions(-) diff --git a/src/skillevaluator/cli.py b/src/skillevaluator/cli.py index 1fa48fbf..b5712683 100644 --- a/src/skillevaluator/cli.py +++ b/src/skillevaluator/cli.py @@ -329,6 +329,28 @@ def _report_options(func): )(func) +_last_validate_json_report: str | None = None + + +def _effective_report_formats(report_formats: tuple[str, ...], *, quiet: bool) -> tuple[str, ...]: + """Resolve the report formats ``validate`` will actually emit.""" + if quiet and not _report_formats_explicit(): + return tuple(dict.fromkeys([fmt for fmt in report_formats if fmt != "cli"] + ["html", "json"])) + return report_formats + + +def _record_validate_json_report(report_name: str | None) -> None: + global _last_validate_json_report + _last_validate_json_report = report_name + + +def _consume_validate_json_report() -> str | None: + global _last_validate_json_report + report_name = _last_validate_json_report + _last_validate_json_report = None + return report_name + + def _report_formats_explicit() -> bool: """True when the user passed ``-r``/``--report`` on the command line. @@ -699,17 +721,6 @@ def _print_catalog_summary(total: int, failures: list[tuple[str, str]], reports_ CATALOG_SUMMARY_FILENAME = "catalog-summary.json" -def _standard_json_report_paths(output_dir: Path) -> set[Path]: - """Return standard JSON reports, excluding SARIF sidecars that share the glob.""" - if not output_dir.is_dir(): - return set() - return { - path - for path in output_dir.glob("skillevaluator-output-*.json") - if path.is_file() and not path.name.endswith(".sarif.json") - } - - def _catalog_child_argv_from_sys(skill_dir: Path, output_dir: Path, parent_argv: list[str]) -> list[str]: """Rebuild ``validate`` argv for one catalog skill from ``sys.argv``.""" argv = list(parent_argv) @@ -854,10 +865,8 @@ def _run_catalog_skill_worker(job: dict[str, Any]) -> tuple[str, bool, str, str os.chdir(workdir) skill_name = str(job["skill_name"]) - output_dir = Path(job["output_dir"]) - existing_reports = _standard_json_report_paths(output_dir) result = CliRunner().invoke(cli, job["argv"]) - json_report_name = _new_skill_json_report_name(output_dir, existing_reports) + json_report_name = _consume_validate_json_report() if result.exit_code == 0: return skill_name, True, "", json_report_name exc = result.exception @@ -870,14 +879,6 @@ def _run_catalog_skill_worker(job: dict[str, Any]) -> tuple[str, bool, str, str return skill_name, False, "validation failed", json_report_name -def _new_skill_json_report_name(output_dir: Path, existing_reports: set[Path]) -> str | None: - """Return the JSON report filename produced during this worker run.""" - new_reports = _standard_json_report_paths(output_dir) - existing_reports - if not new_reports: - return None - return sorted(new_reports, reverse=True)[0].name - - def _catalog_skill_entry( skill_name: str, skill_report_dir: Path, @@ -949,7 +950,6 @@ def _write_catalog_summary(output_dir: Path, skills: list[dict[str, object]]) -> "skills": skills, "generated_at": datetime.now(tz=UTC).isoformat(), } - output_dir.mkdir(parents=True, exist_ok=True) output_path = output_dir / CATALOG_SUMMARY_FILENAME payload = json.dumps(summary, indent=2, default=str, allow_nan=False).encode("utf-8") _write_report_atomically(output_path, payload) @@ -986,7 +986,6 @@ def _validate_catalog( for index, skill_dir in enumerate(skill_dirs, start=1): _print_catalog_divider(index, len(skill_dirs), skill_dir.name) skill_output = output_dir / skill_dir.name - existing_reports = _standard_json_report_paths(skill_output) overrides = { **ctx.params, "target_path": skill_dir, @@ -1002,7 +1001,7 @@ def _validate_catalog( except Exception as exc: # unexpected: keep the catalog running, report it on the scoreboard reason = f"unexpected error: {exc}" failures.append((skill_dir.name, reason)) - skill_reports[skill_dir.name] = _new_skill_json_report_name(skill_output, existing_reports) + skill_reports[skill_dir.name] = _consume_validate_json_report() failure_map = dict(failures) skill_entries = [ _catalog_skill_entry( @@ -1039,7 +1038,6 @@ def _validate_catalog_parallel( return workdir = str(Path.cwd()) - output_dir.mkdir(parents=True, exist_ok=True) make_view_console().print( f"[dim]Validating {len(skill_dirs)} skills with {workers} worker" f"{'s' if workers != 1 else ''} (parallel catalog mode; per-skill pipeline view disabled)[/dim]" @@ -1527,6 +1525,7 @@ def validate( validated against its public contract. Quality/lint/version checks are skill-only and skipped for plugins. """ + _record_validate_json_report(None) if dedup: _reject_linked_tier2_root(target_path) target_path = target_path.resolve() @@ -1815,10 +1814,7 @@ def _on_engine_tail(lines: list[str]) -> None: # the summary; the files carry the findings) and points at them from the # footer. An EXPLICIT -r is a contract and is honored exactly — including # "cli", which renders the full Rich report below the pipeline view. - if quiet and not _report_formats_explicit(): - effective_formats = tuple(dict.fromkeys([f for f in report_formats if f != "cli"] + ["html", "json"])) - else: - effective_formats = report_formats + effective_formats = _effective_report_formats(report_formats, quiet=quiet) report_basename_value = make_timestamped_basename(f"{REPORT_PREFIX}-output") emit_reports( results, @@ -1832,6 +1828,9 @@ def _on_engine_tail(lines: list[str]) -> None: sarif_scan_root=target_path, sarif_repository_root=sarif_repository_root, ) + _record_validate_json_report( + f"{report_basename_value}.json" if "json" in effective_formats else None + ) # BENCHMARK.md is generated compulsorily for skills (matches SkillEvaluator), even on # failure, so the publication card always reflects the latest evaluation -- diff --git a/tests/test_commands.py b/tests/test_commands.py index 4d8d9448..ab077f90 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -481,20 +481,144 @@ def test_validate_catalog_workers_implicit_default_reports() -> None: assert any(Path("out/simple").glob("skillevaluator-output-*.json")) -def test_new_skill_json_report_name_ignores_sarif_sidecar(tmp_path: Path) -> None: - skill_dir = tmp_path / "simple" - skill_dir.mkdir() - existing = { - skill_dir / "skillevaluator-output-20260101T000000.sarif.json", - } - for path in existing: - path.write_text("{}", encoding="utf-8") - standard = skill_dir / "skillevaluator-output-20260101T000001.json" - standard.write_text( - json.dumps({"overall_passed": True, "severity_counts": {"high": 1}}), - encoding="utf-8", +def test_validate_records_json_report_name_for_catalog_binding(monkeypatch) -> None: + from skillevaluator import cli as cli_module + + monkeypatch.setattr(cli_module, "run_validation", lambda *_args, **_kwargs: []) + monkeypatch.setattr(cli_module, "emit_reports", lambda *_args, **_kwargs: True) + + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke( + cli, + ["validate", str(FIXTURE), "--no-llm", "--no-tier2", "--checks", "schema", "-o", "out"], + ) + + assert result.exit_code == 0, result.output + assert cli_module._consume_validate_json_report() is not None + + +def test_catalog_summary_binds_exact_json_report_not_sarif_sidecar(monkeypatch) -> None: + from skillevaluator.models.result import ValidationResult + + def _emit(results, *, report_formats, output_dir, basename, **_kwargs): + output_dir.mkdir(parents=True, exist_ok=True) + standard = output_dir / f"{basename}.json" + sarif = output_dir / f"{basename}.sarif.json" + standard.write_text( + json.dumps({"overall_status": "pass", "severity_counts": {"high": 2}}), + encoding="utf-8", + ) + sarif.write_text("{}", encoding="utf-8") + return all(result.passed for result in results) + + monkeypatch.setattr(cli_module, "run_validation", lambda *_args, **_kwargs: [ValidationResult(validator_name="Schema")]) + monkeypatch.setattr(cli_module, "emit_reports", _emit) + + runner = CliRunner() + with runner.isolated_filesystem(): + catalog = Path("catalog") + shutil.copytree(FIXTURE, catalog / "simple") + result = runner.invoke( + cli, + [ + "validate", + str(catalog.resolve()), + "--no-llm", + "--no-dedup", + "--checks", + "schema", + "-r", + "json", + "-r", + "sarif", + "-o", + "out", + ], + ) + + assert result.exit_code == 0, result.output + summary = json.loads(Path("out/catalog-summary.json").read_text(encoding="utf-8")) + skill = summary["skills"][0] + assert skill["json_report"].endswith(".json") + assert not skill["json_report"].endswith(".sarif.json") + assert skill["severity_counts"] == {"high": 2} + + +def test_catalog_summary_same_second_rerun_overwrites_json(monkeypatch) -> None: + from skillevaluator.models.result import Finding, Severity, ValidationResult + + failing = ValidationResult(validator_name="Schema") + failing.add_structured_finding( + Finding( + category="SCHEMA", + severity=Severity.HIGH, + check_name="author_missing", + message="missing author", + file_path="SKILL.md", + ), + is_error=True, ) - assert cli_module._new_skill_json_report_name(skill_dir, existing) == standard.name + outcomes = [[ValidationResult(validator_name="Schema")], [failing]] + + def _run_validation(_target: Path, **_kwargs): + return outcomes.pop(0) + + def _emit(results, *, report_formats, output_dir, basename, **_kwargs): + output_dir.mkdir(parents=True, exist_ok=True) + report = output_dir / f"{basename}.json" + report.write_text( + json.dumps( + { + "overall_passed": all(result.passed for result in results), + "overall_status": "pass" if all(result.passed for result in results) else "fail", + "severity_counts": {"high": 0 if all(result.passed for result in results) else 3}, + } + ), + encoding="utf-8", + ) + return all(result.passed for result in results) + + monkeypatch.setattr(cli_module, "run_validation", _run_validation) + monkeypatch.setattr(cli_module, "emit_reports", _emit) + monkeypatch.setattr( + "skillevaluator.utils.helpers.make_timestamped_basename", + lambda _prefix: "skillevaluator-output-20260908T000000", + ) + + runner = CliRunner() + with runner.isolated_filesystem(): + catalog = Path("catalog") + shutil.copytree(FIXTURE, catalog / "simple") + first = runner.invoke( + cli, + ["validate", str(catalog.resolve()), "--no-llm", "--no-dedup", "--checks", "schema", "-r", "json", "-o", "out"], + ) + second = runner.invoke( + cli, + ["validate", str(catalog.resolve()), "--no-llm", "--no-dedup", "--checks", "schema", "-r", "json", "-o", "out"], + ) + + assert first.exit_code == 0, first.output + assert second.exit_code != 0, second.output + summary = json.loads(Path("out/catalog-summary.json").read_text(encoding="utf-8")) + skill = summary["skills"][0] + assert skill["json_report"] == "skillevaluator-output-20260908T000000.json" + assert skill["overall_status"] == "fail" + assert skill["severity_counts"] == {"high": 3} + + +def test_write_catalog_summary_rejects_symlinked_parent(tmp_path: Path) -> None: + external = tmp_path / "external" + external.mkdir() + reports = tmp_path / "reports" + reports.symlink_to(external, target_is_directory=True) + output_dir = reports / "new" / "nested" + + with pytest.raises(ValueError, match=r"symlink|reparse"): + cli_module._write_catalog_summary(output_dir, []) + + assert not (external / "new").exists() def test_catalog_skill_entry_skips_stale_json_without_report_name(tmp_path: Path) -> None: From ae23dfb649e7f0b8f7bc874900d6b4415cbdd015 Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Thu, 10 Sep 2026 15:18:25 +0530 Subject: [PATCH 08/12] chore: fix CHANGELOG merge markers after main merge Signed-off-by: Narendran Raghavan Signed-off-by: mimran-khan --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6144b46..0f348282 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,13 +6,11 @@ All notable changes to SkillEvaluator are documented in this file. ### Added -<<<<<<< HEAD - Catalog validation now writes `catalog-summary.json` at the reports root with per-skill pass/fail status, optional severity rollups from child JSON reports, and paths to per-skill report directories. - Catalog `validate` accepts `--workers N` to validate skills in parallel child processes (default 1 preserves the serial per-skill pipeline view). -======= - `SKILL_EVAL_MODEL_CATALOG_ALLOW_HTTP_HOSTS` names hosts whose model catalog may be read over plain HTTP. Catalog reads still require HTTPS for every other non-loopback host. Entries match one whole host as written, with no name @@ -20,7 +18,6 @@ All notable changes to SkillEvaluator are documented in this file. HTTP proxy so its bearer token is not offered to an intermediary. The transport rechecks authorization before dispatch and rejects hosts that are no longer allowed. ->>>>>>> origin/main - SARIF 2.1.0 reporter (`-r sarif`) for GitHub Code Scanning and other SARIF consumers. Findings map to rule IDs, severity levels, and file locations from Tier 1 validation results. From f257265765a84156a30e21be25c91075003794ea Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Sat, 12 Sep 2026 15:56:01 +0530 Subject: [PATCH 09/12] fix(cli): scope validate JSON report handoff with ContextVar Replace module-global report name slot so parallel catalog workers and Ruff PLW0603 stay clean while preserving exact JSON binding after validate. Signed-off-by: mimran-khan Signed-off-by: mimran-khan --- src/skillevaluator/cli.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/skillevaluator/cli.py b/src/skillevaluator/cli.py index b5712683..67f24dbf 100644 --- a/src/skillevaluator/cli.py +++ b/src/skillevaluator/cli.py @@ -10,6 +10,7 @@ import logging import math from concurrent.futures import ProcessPoolExecutor, as_completed +from contextvars import ContextVar from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -329,7 +330,10 @@ def _report_options(func): )(func) -_last_validate_json_report: str | None = None +_validate_json_report_var: ContextVar[str | None] = ContextVar( + "_validate_json_report_var", + default=None, +) def _effective_report_formats(report_formats: tuple[str, ...], *, quiet: bool) -> tuple[str, ...]: @@ -340,14 +344,12 @@ def _effective_report_formats(report_formats: tuple[str, ...], *, quiet: bool) - def _record_validate_json_report(report_name: str | None) -> None: - global _last_validate_json_report - _last_validate_json_report = report_name + _validate_json_report_var.set(report_name) def _consume_validate_json_report() -> str | None: - global _last_validate_json_report - report_name = _last_validate_json_report - _last_validate_json_report = None + report_name = _validate_json_report_var.get() + _validate_json_report_var.set(None) return report_name From 6119cdd7699980a663086291561c99fc34175360 Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Sun, 13 Sep 2026 10:27:56 +0530 Subject: [PATCH 10/12] test(cli): cover validate JSON report handoff isolation Signed-off-by: mimran-khan Signed-off-by: mimran-khan --- tests/test_commands.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/test_commands.py b/tests/test_commands.py index f475362a..09e3a8e8 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -521,6 +521,48 @@ def test_validate_records_json_report_name_for_catalog_binding(monkeypatch) -> N assert cli_module._consume_validate_json_report() is not None +def test_validate_json_report_handoff_isolated_between_invocations(monkeypatch) -> None: + """Back-to-back validate runs must not leak JSON report names through a shared slot.""" + from skillevaluator import cli as cli_module + from skillevaluator.models.result import ValidationResult + + basenames = iter(["skillevaluator-output-alpha", "skillevaluator-output-beta"]) + + def _emit(results, *, report_formats, output_dir, basename, **_kwargs): + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / f"{basename}.json").write_text( + json.dumps({"overall_passed": all(result.passed for result in results)}), + encoding="utf-8", + ) + return all(result.passed for result in results) + + monkeypatch.setattr(cli_module, "run_validation", lambda *_args, **_kwargs: [ValidationResult(validator_name="Schema")]) + monkeypatch.setattr(cli_module, "emit_reports", _emit) + monkeypatch.setattr( + "skillevaluator.utils.helpers.make_timestamped_basename", + lambda _prefix: next(basenames), + ) + + runner = CliRunner() + with runner.isolated_filesystem(): + args = [ + "validate", + str(FIXTURE), + "--no-llm", + "--no-tier2", + "--checks", + "schema", + "-r", + "json", + "-o", + "out", + ] + assert runner.invoke(cli, args).exit_code == 0 + assert cli_module._consume_validate_json_report() == "skillevaluator-output-alpha.json" + assert runner.invoke(cli, args).exit_code == 0 + assert cli_module._consume_validate_json_report() == "skillevaluator-output-beta.json" + + def test_catalog_summary_binds_exact_json_report_not_sarif_sidecar(monkeypatch) -> None: from skillevaluator.models.result import ValidationResult From 8e80909ec9529794152b55a8dce4d6ef164e2a31 Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Mon, 14 Sep 2026 21:40:35 +0530 Subject: [PATCH 11/12] test(cli): prove JSON report handoff under overlapping validate runs Use copy_context and a barrier so two validations record before either consume; a module-global slot would return the wrong basename. Signed-off-by: mimran-khan Signed-off-by: mimran-khan --- tests/test_commands.py | 70 ++++++++++++++++++++++++++++++------------ 1 file changed, 50 insertions(+), 20 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 09e3a8e8..3cc636eb 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -3,8 +3,10 @@ from __future__ import annotations +import contextvars import json import shutil +import threading from pathlib import Path import pytest @@ -521,9 +523,10 @@ def test_validate_records_json_report_name_for_catalog_binding(monkeypatch) -> N assert cli_module._consume_validate_json_report() is not None -def test_validate_json_report_handoff_isolated_between_invocations(monkeypatch) -> None: - """Back-to-back validate runs must not leak JSON report names through a shared slot.""" - from skillevaluator import cli as cli_module +def test_validate_json_report_handoff_isolated_between_invocations( + monkeypatch, tmp_path: Path +) -> None: + """Overlapping validate runs must not leak JSON report names through a shared slot.""" from skillevaluator.models.result import ValidationResult basenames = iter(["skillevaluator-output-alpha", "skillevaluator-output-beta"]) @@ -543,24 +546,51 @@ def _emit(results, *, report_formats, output_dir, basename, **_kwargs): lambda _prefix: next(basenames), ) + args = [ + "validate", + str(FIXTURE), + "--no-llm", + "--no-tier2", + "--checks", + "schema", + "-r", + "json", + ] runner = CliRunner() - with runner.isolated_filesystem(): - args = [ - "validate", - str(FIXTURE), - "--no-llm", - "--no-tier2", - "--checks", - "schema", - "-r", - "json", - "-o", - "out", - ] - assert runner.invoke(cli, args).exit_code == 0 - assert cli_module._consume_validate_json_report() == "skillevaluator-output-alpha.json" - assert runner.invoke(cli, args).exit_code == 0 - assert cli_module._consume_validate_json_report() == "skillevaluator-output-beta.json" + barrier = threading.Barrier(2) + results: dict[str, str | None] = {} + thread_errors: list[BaseException] = [] + + def _run_validate_and_consume(tag: str) -> None: + def _work() -> None: + output_dir = tmp_path / tag + output_dir.mkdir() + invoke_args = [*args, "-o", str(output_dir)] + result = runner.invoke(cli, invoke_args) + assert result.exit_code == 0, result.output + barrier.wait() + results[tag] = cli_module._consume_validate_json_report() + + contextvars.copy_context().run(_work) + + def _worker(tag: str) -> None: + try: + _run_validate_and_consume(tag) + except BaseException as exc: + thread_errors.append(exc) + + threads = [ + threading.Thread(target=_worker, args=("alpha",)), + threading.Thread(target=_worker, args=("beta",)), + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert not thread_errors + assert results["alpha"] == "skillevaluator-output-alpha.json" + assert results["beta"] == "skillevaluator-output-beta.json" def test_catalog_summary_binds_exact_json_report_not_sarif_sidecar(monkeypatch) -> None: From 5c4794fd0f071ce89a24e141375fe7670482de0c Mon Sep 17 00:00:00 2001 From: mimran-khan Date: Mon, 14 Sep 2026 22:06:59 +0530 Subject: [PATCH 12/12] test(cli): isolate JSON handoff in distinct Context instances Overlap record and consume across two threads so a module-global slot cannot pass; add a regression that shows the old global behavior leaks. Signed-off-by: mimran-khan Signed-off-by: mimran-khan --- tests/test_commands.py | 119 +++++++++++++++++++++++------------------ 1 file changed, 68 insertions(+), 51 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 3cc636eb..88d62766 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -7,6 +7,7 @@ import json import shutil import threading +from collections.abc import Callable from pathlib import Path import pytest @@ -523,76 +524,92 @@ def test_validate_records_json_report_name_for_catalog_binding(monkeypatch) -> N assert cli_module._consume_validate_json_report() is not None -def test_validate_json_report_handoff_isolated_between_invocations( - monkeypatch, tmp_path: Path -) -> None: - """Overlapping validate runs must not leak JSON report names through a shared slot.""" - from skillevaluator.models.result import ValidationResult - - basenames = iter(["skillevaluator-output-alpha", "skillevaluator-output-beta"]) - - def _emit(results, *, report_formats, output_dir, basename, **_kwargs): - output_dir.mkdir(parents=True, exist_ok=True) - (output_dir / f"{basename}.json").write_text( - json.dumps({"overall_passed": all(result.passed for result in results)}), - encoding="utf-8", - ) - return all(result.passed for result in results) - - monkeypatch.setattr(cli_module, "run_validation", lambda *_args, **_kwargs: [ValidationResult(validator_name="Schema")]) - monkeypatch.setattr(cli_module, "emit_reports", _emit) - monkeypatch.setattr( - "skillevaluator.utils.helpers.make_timestamped_basename", - lambda _prefix: next(basenames), - ) - - args = [ - "validate", - str(FIXTURE), - "--no-llm", - "--no-tier2", - "--checks", - "schema", - "-r", - "json", - ] - runner = CliRunner() +def _handoff_overlap_results( + *, + record_alpha: str, + record_beta: str, + run_in_context: bool, +) -> dict[str, str | None]: + """Record two handoffs, then consume both after a barrier (overlap window).""" + alpha_ctx = contextvars.Context() + beta_ctx = contextvars.Context() barrier = threading.Barrier(2) results: dict[str, str | None] = {} thread_errors: list[BaseException] = [] - def _run_validate_and_consume(tag: str) -> None: - def _work() -> None: - output_dir = tmp_path / tag - output_dir.mkdir() - invoke_args = [*args, "-o", str(output_dir)] - result = runner.invoke(cli, invoke_args) - assert result.exit_code == 0, result.output - barrier.wait() - results[tag] = cli_module._consume_validate_json_report() + def alpha_lane() -> None: + cli_module._record_validate_json_report(record_alpha) + barrier.wait() + results["alpha"] = cli_module._consume_validate_json_report() - contextvars.copy_context().run(_work) + def beta_lane() -> None: + cli_module._record_validate_json_report(record_beta) + barrier.wait() + results["beta"] = cli_module._consume_validate_json_report() - def _worker(tag: str) -> None: + def run_lane(ctx: contextvars.Context, lane: Callable[[], None]) -> None: try: - _run_validate_and_consume(tag) + if run_in_context: + ctx.run(lane) + else: + lane() except BaseException as exc: thread_errors.append(exc) - threads = [ - threading.Thread(target=_worker, args=("alpha",)), - threading.Thread(target=_worker, args=("beta",)), - ] + threads = ( + threading.Thread(target=run_lane, args=(alpha_ctx, alpha_lane)), + threading.Thread(target=run_lane, args=(beta_ctx, beta_lane)), + ) for thread in threads: thread.start() for thread in threads: thread.join() + if thread_errors: + raise thread_errors[0] + return results + - assert not thread_errors +def test_validate_json_report_handoff_isolated_between_invocations() -> None: + """Distinct Context instances keep each basename when record/consume overlap.""" + results = _handoff_overlap_results( + record_alpha="skillevaluator-output-alpha.json", + record_beta="skillevaluator-output-beta.json", + run_in_context=True, + ) assert results["alpha"] == "skillevaluator-output-alpha.json" assert results["beta"] == "skillevaluator-output-beta.json" +def test_validate_json_report_handoff_module_global_leaks_under_overlap( + monkeypatch, +) -> None: + """A single module-global slot returns the last writer after overlapping records.""" + slot: str | None = None + + def _record(report_name: str | None) -> None: + nonlocal slot + slot = report_name + + def _consume() -> str | None: + nonlocal slot + report_name = slot + slot = None + return report_name + + monkeypatch.setattr(cli_module, "_record_validate_json_report", _record) + monkeypatch.setattr(cli_module, "_consume_validate_json_report", _consume) + + results = _handoff_overlap_results( + record_alpha="skillevaluator-output-alpha.json", + record_beta="skillevaluator-output-beta.json", + run_in_context=False, + ) + assert results != { + "alpha": "skillevaluator-output-alpha.json", + "beta": "skillevaluator-output-beta.json", + } + + def test_catalog_summary_binds_exact_json_report_not_sarif_sidecar(monkeypatch) -> None: from skillevaluator.models.result import ValidationResult