diff --git a/.github/workflows/plugin-validate.yml b/.github/workflows/plugin-validate.yml index 9b4d7fd21..de397cdec 100644 --- a/.github/workflows/plugin-validate.yml +++ b/.github/workflows/plugin-validate.yml @@ -25,6 +25,16 @@ jobs: steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Run repository checks + run: | + python -m pip install --disable-pip-version-check pyyaml pytest + python scripts/check.py + python -m pytest scripts/tests/ -q + - name: Cache Claude Code CLI id: cli-cache uses: actions/cache@v4 diff --git a/scripts/check.py b/scripts/check.py index 23fd8889b..649436ae0 100755 --- a/scripts/check.py +++ b/scripts/check.py @@ -5,10 +5,12 @@ Checks: 1. Every *.yaml under managed-agents/ parses. 2. Every plugin.json / marketplace.json / steering-examples.json parses. - 3. Every /agents/*.md has valid YAML frontmatter with name + description. - 4. Every system.file, skills[].path, callable_agents[].manifest in agent.yaml + 3. Every agent plugin agents/*.md has valid YAML frontmatter with name + description. + 4. Every plugin skill SKILL.md has valid YAML frontmatter with name + description. + 5. Every system.file, skills[].path, callable_agents[].manifest in agent.yaml and subagent yamls resolves to an existing file/dir. - 5. Every managed-agents// has agent.yaml, README.md, steering-examples.json. + 6. Every managed-agent-cookbooks// has agent.yaml, README.md, + steering-examples.json. Exit 0 if clean, 1 otherwise. Requires: pyyaml. """ @@ -88,24 +90,37 @@ def rel(p: Path) -> str: except json.JSONDecodeError as e: err(f"JSON parse: {rel(jf)}: {e}") -# --- 3. agent.md frontmatter ----------------------------------------------- -for md in sorted(PLUGINS.glob("agent-plugins/*/agents/*.md")): - checked += 1 +def check_frontmatter(md: Path, required: tuple[str, ...], label: str) -> None: + """Require a YAML mapping with the specified keys at the start of a file.""" text = md.read_text() if not text.startswith("---"): - err(f"frontmatter: {rel(md)}: missing leading ---") - continue + err(f"{label}: {rel(md)}: missing leading ---") + return try: _, fm, _ = text.split("---", 2) - meta = yaml.safe_load(fm) - for k in ("name", "description"): + meta = yaml.safe_load(fm) or {} + if not isinstance(meta, dict): + err(f"{label}: {rel(md)}: frontmatter must be a YAML mapping") + return + for k in required: if k not in meta: - err(f"frontmatter: {rel(md)}: missing '{k}'") + err(f"{label}: {rel(md)}: missing '{k}'") except (ValueError, yaml.YAMLError) as e: - err(f"frontmatter: {rel(md)}: {e}") + err(f"{label}: {rel(md)}: {e}") + + +# --- 3. agent.md frontmatter ----------------------------------------------- +for md in sorted(PLUGINS.glob("agent-plugins/*/agents/*.md")): + checked += 1 + check_frontmatter(md, ("name", "description"), "frontmatter") + +# --- 4. skill frontmatter -------------------------------------------------- +for md in sorted(PLUGINS.glob("**/skills/*/SKILL.md")): + checked += 1 + check_frontmatter(md, ("name", "description"), "skill-frontmatter") -# --- 4. reference resolution ----------------------------------------------- +# --- 5. reference resolution ----------------------------------------------- def check_refs(yml: Path) -> None: try: data = yaml.safe_load(yml.read_text()) or {} @@ -139,7 +154,7 @@ def check_refs(yml: Path) -> None: for yml in sorted(MANAGED.rglob("*.yaml")): check_refs(yml) -# --- 4b. agent-plugin bundled skills match vertical source ----------------- +# --- 5b. agent-plugin bundled skills match vertical source ----------------- import filecmp # noqa: E402 import re # noqa: E402 @@ -158,7 +173,7 @@ def check_refs(yml: Path) -> None: f"(run scripts/sync-agent-skills.py)" ) -# --- 4b2. agent.md skill references exist in the agent's own bundle -------- +# --- 5b2. agent.md skill references exist in the agent's own bundle -------- for md in sorted(PLUGINS.glob("agent-plugins/*/agents/*.md")): slug = md.parents[1].name sk_dir = PLUGINS / "agent-plugins" / slug / "skills" @@ -170,14 +185,14 @@ def check_refs(yml: Path) -> None: f"plugins/agent-plugins/{slug}/skills/{ref}/ is not bundled" ) -# --- 4c. marketplace source paths resolve ---------------------------------- +# --- 5c. marketplace source paths resolve ---------------------------------- mp = ROOT / ".claude-plugin" / "marketplace.json" for p in json.loads(mp.read_text()).get("plugins", []): src = (ROOT / p["source"]).resolve() if not (src / ".claude-plugin" / "plugin.json").is_file(): err(f"marketplace: {p['name']} source -> {p['source']} (no plugin.json)") -# --- 5. required files per managed-agent ----------------------------------- +# --- 6. required files per managed-agent ----------------------------------- for d in sorted(MANAGED.iterdir()): if not d.is_dir(): continue @@ -185,7 +200,7 @@ def check_refs(yml: Path) -> None: if not (d / req).is_file(): err(f"missing: {rel(d)}/{req}") -# --- 6. PowerShell scripts must be pure ASCII ------------------------------- +# --- 7. PowerShell scripts must be pure ASCII ------------------------------- # Windows PowerShell 5.1 -- still the default shell on managed Windows -- reads # a .ps1 with no BOM using the machine's ANSI code page, not UTF-8. A smart dash # or curly quote then decodes to mojibake that can contain a literal '"', diff --git a/scripts/tests/conftest.py b/scripts/tests/conftest.py new file mode 100644 index 000000000..78af9ef42 --- /dev/null +++ b/scripts/tests/conftest.py @@ -0,0 +1,96 @@ +"""Fixtures for subprocess tests of scripts/check.py.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from collections.abc import Callable +from pathlib import Path + +import pytest + + +REAL_CHECK_PY = Path(__file__).resolve().parents[1] / "check.py" + + +@pytest.fixture +def minimal_repo(tmp_path: Path) -> Path: + """Build the smallest repository that satisfies every current check.""" + root = tmp_path / "repo" + + scripts = root / "scripts" + scripts.mkdir(parents=True) + shutil.copy(REAL_CHECK_PY, scripts / "check.py") + + marketplace = root / ".claude-plugin" + marketplace.mkdir() + (marketplace / "marketplace.json").write_text( + json.dumps( + { + "plugins": [ + { + "name": "test-vertical", + "source": "./plugins/vertical-plugins/test-vertical", + }, + { + "name": "test-agent", + "source": "./plugins/agent-plugins/test-agent", + }, + ] + } + ) + ) + + vertical = root / "plugins" / "vertical-plugins" / "test-vertical" + (vertical / ".claude-plugin").mkdir(parents=True) + (vertical / ".claude-plugin" / "plugin.json").write_text( + json.dumps({"name": "test-vertical"}) + ) + skill_text = "---\nname: shared-skill\ndescription: Test skill\n---\n\nBody\n" + source_skill = vertical / "skills" / "shared-skill" + source_skill.mkdir(parents=True) + (source_skill / "SKILL.md").write_text(skill_text) + + agent = root / "plugins" / "agent-plugins" / "test-agent" + (agent / ".claude-plugin").mkdir(parents=True) + (agent / ".claude-plugin" / "plugin.json").write_text( + json.dumps({"name": "test-agent"}) + ) + (agent / "agents").mkdir() + (agent / "agents" / "test-agent.md").write_text( + "---\nname: test-agent\ndescription: Test agent\n---\n\nBody\n" + ) + bundled_skill = agent / "skills" / "shared-skill" + bundled_skill.mkdir(parents=True) + (bundled_skill / "SKILL.md").write_text(skill_text) + + cookbook = root / "managed-agent-cookbooks" / "test-agent" + cookbook.mkdir(parents=True) + (cookbook / "README.md").write_text("# Test agent\n") + (cookbook / "steering-examples.json").write_text("[]\n") + (cookbook / "agent.yaml").write_text( + "name: test-agent\n" + "system:\n" + " file: ../../plugins/agent-plugins/test-agent/agents/test-agent.md\n" + "skills:\n" + " - from_plugin: ../../plugins/agent-plugins/test-agent\n" + ) + + return root + + +@pytest.fixture +def run_check() -> Callable[[Path], subprocess.CompletedProcess[str]]: + """Run the copied checker exactly as contributors and CI invoke it.""" + + def _run(repo: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(repo / "scripts" / "check.py")], + capture_output=True, + text=True, + check=False, + ) + + return _run diff --git a/scripts/tests/test_check_skill_frontmatter.py b/scripts/tests/test_check_skill_frontmatter.py new file mode 100644 index 000000000..1c221dbde --- /dev/null +++ b/scripts/tests/test_check_skill_frontmatter.py @@ -0,0 +1,81 @@ +"""Regression coverage for Claude skill frontmatter validation.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from subprocess import CompletedProcess + +import pytest + + +def skill_files(repo: Path) -> tuple[Path, Path]: + """Return the canonical skill and its synchronized agent copy.""" + source = ( + repo + / "plugins" + / "vertical-plugins" + / "test-vertical" + / "skills" + / "shared-skill" + / "SKILL.md" + ) + bundled = ( + repo + / "plugins" + / "agent-plugins" + / "test-agent" + / "skills" + / "shared-skill" + / "SKILL.md" + ) + return source, bundled + + +def write_both(repo: Path, content: str) -> None: + """Keep source and bundled skill identical so drift cannot mask failures.""" + for skill in skill_files(repo): + skill.write_text(content) + + +def test_valid_skill_frontmatter_passes( + minimal_repo: Path, + run_check: Callable[[Path], CompletedProcess[str]], +) -> None: + result = run_check(minimal_repo) + + assert result.returncode == 0, result.stderr + + +def test_skill_without_frontmatter_fails( + minimal_repo: Path, + run_check: Callable[[Path], CompletedProcess[str]], +) -> None: + write_both(minimal_repo, "# Shared skill\n\nBody\n") + + result = run_check(minimal_repo) + + assert result.returncode == 1 + assert "skill-frontmatter" in result.stderr + assert "missing leading ---" in result.stderr + + +@pytest.mark.parametrize("missing_key", ["name", "description"]) +def test_skill_frontmatter_requires_metadata( + minimal_repo: Path, + run_check: Callable[[Path], CompletedProcess[str]], + missing_key: str, +) -> None: + metadata = { + "name": "shared-skill", + "description": "Test skill", + } + metadata.pop(missing_key) + body = "\n".join(f"{key}: {value}" for key, value in metadata.items()) + write_both(minimal_repo, f"---\n{body}\n---\n\nBody\n") + + result = run_check(minimal_repo) + + assert result.returncode == 1 + assert "skill-frontmatter" in result.stderr + assert f"missing '{missing_key}'" in result.stderr