test(check): enforce skill frontmatter metadata

.github/workflows/plugin-validate.yml

@@ -25,6 +25,16 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Run repository checks
+        run: |
+          python -m pip install --disable-pip-version-check pyyaml pytest
+          python scripts/check.py
+          python -m pytest scripts/tests/ -q
+
       - name: Cache Claude Code CLI
         id: cli-cache
         uses: actions/cache@v4

scripts/check.py

@@ -5,10 +5,12 @@
 Checks:
   1. Every *.yaml under managed-agents/ parses.
   2. Every plugin.json / marketplace.json / steering-examples.json parses.
-  3. Every <vertical>/agents/*.md has valid YAML frontmatter with name + description.
-  4. Every system.file, skills[].path, callable_agents[].manifest in agent.yaml
+  3. Every agent plugin agents/*.md has valid YAML frontmatter with name + description.
+  4. Every plugin skill SKILL.md has valid YAML frontmatter with name + description.
+  5. Every system.file, skills[].path, callable_agents[].manifest in agent.yaml
      and subagent yamls resolves to an existing file/dir.
-  5. Every managed-agents/<slug>/ has agent.yaml, README.md, steering-examples.json.
+  6. Every managed-agent-cookbooks/<slug>/ has agent.yaml, README.md,
+     steering-examples.json.
 
 Exit 0 if clean, 1 otherwise. Requires: pyyaml.
 """
@@ -88,24 +90,37 @@ def rel(p: Path) -> str:
         except json.JSONDecodeError as e:
             err(f"JSON parse: {rel(jf)}: {e}")
 
-# --- 3. agent.md frontmatter -----------------------------------------------
-for md in sorted(PLUGINS.glob("agent-plugins/*/agents/*.md")):
-    checked += 1
+def check_frontmatter(md: Path, required: tuple[str, ...], label: str) -> None:
+    """Require a YAML mapping with the specified keys at the start of a file."""
     text = md.read_text()
     if not text.startswith("---"):
-        err(f"frontmatter: {rel(md)}: missing leading ---")
-        continue
+        err(f"{label}: {rel(md)}: missing leading ---")
+        return
     try:
         _, fm, _ = text.split("---", 2)
-        meta = yaml.safe_load(fm)
-        for k in ("name", "description"):
+        meta = yaml.safe_load(fm) or {}
+        if not isinstance(meta, dict):
+            err(f"{label}: {rel(md)}: frontmatter must be a YAML mapping")
+            return
+        for k in required:
             if k not in meta:
-                err(f"frontmatter: {rel(md)}: missing '{k}'")
+                err(f"{label}: {rel(md)}: missing '{k}'")
     except (ValueError, yaml.YAMLError) as e:
-        err(f"frontmatter: {rel(md)}: {e}")
+        err(f"{label}: {rel(md)}: {e}")
+
+
+# --- 3. agent.md frontmatter -----------------------------------------------
+for md in sorted(PLUGINS.glob("agent-plugins/*/agents/*.md")):
+    checked += 1
+    check_frontmatter(md, ("name", "description"), "frontmatter")
+
+# --- 4. skill frontmatter --------------------------------------------------
+for md in sorted(PLUGINS.glob("**/skills/*/SKILL.md")):
+    checked += 1
+    check_frontmatter(md, ("name", "description"), "skill-frontmatter")
 
 
-# --- 4. reference resolution -----------------------------------------------
+# --- 5. reference resolution -----------------------------------------------
 def check_refs(yml: Path) -> None:
     try:
         data = yaml.safe_load(yml.read_text()) or {}
@@ -139,7 +154,7 @@ def check_refs(yml: Path) -> None:
 for yml in sorted(MANAGED.rglob("*.yaml")):
     check_refs(yml)
 
-# --- 4b. agent-plugin bundled skills match vertical source -----------------
+# --- 5b. agent-plugin bundled skills match vertical source -----------------
 import filecmp  # noqa: E402
 import re  # noqa: E402
 
@@ -158,7 +173,7 @@ def check_refs(yml: Path) -> None:
             f"(run scripts/sync-agent-skills.py)"
         )
 
-# --- 4b2. agent.md skill references exist in the agent's own bundle --------
+# --- 5b2. agent.md skill references exist in the agent's own bundle --------
 for md in sorted(PLUGINS.glob("agent-plugins/*/agents/*.md")):
     slug = md.parents[1].name
     sk_dir = PLUGINS / "agent-plugins" / slug / "skills"
@@ -170,22 +185,22 @@ def check_refs(yml: Path) -> None:
                 f"plugins/agent-plugins/{slug}/skills/{ref}/ is not bundled"
             )
 
-# --- 4c. marketplace source paths resolve ----------------------------------
+# --- 5c. marketplace source paths resolve ----------------------------------
 mp = ROOT / ".claude-plugin" / "marketplace.json"
 for p in json.loads(mp.read_text()).get("plugins", []):
     src = (ROOT / p["source"]).resolve()
     if not (src / ".claude-plugin" / "plugin.json").is_file():
         err(f"marketplace: {p['name']} source -> {p['source']} (no plugin.json)")
 
-# --- 5. required files per managed-agent -----------------------------------
+# --- 6. required files per managed-agent -----------------------------------
 for d in sorted(MANAGED.iterdir()):
     if not d.is_dir():
         continue
     for req in ("agent.yaml", "README.md", "steering-examples.json"):
         if not (d / req).is_file():
             err(f"missing: {rel(d)}/{req}")
 
-# --- 6. PowerShell scripts must be pure ASCII -------------------------------
+# --- 7. PowerShell scripts must be pure ASCII -------------------------------
 # Windows PowerShell 5.1 -- still the default shell on managed Windows -- reads
 # a .ps1 with no BOM using the machine's ANSI code page, not UTF-8. A smart dash
 # or curly quote then decodes to mojibake that can contain a literal '"',

scripts/tests/conftest.py

@@ -0,0 +1,96 @@
+"""Fixtures for subprocess tests of scripts/check.py."""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+import sys
+from collections.abc import Callable
+from pathlib import Path
+
+import pytest
+
+
+REAL_CHECK_PY = Path(__file__).resolve().parents[1] / "check.py"
+
+
+@pytest.fixture
+def minimal_repo(tmp_path: Path) -> Path:
+    """Build the smallest repository that satisfies every current check."""
+    root = tmp_path / "repo"
+
+    scripts = root / "scripts"
+    scripts.mkdir(parents=True)
+    shutil.copy(REAL_CHECK_PY, scripts / "check.py")
+
+    marketplace = root / ".claude-plugin"
+    marketplace.mkdir()
+    (marketplace / "marketplace.json").write_text(
+        json.dumps(
+            {
+                "plugins": [
+                    {
+                        "name": "test-vertical",
+                        "source": "./plugins/vertical-plugins/test-vertical",
+                    },
+                    {
+                        "name": "test-agent",
+                        "source": "./plugins/agent-plugins/test-agent",
+                    },
+                ]
+            }
+        )
+    )
+
+    vertical = root / "plugins" / "vertical-plugins" / "test-vertical"
+    (vertical / ".claude-plugin").mkdir(parents=True)
+    (vertical / ".claude-plugin" / "plugin.json").write_text(
+        json.dumps({"name": "test-vertical"})
+    )
+    skill_text = "---\nname: shared-skill\ndescription: Test skill\n---\n\nBody\n"
+    source_skill = vertical / "skills" / "shared-skill"
+    source_skill.mkdir(parents=True)
+    (source_skill / "SKILL.md").write_text(skill_text)
+
+    agent = root / "plugins" / "agent-plugins" / "test-agent"
+    (agent / ".claude-plugin").mkdir(parents=True)
+    (agent / ".claude-plugin" / "plugin.json").write_text(
+        json.dumps({"name": "test-agent"})
+    )
+    (agent / "agents").mkdir()
+    (agent / "agents" / "test-agent.md").write_text(
+        "---\nname: test-agent\ndescription: Test agent\n---\n\nBody\n"
+    )
+    bundled_skill = agent / "skills" / "shared-skill"
+    bundled_skill.mkdir(parents=True)
+    (bundled_skill / "SKILL.md").write_text(skill_text)
+
+    cookbook = root / "managed-agent-cookbooks" / "test-agent"
+    cookbook.mkdir(parents=True)
+    (cookbook / "README.md").write_text("# Test agent\n")
+    (cookbook / "steering-examples.json").write_text("[]\n")
+    (cookbook / "agent.yaml").write_text(
+        "name: test-agent\n"
+        "system:\n"
+        "  file: ../../plugins/agent-plugins/test-agent/agents/test-agent.md\n"
+        "skills:\n"
+        "  - from_plugin: ../../plugins/agent-plugins/test-agent\n"
+    )
+
+    return root
+
+
+@pytest.fixture
+def run_check() -> Callable[[Path], subprocess.CompletedProcess[str]]:
+    """Run the copied checker exactly as contributors and CI invoke it."""
+
+    def _run(repo: Path) -> subprocess.CompletedProcess[str]:
+        return subprocess.run(
+            [sys.executable, str(repo / "scripts" / "check.py")],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+
+    return _run

scripts/tests/test_check_skill_frontmatter.py

@@ -0,0 +1,81 @@
+"""Regression coverage for Claude skill frontmatter validation."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from pathlib import Path
+from subprocess import CompletedProcess
+
+import pytest
+
+
+def skill_files(repo: Path) -> tuple[Path, Path]:
+    """Return the canonical skill and its synchronized agent copy."""
+    source = (
+        repo
+        / "plugins"
+        / "vertical-plugins"
+        / "test-vertical"
+        / "skills"
+        / "shared-skill"
+        / "SKILL.md"
+    )
+    bundled = (
+        repo
+        / "plugins"
+        / "agent-plugins"
+        / "test-agent"
+        / "skills"
+        / "shared-skill"
+        / "SKILL.md"
+    )
+    return source, bundled
+
+
+def write_both(repo: Path, content: str) -> None:
+    """Keep source and bundled skill identical so drift cannot mask failures."""
+    for skill in skill_files(repo):
+        skill.write_text(content)
+
+
+def test_valid_skill_frontmatter_passes(
+    minimal_repo: Path,
+    run_check: Callable[[Path], CompletedProcess[str]],
+) -> None:
+    result = run_check(minimal_repo)
+
+    assert result.returncode == 0, result.stderr
+
+
+def test_skill_without_frontmatter_fails(
+    minimal_repo: Path,
+    run_check: Callable[[Path], CompletedProcess[str]],
+) -> None:
+    write_both(minimal_repo, "# Shared skill\n\nBody\n")
+
+    result = run_check(minimal_repo)
+
+    assert result.returncode == 1
+    assert "skill-frontmatter" in result.stderr
+    assert "missing leading ---" in result.stderr
+
+
+@pytest.mark.parametrize("missing_key", ["name", "description"])
+def test_skill_frontmatter_requires_metadata(
+    minimal_repo: Path,
+    run_check: Callable[[Path], CompletedProcess[str]],
+    missing_key: str,
+) -> None:
+    metadata = {
+        "name": "shared-skill",
+        "description": "Test skill",
+    }
+    metadata.pop(missing_key)
+    body = "\n".join(f"{key}: {value}" for key, value in metadata.items())
+    write_both(minimal_repo, f"---\n{body}\n---\n\nBody\n")
+
+    result = run_check(minimal_repo)
+
+    assert result.returncode == 1
+    assert "skill-frontmatter" in result.stderr
+    assert f"missing '{missing_key}'" in result.stderr