fix: keep --no-llm negative cases off-skill

CHANGELOG.md

@@ -6,6 +6,8 @@ All notable changes to SkillEvaluator are documented in this file.
 
 ### Fixed
 
+- `--no-llm` full datasets now generate an off-skill negative prompt instead of
+  asking the agent to describe the skill by name.
 - Tier 3 paired pass@k evidence now respects Python's active integer-string
   conversion limit, preserves nonzero Wilson interval widths and paired-effect
   directions at large case counts, and documents exact-rational omission

src/skillevaluator/tier3/generate_dataset.py

@@ -295,13 +295,13 @@ def _generate_full(skill: dict[str, Any]) -> list[dict[str, Any]]:
             "id": f"{name}-neg-001",
             "question": hint_qs[3]
             if len(hint_qs) > 3
-            else f"What does the {name} skill do and what are its capabilities?",
+            else "What's a good way to organize weekend errands in a new city?",
             "expected_skill": None,
             "expected_script": None,
-            "ground_truth": f"The agent explained the {name} skill's capabilities and when to use it, without executing any scripts",
+            "ground_truth": "The agent answered an unrelated question without loading or applying this skill",
             "expected_behavior": [
-                "The agent responded conversationally without executing tools or scripts",
-                f"The agent's response accurately describes what {name} does",
+                "The agent responded without reading or applying this skill",
+                "The agent did not invoke this skill's tools or scripts",
                 SECURITY_BEHAVIOR,
             ],
         },

tests/tier3/test_generate_dataset_results.py

@@ -12,6 +12,7 @@
 from skillevaluator.tier3 import generate_dataset
 from skillevaluator.tier3.generate_dataset import (
     _discover_trajectories,
+    _generate_full,
     _run_agent_collect_trajectories,
     _to_agentskills_dataset,
 )
@@ -298,3 +299,21 @@ def test_parse_skill_falls_back_to_defaults_on_malformed_frontmatter(tmp_path):
     parsed = _parse(tmp_path, "name: [unclosed\ndescription: broken")
     assert parsed["name"] == "my-skill"
     assert parsed["description"] == ""
+
+
+def test_no_llm_negative_case_does_not_name_the_skill():
+    """Default --no-llm negative prompt must stay off-skill, not ask what the skill does."""
+    skill = {
+        "name": "pdf-extractor",
+        "description": "Extracts tables from PDF files",
+        "scripts": [],
+        "eval_prompt": "",
+    }
+    cases = _generate_full(skill)
+    negative = next(c for c in cases if c["id"] == "pdf-extractor-neg-001")
+    assert negative["expected_skill"] is None
+    assert "pdf-extractor" not in negative["question"]
+    assert "pdf-extractor" not in negative["ground_truth"]
+    for behavior in negative["expected_behavior"]:
+        assert "pdf-extractor" not in behavior
+    assert "without reading or applying this skill" in negative["expected_behavior"][0]