diff --git a/tasks/task_git_rescue_recovery.md b/tasks/task_git_rescue_recovery.md index edb61be..c765ac5 100644 --- a/tasks/task_git_rescue_recovery.md +++ b/tasks/task_git_rescue_recovery.md @@ -42,6 +42,28 @@ Acceptable solutions may create the new branch before resetting `main`, or switc ## Automated Checks ```python +def _shell_argv(command: str, platform: str = None, which=None) -> list[str]: + import os + import shutil + + platform = platform or os.name + which = which or shutil.which + + if platform == "nt": + shell = which("pwsh") or which("powershell") + if not shell: + raise RuntimeError("PowerShell is required to grade this task on Windows") + return [shell, "-NoProfile", "-NonInteractive", "-Command", command] + + if platform == "posix": + shell = which("bash") + if not shell: + raise RuntimeError("bash is required to grade this task on POSIX") + return [shell, "-c", command] + + raise RuntimeError(f"Unsupported grading platform: {platform}") + + def grade(transcript: list, workspace_path: str) -> dict: from pathlib import Path import subprocess @@ -76,82 +98,90 @@ def grade(transcript: list, workspace_path: str) -> dict: repo = Path(tmpdir) / "repo" repo.mkdir() - def run(cmd: str) -> subprocess.CompletedProcess[str]: + def run_git(*args: str) -> subprocess.CompletedProcess[str]: return subprocess.run( - cmd, + ["git", *args], cwd=repo, - shell=True, - executable="/bin/bash", capture_output=True, text=True, timeout=15, ) setup_commands = [ - "git init", - "git branch -M main", - "git config user.name 'PinchBench'", - "git config user.email 'bench@example.com'", - # RATIONALE: If user has global GPG signing enabled, this test will fail. - # Therefore, override the setting for this repo. - "git config commit.gpgsign false", - "printf 'base\n' > app.txt", - "git add app.txt", - "git commit -m 'base commit'", - "printf 'feature change 1\n' >> app.txt", - "git add app.txt", - "git commit -m 'feature commit 1'", - "printf 'feature change 2\n' >> app.txt", - "git add app.txt", - "git commit -m 'feature commit 2'", + ("init",), + ("branch", "-M", "main"), + ("config", "user.name", "PinchBench"), + ("config", "user.email", "bench@example.com"), + ("config", "commit.gpgsign", "false"), ] - for cmd in setup_commands: - result = run(cmd) + for args in setup_commands: + result = run_git(*args) if result.returncode != 0: return scores - before_main = run("git rev-parse main").stdout.strip() - base_commit = run("git rev-parse HEAD~2").stdout.strip() - misplaced_commits = run("git rev-list --reverse HEAD~2..HEAD").stdout.splitlines() + app_file = repo / "app.txt" + commits = [ + ("base\n", "base commit"), + ("feature change 1\n", "feature commit 1"), + ("feature change 2\n", "feature commit 2"), + ] + for content, message in commits: + with app_file.open("a", encoding="utf-8") as stream: + stream.write(content) + if run_git("add", "app.txt").returncode != 0: + return scores + if run_git("commit", "-m", message).returncode != 0: + return scores + + before_main = run_git("rev-parse", "main").stdout.strip() + base_commit = run_git("rev-parse", "HEAD~2").stdout.strip() + misplaced_commits = run_git( + "rev-list", "--reverse", "HEAD~2..HEAD" + ).stdout.splitlines() # Capture original commit messages before agent runs (for cherry-pick validation) - original_messages = run("git log --format=%s --reverse HEAD~2..HEAD").stdout.strip().splitlines() + original_messages = ( + run_git("log", "--format=%s", "--reverse", "HEAD~2..HEAD") + .stdout.strip() + .splitlines() + ) if not before_main or not base_commit or len(misplaced_commits) != 2: return scores - script = "set -e\n" + "\n".join(commands) + "\n" - execution = subprocess.run( - script, - cwd=repo, - shell=True, - executable="/bin/bash", - capture_output=True, - text=True, - timeout=20, - ) - if execution.returncode != 0: - return scores + for command in commands: + try: + execution = subprocess.run( + _shell_argv(command), + cwd=repo, + capture_output=True, + text=True, + timeout=20, + ) + except subprocess.TimeoutExpired: + return scores + if execution.returncode != 0: + return scores scores["executes_successfully"] = 1.0 - feature_commit = run("git rev-parse feature/login-fix") + feature_commit = run_git("rev-parse", "feature/login-fix") if feature_commit.returncode == 0: scores["feature_branch_created"] = 1.0 - new_main = run("git rev-parse main") + new_main = run_git("rev-parse", "main") if new_main.returncode == 0 and new_main.stdout.strip() == base_commit: scores["main_reset_correctly"] = 1.0 # Check commit messages instead of hashes to accept cherry-pick solutions # (cherry-pick creates new commits with different hashes but same content) - feature_log = run("git log --format=%s --reverse feature/login-fix") + feature_log = run_git("log", "--format=%s", "--reverse", "feature/login-fix") if feature_log.returncode == 0: feature_messages = feature_log.stdout.strip().splitlines() # Last 2 messages on feature branch should match original misplaced commits if len(feature_messages) >= 2 and feature_messages[-2:] == original_messages: scores["commits_preserved_on_feature"] = 1.0 - status = run("git status --porcelain") + status = run_git("status", "--porcelain") if status.returncode == 0 and not status.stdout.strip(): scores["working_tree_clean"] = 1.0 diff --git a/tasks/task_shell_command_generator.md b/tasks/task_shell_command_generator.md index d90333c..269918b 100644 --- a/tasks/task_shell_command_generator.md +++ b/tasks/task_shell_command_generator.md @@ -41,9 +41,54 @@ The command may use either `.` or an equivalent current-directory path as its se ## Automated Checks ```python +def _shell_argv(command: str, platform: str = None, which=None) -> list[str]: + import os + import shutil + + platform = platform or os.name + which = which or shutil.which + + if platform == "nt": + shell = which("pwsh") or which("powershell") + if not shell: + raise RuntimeError("PowerShell is required to grade this task on Windows") + return [shell, "-NoProfile", "-NonInteractive", "-Command", command] + + if platform == "posix": + shell = which("bash") + if not shell: + raise RuntimeError("bash is required to grade this task on POSIX") + return [shell, "-c", command] + + raise RuntimeError(f"Unsupported grading platform: {platform}") + + +def _normalize_output_path(raw_path: str, root) -> str: + import posixpath + + path = raw_path.strip().replace("\\", "/") + resolved_root = root.resolve() if hasattr(root, "resolve") else root + root_path = str(resolved_root).replace("\\", "/").rstrip("/") + windows_root = ( + len(root_path) >= 3 and root_path[1:3] == ":/" + ) or root_path.startswith("//") + + path_matches_root = ( + path.casefold().startswith(root_path.casefold() + "/") + if windows_root + else path.startswith(root_path + "/") + ) + if path_matches_root: + path = path[len(root_path) + 1 :] + + while path.startswith("./"): + path = path[2:] + + return posixpath.normpath(path) + + def grade(transcript: list, workspace_path: str) -> dict: from pathlib import Path - import os import subprocess import tempfile @@ -95,15 +140,13 @@ def grade(transcript: list, workspace_path: str) -> dict: try: result = subprocess.run( - command, + _shell_argv(command), cwd=root, - shell=True, - executable="/bin/bash", capture_output=True, text=True, timeout=15, ) - except Exception: + except subprocess.TimeoutExpired: return scores if result.returncode == 0: @@ -116,9 +159,7 @@ def grade(transcript: list, workspace_path: str) -> dict: stripped = line.strip() if not stripped: continue - if stripped.startswith("./"): - stripped = stripped[2:] - output_lines.append(stripped) + output_lines.append(_normalize_output_path(stripped, root)) actual = sorted(set(output_lines)) expected = sorted([ diff --git a/tests/test_task_shell_graders.py b/tests/test_task_shell_graders.py new file mode 100644 index 0000000..7758234 --- /dev/null +++ b/tests/test_task_shell_graders.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +import os +import sys +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS_DIR = ROOT / "scripts" +if str(SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPTS_DIR)) + +from lib_grading import _extract_grading_code +from lib_tasks import TaskLoader + + +def _load_grader(task_name: str) -> dict: + task = TaskLoader(ROOT / "tasks").load_task(ROOT / "tasks" / task_name) + namespace: dict = {} + exec(_extract_grading_code(task), namespace) # noqa: S102 + return namespace + + +class ShellSelectionTests(unittest.TestCase): + def test_selects_powershell_on_windows(self) -> None: + locations = {"pwsh": r"C:\Program Files\PowerShell\7\pwsh.exe"} + + for task_name in ( + "task_shell_command_generator.md", + "task_git_rescue_recovery.md", + ): + with self.subTest(task=task_name): + shell_argv = _load_grader(task_name)["_shell_argv"] + argv = shell_argv("Get-ChildItem", platform="nt", which=locations.get) + + self.assertEqual( + argv, + [ + locations["pwsh"], + "-NoProfile", + "-NonInteractive", + "-Command", + "Get-ChildItem", + ], + ) + + def test_selects_bash_on_posix(self) -> None: + for task_name in ( + "task_shell_command_generator.md", + "task_git_rescue_recovery.md", + ): + with self.subTest(task=task_name): + shell_argv = _load_grader(task_name)["_shell_argv"] + argv = shell_argv( + "find .", + platform="posix", + which=lambda name: f"/bin/{name}", + ) + + self.assertEqual(argv, ["/bin/bash", "-c", "find ."]) + + def test_missing_shell_is_an_infrastructure_error(self) -> None: + for task_name in ( + "task_shell_command_generator.md", + "task_git_rescue_recovery.md", + ): + with self.subTest(task=task_name): + shell_argv = _load_grader(task_name)["_shell_argv"] + with self.assertRaises(RuntimeError): + shell_argv("echo test", platform="nt", which=lambda _name: None) + + +class ShellCommandGeneratorGraderTests(unittest.TestCase): + def test_normalizes_relative_and_absolute_windows_paths(self) -> None: + normalize = _load_grader("task_shell_command_generator.md")[ + "_normalize_output_path" + ] + root = r"c:\fixture" + + self.assertEqual(normalize(r".\app\runtime.log", root), "app/runtime.log") + self.assertEqual( + normalize(r"C:\Fixture\nested\deeper\events.log", root), + "nested/deeper/events.log", + ) + self.assertEqual( + normalize("/tmp/fixture/server.log", "/tmp/fixture"), + "server.log", + ) + + def test_grades_valid_command_by_behavior(self) -> None: + grader = _load_grader("task_shell_command_generator.md") + if os.name == "nt": + command = ( + "Get-ChildItem -Recurse -File -Filter *.log | " + "Select-String -SimpleMatch -CaseSensitive 'FATAL:' | " + "ForEach-Object { $_.Path } | Sort-Object -Unique" + ) + else: + command = "grep -rl --include='*.log' 'FATAL:' ." + + with TemporaryDirectory() as workspace: + Path(workspace, "command.txt").write_text(command, encoding="utf-8") + scores = grader["grade"]([], workspace) + + self.assertTrue(all(score == 1.0 for score in scores.values()), scores) + + def test_rejects_invalid_command(self) -> None: + grader = _load_grader("task_shell_command_generator.md") + + with TemporaryDirectory() as workspace: + Path(workspace, "command.txt").write_text( + "pinchbench-command-that-does-not-exist", + encoding="utf-8", + ) + scores = grader["grade"]([], workspace) + + self.assertEqual(scores["executes_successfully"], 0.0) + self.assertEqual(scores["outputs_correct_matches"], 0.0) + + def test_propagates_missing_shell_infrastructure(self) -> None: + grader = _load_grader("task_shell_command_generator.md") + + with TemporaryDirectory() as workspace: + Path(workspace, "command.txt").write_text("echo test", encoding="utf-8") + with patch.dict( + grader, + {"_shell_argv": lambda _command: (_ for _ in ()).throw(RuntimeError())}, + ), self.assertRaises(RuntimeError): + grader["grade"]([], workspace) + + +class GitRescueGraderTests(unittest.TestCase): + def test_grades_valid_recovery_by_behavior(self) -> None: + grader = _load_grader("task_git_rescue_recovery.md") + + with TemporaryDirectory() as workspace: + Path(workspace, "recovery.sh").write_text( + "git branch feature/login-fix\n" + "git reset --hard HEAD~2\n", + encoding="utf-8", + ) + scores = grader["grade"]([], workspace) + + self.assertTrue(all(score == 1.0 for score in scores.values()), scores) + + def test_rejects_invalid_git_command(self) -> None: + grader = _load_grader("task_git_rescue_recovery.md") + + with TemporaryDirectory() as workspace: + Path(workspace, "recovery.sh").write_text( + "git pinchbench-command-that-does-not-exist\n", + encoding="utf-8", + ) + scores = grader["grade"]([], workspace) + + self.assertEqual(scores["executes_successfully"], 0.0) + self.assertEqual(scores["feature_branch_created"], 0.0) + + def test_propagates_missing_shell_infrastructure(self) -> None: + grader = _load_grader("task_git_rescue_recovery.md") + + with TemporaryDirectory() as workspace: + Path(workspace, "recovery.sh").write_text( + "git status\n", + encoding="utf-8", + ) + with patch.dict( + grader, + {"_shell_argv": lambda _command: (_ for _ in ()).throw(RuntimeError())}, + ), self.assertRaises(RuntimeError): + grader["grade"]([], workspace) + + +if __name__ == "__main__": + unittest.main()