diff --git a/nemo_gym/server_utils.py b/nemo_gym/server_utils.py index a687376dd..fa8d06d3d 100644 --- a/nemo_gym/server_utils.py +++ b/nemo_gym/server_utils.py @@ -658,3 +658,14 @@ class ServerInstanceDisplayConfig(BaseModel): status: Optional[ServerStatus] = None uptime_seconds: Optional[float] = None url: Optional[str] = None + + +def get_server_url(server_name: str) -> str: + global_config_dict = get_global_config_dict() + + model_server_config = get_first_server_config_dict( + global_config_dict, + server_name, + ) + + return f"http://{model_server_config['host']}:{model_server_config['port']}" diff --git a/responses_api_agents/swe_agents/__init__.py b/responses_api_agents/swe_agents/__init__.py index 8579af82b..e69de29bb 100644 --- a/responses_api_agents/swe_agents/__init__.py +++ b/responses_api_agents/swe_agents/__init__.py @@ -1,22 +0,0 @@ -"""SWE-bench wrapper agent for NeMo-Gym. - -This module provides integration between NeMo-Skills' SWE-bench evaluation -capabilities and NeMo-Gym's agent framework. -""" - -from .app import ( - SWEBenchRunRequest, - SWEBenchVerifyRequest, - SWEBenchVerifyResponse, - SWEBenchWrapper, - SWEBenchWrapperConfig, -) - - -__all__ = [ - "SWEBenchWrapper", - "SWEBenchWrapperConfig", - "SWEBenchRunRequest", - "SWEBenchVerifyRequest", - "SWEBenchVerifyResponse", -] diff --git a/responses_api_agents/swe_agents/app.py b/responses_api_agents/swe_agents/app.py index bc27ad071..eac32b944 100644 --- a/responses_api_agents/swe_agents/app.py +++ b/responses_api_agents/swe_agents/app.py @@ -12,23 +12,34 @@ # See the License for the specific language governing permissions and # limitations under the License. import asyncio +import glob import json import os +import re import shlex +import shutil import sys import time import uuid -import warnings from asyncio import Semaphore +from asyncio.subprocess import Process +from contextlib import contextmanager +from fcntl import LOCK_EX, LOCK_UN, flock from pathlib import Path -from typing import Any, Callable, Dict, Optional +from shutil import rmtree +from subprocess import Popen +from subprocess import run as subprocess_run +from typing import Any, Dict, Literal, Optional, Tuple, Union import ray -from pydantic import ConfigDict, Field +import tomlkit +from gprof2dot import main as gprof2dot_main +from openai.types.responses.function_tool import FunctionTool +from pydantic import BaseModel, ConfigDict, Field +from pydot import graph_from_dot_file from nemo_gym.base_resources_server import ( BaseRunRequest, - BaseVerifyRequest, BaseVerifyResponse, ) from nemo_gym.base_responses_api_agent import ( @@ -41,86 +52,20 @@ from nemo_gym.openai_utils import ( NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming, - NeMoGymResponseOutputMessage, - NeMoGymResponseOutputText, ) from nemo_gym.profiling import Profiler -from responses_api_agents.swe_agents.utils import ( - convert_tools_to_function_format, - convert_trajectory_to_output_items, - extract_input_messages_from_trajectory, - extract_problem_info, - get_model_endpoint, - run_swebench_evaluation, - setup_openhands_environment, - setup_r2e_gym_environment, - setup_swebench_environment, -) - - -# There are some mysterious Pydantic serialization warnings related to FunctionTool that are not fatal that clutter up logs. -# At some point we can try continue chasing this one down. Example: -# (NemoGym pid=3160799) (swe_agents_val) PydanticSerializationUnexpectedValue(Expected `general-fields` - serialized value may not be as expected [field_name='tools', input_value=FunctionTool(name='str_re... a single call each.\n'), input_type=FunctionTool]) -warnings.filterwarnings("ignore", message="FunctionTool") - - -@ray.remote -class ConcurrentContainerCounter: - def __init__(self): - self.concurrent_containers = 0 - - def increment(self): - self.concurrent_containers += 1 - return self.concurrent_containers - - def decrement(self): - self.concurrent_containers -= 1 - return self.concurrent_containers - - -@ray.remote( - scheduling_strategy="SPREAD", - runtime_env={ - "py_executable": sys.executable, - }, - num_cpus=1, -) -def runner_ray_remote( - concurrent_container_counter: ConcurrentContainerCounter, runner: Callable, params: dict[str, Any] -) -> Any: - ray_submit_time = time.time() - params["ray_submit_time"] = ray_submit_time - - # This is the first instance so we don't need to load anything - with params["metrics_fpath"].open("w") as f: - json.dump({"ray_queue_time": ray_submit_time - params["ray_queue_time"]}, f) - - if params["debug"]: - concurrent_containers = ray.get(concurrent_container_counter.increment.remote()) - print(f"Concurrent container #{concurrent_containers}", file=sys.stderr) - - instance_id = params["problem_info"].get("instance_id", "unknown") - profiler = Profiler(name=instance_id, base_profile_dir=params["persistent_dir"] / "profiling") - profiler.start() +from responses_api_models.vllm_model.app import VLLMConverter, split_responses_input_output_items - result = asyncio.run(runner(**params)) - if params["debug"]: - profiler.stop() - - ray.get(concurrent_container_counter.decrement.remote()) - - return result +######################################## +# START Configuration +######################################## class SWEBenchWrapperConfig(BaseResponsesAPIAgentConfig): model_server: ModelServerRef # Agent framework configuration - agent_framework: str = Field( - default="swe_agent", - description="Agent framework to use: swe_agent or openhands", - ) agent_config: Optional[str] = Field(default=None, description="Path to agent configuration file") agent_tools_file: Optional[str] = Field( default=None, description="Path to JSON file containing tool definitions in OpenAI format (for SWE-agent)" @@ -151,348 +96,1250 @@ class SWEBenchWrapperConfig(BaseResponsesAPIAgentConfig): # Concurrency control concurrency: int = Field(default=256, description="Maximum number of concurrent SWE-bench runs") - # Pre-built OpenHands directory path (set during initialization) - openhands_setup_dir: Optional[Path] = Field( - default=None, - description="Path to pre-built OpenHands directory (automatically set during initialization)", - exclude=True, - ) - - # Pre-built SWE-bench directory path (set during initialization) - swebench_setup_dir: Optional[Path] = Field( - default=None, - description="Path to pre-built SWE-bench directory (automatically set during initialization)", - exclude=True, - ) - # Pre-built R2E-gym directory path (set during initialization) - r2e_gym_setup_dir: Optional[Path] = Field( - default=None, - description="Path to pre-built R2E-gym directory (automatically set during initialization)", - exclude=True, - ) dataset_path: Optional[str] = Field( default=None, description="Path to the dataset for SWE-bench evaluation", ) - run_session_id: str = Field( - default=None, - description="Session ID for the run", - ) - openhands_should_log: bool = False debug: bool = False -class SWEBenchRunRequest(BaseRunRequest): - """Request format for SWE-bench runs.""" +class SWEBenchWrapperServerConfig(BaseModel): + ng_global_config_dict_str: str + model_server_name: str + openhands_setup_dir: Path + swebench_setup_dir: Path + r2e_gym_setup_dir: Path + run_session_id: str + base_results_dir: Path + + +class ExecuteContainerCommandArgs(BaseModel): + command: str + expected_file_pattern: str + mode: Union[Literal["agent"], Literal["eval"]] + timeout: int + + +class SWEBenchWrapperInstanceConfig(SWEBenchWrapperServerConfig, SWEBenchWrapperConfig): + metrics_fpath: Path + problem_info: Dict[str, Any] + body: NeMoGymResponseCreateParamsNonStreaming + persistent_dir: Path + ray_queue_timestamp: float + inference_params: Dict[str, Any] + agent_run_id: str + instance_dataset_path: Path + trajectories_root: Path + prediction_path: Path + output_for_eval_mounted_path: Path + output_for_eval_path: Path + model_patch_path: Path + container: str + eval_dir_in_openhands: str + openhands_config_file_path: str + agent_script_path: Path + final_eval_apptainer_spinup_timestamp_fpath: Path + final_eval_apptainer_spinup_timestamp_mounted_fpath: Path + generation_apptainer_spinup_timestamp_fpath: Path + generation_apptainer_spinup_timestamp_mounted_fpath: Path + + # Set later + eval_command: Optional[ExecuteContainerCommandArgs] = None + eval_apptainer_command_str: Optional[str] = None + agent_command: Optional[ExecuteContainerCommandArgs] = None + agent_apptainer_command_str: Optional[str] = None + agent_script: Optional[str] = None + + @property + def instance_id(self) -> str: + return self.problem_info["instance_id"] + + +class SWEBenchMetrics(BaseModel): + resolved: Optional[bool] = None + patch_exists: Optional[bool] = None - model_config = {"extra": "allow"} + # Profiling time metrics to report + ray_queue_time: Optional[float] = None + openhands_run_time: Optional[float] = None + generation_apptainer_spinup_time: Optional[float] = None + create_runtime_time: Optional[float] = None + connect_to_runtime_time: Optional[float] = None + initialize_runtime_time: Optional[float] = None + total_command_exec_time: Optional[float] = None + total_model_call_time: Optional[float] = None + final_eval_apptainer_spinup_time: Optional[float] = None + final_eval_time: Optional[float] = None + + +class SWEBenchVerifyResponse(SWEBenchMetrics, BaseVerifyResponse): + instance_config: SWEBenchWrapperInstanceConfig + + +######################################## +# START Dataset and harness handling +######################################## + + +class BaseDatasetHarnessProcessor(BaseModel): + config: SWEBenchWrapperConfig | SWEBenchWrapperInstanceConfig + + ######################################## + # START Setup logic + ######################################## + + @property + def parent_dir(self) -> Path: + return Path(__file__).parent + + def _run_setup_command(self, command: str) -> None: + process = Popen(command, shell=True) + return_code = process.wait() + assert return_code == 0, f"Command failed: {command}" + + @contextmanager + def _setup_directory_lock(self, setup_dir: Path, label: str): + """File-based lock to ensure only one process performs the setup.""" + lock_dir = setup_dir.parent + lock_dir.mkdir(parents=True, exist_ok=True) + lock_path = lock_dir / f".{setup_dir.name}.lock" + + with open(lock_path, "w") as lock_file: + print(f"Acquiring {label} setup lock at {lock_path}", flush=True) + flock(lock_file, LOCK_EX) + try: + yield + finally: + flock(lock_file, LOCK_UN) + + # Setup method is sync for now since there's been no need to concurrently set up + def setup(self) -> Path: + pass + + def get_run_command(self) -> ExecuteContainerCommandArgs: + pass + + def postprocess_after_eval_run(self, report_file: Path) -> None: + pass + + def _get_command_sleep_until_predictions_file(self) -> str: + return f"until [ -f {self.config.output_for_eval_mounted_path} ]; do sleep 5; done" + + +class SweBenchDatasetProcessor(BaseDatasetHarnessProcessor): + def setup(self) -> Path: + swebench_repo = "https://github.com/HeyyyyyyG/SWE-bench.git" + swebench_commit = "HEAD" + + setup_dir = self.parent_dir / "swe_swebench_setup" + setup_dir.mkdir(parents=True, exist_ok=True) + + with self._setup_directory_lock(setup_dir, "SWE-bench"): + swebench_dir = setup_dir / "SWE-bench" + uv_dir = setup_dir / "uv" + python_dir = setup_dir / "python" + + if swebench_dir.exists(): + print(f"SWE-bench already set up at {setup_dir}") + return setup_dir + + print(f"Setting up SWE-bench environment at {setup_dir}...", flush=True) + script_fpath = self.parent_dir / "setup_scripts/swebench.sh" + command = f"""SETUP_DIR={setup_dir} \\ +UV_DIR={uv_dir} \\ +PYTHON_DIR={python_dir} \\ +SWEBENCH_DIR={swebench_dir} \\ +SWEBENCH_REPO={swebench_repo} \\ +SWEBENCH_COMMIT={swebench_commit} \\ + {script_fpath}""" + self._run_setup_command(command) + + return setup_dir + + def get_run_command(self) -> ExecuteContainerCommandArgs: + swebench_cmd = ( + f'date +"%s.%N" > {self.config.final_eval_apptainer_spinup_timestamp_mounted_fpath} && ' + f"{self._get_command_sleep_until_predictions_file()} && " + # Use pre-built SWE-bench + "cd /swebench_setup/SWE-bench && " + # Set UV environment variables to use the mounted portable directories + f'export UV_INSTALL_DIR="{self.config.swebench_setup_dir}/uv" && ' + f'export UV_PYTHON_INSTALL_DIR="{self.config.swebench_setup_dir}/python" && ' + f'export PATH="{self.config.swebench_setup_dir}/uv/bin:$PATH" && ' + f"ls -lrt /root/dataset && " + # Run with clean environment to avoid venv contamination + # Use the pre-built venv directly with its absolute path + f"env -u VIRTUAL_ENV {self.config.swebench_setup_dir}/SWE-bench/venv/bin/python -m swebench.harness.run_local_evaluation " + f" --predictions_path {self.config.output_for_eval_mounted_path} " + f" --instance_ids {self.config.instance_id} " + f" --timeout {self.config.swebench_tests_timeout} " + f" --dataset_name /root/dataset/data.jsonl " + f" --split {self.config.problem_info['split']} " + f" --run_id {self.config.agent_run_id} && " + f"cp -r logs/run_evaluation/{self.config.agent_run_id} /trajectories_mount/ && " + f"rm -rf logs/run_evaluation/{self.config.agent_run_id} && rm -rf *{self.config.agent_run_id}*" + ) + # Execute SWE-bench evaluation command + search_path = os.path.join( + self.config.persistent_dir, + self.config.agent_run_id, + "**", + f"{self.config.instance_id}/report.json", + ) -class SWEBenchVerifyRequest(BaseVerifyRequest): - """Request format for SWE-bench verification.""" + return ExecuteContainerCommandArgs( + command=swebench_cmd, + expected_file_pattern=search_path, + mode="eval", + timeout=self.config.swebench_tests_timeout + 120, + ) - model_config = {"extra": "allow"} +class R2EGymDatasetProcessor(BaseDatasetHarnessProcessor): + def setup(self) -> Path: + eval_harness_repo = "https://github.com/ludwig-n/R2E-Gym.git" + eval_harness_commit = "local-eval" + + setup_dir = self.parent_dir / "swe_r2e_gym_setup" + + with self._setup_directory_lock(setup_dir, "R2E-Gym"): + r2e_gym_dir = setup_dir / "R2E-Gym" + uv_dir = setup_dir / "uv" + python_dir = setup_dir / "python" + + # Check if setup is complete by verifying venv and installed module + venv_dir = r2e_gym_dir / "venv" + python_bin = venv_dir / "bin" / "python" + if r2e_gym_dir.exists() and venv_dir.exists() and python_bin.exists(): + result = subprocess_run([str(python_bin), "-c", "import r2egym"]) + if result.returncode == 0: + print(f"R2E-Gym already set up at {setup_dir}", flush=True) + return setup_dir + + print("R2E-Gym directory exists but module not properly installed, rebuilding...", flush=True) + + print(f"Setting up R2E-Gym environment at {setup_dir}...", flush=True) + setup_dir.mkdir(parents=True, exist_ok=True) + + script_fpath = self.parent_dir / "setup_scripts/r2e_gym.sh" + command = f"""SETUP_DIR={setup_dir} \\ +UV_DIR={uv_dir} \\ +PYTHON_DIR={python_dir} \\ +R2E_GYM_DIR={r2e_gym_dir} \\ +EVAL_HARNESS_REPO={eval_harness_repo} \\ +EVAL_HARNESS_COMMIT={eval_harness_commit} \\ + {script_fpath}""" + self._run_setup_command(command) + + return setup_dir + + def get_run_command(self) -> ExecuteContainerCommandArgs: + r2e_gym_cmd = ( + f'date +"%s.%N" > {self.config.final_eval_apptainer_spinup_timestamp_mounted_fpath} && ' + f"{self._get_command_sleep_until_predictions_file()} && " + # Use mounted directory path for cd + "cd /r2egym_setup/R2E-Gym && " + # Set UV environment variables to use the mounted portable directories + f'export UV_INSTALL_DIR="{self.config.r2e_gym_setup_dir}/uv" && ' + f'export UV_PYTHON_INSTALL_DIR="{self.config.r2e_gym_setup_dir}/python" && ' + f'export PATH="{self.config.r2e_gym_setup_dir}/uv/bin:$PATH" && ' + # Run with clean environment to avoid venv contamination + # Use the pre-built venv directly with its absolute path + f"env -u VIRTUAL_ENV {self.config.r2e_gym_setup_dir}/R2E-Gym/venv/bin/python src/r2egym/agenthub/run/run_local_evaluation.py " + f" --predictions_path {self.config.output_for_eval_mounted_path} " + f" --instance_id {self.config.instance_id} " + f" --timeout {self.config.swebench_tests_timeout} " + f" --dataset /root/dataset/data.jsonl " + f" --output_dir /trajectories_mount/eval-outputs/{self.config.agent_run_id}" + ) -class SWEBenchVerifyResponse(BaseVerifyResponse): - """Response format for SWE-bench verification.""" + search_path = os.path.join( + self.config.persistent_dir, + "eval-outputs", + self.config.agent_run_id, + "report.json", + ) - model_config = {"extra": "allow"} + return ExecuteContainerCommandArgs( + command=r2e_gym_cmd, + expected_file_pattern=search_path, + mode="eval", + timeout=self.config.swebench_tests_timeout + 120, + ) - # Additional SWE-bench specific fields - swebench_metrics: Optional[Dict[str, Any]] = None - # Additional numeric fields for rollout statistics - resolved: Optional[float] = None # 1.0 if resolved, 0.0 otherwise - patch_exists: Optional[float] = None # 1.0 if patch exists, 0.0 otherwise - patch_successfully_applied: Optional[float] = None # 1.0 if patch applied, 0.0 otherwise +class NVInternalDatasetProcessor(BaseDatasetHarnessProcessor): + def get_run_command(self) -> ExecuteContainerCommandArgs: + instance_dict = json.loads(self.config.problem_info["instance_dict"]) + base_dockerfile = instance_dict.get("base_dockerfile", "") + instance_dockerfile = instance_dict.get("instance_dockerfile", "") + + env_lines = [] + for line in (base_dockerfile + "\n" + instance_dockerfile).split("\n"): + line = line.strip() + if line.startswith("ENV "): + # Convert ENV KEY=VALUE or ENV KEY VALUE to export KEY="VALUE" + export_line = line.replace("ENV ", "export ", 1) + # Handle both Docker ENV formats: + # 1. ENV KEY=VALUE (with equals) + # 2. ENV KEY VALUE (space-separated) + if "=" in export_line: + # Format: export KEY=VALUE -> normalize spaces around = + export_line = re.sub(r"\s*=\s*", "=", export_line) + else: + # Format: export KEY VALUE -> convert to export KEY="VALUE" + parts = export_line.split(None, 2) # Split into at most 3 parts + if len(parts) >= 3: # export KEY VALUE + key = parts[1] + value = parts[2] + export_line = f'export {key}="{value}"' + + env_lines.append(export_line) + + env_exports = "\n".join(env_lines) + + # Get repo setup command + repo_cmd = instance_dict.get("before_repo_set_cmd", "").strip() + if repo_cmd: + repo_cmd = repo_cmd.split("\n")[-1] + + # Get test files + test_files_str = instance_dict.get("selected_test_files_to_run", "[]") + if isinstance(test_files_str, str): + test_files = ",".join(eval(test_files_str)) + else: + test_files = ",".join(test_files_str) + + run_script = instance_dict["run_script.sh"] + parsing_script = instance_dict["parsing_script.py"] + run_script_path = self.config.persistent_dir / "run_script.sh" + parsing_script_path = self.config.persistent_dir / "parsing_script.py" + with open(run_script_path, "w") as f: + f.write(run_script) + with open(parsing_script_path, "w") as f: + f.write(parsing_script) + + cmd = f"""#!/bin/bash +set -e + +date +\"%s.%N\" > {self.config.final_eval_apptainer_spinup_timestamp_mounted_fpath} + +{self._get_command_sleep_until_predictions_file()} + +{env_exports} + +# Apply patch +cd /app +git reset --hard {instance_dict.get("base_commit", "")} +git checkout {instance_dict.get("base_commit", "")} + +# Apply patch with rejection to handle conflicts +git apply --ignore-space-change --ignore-whitespace --reject -v /root/patch.diff || true + +# Setup repository +{repo_cmd} + +# Run tests +bash /root/run_script.sh {test_files} > /root/stdout.log 2> /root/stderr.log || true + +# Parse results +python /root/parsing_script.py /root/stdout.log /root/stderr.log /root/output.json + +# Move outputs to the mounted directory +mkdir -p /trajectories_mount/eval_results +cp /root/output.json /trajectories_mount/eval_results/output.json +""" + + search_path = os.path.join( + self.config.persistent_dir, + "eval_results", + "output.json", + ) - # Profiling time metrics to report - ray_queue_time: float - # generation_apptainer_spinup_time: float - # create_runtime_time: float - # container_initialization_time: float - # connect_to_runtime_time: float - # runtime_initialization_fn_time: float - # total_command_exec_time: float - # total_model_call_time: float - # final_eval_apptainer_spinup_time: float - final_eval_time: float - - # Exit condition metrics to report - # TODO add more exit conditions - # hit_sample_timeout: bool - # hit_trajectory_command_exec_timeout: bool - # hit_eval_timeout: bool - hit_empty_trajectory: bool - hit_success: bool - hit_responses_exception: bool + return ExecuteContainerCommandArgs( + command=cmd, + expected_file_pattern=search_path, + mode="eval", + timeout=self.config.swebench_tests_timeout, + ) + def postprocess_after_run(self, report_file: Path) -> None: + instance_dict = json.loads(self.config.problem_info["instance_dict"]) -class SWEBenchWrapper(SimpleResponsesAPIAgent): - """Wrapper for NeMo-Skills SWE-bench evaluation in NeMo-Gym.""" + fail_to_pass_str = instance_dict.get("fail_to_pass_select", instance_dict.get("fail_to_pass", "[]")) + pass_to_pass_str = instance_dict.get("pass_to_pass_select", instance_dict.get("pass_to_pass", "[]")) - config: SWEBenchWrapperConfig - sem: Semaphore = None - _container_counter: ConcurrentContainerCounter = None - _global_config_dict_str: str = None - model_config = ConfigDict(arbitrary_types_allowed=True) + if isinstance(fail_to_pass_str, str): + f2p = set(json.loads(fail_to_pass_str)) + else: + f2p = set(fail_to_pass_str) - def model_post_init(self, __context: Any) -> None: - self.sem = Semaphore(self.config.concurrency) - self._container_counter = ConcurrentContainerCounter.remote() + if isinstance(pass_to_pass_str, str): + p2p = set(json.loads(pass_to_pass_str)) + else: + p2p = set(pass_to_pass_str) - # Pre-build OpenHands environment if using openhands framework - if self.config.agent_framework == "openhands": - self.config.openhands_setup_dir = setup_openhands_environment( - agent_framework_repo=self.config.agent_framework_repo, - agent_framework_commit=self.config.agent_framework_commit, - debug=self.config.debug, + with open(report_file, "r+") as f: + test_results = json.loads(f.read()) + is_resolved = self.check_tests_passed( + test_results, + f2p, + p2p, + ) + report_dict = dict( + resolved=is_resolved, + patch_exists=True, + patch_successfully_applied=is_resolved, + metadata={ + "test_results": test_results, + "f2p": list(f2p), + "p2p": list(p2p), + }, + ) + f.seek(0) + f.write(json.dumps({self.config.instance_id: report_dict}, indent=4)) + + def check_tests_passed( + self, + test_results: dict[str, Any], + f2p: set[str], + p2p: set[str], + ) -> bool: + if not test_results: + return False + + passed_tests = {test["name"] for test in test_results.get("tests", []) if test.get("status") == "PASSED"} + required_tests = f2p.union(p2p) + + # Check if all required tests passed + if len(passed_tests) == 0 or len(required_tests) == 0: + return False + + return required_tests <= passed_tests + + +class OpenHandsHarnessProcessor(BaseDatasetHarnessProcessor): + def setup(self) -> Path: + setup_dir = self.parent_dir / "swe_openhands_setup" + + with self._setup_directory_lock(setup_dir, "OpenHands"): + openhands_dir = setup_dir / "OpenHands" + miniforge_dir = setup_dir / "miniforge3" + + if openhands_dir.exists() and Path(openhands_dir / ".venv" / "bin" / "python").exists(): + print(f"OpenHands already set up at {setup_dir}", flush=True) + return setup_dir + + print(f"Setting up OpenHands environment at {setup_dir}...", flush=True) + rmtree(setup_dir, ignore_errors=True) + setup_dir.mkdir(parents=True, exist_ok=True) + + script_fpath = self.parent_dir / "setup_scripts/openhands.sh" + command = f"""SETUP_DIR={setup_dir} \\ +MINIFORGE_DIR={miniforge_dir} \\ +OPENHANDS_DIR={openhands_dir} \\ +AGENT_FRAMEWORK_REPO={self.config.agent_framework_repo} \\ +AGENT_FRAMEWORK_COMMIT={self.config.agent_framework_commit} \\ + {script_fpath}""" + self._run_setup_command(command) + + return setup_dir + + def get_run_command(self) -> ExecuteContainerCommandArgs: + data_point = self.config.problem_info + agent_run_id = self.config.agent_run_id + + agent_config = os.path.join(os.path.dirname(os.path.abspath(__file__)), "configs/oh_config.toml") + + # Add parameters to config.toml + # TODO(sugam): is there a better way to do this? + with open(agent_config, "r") as f: + config = tomlkit.parse(f.read()) + + config["llm"]["model"] |= { + "model": self.config.body.model, + "base_url": "", # May need to populate this + "temperature": self.config.inference_params["temperature"], + "top_p": self.config.inference_params["top_p"], + } + + config_str = tomlkit.dumps(config) + + eval_dir_in_openhands = self.config.eval_dir_in_openhands + local_dataset_path = "/root/dataset/data.jsonl" + config_file_path = self.config.openhands_config_file_path + + assert self.config.openhands_setup_dir is not None, "OpenHands setup directory is not set" + + if self.config.debug: + profiling_cmd = "export NG_PROFILING_DIR=/trajectories_mount/profiling && " + else: + profiling_cmd = "" + + if self.config.openhands_should_log: + log_cmd = "export LOG_LEVEL=DEBUG && export LOG_TO_FILE=true && export NG_OPENHANDS_SHOULD_LOG=true && " + else: + log_cmd = ( + "export LOG_LEVEL=CRITICAL && " + "export DEBUG=False && " + "export DEBUG_LLM=False && " + "export LOG_TO_FILE=False && " + "export LOG_ALL_EVENTS=False && " + "export DEBUG_RUNTIME=False && " ) - self.config.swebench_setup_dir = setup_swebench_environment() - self.config.r2e_gym_setup_dir = setup_r2e_gym_environment() - print("Dependencies repositories set up complete", flush=True) + agent_main_cmd = ( + "if [ -d /workspace ]; then " + " echo 'Exiting because /workspace is mounted.' && " + " echo 'Please make sure /workspace is not mounted inside of Apptainer before running OpenHands.' && " + " echo 'This is because OpenHands DELETES EVERYTHING in the /workspace folder if it exists.' && " + " exit 1; " + "fi && " + # Add miniforge bin to PATH (for tmux, node, poetry, etc.) + "mkdir -p /tmp/ && " + "export PATH=/openhands_setup/miniforge3/bin:$PATH && " + # Setup tmux socket (OpenHands requirement) + "uid=$(id -ru 2>/dev/null || id -u) && " + "export TMUX_TMPDIR=/tmp && " + "export TMUX=/tmp/tmux-$uid/default && " + "mkdir -p /tmp/tmux-$uid && " + "chown $uid:$uid /tmp/tmux-$uid || true && " + "chmod 700 /tmp/tmux-$uid && " + "tmux -S /tmp/tmux-$uid/default start-server || true && " + # Use pre-built OpenHands + "cd /openhands_setup/OpenHands && " + "export RUNTIME=local && " + f'date +"%s.%N" > {self.config.generation_apptainer_spinup_timestamp_mounted_fpath} && ' + f"{log_cmd}" + f"{profiling_cmd}" + f"export NEMO_GYM_METRICS_FPATH={self.config.metrics_fpath} && " + f"export NEMO_GYM_CONFIG_DICT={self.config.ng_global_config_dict_str} && " + f"export NEMO_GYM_MODEL_SERVER_NAME={self.config.model_server_name} &&" + "export VIRTUAL_ENV=/openhands_setup/OpenHands/.venv && " + "export PATH=$PATH:/openhands_setup/OpenHands/.venv/bin && " + # CRITICAL: Configure poetry to only use the OpenHands venv (ignore external venvs) + "export POETRY_VIRTUALENVS_IN_PROJECT=true && " + "export POETRY_VIRTUALENVS_CREATE=false && " + "export POETRY_VIRTUALENVS_PATH=/openhands_setup/OpenHands && " + f"export TMUX_MEMORY_LIMIT={self.config.apptainer_memory_limit_mb} && " + f"export COMMAND_EXEC_TIMEOUT={self.config.command_exec_timeout} && " + # TODO (sugam): fix cryptography issue + # "override_dir=$(mktemp -d /tmp/cryptography_override.XXXX) && " + # # Reinstall cryptography inside the container (via poetry's venv) using a compatible wheel + # # Clean any broken installs to avoid missing-file errors, then force a wheel-only reinstall + # "site_packages_dir=/openhands_setup/OpenHands/.venv/lib/python3.12/site-packages && " + # 'if [ -d "$site_packages_dir" ]; then ' + # ' find "$site_packages_dir" -maxdepth 1 -name "cryptography*" -exec rm -rf {} +; ' + # "fi && " + # "poetry run python -m pip install --index-url https://pypi.org/simple " + # " --trusted-host pypi.org --trusted-host files.pythonhosted.org " + # " --only-binary cryptography --no-deps --force-reinstall 'cryptography==42.0.8' && " + # disable logging to file in the oh repo + # set up config files + f"echo {shlex.quote(config_str)} >{config_file_path} && " + # f" export EVAL_OUTPUT_DIR={eval_dir_in_openhands} && " + f"./evaluation/benchmarks/swe_bench/scripts/run_infer.sh " + f" llm.model " # name of llm config section in config.toml + f" {self.config.agent_framework_commit} " # openhands commit + f" CodeActAgent " # agent + f" 0 " # Note: this is eval limit which randomly chooses an instance from the dataset + f" {self.config.agent_max_turns} " # max agent iterations + f" 1 " # number of workers + f" {data_point['dataset_name']} " # dataset name + f" {data_point['split']} " # dataset split + f" {eval_dir_in_openhands} " + f" {data_point['instance_id']} " + f" {local_dataset_path} " + f" {config_file_path}" + ) - self.config.run_session_id = f"{int(time.time() * 1000)}_{str(uuid.uuid4())[:8]}" - print(f"Run session ID: {self.config.run_session_id}", flush=True) + agent_script_name = f"agent_script_{agent_run_id}.sh" + agent_script_path = self.config.persistent_dir / agent_script_name + with open(agent_script_path, "w") as f: + f.write("#!/bin/bash\nset -e\n") + f.write(agent_main_cmd) + f.flush() + os.fsync(f.fileno()) + + agent_timeout_seconds = self.config.swebench_agent_timeout + openhands_cmd = ( + f"timeout --signal=TERM --kill-after=30 {agent_timeout_seconds} " + f"bash /trajectories_mount/{agent_script_name}" + ) - self._global_config_dict_str = shlex.quote(OmegaConf.to_yaml(get_global_config_dict())) + search_path = os.path.join( + self.config.openhands_setup_dir / "OpenHands" / eval_dir_in_openhands, + "**", + "output.jsonl", + ) - async def responses(self, body: NeMoGymResponseCreateParamsNonStreaming = Body()) -> NeMoGymResponse: - # Extract problem information from request - problem_info = extract_problem_info( - body, - self.config.container_formatter, + # Execute OpenHands command + return ExecuteContainerCommandArgs( + command=openhands_cmd, + expected_file_pattern=search_path, + mode="agent", + timeout=self.config.swebench_agent_timeout + 60, ) - # Get model endpoint - model_endpoint = get_model_endpoint(self.config.model_server.name) - # Create persistent directory for I/O and logs in local workspace - instance_dir = ( - f"{problem_info.get('instance_id', 'unknown')}_{int(time.time() * 1000)}_{str(uuid.uuid4())[:8]}" - ) - workspace_root = Path(os.path.dirname(os.path.abspath(__file__))) - persistent_dir = workspace_root / f"swebench_results_{self.config.run_session_id}" / instance_dir - persistent_dir.mkdir(parents=True, exist_ok=True) - metrics_fpath = persistent_dir / "nemo_gym_metrics.json" - try: - ray_queue_time = time.time() - params = { - "problem_info": problem_info, - "model_endpoint": model_endpoint, - "body": body, - "agent_framework": self.config.agent_framework, - "agent_config": self.config.agent_config, - "agent_tools_file": self.config.agent_tools_file, - "agent_max_turns": self.config.agent_max_turns, - "swebench_tests_timeout": self.config.swebench_tests_timeout, - "swebench_agent_timeout": self.config.swebench_agent_timeout, - "persistent_dir": persistent_dir, - "metrics_fpath": metrics_fpath, - "agent_framework_repo": self.config.agent_framework_repo, - "agent_framework_commit": self.config.agent_framework_commit, - "openhands_setup_dir": self.config.openhands_setup_dir, - "swebench_setup_dir": self.config.swebench_setup_dir, - "r2e_gym_setup_dir": self.config.r2e_gym_setup_dir, - "dataset_path": self.config.dataset_path, - "ray_queue_time": ray_queue_time, - "openhands_should_log": self.config.openhands_should_log, - "debug": self.config.debug, - "model_server_name": self.config.model_server.name, - "ng_global_config_dict_str": self._global_config_dict_str, - "apptainer_memory_limit_mb": self.config.apptainer_memory_limit_mb, - "command_exec_timeout": self.config.command_exec_timeout, - } +######################################## +# START Ray worker logic +######################################## + + +@ray.remote( + scheduling_strategy="SPREAD", + runtime_env={ + "py_executable": sys.executable, + }, + num_cpus=1, +) +def runner_ray_remote(params_dict: dict[str, Any]) -> Optional[Path]: + # For some reason Ray may not pick up the proper model fields if we don't rebuild the model here. Very strange. + SWEBenchWrapperInstanceConfig.model_rebuild(force=True) + RunOpenHandsAgent.model_rebuild(force=True) - # Run SWE-bench evaluation - future = runner_ray_remote.remote(self._container_counter, run_swebench_evaluation, params) - result = await future + params = SWEBenchWrapperInstanceConfig.model_validate(params_dict) + instance_id = params.instance_id - # Extract trajectory and convert to proper NeMoGym format - output_items = [] - trajectory = result.get("trajectory", []) + params.metrics_fpath.write_text("{}") + update_metrics(params.metrics_fpath, {"ray_queue_time": time.time() - params.ray_queue_timestamp}) - # Convert tools from ChatCompletion format to Response FunctionTool format - raw_tools = result.get("tools", []) - tools = convert_tools_to_function_format(raw_tools) if raw_tools else [] + if params.debug: + profiler = Profiler(name=instance_id, base_profile_dir=params.persistent_dir / "profiling") + profiler.start() - # Convert trajectory to NeMoGym output items - if trajectory: - output_items = convert_trajectory_to_output_items( - trajectory, - self.config.agent_framework, - ) + run_oh = RunOpenHandsAgent(config=params) - # If no trajectory or empty output, create a summary message - if not output_items: - output_items = [ - NeMoGymResponseOutputMessage( - id=f"msg-{problem_info.get('instance_id', 'unknown')}", - content=[ - NeMoGymResponseOutputText( - type="output_text", - text=json.dumps( - {k: v for k, v in result.items() if k not in ["trajectory", "tools"]}, indent=2 - ), - annotations=[], - ) - ], - role="assistant", - status="completed", - type="message", - ) - ] - - # Store the full result in metadata for the verify step - # Note: metadata values must be strings for NeMoGymResponse - metadata = { - "agent_framework": self.config.agent_framework, - "has_trajectory": str(trajectory is not None), - "instance_id": result.get("instance_id", problem_info.get("instance_id", "unknown")), - "instance_dir": instance_dir, - "hit_success_str": json.dumps(bool(output_items)), - "hit_empty_trajectory_str": json.dumps(not trajectory), - "hit_responses_exception_str": json.dumps(False), - } + report_file = asyncio.run(run_oh.process_single_datapoint()) - # Add evaluation results to metadata (convert to strings) - for key in ["resolved", "patch_exists", "patch_successfully_applied"]: - if key in result: - metadata[key] = str(result[key]) - - # For complex metrics, store as JSON string - if "swe-bench-metrics" in result: - metadata["swe-bench-metrics"] = json.dumps(result["swe-bench-metrics"]) - - metadata["timing_metrics"] = metrics_fpath.read_text() - - return NeMoGymResponse( - id=f"swebench-{problem_info.get('instance_id', 'unknown')}", - created_at=int(time.time()), - model=getattr(body, "model", "gpt-4.1-2025-04-14"), - object="response", - output=output_items, - parallel_tool_calls=(False if self.config.agent_framework == "swe_agent" else True), - tool_choice="auto", - tools=tools, - metadata=metadata, - ) + if params.debug: + profiler.stop() + + return report_file + + +def update_metrics(metrics_fpath: Path, update_dict: Dict[str, Any]) -> None: + with metrics_fpath.open() as f: + existing_dict = json.loads(f.read()) - except Exception as e: - print(f"SWE-bench evaluation failed: {str(e)}", flush=True) - # Return error response - error_message = NeMoGymResponseOutputMessage( - id=f"msg-{problem_info.get('instance_id', 'unknown')}-error", - content=[NeMoGymResponseOutputText(type="output_text", text=f"Error: {str(e)}", annotations=[])], - role="assistant", - status="completed", - type="message", + existing_dict = {k: v for k, v in existing_dict.items() if v is not None} + update_dict = {k: v for k, v in update_dict.items() if v is not None} + + with metrics_fpath.open("w") as f: + json.dump(existing_dict | update_dict, f) + + +class ActiveContainerCommand(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) + + process: Process + log_file: Any + log_file_path: Path + + +class RunOpenHandsAgent(BaseModel): + config: SWEBenchWrapperInstanceConfig + + def _openhands_dir_copy_from_host(self, output_file_path: Optional[str]) -> Optional[str]: + data_point = self.config.problem_info + eval_dir_in_openhands = self.config.eval_dir_in_openhands + config_file_path = self.config.openhands_config_file_path + + eval_dir_on_host = Path(self.config.openhands_setup_dir) / "OpenHands" / eval_dir_in_openhands + trajectories_root = self.config.trajectories_root + llm_completions_dir = trajectories_root / "llm_completions" / data_point["instance_id"] + trajectories_root.mkdir(parents=True, exist_ok=True) + llm_completions_dir.mkdir(parents=True, exist_ok=True) + + dest_output: Optional[str] = None + if output_file_path: + source_output = Path(output_file_path) + if not source_output.is_absolute(): + source_output = eval_dir_on_host / source_output + if not source_output.exists(): + output_candidates = sorted(eval_dir_on_host.glob("*/*/*/output.jsonl"), key=os.path.getmtime) + if not output_candidates: + raise FileNotFoundError( + f"No output.jsonl found under {eval_dir_on_host} for {data_point['instance_id']}." + ) + source_output = output_candidates[-1] + + dest_output_path = self.config.prediction_path + shutil.copy2(source_output, dest_output_path) + dest_output = str(dest_output_path) + + completion_candidates = glob.glob(str(eval_dir_on_host / "*/*/*/llm_completions/*/*.json")) + if completion_candidates: + latest_completion = max(completion_candidates, key=os.path.getmtime) + shutil.copy2( + latest_completion, + llm_completions_dir / Path(latest_completion).name, ) - return NeMoGymResponse( - id=f"swebench-{problem_info.get('instance_id', 'unknown')}-error", - created_at=int(time.time()), - model=getattr(body, "model", "gpt-4.1-2025-04-14"), - object="response", - output=[error_message], - parallel_tool_calls=False, - tool_choice="none", - tools=[], - metadata={ - "error": str(e), - "hit_success_str": json.dumps(False), - "hit_empty_trajectory_str": json.dumps((not trajectory) if "trajectory" in dir() else False), - "hit_responses_exception_str": json.dumps(True), - }, + shutil.rmtree(eval_dir_on_host, ignore_errors=True) + try: + Path(config_file_path).unlink() + except OSError: + pass + + return dest_output + + async def _start_container_command( + self, command: ExecuteContainerCommandArgs, apptainer_cmd: str + ) -> ActiveContainerCommand: + # Stream output to log file as it appears + logs_dir = self.config.persistent_dir / "apptainer_logs" + logs_dir.mkdir(exist_ok=True) + log_file_path = logs_dir / f"{self.config.instance_id}_{command.mode}.log" + log_file = open(log_file_path, "w") + + process = await asyncio.create_subprocess_shell(apptainer_cmd, stdout=log_file, stderr=log_file) + + return ActiveContainerCommand(process=process, log_file=log_file, log_file_path=log_file_path) + + async def _finish_container_command( + self, active_command: ActiveContainerCommand, command: ExecuteContainerCommandArgs + ) -> str: + data_point = self.config.problem_info + + try: + # Wait for completion with timeout + await asyncio.wait_for(active_command.process.communicate(), timeout=command.timeout) + except asyncio.TimeoutError: + if active_command.process.returncode is None: + active_command.process.kill() + await active_command.process.wait() + raise ValueError("Command timed out") + finally: + active_command.log_file.close() + + assert active_command.process.returncode == 0, ( + f"Command failed with return code {active_command.process.returncode}. Logs:\n{active_command.log_file_path.read_text()}" + ) + + # Look for the expected file + pred_files = glob.glob(command.expected_file_pattern, recursive=True) + + if len(pred_files) == 1: + return pred_files[0] + elif len(pred_files) > 1: + latest_file = max(pred_files, key=os.path.getmtime) + print( + f"Multiple outputs found for {data_point['instance_id']} " + f"({len(pred_files)}). Using latest: {latest_file}", + flush=True, + ) + return latest_file + else: + raise ValueError( + f"Expected exactly one file matching {command.expected_file_pattern} for {data_point['instance_id']}, " + f"found {len(pred_files)}." ) - async def run(self, body: SWEBenchRunRequest) -> SWEBenchVerifyResponse: - """Run and verify SWE-bench solution.""" - async with self.sem: - if self.config.debug: - print( - f"Semaphore: {self.config.concurrency - self.sem._value} / {self.config.concurrency}", flush=True + async def process_single_datapoint(self) -> Optional[Path]: + metrics = SWEBenchMetrics() + + metrics.openhands_run_time = -time.time() + metrics.generation_apptainer_spinup_time = metrics.openhands_run_time + metrics.final_eval_apptainer_spinup_time = metrics.openhands_run_time + + openhands_active_command = await self._start_container_command( + self.config.agent_command, self.config.agent_apptainer_command_str + ) + eval_active_command = await self._start_container_command( + self.config.eval_command, self.config.eval_apptainer_command_str + ) + + out_file_in_eval = await self._finish_container_command(openhands_active_command, self.config.agent_command) + out_file = self._openhands_dir_copy_from_host(output_file_path=out_file_in_eval) + + generation_apptainer_spinup_timestamp = float( + self.config.generation_apptainer_spinup_timestamp_fpath.read_text() + ) + metrics.generation_apptainer_spinup_time += generation_apptainer_spinup_timestamp + metrics.openhands_run_time += time.time() + + with open(out_file, "r") as f: + out_dict = json.loads(f.read().strip()) + + patch = out_dict["test_result"]["git_patch"] or None + patch = patch + "\n" if patch and not patch.endswith("\n") else patch + + # Create file in the SWE-bench evaluation format + self.config.output_for_eval_path.parent.mkdir(parents=True, exist_ok=True) + with self.config.output_for_eval_path.open("w") as f: + f.write( + json.dumps( + { + "model_name_or_path": out_dict["metadata"]["llm_config"]["model"], + "instance_id": out_dict["instance_id"], + "model_patch": patch, + "oh_time_metrics": out_dict["metrics"], + } ) - body.responses_create_params.metadata["container_concurrency"] = str( - self.config.concurrency - self.sem._value ) - # Fix None values in responses_create_params to use defaults - # This is needed because the pydantic model has non-Optional fields with defaults + # Dump out dot and png files from profiling on OpenHands level + if self.config.debug: + base_profile_dir = self.config.persistent_dir / "profiling" + profiling_name = "openhands" + callgrind_path = base_profile_dir / f"{profiling_name}.callgrind" + callgrind_dotfile_path = base_profile_dir / f"{profiling_name}.dot" + callgrind_graph_path = base_profile_dir / f"{profiling_name}.png" - update_dict = {} - # SWE-agent processes tool calls sequentially, OpenHands can do parallel - update_dict["parallel_tool_calls"] = False if self.config.agent_framework == "swe_agent" else True - if body.responses_create_params.tool_choice is None: - update_dict["tool_choice"] = "auto" - - # Create a copy with the fixed values if needed - fixed_params = ( - body.responses_create_params.model_copy(update=update_dict) - if update_dict - else body.responses_create_params + gprof2dot_main( + argv=f"--format=callgrind --output={callgrind_dotfile_path} -e 5 -n 5 {callgrind_path}".split() ) - # Run the evaluation - response = await self.responses(fixed_params) + (graph,) = graph_from_dot_file(callgrind_dotfile_path) + graph.write_png(callgrind_graph_path) - # Extract initial input messages from the response output and get filtered output - # These are the system/user messages that were actually sent to the agent - input_messages, filtered_output = extract_input_messages_from_trajectory(response.output) + if not patch: + metrics.patch_exists = False + update_metrics(self.config.metrics_fpath, metrics.model_dump()) + return - # Update response with filtered output (system/user messages removed) - response = response.model_copy(update={"output": filtered_output}) + with open(self.config.model_patch_path, "w") as f: + f.write(patch) - # Add the extracted input messages and tools to the params - # Note: tools should already be in the correct format from the response - params_with_input = fixed_params.model_copy( - update={ - "input": input_messages, - "tools": [t.model_dump() for t in response.tools] if response.tools else [], - } - ) + metrics.final_eval_time = -time.time() + report_file = await self._finish_container_command(eval_active_command, self.config.eval_command) + + final_eval_apptainer_spinup_timestamp = float( + self.config.final_eval_apptainer_spinup_timestamp_fpath.read_text() + ) + metrics.final_eval_apptainer_spinup_time += final_eval_apptainer_spinup_timestamp + metrics.final_eval_time += time.time() + + metrics.patch_exists = True + update_metrics(self.config.metrics_fpath, metrics.model_dump()) + + return report_file + + +######################################## +# START Server logic +######################################## + + +class SWEBenchWrapper(SimpleResponsesAPIAgent): + config: SWEBenchWrapperConfig + + _sem: Optional[Semaphore] = None + _vllm_converter: Optional[VLLMConverter] = None + _swe_bench_wrapper_server_config: Optional[SWEBenchWrapperServerConfig] = None + + model_config = ConfigDict(arbitrary_types_allowed=True) + + ######################################## + # START Init + ######################################## + + def model_post_init(self, context: Any) -> None: + run_session_id = f"{int(time.time() * 1000)}_{str(uuid.uuid4())[:8]}" + workspace_root = Path(__file__).parent + self._swe_bench_wrapper_server_config = SWEBenchWrapperServerConfig( + run_session_id=run_session_id, + base_results_dir=workspace_root / f"swebench_results_{run_session_id}", + ng_global_config_dict_str=shlex.quote(OmegaConf.to_yaml(get_global_config_dict())), + model_server_name=self.config.model_server.name, + openhands_setup_dir=OpenHandsHarnessProcessor(config=self.config).setup(), + swebench_setup_dir=SweBenchDatasetProcessor(config=self.config).setup(), + r2e_gym_setup_dir=R2EGymDatasetProcessor(config=self.config).setup(), + ) + + self._sem = Semaphore(self.config.concurrency) + self._vllm_converter = VLLMConverter(return_token_id_information=True) + + return super().model_post_init(context) + + ######################################## + # START Results processing logic + ######################################## + + def get_openhands_trajectory_from_completions(self, trajectories_dir: Path, instance_id: str) -> tuple: + """ + This reads the trajectories directly dumped by OpenHands. + """ + messages, tools = [], [] + + completions_dir = trajectories_dir / instance_id / "llm_completions" / instance_id + if not completions_dir.exists(): + print(f"No llm_completions directory found: {completions_dir}", flush=True) + return messages, tools + + completion_files = sorted(completions_dir.glob("*.json")) + if not completion_files: + print(f"No completion files found in: {completions_dir}", flush=True) + return messages, tools + + last_file = completion_files[-1] + + with open(last_file, "r") as f: + data = json.load(f) + + messages = data["messages"] + provider_specific_fields = data.get("provider_specific_fields", {}) + final_assistant_message = data["response"]["choices"][0]["message"] - # Extract metrics from response metadata - metadata = response.metadata or {} - # Remove metadata from response after extracting metrics - response = response.model_copy(update={"metadata": None}) + for key in ["prompt_token_ids", "generation_token_ids", "generation_log_probs"]: + if key in provider_specific_fields: + final_assistant_message[key] = provider_specific_fields[key] - # Parse metrics from JSON string if present - metrics = json.loads(metadata.get("swe-bench-metrics", "{}")) if "swe-bench-metrics" in metadata else {} + if final_assistant_message.get("content") or final_assistant_message.get("tool_calls"): + messages.append(final_assistant_message) - # Extract individual metrics with proper type conversion - resolved = metrics.get("resolved") or (metadata.get("resolved") == "True") - patch_exists = metrics.get("patch_exists") or (metadata.get("patch_exists") == "True") - patch_applied = metrics.get("patch_successfully_applied") or ( - metadata.get("patch_successfully_applied") == "True" + tools = data.get("kwargs", {}).get("tools", []) + + return messages, tools + + ######################################## + # START Main methods + ######################################## + + def _find_container(self, data_point: dict) -> str: + """Find the container file using multiple strategies (Exact match > Fuzzy match). + + Strategies: + 1. Replace "__" with "_1776_" (Original case, then Lowercase) + 2. Replace "__" with "_s_" (Original case, then Lowercase) + 3. Fuzzy search directory for .sif files matching above patterns. + + Returns: + str: Path to the container file. + + Raises: + FileNotFoundError: If no matching container file is found. + """ + instance_id = data_point["instance_id"] + container_formatters = data_point["container_formatter"] + + if isinstance(container_formatters, str): + container_formatters = [container_formatters] + + if "R2E-Gym" in data_point["dataset_name"]: + instance_id_modified = re.sub( + r"[^_]+__([^-]+)-", lambda m: m.group(1).lower() + "_final_", data_point["instance_id"] ) + for container_formatter in container_formatters: + container_name = container_formatter.format(instance_id=instance_id_modified) + if os.path.exists(container_name): + # print(f"container found: {container_name}", flush=True) + # print(f"container formatter: {container_formatter}", flush=True) + return container_name + + replacements = ["_1776_", "_s_"] + + # Generate all candidate IDs in order of priority + candidate_ids = [instance_id] + for replacement in replacements: + replaced_id = instance_id.replace("__", replacement) + candidate_ids.append(replaced_id) + candidate_ids.append(replaced_id.lower()) + + # Phase 1: Exact Matches - try all container formatters + for container_formatter in container_formatters: + for candidate_id in candidate_ids: + path = container_formatter.format(instance_id=candidate_id) + if os.path.exists(path): + return path + + # Phase 2: Fuzzy Search - try all container formatters + search_terms = [instance_id, instance_id.lower()] + candidate_ids + + for container_formatter in container_formatters: + # Define the default fallback path (Strategy 1, original case) + fallback_path = container_formatter.format(instance_id=instance_id.replace("__", replacements[0])) + container_dir = os.path.dirname(fallback_path) + + if os.path.exists(container_dir): + for term in search_terms: + pattern = os.path.join(container_dir, f"*{term}*.sif") + matches = glob.glob(pattern) + if matches: + return matches[0] + else: + print(f"Container directory {container_dir} does not exist", flush=True) + + # Phase 3: Fallback + tried_paths = [] + for container_formatter in container_formatters: + for candidate_id in candidate_ids: + tried_paths.append(container_formatter.format(instance_id=candidate_id)) + + raise FileNotFoundError( + f"No container file found for instance_id {instance_id}. " + f"Tried the following candidate IDs: {candidate_ids}. " + f"Searched in paths: {tried_paths}." + ) + + def _build_apptainer_command( + self, params: SWEBenchWrapperInstanceConfig, command: ExecuteContainerCommandArgs + ) -> str: + dataset_path_to_mount = str(params.instance_dataset_path) + data_point = params.problem_info + + # Fix localhost URLs not working sometimes + container_commands = [] + container_commands.append("echo '127.0.0.1 localhost' >/etc/hosts") + + # Build mount arguments + mount_args = [ + f"--mount type=bind,src={params.persistent_dir},dst=/trajectories_mount", + ] + + # Add OpenHands setup directory mount if available (for OpenHands) + # Mount the entire setup directory at both /openhands_setup and its original absolute path + # This is needed because poetry and other tools have hardcoded absolute paths + mount_args.append(f"--mount type=bind,src={params.openhands_setup_dir},dst=/openhands_setup,ro") + mount_args.append(f"--mount type=bind,src={params.openhands_setup_dir},dst={params.openhands_setup_dir},ro") + # Mount only the venv and miniforge as read-only to prevent mutation while keeping the rest writable + venv_path = Path(params.openhands_setup_dir) / "OpenHands/.venv" + mount_args.append(f"--mount type=bind,src={venv_path},dst=/openhands_setup/OpenHands/.venv,ro") + mount_args.append(f"--mount type=bind,src={venv_path},dst={venv_path},ro") + + mount_args.extend( + [ + # make everything in OpenHands read-only + f"--mount type=bind,src={params.openhands_setup_dir}/OpenHands,dst=/openhands_setup/OpenHands,ro", + f"--mount type=bind,src={params.openhands_setup_dir}/OpenHands/.eval_sessions,dst=/openhands_setup/OpenHands/.eval_sessions", + f"--mount type=bind,src={params.openhands_setup_dir}/OpenHands/.eval_sessions,dst={params.openhands_setup_dir}/OpenHands/.eval_sessions", + f"--mount type=bind,src={params.openhands_setup_dir}/OpenHands/logs,dst=/openhands_setup/OpenHands/logs", + f"--mount type=bind,src={params.openhands_setup_dir}/OpenHands/logs,dst={params.openhands_setup_dir}/OpenHands/logs", + f"--mount type=bind,src={params.openhands_setup_dir}/OpenHands/evaluation/oh,dst=/openhands_setup/OpenHands/evaluation/oh", + f"--mount type=bind,src={params.openhands_setup_dir}/OpenHands/evaluation/oh,dst={params.openhands_setup_dir}/OpenHands/evaluation/oh", + # Data + f"--mount type=bind,src={dataset_path_to_mount},dst=/root/dataset/data.jsonl", + ] + ) + + miniforge3_path = Path(params.openhands_setup_dir) / "miniforge3" + mount_args.append(f"--mount type=bind,src={miniforge3_path},dst=/openhands_setup/miniforge3,ro") + mount_args.append(f"--mount type=bind,src={miniforge3_path},dst={miniforge3_path},ro") + + # Add SWE-bench setup directory mount if available (for evaluation) + if command.mode == "eval" and data_point["dataset_name"] != "nv-internal-1": + # Mount the entire setup directory at both /swebench_setup and its original absolute path + # This is needed because uv venv has hardcoded absolute paths + # print( + # f"Mounting pre-built SWE-bench from: {self.swebench_setup_dir}", + # flush=True, + # ) + mount_args.append(f"--mount type=bind,src={params.swebench_setup_dir},dst=/swebench_setup") + mount_args.append(f"--mount type=bind,src={params.swebench_setup_dir},dst={params.swebench_setup_dir}") + mount_args.append(f"--mount type=bind,src={dataset_path_to_mount},dst=/root/dataset/data.jsonl") + + if command.mode == "eval" and data_point["dataset_name"] == "nv-internal-1": + run_script_path = params.persistent_dir / "run_script.sh" + parsing_script_path = params.persistent_dir / "parsing_script.py" + model_patch_path = params.persistent_dir / "patch.diff" + + mount_args.append(f"--mount type=bind,src={run_script_path},dst=/root/run_script.sh") + mount_args.append(f"--mount type=bind,src={parsing_script_path},dst=/root/parsing_script.py") + mount_args.append(f"--mount type=bind,src={model_patch_path},dst=/root/patch.diff") + + if command.mode == "eval" and "R2E-Gym" in data_point["dataset_name"]: + # Mount the entire setup directory at both /r2egym_setup and its original absolute path + # This is needed because uv venv has hardcoded absolute paths in its wrappers + # print(f"Mounting R2E-Gym setup directory from: {self.r2e_gym_setup_dir}", flush=True) + mount_args.append(f"--mount type=bind,src={params.r2e_gym_setup_dir},dst=/r2egym_setup") + mount_args.append(f"--mount type=bind,src={params.r2e_gym_setup_dir},dst={params.r2e_gym_setup_dir}") + mount_args.append(f"--mount type=bind,src={dataset_path_to_mount},dst=/root/dataset/data.jsonl") + + if command.mode == "agent" and "R2E-Gym" in data_point["dataset_name"]: + # Remove R2E-Gym test-related files. + for root_dir in ["", "/root", "/testbed"]: + container_commands.append( + # /r2e_tests contains evaluation tests that the agent should not see. + f"rm -rf {root_dir}/r2e_tests && " + # run_tests.sh launches the tests in /r2e_tests, so the agent should not see this either. + # We check that it contains the substring "r2e_tests" + # to avoid accidentally deleting an unrelated file with that name. + f"if grep -qs r2e_tests {root_dir}/run_tests.sh; then rm -rf {root_dir}/run_tests.sh; fi" + ) + container_commands.append(command.command) + combined_command = " && ".join(container_commands) + + mount_str = " ".join(mount_args) + + # Launch Apptainer container and execute the command + apptainer_cmd = ( + f"apptainer exec --writable-tmpfs --cleanenv --pid --no-mount home,tmp,bind-paths " + f"{mount_str} " + f" {params.container} bash -c {shlex.quote(combined_command)}" + ) + memory_limit_mb = params.apptainer_memory_limit_mb + if memory_limit_mb is not None and memory_limit_mb > 0: + memory_limit_kb = int(memory_limit_mb) * 1024 + apptainer_cmd = f"ulimit -v {memory_limit_kb} && {apptainer_cmd}" + + return apptainer_cmd + + def _setup_params( + self, body: NeMoGymResponseCreateParamsNonStreaming + ) -> Tuple[SWEBenchWrapperInstanceConfig, BaseDatasetHarnessProcessor]: + problem_info = body.metadata | {"container_formatter": self.config.container_formatter} + instance_id = problem_info.get("instance_id", "unknown") + + # Create persistent directory for I/O and logs in local workspace + instance_dir = f"{instance_id}_{int(time.time() * 1000)}_{str(uuid.uuid4())[:8]}" + persistent_dir = self._swe_bench_wrapper_server_config.base_results_dir / instance_dir + persistent_dir.mkdir(parents=True, exist_ok=True) - reward = 1.0 if resolved else 0.0 + agent_run_id = f"{instance_id}_{int(time.time())}_{str(uuid.uuid4())[:8]}" + + # To avoid making HF dataset API calls, we write the instance dictionary to a file and mount it in the container. + instance_dataset_dir = persistent_dir / "instance_datasets" + instance_dataset_dir.mkdir(parents=True, exist_ok=True) + instance_dataset_path = instance_dataset_dir / f"{agent_run_id}.jsonl" + instance_dict = json.loads(problem_info["instance_dict"]) + if "repo" in instance_dict and "repo_name" not in instance_dict: + instance_dict["repo_name"] = instance_dict["repo"] + with open(instance_dataset_path, "w") as f: + f.write(json.dumps(instance_dict) + "\n") + + trajectories_root = persistent_dir / "trajectories" / instance_id + output_for_eval_mounted_path = ( + Path("/trajectories_mount") / "trajectories" / instance_id / "output_for_eval.jsonl" + ) + output_for_eval_path = trajectories_root / "output_for_eval.jsonl" + prediction_path = trajectories_root / "output.jsonl" + + # Map from Responses to OpenHands + inference_params = {} + for param, key in [ + ("temperature", "temperature"), + ("top_p", "top_p"), + ("max_output_tokens", "tokens_to_generate"), + ]: + value = getattr(body, param, None) + if value is not None: + inference_params[key] = value + + container = self._find_container(problem_info) + + eval_dir_in_openhands = f"evaluation/oh/{agent_run_id}" + openhands_config_file_path = f"/tmp/config_{agent_run_id}.toml" + + agent_script_name = f"agent_script_{agent_run_id}.sh" + agent_script_path = persistent_dir / agent_script_name + + params: SWEBenchWrapperInstanceConfig = SWEBenchWrapperInstanceConfig( + **self.config.model_dump(), + **self._swe_bench_wrapper_server_config.model_dump(), + problem_info=problem_info, + body=body, + persistent_dir=persistent_dir, + metrics_fpath=persistent_dir / "nemo_gym_metrics.json", + ray_queue_timestamp=time.time(), + inference_params=inference_params, + agent_run_id=agent_run_id, + instance_dataset_path=instance_dataset_path, + trajectories_root=trajectories_root, + output_for_eval_mounted_path=output_for_eval_mounted_path, + output_for_eval_path=output_for_eval_path, + prediction_path=prediction_path, + model_patch_path=persistent_dir / "patch.diff", + container=container, + eval_dir_in_openhands=eval_dir_in_openhands, + openhands_config_file_path=openhands_config_file_path, + agent_script_path=agent_script_path, + final_eval_apptainer_spinup_timestamp_fpath=persistent_dir / "final_eval_apptainer_spinup_timestamp", + final_eval_apptainer_spinup_timestamp_mounted_fpath=Path("/trajectories_mount") + / "final_eval_apptainer_spinup_timestamp", + generation_apptainer_spinup_timestamp_fpath=persistent_dir / "generation_apptainer_spinup_timestamp", + generation_apptainer_spinup_timestamp_mounted_fpath=Path("/trajectories_mount") + / "generation_apptainer_spinup_timestamp", + ) + + if params.problem_info["dataset_name"] == "nv-internal-1": + dataset_processor = NVInternalDatasetProcessor(config=params) + elif "R2E-Gym" in params.problem_info["dataset_name"]: + dataset_processor = R2EGymDatasetProcessor(config=params) + else: + dataset_processor = SweBenchDatasetProcessor(config=params) - hit_metrics = {k.removesuffix("_str"): json.loads(v) for k, v in metadata.items() if k.startswith("hit_")} + params.eval_command = dataset_processor.get_run_command() + params.eval_apptainer_command_str = self._build_apptainer_command(params, params.eval_command) + + params.agent_command = OpenHandsHarnessProcessor(config=params).get_run_command() + params.agent_apptainer_command_str = self._build_apptainer_command(params, params.agent_command) + params.agent_script = params.agent_script_path.read_text() + + return params, dataset_processor + + async def responses(self, body: NeMoGymResponseCreateParamsNonStreaming = Body()) -> NeMoGymResponse: + params, dataset_processor = self._setup_params(body) + + maybe_report_file = await runner_ray_remote.remote(params.model_dump()) + metrics_to_update = dict() + + if maybe_report_file: + dataset_processor.postprocess_after_eval_run(maybe_report_file) + + report = json.loads(Path(maybe_report_file).read_text()) + resolved = report[params.instance_id]["resolved"] + metrics_to_update["resolved"] = resolved + else: + metrics_to_update["resolved"] = False + + trajectories_dir = params.persistent_dir / "trajectories" + chat_completions_trajectory, chat_completions_tools = self.get_openhands_trajectory_from_completions( + trajectories_dir, params.instance_id + ) + + tools = [ + FunctionTool.model_validate(tool["function"] | {"type": "function"}) for tool in chat_completions_tools + ] + responses_items = self._vllm_converter.chat_completions_messages_to_responses_items( + chat_completions_trajectory + ) + input_items, output_items = split_responses_input_output_items(responses_items) + + update_metrics(params.metrics_fpath, metrics_to_update) + + return NeMoGymResponse( + id=f"swebench-{params.instance_id}", + created_at=int(time.time()), + model=body.model, + object="response", + output=output_items, + parallel_tool_calls=body.parallel_tool_calls, + tool_choice=body.tool_choice, + tools=tools, + metadata={ + "input": json.dumps([i.model_dump() for i in input_items]), + "metrics": params.metrics_fpath.read_text(), + "instance_config": params.model_dump_json(), + }, + ) + + async def run(self, body: BaseRunRequest) -> SWEBenchVerifyResponse: + async with self._sem: + body.responses_create_params.parallel_tool_calls = True + body.responses_create_params.tool_choice = "auto" + + response = await self.responses(body.responses_create_params) + + metadata, response.metadata = response.metadata, None + responses_create_params = body.responses_create_params.model_dump() | { + "input": json.loads(metadata["input"]), + "tools": [t.model_dump() for t in response.tools] if response.tools else [], + } + metrics = SWEBenchMetrics.model_validate_json(metadata["metrics"]) - # Build verification response with top-level numeric fields for statistics return SWEBenchVerifyResponse( - responses_create_params=params_with_input, + responses_create_params=responses_create_params, response=response, - reward=reward, - resolved=1.0 if resolved else 0.0, - patch_exists=1.0 if patch_exists else 0.0, - patch_successfully_applied=1.0 if patch_applied else 0.0, - swebench_metrics=metrics, - metadata={ - "instance_id": metadata.get("instance_id", "unknown"), - "agent_framework": self.config.agent_framework, - "patch_exists": patch_exists, - "patch_successfully_applied": patch_applied, - "resolved": resolved, - }, - **json.loads(metadata["timing_metrics"]), - **hit_metrics, + reward=1.0 if metrics.resolved else 0.0, + **metrics.model_dump(), + instance_config=SWEBenchWrapperInstanceConfig.model_validate_json( + metadata["instance_config"] + ).model_dump(), ) diff --git a/responses_api_agents/swe_agents/data/example.jsonl b/responses_api_agents/swe_agents/data/example.jsonl index 28426567b..02a326896 100644 --- a/responses_api_agents/swe_agents/data/example.jsonl +++ b/responses_api_agents/swe_agents/data/example.jsonl @@ -1,5 +1 @@ -{"responses_create_params": {"input": [], "metadata": {"instance_id": "getmoto__moto-7365", "base_commit": "7f6c9cb1deafb280fe7fcc7551c38e397f11a706", "dataset_name": "SWE-Gym/SWE-Gym", "split": "train", "problem_statement": "DynamoDB's `update_item` performs floating-point arithmetic with mock table created via `boto3`\nWhen using `moto.mock_aws` to create a `pytest` fixture for a DynamoDB table created with `boto3`, it appears that the `update_item` operation called with an `ADD` expression performs floating-point arithmetic rather than `Decimal` arithmetic.\r\n\r\nI've created a repo at https://github.com/jtherrmann/moto-issue with a minimal reproducible example of this issue. The mock table is configured in [`conftest.py`](https://github.com/jtherrmann/moto-issue/blob/main/tests/conftest.py) and the unit tests are in [`test_update_item.py`](https://github.com/jtherrmann/moto-issue/blob/main/tests/test_update_item.py).\r\n\r\nThe `test_update_item_bad` unit test fails with:\r\n\r\n```\r\n{'id': 'foo', 'amount': Decimal('11.700000000000003')} != {'id': 'foo', 'amount': Decimal('11.7')}\r\n```\r\n\r\nThis demonstrates that the mocked `update_item` operation appears to be performing floating-point arithmetic and then rounding the result, given that `Decimal(100 - 88.3)` evaluates to `Decimal('11.7000000000000028421709430404007434844970703125')`, which rounds to `Decimal('11.700000000000003')`.\r\n\r\nNote that the `test_update_item_good` unit test passes. I would guess that arithmetic performed with smaller quantities avoids the error, though I'm not sure.\r\n\r\nThe repo also provides [`create_table.py`](https://github.com/jtherrmann/moto-issue/blob/main/create_table.py) and [`update_item.py`](https://github.com/jtherrmann/moto-issue/blob/main/update_item.py) scripts that can be run to create a real DynamoDB table and perform the same `update_item` operation as the failing unit test, demonstrating that this issue does not occur with real DynamoDB operations.\r\n\r\nI reproduced the issue using Python 3.9.18 on Debian GNU/Linux 12 (bookworm), in a `mamba` environment with requirements installed via `pip` from PyPI. Output of `mamba list | grep -e boto -e moto -e pytest`:\r\n\r\n```\r\nboto3 1.34.43 pypi_0 pypi\r\nbotocore 1.34.44 pypi_0 pypi\r\nmoto 5.0.1 pypi_0 pypi\r\npytest 8.0.0 pypi_0 pypi\r\n```\r\n\r\nThe [README](https://github.com/jtherrmann/moto-issue?tab=readme-ov-file#moto-issue) included with my repo provides instructions for installing dependencies and running the example code.\n", "golden_patch": "diff --git a/moto/dynamodb/models/dynamo_type.py b/moto/dynamodb/models/dynamo_type.py\n--- a/moto/dynamodb/models/dynamo_type.py\n+++ b/moto/dynamodb/models/dynamo_type.py\n@@ -1,6 +1,6 @@\n import base64\n import copy\n-import decimal\n+from decimal import Decimal\n from typing import Any, Dict, List, Optional, Union\n \n from boto3.dynamodb.types import TypeDeserializer, TypeSerializer\n@@ -100,9 +100,14 @@ def __add__(self, other: \"DynamoType\") -> \"DynamoType\":\n if self.type != other.type:\n raise TypeError(\"Different types of operandi is not allowed.\")\n if self.is_number():\n- self_value = float(self.value) if \".\" in self.value else int(self.value)\n- other_value = float(other.value) if \".\" in other.value else int(other.value)\n- return DynamoType({DDBType.NUMBER: f\"{self_value + other_value}\"})\n+ self_value: Union[Decimal, int] = (\n+ Decimal(self.value) if \".\" in self.value else int(self.value)\n+ )\n+ other_value: Union[Decimal, int] = (\n+ Decimal(other.value) if \".\" in other.value else int(other.value)\n+ )\n+ total = self_value + other_value\n+ return DynamoType({DDBType.NUMBER: f\"{total}\"})\n else:\n raise IncorrectDataType()\n \n@@ -385,12 +390,7 @@ def update_with_attribute_updates(self, attribute_updates: Dict[str, Any]) -> No\n if set(update_action[\"Value\"].keys()) == set([\"N\"]):\n existing = self.attrs.get(attribute_name, DynamoType({\"N\": \"0\"}))\n self.attrs[attribute_name] = DynamoType(\n- {\n- \"N\": str(\n- decimal.Decimal(existing.value)\n- + decimal.Decimal(new_value)\n- )\n- }\n+ {\"N\": str(Decimal(existing.value) + Decimal(new_value))}\n )\n elif set(update_action[\"Value\"].keys()) == set([\"SS\"]):\n existing = self.attrs.get(attribute_name, DynamoType({\"SS\": {}}))\n", "hints_text": "", "test_patch": "diff --git a/tests/test_dynamodb/test_dynamodb_update_expressions.py b/tests/test_dynamodb/test_dynamodb_update_expressions.py\n--- a/tests/test_dynamodb/test_dynamodb_update_expressions.py\n+++ b/tests/test_dynamodb/test_dynamodb_update_expressions.py\n@@ -1,3 +1,5 @@\n+from decimal import Decimal\n+\n import boto3\n import pytest\n \n@@ -40,3 +42,50 @@ def test_update_different_map_elements_in_single_request(table_name=None):\n ExpressionAttributeValues={\":MyCount\": 5},\n )\n assert table.get_item(Key={\"pk\": \"example_id\"})[\"Item\"][\"MyTotalCount\"] == 5\n+\n+\n+@pytest.mark.aws_verified\n+@dynamodb_aws_verified()\n+def test_update_item_add_float(table_name=None):\n+ table = boto3.resource(\"dynamodb\", \"us-east-1\").Table(table_name)\n+\n+ # DECIMAL - DECIMAL\n+ table.put_item(Item={\"pk\": \"foo\", \"amount\": Decimal(100), \"nr\": 5})\n+ table.update_item(\n+ Key={\"pk\": \"foo\"},\n+ UpdateExpression=\"ADD amount :delta\",\n+ ExpressionAttributeValues={\":delta\": -Decimal(\"88.3\")},\n+ )\n+ assert table.scan()[\"Items\"][0][\"amount\"] == Decimal(\"11.7\")\n+\n+ # DECIMAL + DECIMAL\n+ table.update_item(\n+ Key={\"pk\": \"foo\"},\n+ UpdateExpression=\"ADD amount :delta\",\n+ ExpressionAttributeValues={\":delta\": Decimal(\"25.41\")},\n+ )\n+ assert table.scan()[\"Items\"][0][\"amount\"] == Decimal(\"37.11\")\n+\n+ # DECIMAL + INT\n+ table.update_item(\n+ Key={\"pk\": \"foo\"},\n+ UpdateExpression=\"ADD amount :delta\",\n+ ExpressionAttributeValues={\":delta\": 6},\n+ )\n+ assert table.scan()[\"Items\"][0][\"amount\"] == Decimal(\"43.11\")\n+\n+ # INT + INT\n+ table.update_item(\n+ Key={\"pk\": \"foo\"},\n+ UpdateExpression=\"ADD nr :delta\",\n+ ExpressionAttributeValues={\":delta\": 1},\n+ )\n+ assert table.scan()[\"Items\"][0][\"nr\"] == Decimal(\"6\")\n+\n+ # INT + DECIMAL\n+ table.update_item(\n+ Key={\"pk\": \"foo\"},\n+ UpdateExpression=\"ADD nr :delta\",\n+ ExpressionAttributeValues={\":delta\": Decimal(\"25.41\")},\n+ )\n+ assert table.scan()[\"Items\"][0][\"nr\"] == Decimal(\"31.41\")\n", "repo": "getmoto/moto", "version": "5.0", "created_at": "2024-02-19 20:29:03"}, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "temperature": 0.6, "top_p": 0.95}} -{"responses_create_params": {"input": [], "metadata": {"instance_id": "getmoto__moto-6920", "base_commit": "2021e564fafcdaa701b53de49bd580c8691a5fcc", "dataset_name": "SWE-Gym/SWE-Gym", "split": "train", "problem_statement": "Lambda publish_layer_version function failed due to the wrong implementation\n## Reporting Bugs\r\n\r\nWhen you run ``publish_layer_version``\r\n\r\n```\r\nlambda_client.publish_layer_version(\r\n LayerName=\"my_layer\",\r\n Content=dict(\r\n S3Bucket=\"my-bucket\",\r\n S3Key=\"my-key.zip\",\r\n )\r\n)\r\n```\r\n\r\nIt raises this error:\r\n\r\n```\r\n File \"/Users/myusername/Documents/GitHub/aws_resource_search-project/.venv/lib/python3.8/site-packages/moto/core/botocore_stubber.py\", line 61, in __call__\r\n status, headers, body = response_callback(\r\n File \"/Users/myusername/Documents/GitHub/aws_resource_search-project/.venv/lib/python3.8/site-packages/moto/core/responses.py\", line 261, in _inner\r\n return getattr(cls(), to_call.__name__)(request, full_url, headers)\r\n File \"/Users/myusername/Documents/GitHub/aws_resource_search-project/.venv/lib/python3.8/site-packages/moto/awslambda/responses.py\", line 101, in layers_versions\r\n return self._publish_layer_version()\r\n File \"/Users/myusername/Documents/GitHub/aws_resource_search-project/.venv/lib/python3.8/site-packages/moto/awslambda/responses.py\", line 548, in _publish_layer_version\r\n config = layer_version.get_layer_version()\r\n File \"/Users/myusername/Documents/GitHub/aws_resource_search-project/.venv/lib/python3.8/site-packages/moto/awslambda/models.py\", line 376, in get_layer_version\r\n \"CodeSha256\": self.code_sha_256,\r\nAttributeError: 'LayerVersion' object has no attribute 'code_sha_256'\r\n```\r\n\r\nIt is because ``moto`` uses the ``get_layer_version`` function to create the response for ``publish_layer_version``. However, the ``publish_layer_version`` failed to calculate code_sha_256. I checked the ``publish_layer_version`` logic, there's no such logic that get the content from the fake s3 bucket then calculate the sha_256 of the content. I think we should add the code_sha_256 logic to [THIS function](https://github.com/getmoto/moto/blob/master/moto/awslambda/models.py#L1846)\r\n\r\n\n", "golden_patch": "diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py\n--- a/moto/awslambda/models.py\n+++ b/moto/awslambda/models.py\n@@ -371,6 +371,11 @@ def __init__(self, spec: Dict[str, Any], account_id: str, region: str):\n self.code_sha_256,\n self.code_digest,\n ) = _s3_content(key)\n+ else:\n+ self.code_bytes = b\"\"\n+ self.code_size = 0\n+ self.code_sha_256 = \"\"\n+ self.code_digest = \"\"\n \n @property\n def arn(self) -> str:\n", "hints_text": "Hi @MacHu-GWU, that attribute should be calculated inside the `LayerVersion`-class:\r\nhttps://github.com/getmoto/moto/blob/368fa07ec35aa6806c839a1f4883426159179127/moto/awslambda/models.py#L371\r\n\r\nIf the S3 file exists, it will use that information.\r\nIf it does not exist, it will throw an error (`The specified bucket does not exist`)\r\n\r\nBut I'm guessing you're running this code with `VALIDATE_LAMBDA_S3=false`? Then it won't throw an error, and it will try to continue.\r\n\r\nI'll raise a PR to just set these attributes to `b\"\"` if there the S3-file does not exist (and `VALIDATE_LAMBDA_S3` is not set).", "test_patch": "diff --git a/tests/test_awslambda/test_lambda_layers.py b/tests/test_awslambda/test_lambda_layers.py\n--- a/tests/test_awslambda/test_lambda_layers.py\n+++ b/tests/test_awslambda/test_lambda_layers.py\n@@ -1,10 +1,12 @@\n import boto3\n+import os\n import pytest\n \n from botocore.exceptions import ClientError\n from freezegun import freeze_time\n-from moto import mock_lambda, mock_s3\n+from moto import mock_lambda, mock_s3, settings\n from moto.core import DEFAULT_ACCOUNT_ID as ACCOUNT_ID\n+from unittest import mock, SkipTest\n from uuid import uuid4\n \n from .utilities import get_role_name, get_test_zip_file1\n@@ -31,6 +33,20 @@ def test_publish_lambda_layers__without_content():\n assert err[\"Message\"] == \"Missing Content\"\n \n \n+@mock_lambda\n+@mock.patch.dict(os.environ, {\"VALIDATE_LAMBDA_S3\": \"false\"})\n+def test_publish_layer_with_unknown_s3_file():\n+ if not settings.TEST_DECORATOR_MODE:\n+ raise SkipTest(\"Can only set env var in DecoratorMode\")\n+ conn = boto3.client(\"lambda\", _lambda_region)\n+ content = conn.publish_layer_version(\n+ LayerName=str(uuid4())[0:6],\n+ Content=dict(S3Bucket=\"my-bucket\", S3Key=\"my-key.zip\"),\n+ )[\"Content\"]\n+ assert content[\"CodeSha256\"] == \"\"\n+ assert content[\"CodeSize\"] == 0\n+\n+\n @mock_lambda\n @mock_s3\n @freeze_time(\"2015-01-01 00:00:00\")\n", "repo": "getmoto/moto", "version": "4.2", "created_at": "2023-10-15 20:33:23"}, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "temperature": 0.6, "top_p": 0.95}} -{"responses_create_params": {"input": [], "metadata": {"instance_id": "getmoto__moto-5876", "base_commit": "6d41ad72e09b49f61e54d47880f8a65026e7c0e4", "dataset_name": "SWE-Gym/SWE-Gym", "split": "train", "problem_statement": "Cognito - No validation that there isn't already an existing user with the same username in admin_update_user_attributes\nHi,\r\n\r\nSorry for the spam, just raising another issue for a potential enhancement. There is currently no validation on the `admin_update_user_attributes` function to check that the email address we are trying to update for a user isn't going to cause a conflict.\r\n\r\nIf you try to update the email address of a user to one that already exists in the user pool, a `ClientError` exception should be raised with the code `AliasExistsException`.\r\n\r\nThis piece of code should raise the exception:\r\n```\r\ncognito_client.admin_update_user_attributes(\r\n UserPoolId=user_pool_id,\r\n Username=user_sub,\r\n UserAttributes=[{\"Name\": \"email\", \"Value\": email_address_of_existing_user}],\r\n)\r\n```\r\n\r\nConsidering how bad the Cognito service is, I have a feeling it might be dependent on the configuration of the User Pool and won't always raise an exception depending on how it's configured. You might require your user pool to be configured with the following to throw this type of exception: `UsernameAttributes=[\"email\"]`. Not 100% sure though.\n", "golden_patch": "diff --git a/moto/cognitoidp/exceptions.py b/moto/cognitoidp/exceptions.py\n--- a/moto/cognitoidp/exceptions.py\n+++ b/moto/cognitoidp/exceptions.py\n@@ -2,6 +2,13 @@\n from typing import Optional\n \n \n+class AliasExistsException(JsonRESTError):\n+ def __init__(self) -> None:\n+ super().__init__(\n+ \"AliasExistsException\", \"An account with the given email already exists.\"\n+ )\n+\n+\n class ResourceNotFoundError(JsonRESTError):\n def __init__(self, message: Optional[str]):\n super().__init__(error_type=\"ResourceNotFoundException\", message=message or \"\")\ndiff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py\n--- a/moto/cognitoidp/models.py\n+++ b/moto/cognitoidp/models.py\n@@ -11,6 +11,7 @@\n from moto.core import BaseBackend, BackendDict, BaseModel\n from moto.moto_api._internal import mock_random as random\n from .exceptions import (\n+ AliasExistsException,\n GroupExistsException,\n NotAuthorizedError,\n ResourceNotFoundError,\n@@ -1636,6 +1637,9 @@ def admin_update_user_attributes(\n ) -> None:\n user = self.admin_get_user(user_pool_id, username)\n \n+ email = self._find_attr(\"email\", attributes)\n+ self._verify_email_is_not_used(user_pool_id, email)\n+\n user.update_attributes(attributes)\n \n def admin_delete_user_attributes(\n@@ -2031,11 +2035,32 @@ def update_user_attributes(\n _, username = user_pool.access_tokens[access_token]\n user = self.admin_get_user(user_pool.id, username)\n \n+ email = self._find_attr(\"email\", attributes)\n+ self._verify_email_is_not_used(user_pool.id, email)\n+\n user.update_attributes(attributes)\n return\n \n raise NotAuthorizedError(access_token)\n \n+ def _find_attr(self, name: str, attrs: List[Dict[str, str]]) -> Optional[str]:\n+ return next((a[\"Value\"] for a in attrs if a[\"Name\"] == name), None)\n+\n+ def _verify_email_is_not_used(\n+ self, user_pool_id: str, email: Optional[str]\n+ ) -> None:\n+ if not email:\n+ # We're not updating emails\n+ return\n+ user_pool = self.describe_user_pool(user_pool_id)\n+ if \"email\" not in user_pool.extended_config.get(\"UsernameAttributes\", []):\n+ # email is not used as a username - duplicate emails are allowed\n+ return\n+\n+ for user in user_pool.users.values():\n+ if user.attribute_lookup.get(\"email\", \"\") == email:\n+ raise AliasExistsException\n+\n \n class RegionAgnosticBackend:\n # Some operations are unauthenticated\n", "hints_text": "All good @JorisLimousin - every enhancement is useful!\nhi, I am interested in fixing this issue. it will be a great opportunity to fix this issue and contribute to this project if you assign me this issue . @JorisLimousin @bblommers @corasaurus-hex @olleolleolle @JackDanger \nDone @ArpanShah2k! We have some documentation on how to get started: http://docs.getmoto.org/en/latest/docs/contributing/index.html\r\nPlease let us know if you run into any issues.\nThank you sir for your kind consideration. I will go through this documentation and start working on the enhancement. I'll approach if I need help.\nRespected sir,\nI have read the documentation and all. but i am facing issues in\ninstallation of moto in my laptop.\n\nthe path i went through is :\n1) install python 3.10.8 will all its dependencies like pip, idle , etc.\n2) install docker ( facing issues).\n2) set path in cmd.\n3) run commands in python and cmd to install moto. ( facing issues).\n\n\n\ncan you please help me out with this .\n\n\n\nOn Mon, Sep 12, 2022 at 2:55 PM Bert Blommers ***@***.***>\nwrote:\n\n> Done @ArpanShah2k ! We have some\n> documentation on how to get started:\n> http://docs.getmoto.org/en/latest/docs/contributing/index.html\n> Please let us know if you run into any issues.\n>\n> \u2014\n> Reply to this email directly, view it on GitHub\n> , or\n> unsubscribe\n> \n> .\n> You are receiving this because you were mentioned.Message ID:\n> ***@***.***>\n>\n\n-- \nThe information contained in this electronic communication is intended \nsolely for the individual(s) or entity to which it is addressed. It may \ncontain proprietary, confidential and/or legally privileged information. \nAny review, retransmission, dissemination, printing, copying or other use \nof, or taking any action in reliance on the contents of this information by \nperson(s) or entities other than the intended recipient is strictly \nprohibited and may be unlawful. If you have received this communication in \nerror, please notify us by responding to this email or telephone and \nimmediately and permanently delete all copies of this message and any \nattachments from your system(s). The contents of this message do not \nnecessarily represent the views or policies of BITS Pilani.\n\nDon't worry about the Docker issues @ArpanShah2k - a working Docker installation is not a requirement for Cognito. (Only for other services.)\r\n\r\n> 3) run commands in python and cmd to install moto. ( facing issues). \r\n>\r\n\r\nJust to verify: you have forked Moto, and checked out your copy, before installing?\r\n\r\nWhich commands are you running, and what are the errors that you see?\r\n\nI have solved\r\n\r\n> Don't worry about the Docker issues @ArpanShah2k - a working Docker installation is not a requirement for Cognito. (Only for other services.)\r\n> \r\n> > 3. run commands in python and cmd to install moto. ( facing issues).\r\n> \r\n> Just to verify: you have forked Moto, and checked out your copy, before installing?\r\n> \r\n> Which commands are you running, and what are the errors that you see?\r\n\r\nI have solved this errors that i was getting while setup now.\nsir i have created PR for this Issue. I request you to review it and merge it if all the test cases are cleared. ", "test_patch": "diff --git a/tests/test_cognitoidp/test_cognitoidp_exceptions.py b/tests/test_cognitoidp/test_cognitoidp_exceptions.py\n--- a/tests/test_cognitoidp/test_cognitoidp_exceptions.py\n+++ b/tests/test_cognitoidp/test_cognitoidp_exceptions.py\n@@ -1,6 +1,8 @@\n from unittest import TestCase\n \n import boto3\n+import pytest\n+\n from moto import mock_cognitoidp\n from botocore.exceptions import ClientError\n \n@@ -49,3 +51,47 @@ def test_authenticate_with_signed_out_user(self):\n },\n )\n exc.exception.response[\"Error\"][\"Code\"].should.equal(\"NotAuthorizedException\")\n+\n+\n+@mock_cognitoidp\n+class TestCognitoUserPoolDuplidateEmails(TestCase):\n+ def setUp(self) -> None:\n+ self.client = boto3.client(\"cognito-idp\", \"us-east-1\")\n+\n+ self.pool_id1 = self.client.create_user_pool(PoolName=\"test\")[\"UserPool\"][\"Id\"]\n+ self.pool_id2 = self.client.create_user_pool(\n+ PoolName=\"test\", UsernameAttributes=[\"email\"]\n+ )[\"UserPool\"][\"Id\"]\n+\n+ # create two users\n+ for user in [\"user1\", \"user2\"]:\n+ self.client.admin_create_user(\n+ UserPoolId=self.pool_id1,\n+ Username=user,\n+ UserAttributes=[{\"Name\": \"email\", \"Value\": f\"{user}@test.com\"}],\n+ )\n+ self.client.admin_create_user(\n+ UserPoolId=self.pool_id2,\n+ Username=f\"{user}@test.com\",\n+ UserAttributes=[{\"Name\": \"email\", \"Value\": f\"{user}@test.com\"}],\n+ )\n+\n+ def test_use_existing_email__when_email_is_login(self):\n+ with pytest.raises(ClientError) as exc:\n+ self.client.admin_update_user_attributes(\n+ UserPoolId=self.pool_id2,\n+ Username=\"user1@test.com\",\n+ UserAttributes=[{\"Name\": \"email\", \"Value\": \"user2@test.com\"}],\n+ )\n+ err = exc.value.response[\"Error\"]\n+ err[\"Code\"].should.equal(\"AliasExistsException\")\n+ err[\"Message\"].should.equal(\"An account with the given email already exists.\")\n+\n+ def test_use_existing_email__when_username_is_login(self):\n+ # Because we cannot use the email as username,\n+ # multiple users can have the same email address\n+ self.client.admin_update_user_attributes(\n+ UserPoolId=self.pool_id1,\n+ Username=\"user1\",\n+ UserAttributes=[{\"Name\": \"email\", \"Value\": \"user2@test.com\"}],\n+ )\n", "repo": "getmoto/moto", "version": "4.1", "created_at": "2023-01-24 23:37:57"}, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "temperature": 0.6, "top_p": 0.95}} -{"responses_create_params": {"input": [], "metadata": {"instance_id": "getmoto__moto-5085", "base_commit": "6b70cd1b6b1cf493b66b6fcaaea9d1041331e836", "dataset_name": "SWE-Gym/SWE-Gym", "split": "train", "problem_statement": "When creating ec2 instances from launch template via run_instances, the instances aren't tagged\nI'm using moto in pytest. I have created a launch template using `create_launch_template`. This template is created with `TagSpecifications` for instance and volume.\r\n\r\nUpon using `run_instances` to create new instances based on this launch template, their tags are empty. Is this to be expected?\n", "golden_patch": "diff --git a/moto/core/responses.py b/moto/core/responses.py\n--- a/moto/core/responses.py\n+++ b/moto/core/responses.py\n@@ -725,20 +725,6 @@ def _get_map_prefix(self, param_prefix, key_end=\".key\", value_end=\".value\"):\n \n return results\n \n- def _parse_tag_specification(self):\n- # [{\"ResourceType\": _type, \"Tag\": [{\"Key\": k, \"Value\": v}, ..]}]\n- tag_spec = self._get_multi_param(\"TagSpecification\")\n- # {_type: {k: v, ..}}\n- tags = {}\n- for spec in tag_spec:\n- if spec[\"ResourceType\"] not in tags:\n- tags[spec[\"ResourceType\"]] = {}\n- tags[spec[\"ResourceType\"]].update(\n- {tag[\"Key\"]: tag[\"Value\"] for tag in spec[\"Tag\"]}\n- )\n-\n- return tags\n-\n def _get_object_map(self, prefix, name=\"Name\", value=\"Value\"):\n \"\"\"\n Given a query dict like\ndiff --git a/moto/ec2/_models/instances.py b/moto/ec2/_models/instances.py\n--- a/moto/ec2/_models/instances.py\n+++ b/moto/ec2/_models/instances.py\n@@ -22,6 +22,7 @@\n random_reservation_id,\n filter_reservations,\n utc_date_and_time,\n+ convert_tag_spec,\n )\n \n \n@@ -70,6 +71,13 @@ def __init__(self, ec2_backend, image_id, user_data, security_groups, **kwargs):\n self.image_id = template_version.image_id\n else:\n self.image_id = image_id\n+ # Check if we have tags to process\n+ if launch_template_arg:\n+ template_version = ec2_backend._get_template_from_args(launch_template_arg)\n+ tag_spec_set = template_version.data.get(\"TagSpecification\", {})\n+ tags = convert_tag_spec(tag_spec_set)\n+ instance_tags = tags.get(\"instance\", {})\n+ self.add_tags(instance_tags)\n \n self._state = InstanceState(\"running\", 16)\n self._reason = \"\"\ndiff --git a/moto/ec2/_models/spot_requests.py b/moto/ec2/_models/spot_requests.py\n--- a/moto/ec2/_models/spot_requests.py\n+++ b/moto/ec2/_models/spot_requests.py\n@@ -11,6 +11,7 @@\n random_spot_fleet_request_id,\n random_spot_request_id,\n generic_filter,\n+ convert_tag_spec,\n )\n \n \n@@ -249,7 +250,8 @@ def __init__(\n launch_specs_from_config.append(new_launch_template)\n \n for spec in (launch_specs or []) + launch_specs_from_config:\n- tags = self._extract_tags(spec)\n+ tag_spec_set = spec.get(\"TagSpecificationSet\", [])\n+ tags = convert_tag_spec(tag_spec_set)\n self.launch_specs.append(\n SpotFleetLaunchSpec(\n ebs_optimized=spec.get(\"EbsOptimized\"),\n@@ -270,19 +272,6 @@ def __init__(\n self.spot_requests = []\n self.create_spot_requests(self.target_capacity)\n \n- def _extract_tags(self, spec):\n- # IN: [{\"ResourceType\": _type, \"Tag\": [{\"Key\": k, \"Value\": v}, ..]}]\n- # OUT: {_type: {k: v, ..}}\n- tag_spec_set = spec.get(\"TagSpecificationSet\", [])\n- tags = {}\n- for tag_spec in tag_spec_set:\n- if tag_spec[\"ResourceType\"] not in tags:\n- tags[tag_spec[\"ResourceType\"]] = {}\n- tags[tag_spec[\"ResourceType\"]].update(\n- {tag[\"Key\"]: tag[\"Value\"] for tag in tag_spec[\"Tag\"]}\n- )\n- return tags\n-\n @property\n def physical_resource_id(self):\n return self.id\ndiff --git a/moto/ec2/responses/_base_response.py b/moto/ec2/responses/_base_response.py\n--- a/moto/ec2/responses/_base_response.py\n+++ b/moto/ec2/responses/_base_response.py\n@@ -1,4 +1,5 @@\n from moto.core.responses import BaseResponse\n+from ..utils import convert_tag_spec\n \n \n class EC2BaseResponse(BaseResponse):\n@@ -7,3 +8,9 @@ def _filters_from_querystring(self):\n _filters = self._get_multi_param(\"Filter.\")\n # return {x1: y1, ...}\n return {f[\"Name\"]: f[\"Value\"] for f in _filters}\n+\n+ def _parse_tag_specification(self):\n+ # [{\"ResourceType\": _type, \"Tag\": [{\"Key\": k, \"Value\": v}, ..]}]\n+ tag_spec_set = self._get_multi_param(\"TagSpecification\")\n+ # {_type: {k: v, ..}}\n+ return convert_tag_spec(tag_spec_set)\ndiff --git a/moto/ec2/utils.py b/moto/ec2/utils.py\n--- a/moto/ec2/utils.py\n+++ b/moto/ec2/utils.py\n@@ -773,3 +773,16 @@ def gen_moto_amis(described_images, drop_images_missing_keys=True):\n raise err\n \n return result\n+\n+\n+def convert_tag_spec(tag_spec_set):\n+ # IN: [{\"ResourceType\": _type, \"Tag\": [{\"Key\": k, \"Value\": v}, ..]}]\n+ # OUT: {_type: {k: v, ..}}\n+ tags = {}\n+ for tag_spec in tag_spec_set:\n+ if tag_spec[\"ResourceType\"] not in tags:\n+ tags[tag_spec[\"ResourceType\"]] = {}\n+ tags[tag_spec[\"ResourceType\"]].update(\n+ {tag[\"Key\"]: tag[\"Value\"] for tag in tag_spec[\"Tag\"]}\n+ )\n+ return tags\n", "hints_text": "Hi @dkatzbuc, thanks for raising this - doesn't look like this behaviour is implemented yet. Marking it as an enhancement.", "test_patch": "diff --git a/tests/test_ec2/test_instances.py b/tests/test_ec2/test_instances.py\n--- a/tests/test_ec2/test_instances.py\n+++ b/tests/test_ec2/test_instances.py\n@@ -2170,6 +2170,29 @@ def test_create_instance_with_launch_template_id_produces_no_warning(\n assert len(captured_warnings) == 0\n \n \n+@mock_ec2\n+def test_create_instance_from_launch_template__process_tags():\n+ client = boto3.client(\"ec2\", region_name=\"us-west-1\")\n+\n+ template = client.create_launch_template(\n+ LaunchTemplateName=str(uuid4()),\n+ LaunchTemplateData={\n+ \"ImageId\": EXAMPLE_AMI_ID,\n+ \"TagSpecifications\": [\n+ {\"ResourceType\": \"instance\", \"Tags\": [{\"Key\": \"k\", \"Value\": \"v\"}]}\n+ ],\n+ },\n+ )[\"LaunchTemplate\"]\n+\n+ instance = client.run_instances(\n+ MinCount=1,\n+ MaxCount=1,\n+ LaunchTemplate={\"LaunchTemplateId\": template[\"LaunchTemplateId\"]},\n+ )[\"Instances\"][0]\n+\n+ instance.should.have.key(\"Tags\").equals([{\"Key\": \"k\", \"Value\": \"v\"}])\n+\n+\n @mock_ec2\n def test_run_instance_and_associate_public_ip():\n ec2 = boto3.resource(\"ec2\", \"us-west-1\")\n", "repo": "getmoto/moto", "version": "3.1", "created_at": "2022-05-01 18:07:16"}, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "temperature": 0.6, "top_p": 0.95}} -{"responses_create_params": {"input": [], "metadata": {"instance_id": "getmoto__moto-6709", "base_commit": "78c518ddc832a30e1cf20015bc5c3b1850a1c797", "dataset_name": "SWE-Gym/SWE-Gym", "split": "train", "problem_statement": "DynamoDB: special characters in get_item() projection expression not handled correctly\nHi!\r\n\r\nI have a nested attribute inside a dynamodb table like so:\r\n````json\r\n{\r\n \"device\": {\r\n \"N\": \"123456\"\r\n },\r\n \"software\": {\r\n \"M\": {\r\n \"python3.10\": {\r\n \"M\": {\r\n \"lorem\": {\r\n \"S\": \"asdf\"\r\n },\r\n \"ipsum\": {\r\n \"S\": \"asdf\"\r\n }\r\n }\r\n },\r\n \"curl\": {\r\n \"M\": {\r\n \"lorem\": {\r\n \"S\": \"asdf\"\r\n },\r\n \"ipsum\": {\r\n \"S\": \"asdf\"\r\n }\r\n }\r\n }\r\n }\r\n }\r\n}\r\n````\r\nNow I want to use the `get_item()` function of a dynamodb resource to only get the data of the \"python3.10\" entry:\r\n\r\n````python\r\nresult = table.get_item(\r\n Key={\"device\": 123456},\r\n ProjectionExpression=\"software.#python3_10\",\r\n ExpressionAttributeNames={\"#python3_10\": \"python3.10\"}\r\n)\r\n````\r\nBut I only get an empty result set (`Item: {}`).\r\n_It works when I do this via the AWS CLI_. That leads me to believe, that this might be a moto issue. I would be very happy if someone could verify this assumption.\r\n\r\nThanks in advance,\r\nMats\n", "golden_patch": "diff --git a/moto/dynamodb/models/__init__.py b/moto/dynamodb/models/__init__.py\n--- a/moto/dynamodb/models/__init__.py\n+++ b/moto/dynamodb/models/__init__.py\n@@ -301,11 +301,11 @@ def get_item(\n self,\n table_name: str,\n keys: Dict[str, Any],\n- projection_expression: Optional[str] = None,\n+ projection_expressions: Optional[List[List[str]]] = None,\n ) -> Optional[Item]:\n table = self.get_table(table_name)\n hash_key, range_key = self.get_keys_value(table, keys)\n- return table.get_item(hash_key, range_key, projection_expression)\n+ return table.get_item(hash_key, range_key, projection_expressions)\n \n def query(\n self,\n@@ -316,7 +316,7 @@ def query(\n limit: int,\n exclusive_start_key: Dict[str, Any],\n scan_index_forward: bool,\n- projection_expression: Optional[str],\n+ projection_expressions: Optional[List[List[str]]],\n index_name: Optional[str] = None,\n expr_names: Optional[Dict[str, str]] = None,\n expr_values: Optional[Dict[str, str]] = None,\n@@ -339,7 +339,7 @@ def query(\n limit,\n exclusive_start_key,\n scan_index_forward,\n- projection_expression,\n+ projection_expressions,\n index_name,\n filter_expression_op,\n **filter_kwargs,\n@@ -355,7 +355,7 @@ def scan(\n expr_names: Dict[str, Any],\n expr_values: Dict[str, Any],\n index_name: str,\n- projection_expression: Optional[str],\n+ projection_expression: Optional[List[List[str]]],\n ) -> Tuple[List[Item], int, Optional[Dict[str, Any]]]:\n table = self.get_table(table_name)\n \ndiff --git a/moto/dynamodb/models/dynamo_type.py b/moto/dynamodb/models/dynamo_type.py\n--- a/moto/dynamodb/models/dynamo_type.py\n+++ b/moto/dynamodb/models/dynamo_type.py\n@@ -418,13 +418,12 @@ def update_with_attribute_updates(self, attribute_updates: Dict[str, Any]) -> No\n f\"{action} action not support for update_with_attribute_updates\"\n )\n \n- def project(self, projection_expression: str) -> \"Item\":\n+ def project(self, projection_expressions: List[List[str]]) -> \"Item\":\n # Returns a new Item with only the dictionary-keys that match the provided projection_expression\n # Will return an empty Item if the expression does not match anything\n result: Dict[str, Any] = dict()\n- expressions = [x.strip() for x in projection_expression.split(\",\")]\n- for expr in expressions:\n- x = find_nested_key(expr.split(\".\"), self.to_regular_json())\n+ for expr in projection_expressions:\n+ x = find_nested_key(expr, self.to_regular_json())\n merge_dicts(result, x)\n \n return Item(\ndiff --git a/moto/dynamodb/models/table.py b/moto/dynamodb/models/table.py\n--- a/moto/dynamodb/models/table.py\n+++ b/moto/dynamodb/models/table.py\n@@ -50,12 +50,18 @@ def project(self, item: Item) -> Item:\n ]\n \n if projection_type == \"KEYS_ONLY\":\n- item = item.project(\",\".join(key_attributes))\n+ # 'project' expects lists of lists of strings\n+ # project([[\"attr1\"], [\"nested\", \"attr2\"]]\n+ #\n+ # In our case, we need to convert\n+ # [\"key1\", \"key2\"]\n+ # into\n+ # [[\"key1\"], [\"key2\"]]\n+ item = item.project([[attr] for attr in key_attributes])\n elif projection_type == \"INCLUDE\":\n- allowed_attributes = key_attributes + self.projection.get(\n- \"NonKeyAttributes\", []\n- )\n- item = item.project(\",\".join(allowed_attributes))\n+ allowed_attributes = key_attributes\n+ allowed_attributes.extend(self.projection.get(\"NonKeyAttributes\", []))\n+ item = item.project([[attr] for attr in allowed_attributes])\n # ALL is handled implicitly by not filtering\n return item\n \n@@ -592,7 +598,7 @@ def get_item(\n self,\n hash_key: DynamoType,\n range_key: Optional[DynamoType] = None,\n- projection_expression: Optional[str] = None,\n+ projection_expression: Optional[List[List[str]]] = None,\n ) -> Optional[Item]:\n if self.has_range_key and not range_key:\n raise MockValidationException(\n@@ -637,7 +643,7 @@ def query(\n limit: int,\n exclusive_start_key: Dict[str, Any],\n scan_index_forward: bool,\n- projection_expression: Optional[str],\n+ projection_expressions: Optional[List[List[str]]],\n index_name: Optional[str] = None,\n filter_expression: Any = None,\n **filter_kwargs: Any,\n@@ -754,8 +760,8 @@ def conv(x: DynamoType) -> Any:\n if filter_expression is not None:\n results = [item for item in results if filter_expression.expr(item)]\n \n- if projection_expression:\n- results = [r.project(projection_expression) for r in results]\n+ if projection_expressions:\n+ results = [r.project(projection_expressions) for r in results]\n \n return results, scanned_count, last_evaluated_key\n \n@@ -799,7 +805,7 @@ def scan(\n exclusive_start_key: Dict[str, Any],\n filter_expression: Any = None,\n index_name: Optional[str] = None,\n- projection_expression: Optional[str] = None,\n+ projection_expression: Optional[List[List[str]]] = None,\n ) -> Tuple[List[Item], int, Optional[Dict[str, Any]]]:\n results = []\n scanned_count = 0\ndiff --git a/moto/dynamodb/responses.py b/moto/dynamodb/responses.py\n--- a/moto/dynamodb/responses.py\n+++ b/moto/dynamodb/responses.py\n@@ -556,11 +556,11 @@ def get_item(self) -> str:\n )\n \n expression_attribute_names = expression_attribute_names or {}\n- projection_expression = self._adjust_projection_expression(\n+ projection_expressions = self._adjust_projection_expression(\n projection_expression, expression_attribute_names\n )\n \n- item = self.dynamodb_backend.get_item(name, key, projection_expression)\n+ item = self.dynamodb_backend.get_item(name, key, projection_expressions)\n if item:\n item_dict = item.describe_attrs(attributes=None)\n return dynamo_json_dump(item_dict)\n@@ -608,14 +608,14 @@ def batch_get_item(self) -> str:\n \"ExpressionAttributeNames\", {}\n )\n \n- projection_expression = self._adjust_projection_expression(\n+ projection_expressions = self._adjust_projection_expression(\n projection_expression, expression_attribute_names\n )\n \n results[\"Responses\"][table_name] = []\n for key in keys:\n item = self.dynamodb_backend.get_item(\n- table_name, key, projection_expression\n+ table_name, key, projection_expressions\n )\n if item:\n # A single operation can retrieve up to 16 MB of data [and] returns a partial result if the response size limit is exceeded\n@@ -652,7 +652,7 @@ def query(self) -> str:\n filter_expression = self._get_filter_expression()\n expression_attribute_values = self.body.get(\"ExpressionAttributeValues\", {})\n \n- projection_expression = self._adjust_projection_expression(\n+ projection_expressions = self._adjust_projection_expression(\n projection_expression, expression_attribute_names\n )\n \n@@ -720,7 +720,7 @@ def query(self) -> str:\n limit,\n exclusive_start_key,\n scan_index_forward,\n- projection_expression,\n+ projection_expressions,\n index_name=index_name,\n expr_names=expression_attribute_names,\n expr_values=expression_attribute_values,\n@@ -743,27 +743,24 @@ def query(self) -> str:\n \n def _adjust_projection_expression(\n self, projection_expression: Optional[str], expr_attr_names: Dict[str, str]\n- ) -> Optional[str]:\n+ ) -> List[List[str]]:\n+ \"\"\"\n+ lvl1.lvl2.attr1,lvl1.attr2 --> [[\"lvl1\", \"lvl2\", \"attr1\"], [\"lvl1\", \"attr2]]\n+ \"\"\"\n+\n def _adjust(expression: str) -> str:\n- return (\n- expr_attr_names[expression]\n- if expression in expr_attr_names\n- else expression\n- )\n+ return (expr_attr_names or {}).get(expression, expression)\n \n if projection_expression:\n expressions = [x.strip() for x in projection_expression.split(\",\")]\n for expression in expressions:\n check_projection_expression(expression)\n- if expr_attr_names:\n- return \",\".join(\n- [\n- \".\".join([_adjust(expr) for expr in nested_expr.split(\".\")])\n- for nested_expr in expressions\n- ]\n- )\n+ return [\n+ [_adjust(expr) for expr in nested_expr.split(\".\")]\n+ for nested_expr in expressions\n+ ]\n \n- return projection_expression\n+ return []\n \n @include_consumed_capacity()\n def scan(self) -> str:\n@@ -786,7 +783,7 @@ def scan(self) -> str:\n limit = self.body.get(\"Limit\")\n index_name = self.body.get(\"IndexName\")\n \n- projection_expression = self._adjust_projection_expression(\n+ projection_expressions = self._adjust_projection_expression(\n projection_expression, expression_attribute_names\n )\n \n@@ -800,7 +797,7 @@ def scan(self) -> str:\n expression_attribute_names,\n expression_attribute_values,\n index_name,\n- projection_expression,\n+ projection_expressions,\n )\n except ValueError as err:\n raise MockValidationException(f\"Bad Filter Expression: {err}\")\n", "hints_text": "The Dynamo item has `software`, but the query looks for `packages` - could that be the problem?\r\n\r\nNote that I haven't verified this in Moto.\n> The Dynamo item has `software`, but the query looks for `packages` - could that be the problem?\r\n> \r\n> Note that I haven't verified this in Moto.\r\n\r\nNo sorry, that was a mistake by me when I was constructing the example.\nAh, found it. Moto doesn't play nice with attributes that contain a `.` - presumably because it assumes that it should be a map. Marking it as a bug!\nAlright, thank you so much for the quick reply. ", "test_patch": "diff --git a/tests/test_dynamodb/models/test_item.py b/tests/test_dynamodb/models/test_item.py\n--- a/tests/test_dynamodb/models/test_item.py\n+++ b/tests/test_dynamodb/models/test_item.py\n@@ -34,17 +34,17 @@ def _project(self, expression, result):\n assert x == y\n \n def test_find_nothing(self):\n- self._project(\"\", result={})\n+ self._project([[\"\"]], result={})\n \n def test_find_unknown_key(self):\n- self._project(\"unknown\", result={})\n+ self._project([[\"unknown\"]], result={})\n \n def test_project_single_key_string(self):\n- self._project(\"simplestring\", result={\"simplestring\": \"val\"})\n+ self._project([[\"simplestring\"]], result={\"simplestring\": \"val\"})\n \n def test_project_single_key_dict(self):\n self._project(\n- \"nesteddict\",\n+ [[\"nesteddict\"]],\n result={\n \"nesteddict\": {\n \"level21\": {\"ll31\": \"val\", \"ll32\": \"val\"},\n@@ -59,31 +59,31 @@ def test_project_single_key_dict(self):\n \n def test_project_nested_key(self):\n self._project(\n- \"nesteddict.level21\",\n+ [[\"nesteddict\", \"level21\"]],\n result={\"nesteddict\": {\"level21\": {\"ll31\": \"val\", \"ll32\": \"val\"}}},\n )\n \n def test_project_multi_level_nested_key(self):\n self._project(\n- \"nesteddict.level21.ll32\",\n+ [[\"nesteddict\", \"level21\", \"ll32\"]],\n result={\"nesteddict\": {\"level21\": {\"ll32\": \"val\"}}},\n )\n \n def test_project_nested_key__partial_fix(self):\n- self._project(\"nesteddict.levelunknown\", result={})\n+ self._project([[\"nesteddict\", \"levelunknown\"]], result={})\n \n def test_project_nested_key__partial_fix2(self):\n- self._project(\"nesteddict.unknown.unknown2\", result={})\n+ self._project([[\"nesteddict\", \"unknown\", \"unknown2\"]], result={})\n \n def test_list_index(self):\n self._project(\n- \"rootlist[0]\",\n+ [[\"rootlist[0]\"]],\n result={\"rootlist\": [{\"ll21\": {\"ll31\": \"val\", \"ll32\": \"val\"}}]},\n )\n \n def test_nested_list_index(self):\n self._project(\n- \"nesteddict.nestedlist[1]\",\n+ [[\"nesteddict\", \"nestedlist[1]\"]],\n result={\n \"nesteddict\": {\"nestedlist\": [{\"ll22\": {\"ll31\": \"val\", \"ll32\": \"val\"}}]}\n },\n@@ -91,16 +91,16 @@ def test_nested_list_index(self):\n \n def test_nested_obj_in_list(self):\n self._project(\n- \"nesteddict.nestedlist[1].ll22.ll31\",\n+ [[\"nesteddict\", \"nestedlist[1]\", \"ll22\", \"ll31\"]],\n result={\"nesteddict\": {\"nestedlist\": [{\"ll22\": {\"ll31\": \"val\"}}]}},\n )\n \n def test_list_unknown_indexes(self):\n- self._project(\"nesteddict.nestedlist[25]\", result={})\n+ self._project([[\"nesteddict\", \"nestedlist[25]\"]], result={})\n \n def test_multiple_projections(self):\n self._project(\n- \"nesteddict.nestedlist[1].ll22,rootlist[0]\",\n+ [[\"nesteddict\", \"nestedlist[1]\", \"ll22\"], [\"rootlist[0]\"]],\n result={\n \"nesteddict\": {\n \"nestedlist\": [{\"ll22\": {\"ll31\": \"val\", \"ll32\": \"val\"}}]\ndiff --git a/tests/test_dynamodb/test_dynamodb.py b/tests/test_dynamodb/test_dynamodb.py\n--- a/tests/test_dynamodb/test_dynamodb.py\n+++ b/tests/test_dynamodb/test_dynamodb.py\n@@ -886,7 +886,7 @@ def test_nested_projection_expression_using_get_item_with_attr_expression():\n \"forum_name\": \"key1\",\n \"nested\": {\n \"level1\": {\"id\": \"id1\", \"att\": \"irrelevant\"},\n- \"level2\": {\"id\": \"id2\", \"include\": \"all\"},\n+ \"level.2\": {\"id\": \"id2\", \"include\": \"all\"},\n \"level3\": {\n \"id\": \"irrelevant\",\n \"children\": [{\"Name\": \"child_a\"}, {\"Name\": \"child_b\"}],\n@@ -907,10 +907,10 @@ def test_nested_projection_expression_using_get_item_with_attr_expression():\n result = table.get_item(\n Key={\"forum_name\": \"key1\"},\n ProjectionExpression=\"#nst.level1.id, #nst.#lvl2\",\n- ExpressionAttributeNames={\"#nst\": \"nested\", \"#lvl2\": \"level2\"},\n+ ExpressionAttributeNames={\"#nst\": \"nested\", \"#lvl2\": \"level.2\"},\n )[\"Item\"]\n assert result == {\n- \"nested\": {\"level1\": {\"id\": \"id1\"}, \"level2\": {\"id\": \"id2\", \"include\": \"all\"}}\n+ \"nested\": {\"level1\": {\"id\": \"id1\"}, \"level.2\": {\"id\": \"id2\", \"include\": \"all\"}}\n }\n # Assert actual data has not been deleted\n result = table.get_item(Key={\"forum_name\": \"key1\"})[\"Item\"]\n@@ -919,7 +919,7 @@ def test_nested_projection_expression_using_get_item_with_attr_expression():\n \"forum_name\": \"key1\",\n \"nested\": {\n \"level1\": {\"id\": \"id1\", \"att\": \"irrelevant\"},\n- \"level2\": {\"id\": \"id2\", \"include\": \"all\"},\n+ \"level.2\": {\"id\": \"id2\", \"include\": \"all\"},\n \"level3\": {\n \"id\": \"irrelevant\",\n \"children\": [{\"Name\": \"child_a\"}, {\"Name\": \"child_b\"}],\n", "repo": "getmoto/moto", "version": "4.1", "created_at": "2023-08-21 18:57:36"}, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "temperature": 0.6, "top_p": 0.95}} \ No newline at end of file +{"responses_create_params": {"input": [], "metadata": {"instance_id": "astropy__astropy-12907", "base_commit": "d16bfe05a744909de4b27f5875fe0d4ed41ce607", "dataset_name": "princeton-nlp/SWE-bench_Verified", "split": "test", "problem_statement": "Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\nConsider the following model:\r\n\r\n```python\r\nfrom astropy.modeling import models as m\r\nfrom astropy.modeling.separable import separability_matrix\r\n\r\ncm = m.Linear1D(10) & m.Linear1D(5)\r\n```\r\n\r\nIt's separability matrix as you might expect is a diagonal:\r\n\r\n```python\r\n>>> separability_matrix(cm)\r\narray([[ True, False],\r\n [False, True]])\r\n```\r\n\r\nIf I make the model more complex:\r\n```python\r\n>>> separability_matrix(m.Pix2Sky_TAN() & m.Linear1D(10) & m.Linear1D(5))\r\narray([[ True, True, False, False],\r\n [ True, True, False, False],\r\n [False, False, True, False],\r\n [False, False, False, True]])\r\n```\r\n\r\nThe output matrix is again, as expected, the outputs and inputs to the linear models are separable and independent of each other.\r\n\r\nIf however, I nest these compound models:\r\n```python\r\n>>> separability_matrix(m.Pix2Sky_TAN() & cm)\r\narray([[ True, True, False, False],\r\n [ True, True, False, False],\r\n [False, False, True, True],\r\n [False, False, True, True]])\r\n```\r\nSuddenly the inputs and outputs are no longer separable?\r\n\r\nThis feels like a bug to me, but I might be missing something?\n", "golden_patch": "diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\n--- a/astropy/modeling/separable.py\n+++ b/astropy/modeling/separable.py\n@@ -242,7 +242,7 @@ def _cstack(left, right):\n cright = _coord_matrix(right, 'right', noutp)\n else:\n cright = np.zeros((noutp, right.shape[1]))\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\n \n return np.hstack([cleft, cright])\n \n", "instance_dict": "{\"instance_id\": \"astropy__astropy-12907\", \"base_commit\": \"d16bfe05a744909de4b27f5875fe0d4ed41ce607\", \"dataset_name\": \"princeton-nlp/SWE-bench_Verified\", \"split\": \"test\", \"problem_statement\": \"Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\\nConsider the following model:\\r\\n\\r\\n```python\\r\\nfrom astropy.modeling import models as m\\r\\nfrom astropy.modeling.separable import separability_matrix\\r\\n\\r\\ncm = m.Linear1D(10) & m.Linear1D(5)\\r\\n```\\r\\n\\r\\nIt's separability matrix as you might expect is a diagonal:\\r\\n\\r\\n```python\\r\\n>>> separability_matrix(cm)\\r\\narray([[ True, False],\\r\\n [False, True]])\\r\\n```\\r\\n\\r\\nIf I make the model more complex:\\r\\n```python\\r\\n>>> separability_matrix(m.Pix2Sky_TAN() & m.Linear1D(10) & m.Linear1D(5))\\r\\narray([[ True, True, False, False],\\r\\n [ True, True, False, False],\\r\\n [False, False, True, False],\\r\\n [False, False, False, True]])\\r\\n```\\r\\n\\r\\nThe output matrix is again, as expected, the outputs and inputs to the linear models are separable and independent of each other.\\r\\n\\r\\nIf however, I nest these compound models:\\r\\n```python\\r\\n>>> separability_matrix(m.Pix2Sky_TAN() & cm)\\r\\narray([[ True, True, False, False],\\r\\n [ True, True, False, False],\\r\\n [False, False, True, True],\\r\\n [False, False, True, True]])\\r\\n```\\r\\nSuddenly the inputs and outputs are no longer separable?\\r\\n\\r\\nThis feels like a bug to me, but I might be missing something?\\n\", \"golden_patch\": \"diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\\n--- a/astropy/modeling/separable.py\\n+++ b/astropy/modeling/separable.py\\n@@ -242,7 +242,7 @@ def _cstack(left, right):\\n cright = _coord_matrix(right, 'right', noutp)\\n else:\\n cright = np.zeros((noutp, right.shape[1]))\\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\\n \\n return np.hstack([cleft, cright])\\n \\n\", \"repo\": \"astropy/astropy\", \"patch\": \"diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\\n--- a/astropy/modeling/separable.py\\n+++ b/astropy/modeling/separable.py\\n@@ -242,7 +242,7 @@ def _cstack(left, right):\\n cright = _coord_matrix(right, 'right', noutp)\\n else:\\n cright = np.zeros((noutp, right.shape[1]))\\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\\n \\n return np.hstack([cleft, cright])\\n \\n\", \"test_patch\": \"diff --git a/astropy/modeling/tests/test_separable.py b/astropy/modeling/tests/test_separable.py\\n--- a/astropy/modeling/tests/test_separable.py\\n+++ b/astropy/modeling/tests/test_separable.py\\n@@ -28,6 +28,13 @@\\n p1 = models.Polynomial1D(1, name='p1')\\n \\n \\n+cm_4d_expected = (np.array([False, False, True, True]),\\n+ np.array([[True, True, False, False],\\n+ [True, True, False, False],\\n+ [False, False, True, False],\\n+ [False, False, False, True]]))\\n+\\n+\\n compound_models = {\\n 'cm1': (map3 & sh1 | rot & sh1 | sh1 & sh2 & sh1,\\n (np.array([False, False, True]),\\n@@ -52,7 +59,17 @@\\n 'cm7': (map2 | p2 & sh1,\\n (np.array([False, True]),\\n np.array([[True, False], [False, True]]))\\n- )\\n+ ),\\n+ 'cm8': (rot & (sh1 & sh2), cm_4d_expected),\\n+ 'cm9': (rot & sh1 & sh2, cm_4d_expected),\\n+ 'cm10': ((rot & sh1) & sh2, cm_4d_expected),\\n+ 'cm11': (rot & sh1 & (scl1 & scl2),\\n+ (np.array([False, False, True, True, True]),\\n+ np.array([[True, True, False, False, False],\\n+ [True, True, False, False, False],\\n+ [False, False, True, False, False],\\n+ [False, False, False, True, False],\\n+ [False, False, False, False, True]]))),\\n }\\n \\n \\n\", \"hints_text\": \"\", \"created_at\": \"2022-03-03T15:14:54Z\", \"version\": \"4.3\", \"FAIL_TO_PASS\": \"[\\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model6-result6]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model9-result9]\\\"]\", \"PASS_TO_PASS\": \"[\\\"astropy/modeling/tests/test_separable.py::test_coord_matrix\\\", \\\"astropy/modeling/tests/test_separable.py::test_cdot\\\", \\\"astropy/modeling/tests/test_separable.py::test_cstack\\\", \\\"astropy/modeling/tests/test_separable.py::test_arith_oper\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model0-result0]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model1-result1]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model2-result2]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model3-result3]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model4-result4]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model5-result5]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model7-result7]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model8-result8]\\\", \\\"astropy/modeling/tests/test_separable.py::test_custom_model_separable\\\"]\", \"environment_setup_commit\": \"298ccb478e6bf092953bca67a3d29dc6c35f6752\", \"difficulty\": \"15 min - 1 hour\"}"}, "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", "temperature": 0.7, "top_p": 0.8, "max_output_tokens": 12288}, "agent_ref": {"type": "responses_api_agents", "name": "swe_agents_val"}, "repo": "astropy/astropy", "instance_id": "astropy__astropy-12907", "base_commit": "d16bfe05a744909de4b27f5875fe0d4ed41ce607", "patch": "diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\n--- a/astropy/modeling/separable.py\n+++ b/astropy/modeling/separable.py\n@@ -242,7 +242,7 @@ def _cstack(left, right):\n cright = _coord_matrix(right, 'right', noutp)\n else:\n cright = np.zeros((noutp, right.shape[1]))\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\n \n return np.hstack([cleft, cright])\n \n", "test_patch": "diff --git a/astropy/modeling/tests/test_separable.py b/astropy/modeling/tests/test_separable.py\n--- a/astropy/modeling/tests/test_separable.py\n+++ b/astropy/modeling/tests/test_separable.py\n@@ -28,6 +28,13 @@\n p1 = models.Polynomial1D(1, name='p1')\n \n \n+cm_4d_expected = (np.array([False, False, True, True]),\n+ np.array([[True, True, False, False],\n+ [True, True, False, False],\n+ [False, False, True, False],\n+ [False, False, False, True]]))\n+\n+\n compound_models = {\n 'cm1': (map3 & sh1 | rot & sh1 | sh1 & sh2 & sh1,\n (np.array([False, False, True]),\n@@ -52,7 +59,17 @@\n 'cm7': (map2 | p2 & sh1,\n (np.array([False, True]),\n np.array([[True, False], [False, True]]))\n- )\n+ ),\n+ 'cm8': (rot & (sh1 & sh2), cm_4d_expected),\n+ 'cm9': (rot & sh1 & sh2, cm_4d_expected),\n+ 'cm10': ((rot & sh1) & sh2, cm_4d_expected),\n+ 'cm11': (rot & sh1 & (scl1 & scl2),\n+ (np.array([False, False, True, True, True]),\n+ np.array([[True, True, False, False, False],\n+ [True, True, False, False, False],\n+ [False, False, True, False, False],\n+ [False, False, False, True, False],\n+ [False, False, False, False, True]]))),\n }\n \n \n", "problem_statement": "Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\nConsider the following model:\r\n\r\n```python\r\nfrom astropy.modeling import models as m\r\nfrom astropy.modeling.separable import separability_matrix\r\n\r\ncm = m.Linear1D(10) & m.Linear1D(5)\r\n```\r\n\r\nIt's separability matrix as you might expect is a diagonal:\r\n\r\n```python\r\n>>> separability_matrix(cm)\r\narray([[ True, False],\r\n [False, True]])\r\n```\r\n\r\nIf I make the model more complex:\r\n```python\r\n>>> separability_matrix(m.Pix2Sky_TAN() & m.Linear1D(10) & m.Linear1D(5))\r\narray([[ True, True, False, False],\r\n [ True, True, False, False],\r\n [False, False, True, False],\r\n [False, False, False, True]])\r\n```\r\n\r\nThe output matrix is again, as expected, the outputs and inputs to the linear models are separable and independent of each other.\r\n\r\nIf however, I nest these compound models:\r\n```python\r\n>>> separability_matrix(m.Pix2Sky_TAN() & cm)\r\narray([[ True, True, False, False],\r\n [ True, True, False, False],\r\n [False, False, True, True],\r\n [False, False, True, True]])\r\n```\r\nSuddenly the inputs and outputs are no longer separable?\r\n\r\nThis feels like a bug to me, but I might be missing something?\n", "hints_text": "", "created_at": "2022-03-03T15:14:54Z", "version": "4.3", "FAIL_TO_PASS": "[\"astropy/modeling/tests/test_separable.py::test_separable[compound_model6-result6]\", \"astropy/modeling/tests/test_separable.py::test_separable[compound_model9-result9]\"]", "PASS_TO_PASS": "[\"astropy/modeling/tests/test_separable.py::test_coord_matrix\", \"astropy/modeling/tests/test_separable.py::test_cdot\", \"astropy/modeling/tests/test_separable.py::test_cstack\", \"astropy/modeling/tests/test_separable.py::test_arith_oper\", \"astropy/modeling/tests/test_separable.py::test_separable[compound_model0-result0]\", \"astropy/modeling/tests/test_separable.py::test_separable[compound_model1-result1]\", \"astropy/modeling/tests/test_separable.py::test_separable[compound_model2-result2]\", \"astropy/modeling/tests/test_separable.py::test_separable[compound_model3-result3]\", \"astropy/modeling/tests/test_separable.py::test_separable[compound_model4-result4]\", \"astropy/modeling/tests/test_separable.py::test_separable[compound_model5-result5]\", \"astropy/modeling/tests/test_separable.py::test_separable[compound_model7-result7]\", \"astropy/modeling/tests/test_separable.py::test_separable[compound_model8-result8]\", \"astropy/modeling/tests/test_separable.py::test_custom_model_separable\"]", "environment_setup_commit": "298ccb478e6bf092953bca67a3d29dc6c35f6752", "difficulty": "15 min - 1 hour"} diff --git a/responses_api_agents/swe_agents/data/example_dummy_swebench_response.json b/responses_api_agents/swe_agents/data/example_dummy_swebench_response.json new file mode 100644 index 000000000..34434386e --- /dev/null +++ b/responses_api_agents/swe_agents/data/example_dummy_swebench_response.json @@ -0,0 +1,189 @@ +{ + "responses_create_params": { + "background": null, + "include": null, + "input": [], + "instructions": null, + "max_output_tokens": 12288, + "max_tool_calls": null, + "metadata": { + "instance_id": "astropy__astropy-12907", + "base_commit": "d16bfe05a744909de4b27f5875fe0d4ed41ce607", + "dataset_name": "princeton-nlp/SWE-bench_Verified", + "split": "test", + "problem_statement": "Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\nConsider the following model:\r\n\r\n```python\r\nfrom astropy.modeling import models as m\r\nfrom astropy.modeling.separable import separability_matrix\r\n\r\ncm = m.Linear1D(10) & m.Linear1D(5)\r\n```\r\n\r\nIt's separability matrix as you might expect is a diagonal:\r\n\r\n```python\r\n>>> separability_matrix(cm)\r\narray([[ True, False],\r\n [False, True]])\r\n```\r\n\r\nIf I make the model more complex:\r\n```python\r\n>>> separability_matrix(m.Pix2Sky_TAN() & m.Linear1D(10) & m.Linear1D(5))\r\narray([[ True, True, False, False],\r\n [ True, True, False, False],\r\n [False, False, True, False],\r\n [False, False, False, True]])\r\n```\r\n\r\nThe output matrix is again, as expected, the outputs and inputs to the linear models are separable and independent of each other.\r\n\r\nIf however, I nest these compound models:\r\n```python\r\n>>> separability_matrix(m.Pix2Sky_TAN() & cm)\r\narray([[ True, True, False, False],\r\n [ True, True, False, False],\r\n [False, False, True, True],\r\n [False, False, True, True]])\r\n```\r\nSuddenly the inputs and outputs are no longer separable?\r\n\r\nThis feels like a bug to me, but I might be missing something?\n", + "golden_patch": "diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\n--- a/astropy/modeling/separable.py\n+++ b/astropy/modeling/separable.py\n@@ -242,7 +242,7 @@ def _cstack(left, right):\n cright = _coord_matrix(right, 'right', noutp)\n else:\n cright = np.zeros((noutp, right.shape[1]))\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\n \n return np.hstack([cleft, cright])\n \n", + "instance_dict": "{\"instance_id\": \"astropy__astropy-12907\", \"base_commit\": \"d16bfe05a744909de4b27f5875fe0d4ed41ce607\", \"dataset_name\": \"princeton-nlp/SWE-bench_Verified\", \"split\": \"test\", \"problem_statement\": \"Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\\nConsider the following model:\\r\\n\\r\\n```python\\r\\nfrom astropy.modeling import models as m\\r\\nfrom astropy.modeling.separable import separability_matrix\\r\\n\\r\\ncm = m.Linear1D(10) & m.Linear1D(5)\\r\\n```\\r\\n\\r\\nIt's separability matrix as you might expect is a diagonal:\\r\\n\\r\\n```python\\r\\n>>> separability_matrix(cm)\\r\\narray([[ True, False],\\r\\n [False, True]])\\r\\n```\\r\\n\\r\\nIf I make the model more complex:\\r\\n```python\\r\\n>>> separability_matrix(m.Pix2Sky_TAN() & m.Linear1D(10) & m.Linear1D(5))\\r\\narray([[ True, True, False, False],\\r\\n [ True, True, False, False],\\r\\n [False, False, True, False],\\r\\n [False, False, False, True]])\\r\\n```\\r\\n\\r\\nThe output matrix is again, as expected, the outputs and inputs to the linear models are separable and independent of each other.\\r\\n\\r\\nIf however, I nest these compound models:\\r\\n```python\\r\\n>>> separability_matrix(m.Pix2Sky_TAN() & cm)\\r\\narray([[ True, True, False, False],\\r\\n [ True, True, False, False],\\r\\n [False, False, True, True],\\r\\n [False, False, True, True]])\\r\\n```\\r\\nSuddenly the inputs and outputs are no longer separable?\\r\\n\\r\\nThis feels like a bug to me, but I might be missing something?\\n\", \"golden_patch\": \"diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\\n--- a/astropy/modeling/separable.py\\n+++ b/astropy/modeling/separable.py\\n@@ -242,7 +242,7 @@ def _cstack(left, right):\\n cright = _coord_matrix(right, 'right', noutp)\\n else:\\n cright = np.zeros((noutp, right.shape[1]))\\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\\n \\n return np.hstack([cleft, cright])\\n \\n\", \"repo\": \"astropy/astropy\", \"patch\": \"diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\\n--- a/astropy/modeling/separable.py\\n+++ b/astropy/modeling/separable.py\\n@@ -242,7 +242,7 @@ def _cstack(left, right):\\n cright = _coord_matrix(right, 'right', noutp)\\n else:\\n cright = np.zeros((noutp, right.shape[1]))\\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\\n \\n return np.hstack([cleft, cright])\\n \\n\", \"test_patch\": \"diff --git a/astropy/modeling/tests/test_separable.py b/astropy/modeling/tests/test_separable.py\\n--- a/astropy/modeling/tests/test_separable.py\\n+++ b/astropy/modeling/tests/test_separable.py\\n@@ -28,6 +28,13 @@\\n p1 = models.Polynomial1D(1, name='p1')\\n \\n \\n+cm_4d_expected = (np.array([False, False, True, True]),\\n+ np.array([[True, True, False, False],\\n+ [True, True, False, False],\\n+ [False, False, True, False],\\n+ [False, False, False, True]]))\\n+\\n+\\n compound_models = {\\n 'cm1': (map3 & sh1 | rot & sh1 | sh1 & sh2 & sh1,\\n (np.array([False, False, True]),\\n@@ -52,7 +59,17 @@\\n 'cm7': (map2 | p2 & sh1,\\n (np.array([False, True]),\\n np.array([[True, False], [False, True]]))\\n- )\\n+ ),\\n+ 'cm8': (rot & (sh1 & sh2), cm_4d_expected),\\n+ 'cm9': (rot & sh1 & sh2, cm_4d_expected),\\n+ 'cm10': ((rot & sh1) & sh2, cm_4d_expected),\\n+ 'cm11': (rot & sh1 & (scl1 & scl2),\\n+ (np.array([False, False, True, True, True]),\\n+ np.array([[True, True, False, False, False],\\n+ [True, True, False, False, False],\\n+ [False, False, True, False, False],\\n+ [False, False, False, True, False],\\n+ [False, False, False, False, True]]))),\\n }\\n \\n \\n\", \"hints_text\": \"\", \"created_at\": \"2022-03-03T15:14:54Z\", \"version\": \"4.3\", \"FAIL_TO_PASS\": \"[\\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model6-result6]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model9-result9]\\\"]\", \"PASS_TO_PASS\": \"[\\\"astropy/modeling/tests/test_separable.py::test_coord_matrix\\\", \\\"astropy/modeling/tests/test_separable.py::test_cdot\\\", \\\"astropy/modeling/tests/test_separable.py::test_cstack\\\", \\\"astropy/modeling/tests/test_separable.py::test_arith_oper\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model0-result0]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model1-result1]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model2-result2]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model3-result3]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model4-result4]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model5-result5]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model7-result7]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model8-result8]\\\", \\\"astropy/modeling/tests/test_separable.py::test_custom_model_separable\\\"]\", \"environment_setup_commit\": \"298ccb478e6bf092953bca67a3d29dc6c35f6752\", \"difficulty\": \"15 min - 1 hour\"}" + }, + "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", + "parallel_tool_calls": true, + "previous_response_id": null, + "prompt": null, + "reasoning": null, + "service_tier": null, + "store": null, + "temperature": 0.7, + "text": null, + "tool_choice": "auto", + "tools": [], + "top_logprobs": null, + "top_p": 0.8, + "truncation": null, + "user": null, + "stream": null + }, + "response": { + "id": "swebench-astropy__astropy-12907", + "created_at": 1770242297.0, + "error": null, + "incomplete_details": null, + "instructions": null, + "metadata": null, + "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", + "object": "response", + "output": [], + "parallel_tool_calls": true, + "temperature": null, + "tool_choice": "auto", + "tools": [], + "top_p": null, + "background": null, + "conversation": null, + "max_output_tokens": null, + "max_tool_calls": null, + "previous_response_id": null, + "prompt": null, + "prompt_cache_key": null, + "reasoning": null, + "safety_identifier": null, + "service_tier": null, + "status": null, + "text": null, + "top_logprobs": null, + "truncation": null, + "usage": null, + "user": null + }, + "reward": 0.0, + "instance_id": "", + "instance_dir": "", + "resolved": false, + "patch_exists": false, + "patch_successfully_applied": false, + "ray_queue_time": 0.0, + "final_eval_time": 0.0, + "hit_empty_trajectory": false, + "hit_success": false, + "hit_responses_exception": false, + "instance_config": { + "host": "localhost", + "port": 9003, + "num_workers": null, + "entrypoint": "responses_api_agents/swe_agents", + "domain": null, + "name": "test_swe_agent", + "model_server": { + "type": "responses_api_models", + "name": "test_model" + }, + "agent_config": "custom/config", + "agent_tools_file": "tools.json", + "agent_max_turns": 50, + "agent_framework_repo": null, + "agent_framework_commit": "HEAD", + "container_formatter": [ + "docker://custom/{instance_id}" + ], + "swebench_tests_timeout": 900, + "swebench_agent_timeout": 2700, + "apptainer_memory_limit_mb": 32768, + "command_exec_timeout": 300, + "concurrency": 256, + "dataset_path": null, + "openhands_should_log": false, + "debug": false, + "ng_global_config_dict_str": "'{}\n'", + "model_server_name": "test_model", + "openhands_setup_dir": "/Users/bxyu/Documents/nemo-gym/responses_api_agents/swe_agents/swe_openhands_setup", + "swebench_setup_dir": "/Users/bxyu/Documents/nemo-gym/responses_api_agents/swe_agents/swe_swebench_setup", + "r2e_gym_setup_dir": "/Users/bxyu/Documents/nemo-gym/responses_api_agents/swe_agents/swe_r2e_gym_setup", + "run_session_id": "1770242297263_f7e9a1fe", + "base_results_dir": "/Users/bxyu/Documents/nemo-gym/responses_api_agents/swe_agents/swebench_results_1770242297263_f7e9a1fe", + "metrics_fpath": "/Users/bxyu/Documents/nemo-gym/responses_api_agents/swe_agents/swebench_results_1770242297263_f7e9a1fe/astropy__astropy-12907_1770242297265_1d3795a7/nemo_gym_metrics.json", + "problem_info": { + "instance_id": "astropy__astropy-12907", + "base_commit": "d16bfe05a744909de4b27f5875fe0d4ed41ce607", + "dataset_name": "princeton-nlp/SWE-bench_Verified", + "split": "test", + "problem_statement": "Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\nConsider the following model:\r\n\r\n```python\r\nfrom astropy.modeling import models as m\r\nfrom astropy.modeling.separable import separability_matrix\r\n\r\ncm = m.Linear1D(10) & m.Linear1D(5)\r\n```\r\n\r\nIt's separability matrix as you might expect is a diagonal:\r\n\r\n```python\r\n>>> separability_matrix(cm)\r\narray([[ True, False],\r\n [False, True]])\r\n```\r\n\r\nIf I make the model more complex:\r\n```python\r\n>>> separability_matrix(m.Pix2Sky_TAN() & m.Linear1D(10) & m.Linear1D(5))\r\narray([[ True, True, False, False],\r\n [ True, True, False, False],\r\n [False, False, True, False],\r\n [False, False, False, True]])\r\n```\r\n\r\nThe output matrix is again, as expected, the outputs and inputs to the linear models are separable and independent of each other.\r\n\r\nIf however, I nest these compound models:\r\n```python\r\n>>> separability_matrix(m.Pix2Sky_TAN() & cm)\r\narray([[ True, True, False, False],\r\n [ True, True, False, False],\r\n [False, False, True, True],\r\n [False, False, True, True]])\r\n```\r\nSuddenly the inputs and outputs are no longer separable?\r\n\r\nThis feels like a bug to me, but I might be missing something?\n", + "golden_patch": "diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\n--- a/astropy/modeling/separable.py\n+++ b/astropy/modeling/separable.py\n@@ -242,7 +242,7 @@ def _cstack(left, right):\n cright = _coord_matrix(right, 'right', noutp)\n else:\n cright = np.zeros((noutp, right.shape[1]))\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\n \n return np.hstack([cleft, cright])\n \n", + "instance_dict": "{\"instance_id\": \"astropy__astropy-12907\", \"base_commit\": \"d16bfe05a744909de4b27f5875fe0d4ed41ce607\", \"dataset_name\": \"princeton-nlp/SWE-bench_Verified\", \"split\": \"test\", \"problem_statement\": \"Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\\nConsider the following model:\\r\\n\\r\\n```python\\r\\nfrom astropy.modeling import models as m\\r\\nfrom astropy.modeling.separable import separability_matrix\\r\\n\\r\\ncm = m.Linear1D(10) & m.Linear1D(5)\\r\\n```\\r\\n\\r\\nIt's separability matrix as you might expect is a diagonal:\\r\\n\\r\\n```python\\r\\n>>> separability_matrix(cm)\\r\\narray([[ True, False],\\r\\n [False, True]])\\r\\n```\\r\\n\\r\\nIf I make the model more complex:\\r\\n```python\\r\\n>>> separability_matrix(m.Pix2Sky_TAN() & m.Linear1D(10) & m.Linear1D(5))\\r\\narray([[ True, True, False, False],\\r\\n [ True, True, False, False],\\r\\n [False, False, True, False],\\r\\n [False, False, False, True]])\\r\\n```\\r\\n\\r\\nThe output matrix is again, as expected, the outputs and inputs to the linear models are separable and independent of each other.\\r\\n\\r\\nIf however, I nest these compound models:\\r\\n```python\\r\\n>>> separability_matrix(m.Pix2Sky_TAN() & cm)\\r\\narray([[ True, True, False, False],\\r\\n [ True, True, False, False],\\r\\n [False, False, True, True],\\r\\n [False, False, True, True]])\\r\\n```\\r\\nSuddenly the inputs and outputs are no longer separable?\\r\\n\\r\\nThis feels like a bug to me, but I might be missing something?\\n\", \"golden_patch\": \"diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\\n--- a/astropy/modeling/separable.py\\n+++ b/astropy/modeling/separable.py\\n@@ -242,7 +242,7 @@ def _cstack(left, right):\\n cright = _coord_matrix(right, 'right', noutp)\\n else:\\n cright = np.zeros((noutp, right.shape[1]))\\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\\n \\n return np.hstack([cleft, cright])\\n \\n\", \"repo\": \"astropy/astropy\", \"patch\": \"diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\\n--- a/astropy/modeling/separable.py\\n+++ b/astropy/modeling/separable.py\\n@@ -242,7 +242,7 @@ def _cstack(left, right):\\n cright = _coord_matrix(right, 'right', noutp)\\n else:\\n cright = np.zeros((noutp, right.shape[1]))\\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\\n \\n return np.hstack([cleft, cright])\\n \\n\", \"test_patch\": \"diff --git a/astropy/modeling/tests/test_separable.py b/astropy/modeling/tests/test_separable.py\\n--- a/astropy/modeling/tests/test_separable.py\\n+++ b/astropy/modeling/tests/test_separable.py\\n@@ -28,6 +28,13 @@\\n p1 = models.Polynomial1D(1, name='p1')\\n \\n \\n+cm_4d_expected = (np.array([False, False, True, True]),\\n+ np.array([[True, True, False, False],\\n+ [True, True, False, False],\\n+ [False, False, True, False],\\n+ [False, False, False, True]]))\\n+\\n+\\n compound_models = {\\n 'cm1': (map3 & sh1 | rot & sh1 | sh1 & sh2 & sh1,\\n (np.array([False, False, True]),\\n@@ -52,7 +59,17 @@\\n 'cm7': (map2 | p2 & sh1,\\n (np.array([False, True]),\\n np.array([[True, False], [False, True]]))\\n- )\\n+ ),\\n+ 'cm8': (rot & (sh1 & sh2), cm_4d_expected),\\n+ 'cm9': (rot & sh1 & sh2, cm_4d_expected),\\n+ 'cm10': ((rot & sh1) & sh2, cm_4d_expected),\\n+ 'cm11': (rot & sh1 & (scl1 & scl2),\\n+ (np.array([False, False, True, True, True]),\\n+ np.array([[True, True, False, False, False],\\n+ [True, True, False, False, False],\\n+ [False, False, True, False, False],\\n+ [False, False, False, True, False],\\n+ [False, False, False, False, True]]))),\\n }\\n \\n \\n\", \"hints_text\": \"\", \"created_at\": \"2022-03-03T15:14:54Z\", \"version\": \"4.3\", \"FAIL_TO_PASS\": \"[\\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model6-result6]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model9-result9]\\\"]\", \"PASS_TO_PASS\": \"[\\\"astropy/modeling/tests/test_separable.py::test_coord_matrix\\\", \\\"astropy/modeling/tests/test_separable.py::test_cdot\\\", \\\"astropy/modeling/tests/test_separable.py::test_cstack\\\", \\\"astropy/modeling/tests/test_separable.py::test_arith_oper\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model0-result0]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model1-result1]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model2-result2]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model3-result3]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model4-result4]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model5-result5]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model7-result7]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model8-result8]\\\", \\\"astropy/modeling/tests/test_separable.py::test_custom_model_separable\\\"]\", \"environment_setup_commit\": \"298ccb478e6bf092953bca67a3d29dc6c35f6752\", \"difficulty\": \"15 min - 1 hour\"}", + "container_formatter": [ + "docker://custom/{instance_id}" + ] + }, + "body": { + "background": null, + "include": null, + "input": [], + "instructions": null, + "max_output_tokens": 12288, + "max_tool_calls": null, + "metadata": { + "instance_id": "astropy__astropy-12907", + "base_commit": "d16bfe05a744909de4b27f5875fe0d4ed41ce607", + "dataset_name": "princeton-nlp/SWE-bench_Verified", + "split": "test", + "problem_statement": "Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\nConsider the following model:\r\n\r\n```python\r\nfrom astropy.modeling import models as m\r\nfrom astropy.modeling.separable import separability_matrix\r\n\r\ncm = m.Linear1D(10) & m.Linear1D(5)\r\n```\r\n\r\nIt's separability matrix as you might expect is a diagonal:\r\n\r\n```python\r\n>>> separability_matrix(cm)\r\narray([[ True, False],\r\n [False, True]])\r\n```\r\n\r\nIf I make the model more complex:\r\n```python\r\n>>> separability_matrix(m.Pix2Sky_TAN() & m.Linear1D(10) & m.Linear1D(5))\r\narray([[ True, True, False, False],\r\n [ True, True, False, False],\r\n [False, False, True, False],\r\n [False, False, False, True]])\r\n```\r\n\r\nThe output matrix is again, as expected, the outputs and inputs to the linear models are separable and independent of each other.\r\n\r\nIf however, I nest these compound models:\r\n```python\r\n>>> separability_matrix(m.Pix2Sky_TAN() & cm)\r\narray([[ True, True, False, False],\r\n [ True, True, False, False],\r\n [False, False, True, True],\r\n [False, False, True, True]])\r\n```\r\nSuddenly the inputs and outputs are no longer separable?\r\n\r\nThis feels like a bug to me, but I might be missing something?\n", + "golden_patch": "diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\n--- a/astropy/modeling/separable.py\n+++ b/astropy/modeling/separable.py\n@@ -242,7 +242,7 @@ def _cstack(left, right):\n cright = _coord_matrix(right, 'right', noutp)\n else:\n cright = np.zeros((noutp, right.shape[1]))\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\n \n return np.hstack([cleft, cright])\n \n", + "instance_dict": "{\"instance_id\": \"astropy__astropy-12907\", \"base_commit\": \"d16bfe05a744909de4b27f5875fe0d4ed41ce607\", \"dataset_name\": \"princeton-nlp/SWE-bench_Verified\", \"split\": \"test\", \"problem_statement\": \"Modeling's `separability_matrix` does not compute separability correctly for nested CompoundModels\\nConsider the following model:\\r\\n\\r\\n```python\\r\\nfrom astropy.modeling import models as m\\r\\nfrom astropy.modeling.separable import separability_matrix\\r\\n\\r\\ncm = m.Linear1D(10) & m.Linear1D(5)\\r\\n```\\r\\n\\r\\nIt's separability matrix as you might expect is a diagonal:\\r\\n\\r\\n```python\\r\\n>>> separability_matrix(cm)\\r\\narray([[ True, False],\\r\\n [False, True]])\\r\\n```\\r\\n\\r\\nIf I make the model more complex:\\r\\n```python\\r\\n>>> separability_matrix(m.Pix2Sky_TAN() & m.Linear1D(10) & m.Linear1D(5))\\r\\narray([[ True, True, False, False],\\r\\n [ True, True, False, False],\\r\\n [False, False, True, False],\\r\\n [False, False, False, True]])\\r\\n```\\r\\n\\r\\nThe output matrix is again, as expected, the outputs and inputs to the linear models are separable and independent of each other.\\r\\n\\r\\nIf however, I nest these compound models:\\r\\n```python\\r\\n>>> separability_matrix(m.Pix2Sky_TAN() & cm)\\r\\narray([[ True, True, False, False],\\r\\n [ True, True, False, False],\\r\\n [False, False, True, True],\\r\\n [False, False, True, True]])\\r\\n```\\r\\nSuddenly the inputs and outputs are no longer separable?\\r\\n\\r\\nThis feels like a bug to me, but I might be missing something?\\n\", \"golden_patch\": \"diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\\n--- a/astropy/modeling/separable.py\\n+++ b/astropy/modeling/separable.py\\n@@ -242,7 +242,7 @@ def _cstack(left, right):\\n cright = _coord_matrix(right, 'right', noutp)\\n else:\\n cright = np.zeros((noutp, right.shape[1]))\\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\\n \\n return np.hstack([cleft, cright])\\n \\n\", \"repo\": \"astropy/astropy\", \"patch\": \"diff --git a/astropy/modeling/separable.py b/astropy/modeling/separable.py\\n--- a/astropy/modeling/separable.py\\n+++ b/astropy/modeling/separable.py\\n@@ -242,7 +242,7 @@ def _cstack(left, right):\\n cright = _coord_matrix(right, 'right', noutp)\\n else:\\n cright = np.zeros((noutp, right.shape[1]))\\n- cright[-right.shape[0]:, -right.shape[1]:] = 1\\n+ cright[-right.shape[0]:, -right.shape[1]:] = right\\n \\n return np.hstack([cleft, cright])\\n \\n\", \"test_patch\": \"diff --git a/astropy/modeling/tests/test_separable.py b/astropy/modeling/tests/test_separable.py\\n--- a/astropy/modeling/tests/test_separable.py\\n+++ b/astropy/modeling/tests/test_separable.py\\n@@ -28,6 +28,13 @@\\n p1 = models.Polynomial1D(1, name='p1')\\n \\n \\n+cm_4d_expected = (np.array([False, False, True, True]),\\n+ np.array([[True, True, False, False],\\n+ [True, True, False, False],\\n+ [False, False, True, False],\\n+ [False, False, False, True]]))\\n+\\n+\\n compound_models = {\\n 'cm1': (map3 & sh1 | rot & sh1 | sh1 & sh2 & sh1,\\n (np.array([False, False, True]),\\n@@ -52,7 +59,17 @@\\n 'cm7': (map2 | p2 & sh1,\\n (np.array([False, True]),\\n np.array([[True, False], [False, True]]))\\n- )\\n+ ),\\n+ 'cm8': (rot & (sh1 & sh2), cm_4d_expected),\\n+ 'cm9': (rot & sh1 & sh2, cm_4d_expected),\\n+ 'cm10': ((rot & sh1) & sh2, cm_4d_expected),\\n+ 'cm11': (rot & sh1 & (scl1 & scl2),\\n+ (np.array([False, False, True, True, True]),\\n+ np.array([[True, True, False, False, False],\\n+ [True, True, False, False, False],\\n+ [False, False, True, False, False],\\n+ [False, False, False, True, False],\\n+ [False, False, False, False, True]]))),\\n }\\n \\n \\n\", \"hints_text\": \"\", \"created_at\": \"2022-03-03T15:14:54Z\", \"version\": \"4.3\", \"FAIL_TO_PASS\": \"[\\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model6-result6]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model9-result9]\\\"]\", \"PASS_TO_PASS\": \"[\\\"astropy/modeling/tests/test_separable.py::test_coord_matrix\\\", \\\"astropy/modeling/tests/test_separable.py::test_cdot\\\", \\\"astropy/modeling/tests/test_separable.py::test_cstack\\\", \\\"astropy/modeling/tests/test_separable.py::test_arith_oper\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model0-result0]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model1-result1]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model2-result2]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model3-result3]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model4-result4]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model5-result5]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model7-result7]\\\", \\\"astropy/modeling/tests/test_separable.py::test_separable[compound_model8-result8]\\\", \\\"astropy/modeling/tests/test_separable.py::test_custom_model_separable\\\"]\", \"environment_setup_commit\": \"298ccb478e6bf092953bca67a3d29dc6c35f6752\", \"difficulty\": \"15 min - 1 hour\"}" + }, + "model": "Qwen/Qwen3-Coder-30B-A3B-Instruct", + "parallel_tool_calls": true, + "previous_response_id": null, + "prompt": null, + "reasoning": null, + "service_tier": null, + "store": null, + "temperature": 0.7, + "text": null, + "tool_choice": "auto", + "tools": [], + "top_logprobs": null, + "top_p": 0.8, + "truncation": null, + "user": null, + "stream": null + }, + "persistent_dir": "/Users/bxyu/Documents/nemo-gym/responses_api_agents/swe_agents/swebench_results_1770242297263_f7e9a1fe/astropy__astropy-12907_1770242297265_1d3795a7", + "ray_queue_timestamp": 1770242297.266449, + "inference_params": { + "temperature": 0.7, + "top_p": 0.8, + "tokens_to_generate": 12288 + }, + "agent_run_id": "astropy__astropy-12907_1770242297_f908b866", + "instance_dataset_path": "/Users/bxyu/Documents/nemo-gym/responses_api_agents/swe_agents/swebench_results_1770242297263_f7e9a1fe/astropy__astropy-12907_1770242297265_1d3795a7/instance_datasets/astropy__astropy-12907_1770242297_f908b866.jsonl", + "trajectories_root": "/Users/bxyu/Documents/nemo-gym/responses_api_agents/swe_agents/swebench_results_1770242297263_f7e9a1fe/astropy__astropy-12907_1770242297265_1d3795a7/trajectories/astropy__astropy-12907", + "prediction_path": "/Users/bxyu/Documents/nemo-gym/responses_api_agents/swe_agents/swebench_results_1770242297263_f7e9a1fe/astropy__astropy-12907_1770242297265_1d3795a7/trajectories/astropy__astropy-12907/output.jsonl", + "prediction_mounted_path": "/trajectories_mount/trajectories/astropy__astropy-12907", + "model_patch_path": "/Users/bxyu/Documents/nemo-gym/responses_api_agents/swe_agents/swebench_results_1770242297263_f7e9a1fe/astropy__astropy-12907_1770242297265_1d3795a7/patch.diff", + "container": "test_container", + "eval_dir_in_openhands": "evaluation/oh/astropy__astropy-12907_1770242297_f908b866", + "openhands_config_file_path": "/tmp/config_astropy__astropy-12907_1770242297_f908b866.toml", + "agent_script_path": "/Users/bxyu/Documents/nemo-gym/responses_api_agents/swe_agents/swebench_results_1770242297263_f7e9a1fe/astropy__astropy-12907_1770242297265_1d3795a7/agent_script_astropy__astropy-12907_1770242297_f908b866.sh", + "eval_command": { + "command": "cd /swebench_setup/SWE-bench && export UV_INSTALL_DIR=\"/Users/bxyu/Documents/nemo-gym/responses_api_agents/swe_agents/swe_swebench_setup/uv\" && export UV_PYTHON_INSTALL_DIR=\"/Users/bxyu/Documents/nemo-gym/responses_api_agents/swe_agents/swe_swebench_setup/python\" && export PATH=\"/Users/bxyu/Documents/nemo-gym/responses_api_agents/swe_agents/swe_swebench_setup/uv/bin:$PATH\" && ls -lrt /root/dataset && env -u VIRTUAL_ENV /Users/bxyu/Documents/nemo-gym/responses_api_agents/swe_agents/swe_swebench_setup/SWE-bench/venv/bin/python -m swebench.harness.run_local_evaluation --predictions_path /trajectories_mount/trajectories/astropy__astropy-12907 --instance_ids astropy__astropy-12907 --timeout 900 --dataset_name /root/dataset/data.jsonl --split test --run_id astropy__astropy-12907_1770242297_f908b866 && cp -r logs/run_evaluation/astropy__astropy-12907_1770242297_f908b866 /trajectories_mount/ && rm -rf logs/run_evaluation/astropy__astropy-12907_1770242297_f908b866 && rm -rf *astropy__astropy-12907_1770242297_f908b866*", + "expected_file_pattern": "/Users/bxyu/Documents/nemo-gym/responses_api_agents/swe_agents/swebench_results_1770242297263_f7e9a1fe/astropy__astropy-12907_1770242297265_1d3795a7/astropy__astropy-12907_1770242297_f908b866/**/astropy__astropy-12907/report.json", + "mode": "eval", + "timeout": 1020 + }, + "agent_command": { + "command": "timeout --signal=TERM --kill-after=30 2700 bash /trajectories_mount/agent_script_astropy__astropy-12907_1770242297_f908b866.sh", + "expected_file_pattern": "/Users/bxyu/Documents/nemo-gym/responses_api_agents/swe_agents/swe_openhands_setup/OpenHands/evaluation/oh/astropy__astropy-12907_1770242297_f908b866/**/output.jsonl", + "mode": "agent", + "timeout": 2760 + }, + "agent_script": "#!/bin/bash\nset -e\nif [ -d /workspace ]; then echo 'Exiting because /workspace is mounted.' && echo 'Please make sure /workspace is not mounted inside of Apptainer before running OpenHands.' && echo 'This is because OpenHands DELETES EVERYTHING in the /workspace folder if it exists.' && exit 1; fi && mkdir -p /tmp/ && export PATH=/openhands_setup/miniforge3/bin:$PATH && uid=$(id -ru 2>/dev/null || id -u) && export TMUX_TMPDIR=/tmp && export TMUX=/tmp/tmux-$uid/default && mkdir -p /tmp/tmux-$uid && chown $uid:$uid /tmp/tmux-$uid || true && chmod 700 /tmp/tmux-$uid && tmux -S /tmp/tmux-$uid/default start-server || true && cd /openhands_setup/OpenHands && export RUNTIME=local && export LOG_LEVEL=CRITICAL && export DEBUG=False && export DEBUG_LLM=False && export LOG_TO_FILE=False && export LOG_ALL_EVENTS=False && export DEBUG_RUNTIME=False && export NEMO_GYM_METRICS_FPATH=/Users/bxyu/Documents/nemo-gym/responses_api_agents/swe_agents/swebench_results_1770242297263_f7e9a1fe/astropy__astropy-12907_1770242297265_1d3795a7/nemo_gym_metrics.json && export NEMO_GYM_CONFIG_DICT='{}\n' && export NEMO_GYM_MODEL_SERVER_NAME=test_model &&export VIRTUAL_ENV=/openhands_setup/OpenHands/.venv && export PATH=$PATH:/openhands_setup/OpenHands/.venv/bin && export POETRY_VIRTUALENVS_IN_PROJECT=true && export POETRY_VIRTUALENVS_CREATE=false && export POETRY_VIRTUALENVS_PATH=/openhands_setup/OpenHands && export TMUX_MEMORY_LIMIT=32768 && export COMMAND_EXEC_TIMEOUT=300 && echo '[llm.model]\n# The following parameters are overridden by Nemo-Skills:\n# model, base_url, temperature, top_p.\n# Specifying them here will have no effect! Use Nemo-Skills options instead.\napi_key = \"EMPTY\" # pragma: allowlist secret\ncustom_llm_provider = \"openai\"\nnative_tool_calling = true\nmodel = \"Qwen/Qwen3-Coder-30B-A3B-Instruct\"\nbase_url = \"\"\ntemperature = 0.7\ntop_p = 0.8\n' >/tmp/config_astropy__astropy-12907_1770242297_f908b866.toml && ./evaluation/benchmarks/swe_bench/scripts/run_infer.sh llm.model HEAD CodeActAgent 0 50 1 princeton-nlp/SWE-bench_Verified test evaluation/oh/astropy__astropy-12907_1770242297_f908b866 astropy__astropy-12907 /root/dataset/data.jsonl /tmp/config_astropy__astropy-12907_1770242297_f908b866.toml" + } +} \ No newline at end of file diff --git a/responses_api_agents/swe_agents/run_openhands.py b/responses_api_agents/swe_agents/run_openhands.py deleted file mode 100644 index 8007cd492..000000000 --- a/responses_api_agents/swe_agents/run_openhands.py +++ /dev/null @@ -1,1125 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import asyncio -import glob -import json -import os -import re -import shlex -import shutil -import time -import uuid -from dataclasses import dataclass, field -from enum import Enum -from pathlib import Path -from typing import Any, Optional - -import tomlkit -from gprof2dot import main as gprof2dot_main -from pydot import graph_from_dot_file - - -class SupportedAgentFrameworks(str, Enum): - swe_agent = "swe_agent" - openhands = "openhands" - - -SUPPORTED_DATASETS = [ - "SWE-Gym/SWE-Gym", - "R2E-Gym/R2E-Gym-Subset", - "princeton-nlp/SWE-bench_Verified", - "nv-internal-1", -] - - -@dataclass -class SweBenchInferenceConfig: - temperature: float = 1.0 - top_k: int | None = None - top_p: float = 1.0 - min_p: float | None = None - random_seed: int | None = None - tokens_to_generate: int | None = None - repetition_penalty: float | None = None - top_logprobs: int | None = None - - -@dataclass -class SweBenchGenerationConfig: - output_file: Path - agent_framework: SupportedAgentFrameworks - agent_framework_repo: str | None = None - agent_framework_commit: str = "HEAD" - agent_config: str | None = None - agent_max_turns: int = 100 - swebench_tests_timeout: int = 30 * 60 - swebench_agent_timeout: int = 45 * 60 - apptainer_memory_limit_mb: int = 32 * 1024 - command_exec_timeout: int = 5 * 60 - inference: SweBenchInferenceConfig = field(default_factory=SweBenchInferenceConfig) - server: dict = field(default_factory=dict) - - -# Converts the parameter names above to the corresponding OpenAI parameter names. -NS_TO_OPENAI_PARAM = { - "tokens_to_generate": "max_tokens", - "top_logprobs": "top_logprobs", - "random_seed": "seed", - "top_k": "top_k", - "min_p": "min_p", - "repetition_penalty": "repetition_penalty", -} - - -# Converts the parameter names above to the corresponding parameters in OpenHands's LLM config. -# https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/core/config/llm_config.py#L12 -NS_TO_OPENHANDS_PARAM = { - "tokens_to_generate": "max_output_tokens", - "top_k": "top_k", - "random_seed": "seed", - "min_p": None, - "repetition_penalty": None, - "top_logprobs": None, -} - - -@dataclass -class RunOpenHandsAgent: - cfg: SweBenchGenerationConfig - ng_global_config_dict_str: str - model_server_name: str - output_dir: str = None - openhands_setup_dir: Path | None = None - swebench_setup_dir: Path | None = None - r2e_gym_setup_dir: Path | None = None - dataset_path: str | None = None - openhands_should_log: bool = False - debug: bool = False - metrics_fpath: Path - - async def _run_swe_agent(self, data_point, api_base): - """ - Runs SWE-agent on one instance. - Returns the absolute (not mounted) path to a .jsonl file in the SWE-bench evaluation format. - """ - if self.cfg.agent_config is None: - self.cfg.agent_config = "eval/swe-bench/swe-agent/default" - if self.cfg.agent_framework_repo is None: - self.cfg.agent_framework_repo = "https://github.com/SWE-agent/SWE-agent.git" - - completion_kwargs = { - openai_param: getattr(self.cfg.inference, ns_param) - for ns_param, openai_param in NS_TO_OPENAI_PARAM.items() - if getattr(self.cfg.inference, ns_param) is not None - } - if "top_logprobs" in completion_kwargs: - completion_kwargs["logprobs"] = True - - swe_agent_cmd = ( - # first installing swe-agent repo - "curl -LsSf https://astral.sh/uv/install.sh | sh && " - "source /root/.local/bin/env && " - "cd /root && " - "mkdir SWE-agent && " - "cd SWE-agent && " - f"git clone {self.cfg.agent_framework_repo} . && " - f"git checkout {self.cfg.agent_framework_commit} && " - "uv venv --python 3.12 venv && " - # do not activate venv, use uv pip with -p flag instead - # "source venv/bin/activate && " - # "uv pip install -e . && " - "uv pip install -p /root/SWE-agent/venv/bin/python -e . && " - # then running the agent - f"/root/SWE-agent/venv/bin/python -m sweagent run " - f" --config {self.cfg.agent_config} " - f" --agent.model.name hosted_vllm/{self.cfg.server.model} " - f" --agent.model.api_base {api_base} " - f" --agent.model.temperature {self.cfg.inference.temperature} " - f" --agent.model.top_p {self.cfg.inference.top_p} " - f" --agent.model.completion_kwargs {shlex.quote(json.dumps(completion_kwargs))} " - f" --agent.model.per_instance_call_limit {self.cfg.agent_max_turns} " - f" --env.deployment.type local " - f" --env.repo.type preexisting " - f" --env.repo.repo_name testbed " - f" --env.repo.base_commit {data_point['base_commit']} " - f" --problem_statement.text {shlex.quote(data_point['problem_statement'])} " - f" --problem_statement.id {data_point['instance_id']} && " - # move trajectories to the mounted directory - f"cp -r trajectories /trajectories_mount/" - ) - - # Execute SWE-agent command - search_path = os.path.join( - self.output_dir / "trajectories", - "**", - f"{data_point['instance_id']}.pred", - ) - pred_file = await self._execute_container_command( - data_point, - swe_agent_cmd, - search_path, - mode="agent", - ) - - with open(pred_file, "r") as f: - trajectory_dict = json.loads(f.read().strip()) - - # need to rename .pred to .jsonl - pred_jsonl_file = pred_file.replace(".pred", ".jsonl") - with open(pred_jsonl_file, "w") as f: - f.write(json.dumps(trajectory_dict)) - - # TODO: get num_generated_tokens and other stats from .traj file - # looks like data['info']['model_stats'] - # {'instance_cost': 0, 'tokens_sent': 40858, 'tokens_received': 1775, 'api_calls': 9} - - return pred_jsonl_file - - async def _run_openhands( - self, - data_point: dict[str, Any], - api_base: str, - agent_run_id: str, - dataset_mount_path: Optional[str] = None, - ): - """ - Runs OpenHands on one instance. - Returns the absolute (not mounted) path to a .jsonl file in the SWE-bench evaluation format. - """ - agent_config = os.path.join(os.path.dirname(os.path.abspath(__file__)), "configs/oh_config.toml") - - # Add parameters to config.toml - # TODO(sugam): is there a better way to do this? - with open(agent_config, "r") as f: - config = tomlkit.parse(f.read()) - - config["llm"]["model"] |= { - "model": self.cfg.server["model"], - "base_url": api_base, - "temperature": self.cfg.inference.temperature, - "top_p": self.cfg.inference.top_p, - } - - for ns_param, oh_param in NS_TO_OPENHANDS_PARAM.items(): - if not getattr(self.cfg.inference, ns_param): - continue - if oh_param: - config["llm"]["model"][oh_param] = getattr(self.cfg.inference, ns_param) - else: - supported_params = [key for key, value in NS_TO_OPENHANDS_PARAM.items() if value is not None] - raise ValueError( - f"Inference parameter {ns_param} is not supported by OpenHands. " - f"Supported inference parameters: temperature, top_p, {', '.join(supported_params)}." - ) - - config_str = tomlkit.dumps(config) - - eval_dir_in_openhands = f"evaluation/oh/{agent_run_id}" - local_dataset_path = "/root/dataset/data.jsonl" - config_file_path = f"/tmp/config_{agent_run_id}.toml" - - assert self.openhands_setup_dir is not None, "OpenHands setup directory is not set" - - agent_script_name = f"agent_script_{agent_run_id}.sh" - - if self.debug: - profiling_cmd = "export NG_PROFILING_DIR=/trajectories_mount/profiling && " - else: - profiling_cmd = "" - - if self.openhands_should_log: - log_cmd = "export LOG_LEVEL=DEBUG && export LOG_TO_FILE=true && export NG_OPENHANDS_SHOULD_LOG=true && " - else: - log_cmd = ( - "export LOG_LEVEL=CRITICAL && " - "export DEBUG=False && " - "export DEBUG_LLM=False && " - "export LOG_TO_FILE=False && " - "export LOG_ALL_EVENTS=False && " - "export DEBUG_RUNTIME=False && " - ) - - agent_main_cmd = ( - "if [ -d /workspace ]; then " - " echo 'Exiting because /workspace is mounted.' && " - " echo 'Please make sure /workspace is not mounted inside of Apptainer before running OpenHands.' && " - " echo 'This is because OpenHands DELETES EVERYTHING in the /workspace folder if it exists.' && " - " exit 1; " - "fi && " - # Add miniforge bin to PATH (for tmux, node, poetry, etc.) - "mkdir -p /tmp/ && " - "export PATH=/openhands_setup/miniforge3/bin:$PATH && " - # Setup tmux socket (OpenHands requirement) - "uid=$(id -ru 2>/dev/null || id -u) && " - "export TMUX_TMPDIR=/tmp && " - "export TMUX=/tmp/tmux-$uid/default && " - "mkdir -p /tmp/tmux-$uid && " - "chown $uid:$uid /tmp/tmux-$uid || true && " - "chmod 700 /tmp/tmux-$uid && " - "tmux -S /tmp/tmux-$uid/default start-server || true && " - # Use pre-built OpenHands - "cd /openhands_setup/OpenHands && " - "export RUNTIME=local && " - f"{log_cmd}" - f"{profiling_cmd}" - f"export NEMO_GYM_METRICS_FPATH={self.metrics_fpath} && " - f"export NEMO_GYM_CONFIG_DICT={self.ng_global_config_dict_str} && " - f"export NEMO_GYM_MODEL_SERVER_NAME={self.model_server_name} &&" - "export VIRTUAL_ENV=/openhands_setup/OpenHands/.venv && " - "export PATH=$PATH:/openhands_setup/OpenHands/.venv/bin && " - # CRITICAL: Configure poetry to only use the OpenHands venv (ignore external venvs) - "export POETRY_VIRTUALENVS_IN_PROJECT=true && " - "export POETRY_VIRTUALENVS_CREATE=false && " - "export POETRY_VIRTUALENVS_PATH=/openhands_setup/OpenHands && " - f"export TMUX_MEMORY_LIMIT={self.cfg.apptainer_memory_limit_mb} && " - f"export COMMAND_EXEC_TIMEOUT={self.cfg.command_exec_timeout} && " - # TODO (sugam): fix cryptography issue - # "override_dir=$(mktemp -d /tmp/cryptography_override.XXXX) && " - # # Reinstall cryptography inside the container (via poetry's venv) using a compatible wheel - # # Clean any broken installs to avoid missing-file errors, then force a wheel-only reinstall - # "site_packages_dir=/openhands_setup/OpenHands/.venv/lib/python3.12/site-packages && " - # 'if [ -d "$site_packages_dir" ]; then ' - # ' find "$site_packages_dir" -maxdepth 1 -name "cryptography*" -exec rm -rf {} +; ' - # "fi && " - # "poetry run python -m pip install --index-url https://pypi.org/simple " - # " --trusted-host pypi.org --trusted-host files.pythonhosted.org " - # " --only-binary cryptography --no-deps --force-reinstall 'cryptography==42.0.8' && " - # disable logging to file in the oh repo - # set up config files - f"echo {shlex.quote(config_str)} >{config_file_path} && " - # f" export EVAL_OUTPUT_DIR={eval_dir_in_openhands} && " - f"./evaluation/benchmarks/swe_bench/scripts/run_infer.sh " - f" llm.model " # name of llm config section in config.toml - f" {self.cfg.agent_framework_commit} " # openhands commit - f" CodeActAgent " # agent - f" 0 " # Note: this is eval limit which randomly chooses an instance from the dataset - f" {self.cfg.agent_max_turns} " # max agent iterations - f" 1 " # number of workers - f" {data_point['dataset_name']} " # dataset name - f" {data_point['split']} " # dataset split - f" {eval_dir_in_openhands} " - f" {data_point['instance_id']} " - f" {local_dataset_path} " - f" {config_file_path}" - ) - - agent_script_path = Path(self.output_dir) / agent_script_name - with open(agent_script_path, "w") as f: - f.write("#!/bin/bash\nset -e\n") - f.write(agent_main_cmd) - f.flush() - os.fsync(f.fileno()) - - for _ in range(10): - if agent_script_path.exists(): - break - time.sleep(0.5) - - if not agent_script_path.exists(): - raise FileNotFoundError(f"Failed to create agent script at {agent_script_path}") - - agent_timeout_seconds = self.cfg.swebench_agent_timeout - openhands_cmd = ( - f"timeout --signal=TERM --kill-after=30 {agent_timeout_seconds} " - f"bash /trajectories_mount/{agent_script_name}" - ) - - search_path = os.path.join( - self.openhands_setup_dir / "OpenHands" / eval_dir_in_openhands, - "**", - "output.jsonl", - ) - - try: - # Execute OpenHands command - out_file_in_eval = await self._execute_container_command( - data_point=data_point, - command=openhands_cmd, - expected_file_pattern=search_path, - mode="agent", - max_retries=1, - timeout=self.cfg.swebench_agent_timeout + 60, - dataset_mount_path=dataset_mount_path, - ) - out_file = self._openhands_dir_copy_from_host( - data_point=data_point, - eval_dir_in_openhands=eval_dir_in_openhands, - config_file_path=config_file_path, - output_file_path=out_file_in_eval, - ) - - with open(out_file, "r") as f: - out_dict = json.loads(f.read().strip()) - - patch = out_dict["test_result"]["git_patch"] - if not patch: - patch = None - - # Create file in the SWE-bench evaluation format - pred_file = out_file.replace("output.jsonl", "output_for_eval.jsonl") - with open(pred_file, "w") as f: - f.write( - json.dumps( - { - "model_name_or_path": out_dict["metadata"]["llm_config"]["model"], - "instance_id": out_dict["instance_id"], - "model_patch": patch + "\n" if patch and not patch.endswith("\n") else patch, - "oh_time_metrics": out_dict["metrics"], - } - ) - ) - - # Dump out dot and png files from profiling on OpenHands level - if self.debug: - base_profile_dir = Path(self.output_dir) / "profiling" - profiling_name = "openhands" - callgrind_path = base_profile_dir / f"{profiling_name}.callgrind" - callgrind_dotfile_path = base_profile_dir / f"{profiling_name}.dot" - callgrind_graph_path = base_profile_dir / f"{profiling_name}.png" - - gprof2dot_main( - argv=f"--format=callgrind --output={callgrind_dotfile_path} -e 5 -n 5 {callgrind_path}".split() - ) - - (graph,) = graph_from_dot_file(callgrind_dotfile_path) - graph.write_png(callgrind_graph_path) - except Exception as e: - self._openhands_dir_copy_from_host( - data_point=data_point, - eval_dir_in_openhands=eval_dir_in_openhands, - config_file_path=config_file_path, - output_file_path=None, - ) - print(f"Running OpenHands failed: {e}", flush=True) - return None - return pred_file - - def _openhands_dir_copy_from_host( - self, - data_point: dict[str, Any], - eval_dir_in_openhands: str, - config_file_path: str, - output_file_path: Optional[str], - ) -> Optional[str]: - eval_dir_on_host = Path(self.openhands_setup_dir) / "OpenHands" / eval_dir_in_openhands - trajectories_root = Path(self.output_dir) / "trajectories" / data_point["instance_id"] - llm_completions_dir = trajectories_root / "llm_completions" / data_point["instance_id"] - trajectories_root.mkdir(parents=True, exist_ok=True) - llm_completions_dir.mkdir(parents=True, exist_ok=True) - - dest_output: Optional[str] = None - if output_file_path: - source_output = Path(output_file_path) - if not source_output.is_absolute(): - source_output = eval_dir_on_host / source_output - if not source_output.exists(): - output_candidates = sorted(eval_dir_on_host.glob("*/*/*/output.jsonl"), key=os.path.getmtime) - if not output_candidates: - raise FileNotFoundError( - f"No output.jsonl found under {eval_dir_on_host} for {data_point['instance_id']}." - ) - source_output = output_candidates[-1] - - dest_output_path = trajectories_root / "output.jsonl" - shutil.copy2(source_output, dest_output_path) - dest_output = str(dest_output_path) - - completion_candidates = glob.glob(str(eval_dir_on_host / "*/*/*/llm_completions/*/*.json")) - if completion_candidates: - latest_completion = max(completion_candidates, key=os.path.getmtime) - shutil.copy2( - latest_completion, - llm_completions_dir / Path(latest_completion).name, - ) - - shutil.rmtree(eval_dir_on_host, ignore_errors=True) - try: - Path(config_file_path).unlink() - except OSError: - pass - - return dest_output - - def _write_instance_dataset(self, data_point: dict[str, Any], agent_run_id: str) -> Path: - """ - To avoid making HF dataset API calls, we write the instance dictionary to a file and mount it in the container. - """ - instance_dataset_dir = Path(self.output_dir) / "instance_datasets" - instance_dataset_dir.mkdir(parents=True, exist_ok=True) - instance_dataset_path = instance_dataset_dir / f"{agent_run_id}.jsonl" - - # Parse instance_dict to ensure repo_name field exists - instance_dict = json.loads(data_point["instance_dict"]) - if "repo" in instance_dict and "repo_name" not in instance_dict: - instance_dict["repo_name"] = instance_dict["repo"] - - with open(instance_dataset_path, "w") as f: - f.write(json.dumps(instance_dict) + "\n") - return instance_dataset_path - - def _cleanup_instance_dataset(self, dataset_path): - if dataset_path is None: - return - try: - Path(dataset_path).unlink(missing_ok=True) - except OSError: - pass - try: - parent_dir = Path(dataset_path).parent - if parent_dir.exists() and not any(parent_dir.iterdir()): - parent_dir.rmdir() - except OSError: - pass - - def _find_container(self, data_point: dict) -> str: - """Find the container file using multiple strategies (Exact match > Fuzzy match). - - Strategies: - 1. Replace "__" with "_1776_" (Original case, then Lowercase) - 2. Replace "__" with "_s_" (Original case, then Lowercase) - 3. Fuzzy search directory for .sif files matching above patterns. - - Returns: - str: Path to the container file. - - Raises: - FileNotFoundError: If no matching container file is found. - """ - instance_id = data_point["instance_id"] - container_formatters = data_point["container_formatter"] - - if isinstance(container_formatters, str): - container_formatters = [container_formatters] - - if "R2E-Gym" in data_point["dataset_name"]: - instance_id_modified = re.sub( - r"[^_]+__([^-]+)-", lambda m: m.group(1).lower() + "_final_", data_point["instance_id"] - ) - for container_formatter in container_formatters: - container_name = container_formatter.format(instance_id=instance_id_modified) - if os.path.exists(container_name): - # print(f"container found: {container_name}", flush=True) - # print(f"container formatter: {container_formatter}", flush=True) - return container_name - - replacements = ["_1776_", "_s_"] - - # Generate all candidate IDs in order of priority - candidate_ids = [instance_id] - for replacement in replacements: - replaced_id = instance_id.replace("__", replacement) - candidate_ids.append(replaced_id) - candidate_ids.append(replaced_id.lower()) - - # Phase 1: Exact Matches - try all container formatters - for container_formatter in container_formatters: - for candidate_id in candidate_ids: - path = container_formatter.format(instance_id=candidate_id) - if os.path.exists(path): - return path - - # Phase 2: Fuzzy Search - try all container formatters - search_terms = [instance_id, instance_id.lower()] + candidate_ids - - for container_formatter in container_formatters: - # Define the default fallback path (Strategy 1, original case) - fallback_path = container_formatter.format(instance_id=instance_id.replace("__", replacements[0])) - container_dir = os.path.dirname(fallback_path) - - if os.path.exists(container_dir): - for term in search_terms: - pattern = os.path.join(container_dir, f"*{term}*.sif") - matches = glob.glob(pattern) - if matches: - return matches[0] - else: - print(f"Container directory {container_dir} does not exist", flush=True) - - # Phase 3: Fallback - tried_paths = [] - for container_formatter in container_formatters: - for candidate_id in candidate_ids: - tried_paths.append(container_formatter.format(instance_id=candidate_id)) - - raise FileNotFoundError( - f"No container file found for instance_id {instance_id}. " - f"Tried the following candidate IDs: {candidate_ids}. " - f"Searched in paths: {tried_paths}." - ) - - async def _execute_container_command( - self, - data_point: dict[str, Any], - command: str, - expected_file_pattern: str, - mode: str, - max_retries: int = 2, - timeout: int = 45 * 60, # 45 minutes - dataset_mount_path: Optional[str] = None, - ): - """Execute a command in an Apptainer container with retry logic.""" - # Find the container using multiple strategies - container_name = self._find_container(data_point) - - dataset_path_to_mount = dataset_mount_path or self.dataset_path - if dataset_path_to_mount is None: - raise ValueError("Dataset path is not set") - dataset_path_to_mount = str(dataset_path_to_mount) - - logs_dir = self.output_dir / "apptainer_logs" - logs_dir.mkdir(exist_ok=True) - log_file_path = logs_dir / f"{data_point['instance_id']}_{mode}.log" - # print( - # f"Starting execution of an apptainer command. Logs are available at {log_file_path}", - # ) - - # Fix localhost URLs not working sometimes - container_commands = [] - container_commands.append("echo '127.0.0.1 localhost' >/etc/hosts") - - # Build mount arguments - mount_args = [ - f"--mount type=bind,src={self.output_dir},dst=/trajectories_mount", - ] - - # Add OpenHands setup directory mount if available (for OpenHands) - if mode == "agent" and self.cfg.agent_framework == SupportedAgentFrameworks.openhands: - # Mount the entire setup directory at both /openhands_setup and its original absolute path - # This is needed because poetry and other tools have hardcoded absolute paths - # print( - # f"Mounting pre-built OpenHands from: {self.openhands_setup_dir}", - # flush=True, - # ) - mount_args.append(f"--mount type=bind,src={self.openhands_setup_dir},dst=/openhands_setup,ro") - mount_args.append(f"--mount type=bind,src={self.openhands_setup_dir},dst={self.openhands_setup_dir},ro") - # Mount only the venv and miniforge as read-only to prevent mutation while keeping the rest writable - venv_path = Path(self.openhands_setup_dir) / "OpenHands/.venv" - mount_args.append(f"--mount type=bind,src={venv_path},dst=/openhands_setup/OpenHands/.venv,ro") - mount_args.append(f"--mount type=bind,src={venv_path},dst={venv_path},ro") - - mount_args.extend( - [ - # make everything in OpenHands read-only - f"--mount type=bind,src={self.openhands_setup_dir}/OpenHands,dst=/openhands_setup/OpenHands,ro", - f"--mount type=bind,src={self.openhands_setup_dir}/OpenHands/.eval_sessions,dst=/openhands_setup/OpenHands/.eval_sessions", - f"--mount type=bind,src={self.openhands_setup_dir}/OpenHands/.eval_sessions,dst={self.openhands_setup_dir}/OpenHands/.eval_sessions", - f"--mount type=bind,src={self.openhands_setup_dir}/OpenHands/logs,dst=/openhands_setup/OpenHands/logs", - f"--mount type=bind,src={self.openhands_setup_dir}/OpenHands/logs,dst={self.openhands_setup_dir}/OpenHands/logs", - f"--mount type=bind,src={self.openhands_setup_dir}/OpenHands/evaluation/oh,dst=/openhands_setup/OpenHands/evaluation/oh", - f"--mount type=bind,src={self.openhands_setup_dir}/OpenHands/evaluation/oh,dst={self.openhands_setup_dir}/OpenHands/evaluation/oh", - # Data - f"--mount type=bind,src={dataset_path_to_mount},dst=/root/dataset/data.jsonl", - ] - ) - - miniforge3_path = Path(self.openhands_setup_dir) / "miniforge3" - mount_args.append(f"--mount type=bind,src={miniforge3_path},dst=/openhands_setup/miniforge3,ro") - mount_args.append(f"--mount type=bind,src={miniforge3_path},dst={miniforge3_path},ro") - - # Add SWE-bench setup directory mount if available (for evaluation) - if mode == "eval" and data_point["dataset_name"] != "nv-internal-1": - # Mount the entire setup directory at both /swebench_setup and its original absolute path - # This is needed because uv venv has hardcoded absolute paths - # print( - # f"Mounting pre-built SWE-bench from: {self.swebench_setup_dir}", - # flush=True, - # ) - mount_args.append(f"--mount type=bind,src={self.swebench_setup_dir},dst=/swebench_setup") - mount_args.append(f"--mount type=bind,src={self.swebench_setup_dir},dst={self.swebench_setup_dir}") - mount_args.append(f"--mount type=bind,src={dataset_path_to_mount},dst=/root/dataset/data.jsonl") - - if mode == "eval" and data_point["dataset_name"] == "nv-internal-1": - run_script_path = self.output_dir / "run_script.sh" - parsing_script_path = self.output_dir / "parsing_script.py" - model_patch_path = self.output_dir / "patch.diff" - - mount_args.append(f"--mount type=bind,src={run_script_path},dst=/root/run_script.sh") - mount_args.append(f"--mount type=bind,src={parsing_script_path},dst=/root/parsing_script.py") - mount_args.append(f"--mount type=bind,src={model_patch_path},dst=/root/patch.diff") - - if mode == "eval" and "R2E-Gym" in data_point["dataset_name"]: - # Mount the entire setup directory at both /r2egym_setup and its original absolute path - # This is needed because uv venv has hardcoded absolute paths in its wrappers - # print(f"Mounting R2E-Gym setup directory from: {self.r2e_gym_setup_dir}", flush=True) - mount_args.append(f"--mount type=bind,src={self.r2e_gym_setup_dir},dst=/r2egym_setup") - mount_args.append(f"--mount type=bind,src={self.r2e_gym_setup_dir},dst={self.r2e_gym_setup_dir}") - mount_args.append(f"--mount type=bind,src={dataset_path_to_mount},dst=/root/dataset/data.jsonl") - - if mode == "agent" and "R2E-Gym" in data_point["dataset_name"]: - # Remove R2E-Gym test-related files. - for root_dir in ["", "/root", "/testbed"]: - container_commands.append( - # /r2e_tests contains evaluation tests that the agent should not see. - f"rm -rf {root_dir}/r2e_tests && " - # run_tests.sh launches the tests in /r2e_tests, so the agent should not see this either. - # We check that it contains the substring "r2e_tests" - # to avoid accidentally deleting an unrelated file with that name. - f"if grep -qs r2e_tests {root_dir}/run_tests.sh; then rm -rf {root_dir}/run_tests.sh; fi" - ) - container_commands.append(command) - combined_command = " && ".join(container_commands) - - mount_str = " ".join(mount_args) - - # Launch Apptainer container and execute the command - apptainer_cmd = ( - f"apptainer exec --writable-tmpfs --cleanenv --pid --no-mount home,tmp,bind-paths " - f"{mount_str} " - f" {container_name} bash -c {shlex.quote(combined_command)}" - ) - memory_limit_mb = self.cfg.apptainer_memory_limit_mb - if memory_limit_mb is not None and memory_limit_mb > 0: - memory_limit_kb = int(memory_limit_mb) * 1024 - apptainer_cmd = f"ulimit -v {memory_limit_kb} && {apptainer_cmd}" - - # Retry apptainer command up to max_retries times - for attempt in range(max_retries): - try: - # Stream output to log file as it appears - with open(log_file_path, "w") as log_file: - try: - # Create async subprocess - process = await asyncio.create_subprocess_shell( - apptainer_cmd, stdout=log_file, stderr=log_file - ) - # Wait for completion with timeout - await asyncio.wait_for(process.communicate(), timeout=timeout) - - if process.returncode != 0: - raise ValueError(f"Command failed with return code {process.returncode}") - - except asyncio.TimeoutError: - if process.returncode is None: - process.terminate() - try: - await asyncio.wait_for(process.wait(), timeout=10) - except asyncio.TimeoutError: - # Force kill if still running - process.kill() - await process.wait() - attempt = max_retries # Force exit the loop on timeout - raise ValueError("Command timed out") - - # Look for the expected file - pred_files = glob.glob(expected_file_pattern, recursive=True) - - if len(pred_files) == 1: - return pred_files[0] - elif len(pred_files) > 1: - latest_file = max(pred_files, key=os.path.getmtime) - print( - f"Multiple outputs found for {data_point['instance_id']} " - f"({len(pred_files)}). Using latest: {latest_file}", - flush=True, - ) - return latest_file - else: - raise ValueError( - f"Expected exactly one file matching {expected_file_pattern} for {data_point['instance_id']}, " - f"found {len(pred_files)}." - ) - except Exception as e: - if attempt < max_retries - 1: - print( - f"Attempt {attempt + 1} failed for instance {data_point['instance_id']}. Retrying... Error: {repr(e)}", - flush=True, - ) - continue - else: - print( - f"All {max_retries} attempts failed for instance {data_point['instance_id']}. Error: {repr(e)}", - flush=True, - ) - print( - f"Apptainer command failed. Check logs at: {log_file_path}. Error: {repr(e)}", - flush=True, - ) - raise ValueError( - f"Job failed for {data_point['instance_id']}. Check logs at: {log_file_path}. Error: {repr(e)}. " - f"Expected exactly one file matching {expected_file_pattern}, " - f"found {len(pred_files) if 'pred_files' in locals() else 'unknown'}." - ) - - async def _run_r2e_gym_eval( - self, - pred_mounted_path: str, - data_point: dict[str, Any], - agent_run_id: str, - instance_dataset_path: str, - ): - assert self.r2e_gym_setup_dir is not None, "R2E-Gym setup directory is not set" - assert self.dataset_path is not None, "Dataset path is not set" - - r2e_gym_cmd = ( - # Use mounted directory path for cd - "cd /r2egym_setup/R2E-Gym && " - # Set UV environment variables to use the mounted portable directories - f'export UV_INSTALL_DIR="{self.r2e_gym_setup_dir}/uv" && ' - f'export UV_PYTHON_INSTALL_DIR="{self.r2e_gym_setup_dir}/python" && ' - f'export PATH="{self.r2e_gym_setup_dir}/uv/bin:$PATH" && ' - # Run with clean environment to avoid venv contamination - # Use the pre-built venv directly with its absolute path - f"env -u VIRTUAL_ENV {self.r2e_gym_setup_dir}/R2E-Gym/venv/bin/python src/r2egym/agenthub/run/run_local_evaluation.py " - f" --predictions_path {pred_mounted_path} " - f" --instance_id {data_point['instance_id']} " - f" --timeout {self.cfg.swebench_tests_timeout} " - f" --dataset /root/dataset/data.jsonl " - f" --output_dir /trajectories_mount/eval-outputs/{agent_run_id}" - ) - - search_path = os.path.join( - self.output_dir, - "eval-outputs", - agent_run_id, - "report.json", - ) - report_file = await self._execute_container_command( - data_point, - r2e_gym_cmd, - search_path, - mode="eval", - timeout=self.cfg.swebench_tests_timeout + 120, - dataset_mount_path=instance_dataset_path, - ) - return report_file - - async def _run_swebench_eval( - self, - pred_mounted_path: str, - data_point: dict[str, Any], - agent_run_id: str, - instance_dataset_path: str, - ): - assert self.swebench_setup_dir is not None, "SWE-bench setup directory is not set" - assert self.dataset_path is not None, "Dataset path is not set" - - swebench_cmd = ( - # Use pre-built SWE-bench - "cd /swebench_setup/SWE-bench && " - # Set UV environment variables to use the mounted portable directories - f'export UV_INSTALL_DIR="{self.swebench_setup_dir}/uv" && ' - f'export UV_PYTHON_INSTALL_DIR="{self.swebench_setup_dir}/python" && ' - f'export PATH="{self.swebench_setup_dir}/uv/bin:$PATH" && ' - f"ls -lrt /root/dataset && " - # Run with clean environment to avoid venv contamination - # Use the pre-built venv directly with its absolute path - f"env -u VIRTUAL_ENV {self.swebench_setup_dir}/SWE-bench/venv/bin/python -m swebench.harness.run_local_evaluation " - f" --predictions_path {pred_mounted_path} " - f" --instance_ids {data_point['instance_id']} " - f" --timeout {self.cfg.swebench_tests_timeout} " - f" --dataset_name /root/dataset/data.jsonl " - f" --split {data_point['split']} " - f" --run_id {agent_run_id} && " - f"cp -r logs/run_evaluation/{agent_run_id} /trajectories_mount/ && " - f"rm -rf logs/run_evaluation/{agent_run_id} && rm -rf *{agent_run_id}*" - ) - - # Execute SWE-bench evaluation command - search_path = os.path.join( - self.output_dir, - agent_run_id, - "**", - f"{data_point['instance_id']}/report.json", - ) - - report_file = await self._execute_container_command( - data_point, - swebench_cmd, - search_path, - mode="eval", - timeout=self.cfg.swebench_tests_timeout + 120, - dataset_mount_path=instance_dataset_path, - ) - - return report_file - - async def _run_nv_internal_eval( - self, data_point: dict[str, Any], model_patch: str, instance_dataset_path: str - ) -> str: - nv_internal_eval_cmd = await self.prepare_nv_internal_eval(data_point, model_patch) - instance_dict = json.loads(data_point["instance_dict"]) - - fail_to_pass_str = instance_dict.get("fail_to_pass_select", instance_dict.get("fail_to_pass", "[]")) - pass_to_pass_str = instance_dict.get("pass_to_pass_select", instance_dict.get("pass_to_pass", "[]")) - - if isinstance(fail_to_pass_str, str): - f2p = set(json.loads(fail_to_pass_str)) - else: - f2p = set(fail_to_pass_str) - - if isinstance(pass_to_pass_str, str): - p2p = set(json.loads(pass_to_pass_str)) - else: - p2p = set(pass_to_pass_str) - - search_path = os.path.join( - self.output_dir, - "eval_results", - "output.json", - ) - report_file = await self._execute_container_command( - data_point, - nv_internal_eval_cmd, - search_path, - mode="eval", - timeout=self.cfg.swebench_tests_timeout + 120, - dataset_mount_path=instance_dataset_path, - ) - - with open(report_file, "r+") as f: - test_results = json.loads(f.read()) - is_resolved = self.check_tests_passed( - test_results, - f2p, - p2p, - ) - report_dict = dict( - resolved=is_resolved, - patch_exists=True, - patch_successfully_applied=is_resolved, - metadata={ - "test_results": test_results, - "f2p": list(f2p), - "p2p": list(p2p), - }, - ) - f.seek(0) - f.write(json.dumps({data_point["instance_id"]: report_dict}, indent=4)) - return report_file - - async def prepare_nv_internal_eval(self, data_point: dict[str, Any], model_patch: str): - instance_dict = json.loads(data_point["instance_dict"]) - base_dockerfile = instance_dict.get("base_dockerfile", "") - instance_dockerfile = instance_dict.get("instance_dockerfile", "") - - env_lines = [] - for line in (base_dockerfile + "\n" + instance_dockerfile).split("\n"): - line = line.strip() - if line.startswith("ENV "): - # Convert ENV KEY=VALUE or ENV KEY VALUE to export KEY="VALUE" - export_line = line.replace("ENV ", "export ", 1) - # Handle both Docker ENV formats: - # 1. ENV KEY=VALUE (with equals) - # 2. ENV KEY VALUE (space-separated) - if "=" in export_line: - # Format: export KEY=VALUE -> normalize spaces around = - export_line = re.sub(r"\s*=\s*", "=", export_line) - else: - # Format: export KEY VALUE -> convert to export KEY="VALUE" - parts = export_line.split(None, 2) # Split into at most 3 parts - if len(parts) >= 3: # export KEY VALUE - key = parts[1] - value = parts[2] - export_line = f'export {key}="{value}"' - - env_lines.append(export_line) - - env_exports = "\n".join(env_lines) - - # Get repo setup command - repo_cmd = instance_dict.get("before_repo_set_cmd", "").strip() - if repo_cmd: - repo_cmd = repo_cmd.split("\n")[-1] - - # Get test files - test_files_str = instance_dict.get("selected_test_files_to_run", "[]") - if isinstance(test_files_str, str): - test_files = ",".join(eval(test_files_str)) - else: - test_files = ",".join(test_files_str) - - run_script = instance_dict["run_script.sh"] - parsing_script = instance_dict["parsing_script.py"] - run_script_path = self.output_dir / "run_script.sh" - parsing_script_path = self.output_dir / "parsing_script.py" - model_patch_path = self.output_dir / "patch.diff" - with open(model_patch_path, "w") as f: - # Add a newline to the end of the patch if it doesn't have one - model_patch = model_patch + "\n" if not model_patch.endswith("\n") else model_patch - f.write(model_patch) - with open(run_script_path, "w") as f: - f.write(run_script) - with open(parsing_script_path, "w") as f: - f.write(parsing_script) - - cmd = f"""#!/bin/bash -set -e - -{env_exports} - -# Apply patch -cd /app -git reset --hard {instance_dict.get("base_commit", "")} -git checkout {instance_dict.get("base_commit", "")} - -# Apply patch with rejection to handle conflicts -git apply --ignore-space-change --ignore-whitespace --reject -v /root/patch.diff || true - -# Setup repository -{repo_cmd} - -# Run tests -bash /root/run_script.sh {test_files} > /root/stdout.log 2> /root/stderr.log || true - -# Parse results -python /root/parsing_script.py /root/stdout.log /root/stderr.log /root/output.json - -# Move outputs to the mounted directory -mkdir -p /trajectories_mount/eval_results -cp /root/output.json /trajectories_mount/eval_results/output.json -""" - - return cmd - - def check_tests_passed( - self, - test_results: dict[str, Any], - f2p: set[str], - p2p: set[str], - ) -> bool: - if not test_results: - return False - - passed_tests = {test["name"] for test in test_results.get("tests", []) if test.get("status") == "PASSED"} - required_tests = f2p.union(p2p) - - # Check if all required tests passed - if len(passed_tests) == 0 or len(required_tests) == 0: - return False - - return required_tests <= passed_tests - - async def process_single_datapoint(self, data_point: dict[str, Any], persistent_dir: Path): - self.output_dir = Path(self.cfg.output_file).parent - - agent_run_id = f"{data_point['instance_id']}_{int(time.time())}_{str(uuid.uuid4())[:8]}" - instance_dataset_path = self._write_instance_dataset(data_point, agent_run_id) - api_base = self.cfg.server["base_url"] - - start_time = asyncio.get_running_loop().time() - generation_time = None - evaluation_time = None - trajectory_dict = None - try: - if self.cfg.agent_framework == SupportedAgentFrameworks.swe_agent: - pred_file = await self._run_swe_agent( - data_point, - api_base, - instance_dataset_path, - ) - elif self.cfg.agent_framework == SupportedAgentFrameworks.openhands: - pred_file = await self._run_openhands( - data_point, - api_base, - agent_run_id, - instance_dataset_path, - ) - else: - raise ValueError( - f"Unsupported agent framework: {self.cfg.agent_framework}. " - f"Supported frameworks: {', '.join(SupportedAgentFrameworks)}." - ) - - generation_time = asyncio.get_running_loop().time() - start_time - - if pred_file is None: - report_json = { - data_point["instance_id"]: { - "resolved": False, - "patch_exists": False, - "patch_successfully_applied": False, - "generation_time": generation_time, - "evaluation_time": evaluation_time, - } - } - else: - pred_mounted_path = pred_file.replace(str(self.output_dir), "/trajectories_mount") - with open(pred_file, "r") as f: - trajectory_dict = json.loads(f.read()) - - # Check if the trajectory has an empty patch before running evaluation - has_patch = trajectory_dict["model_patch"] is not None - - if not has_patch: - report_json = { - data_point["instance_id"]: { - "resolved": False, - "patch_exists": False, - "patch_successfully_applied": False, - "generation_time": generation_time, - "evaluation_time": evaluation_time, - } - } - - else: - # Run full evaluation with streaming output - # TODO: should we fail on errors here? Seems that json isn't always generated - try: - start_time = asyncio.get_running_loop().time() - if data_point["dataset_name"] == "nv-internal-1": - report_file = await self._run_nv_internal_eval( - data_point, - trajectory_dict["model_patch"], - instance_dataset_path, - ) - elif "R2E-Gym" in data_point["dataset_name"]: - report_file = await self._run_r2e_gym_eval( - pred_mounted_path, - data_point, - agent_run_id, - instance_dataset_path, - ) - else: - report_file = await self._run_swebench_eval( - pred_mounted_path, - data_point, - agent_run_id, - instance_dataset_path, - ) - evaluation_time = asyncio.get_running_loop().time() - start_time - except ValueError: - print( - f"Failed to execute SWE-bench evaluation command for {data_point['instance_id']}", - flush=True, - ) - report_json = { - data_point["instance_id"]: { - "resolved": False, - "patch_exists": True, - "patch_successfully_applied": False, - "generation_time": generation_time, - "evaluation_time": evaluation_time, - } - } - report_file = None - - if report_file is not None: - with open(report_file, "r") as f: - report_json = json.loads(f.read().strip()) - - output_dict = { - "swe-bench-metrics": report_json[data_point["instance_id"]], - "oh_time_metrics": trajectory_dict.get("oh_time_metrics", None) if trajectory_dict else {}, - "generation": "", # required TODO: we should fix this - "generation_time": generation_time, - "evaluation_time": evaluation_time, - } - - nemo_gym_metrics = json.loads(self.metrics_fpath.read_text()) - with self.metrics_fpath.open("w") as f: - json.dump(nemo_gym_metrics | {"final_eval_time": evaluation_time}, f) - - return output_dict - finally: - self._cleanup_instance_dataset(instance_dataset_path) diff --git a/responses_api_agents/swe_agents/setup_scripts/openhands.sh b/responses_api_agents/swe_agents/setup_scripts/openhands.sh new file mode 100755 index 000000000..6e3d241a5 --- /dev/null +++ b/responses_api_agents/swe_agents/setup_scripts/openhands.sh @@ -0,0 +1,113 @@ +#!/bin/bash +set -e +set -x # Enable debug output + +# Variables +setup_dir=$SETUP_DIR +miniforge_dir=$MINIFORGE_DIR +openhands_dir=$OPENHANDS_DIR +agent_framework_repo=$AGENT_FRAMEWORK_REPO +agent_framework_commit=$AGENT_FRAMEWORK_COMMIT + +cd $setup_dir + +# Install miniforge if not properly installed +if [ ! -f "$miniforge_dir/bin/conda" ] || [ ! -f "$miniforge_dir/bin/mamba" ]; then + echo "Installing miniforge..." + # Clean up any partial installation + rm -rf "$miniforge_dir" + rm -f Miniforge3-*.sh + + echo "Downloading miniforge..." + curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" + + echo "Running miniforge installer..." + bash Miniforge3-$(uname)-$(uname -m).sh -b -p $miniforge_dir + + echo "Cleaning up installer..." + rm Miniforge3-$(uname)-$(uname -m).sh +else + echo "Miniforge already installed at $miniforge_dir" +fi + +# Add conda to PATH and source conda setup +echo "Setting up conda environment..." +export PATH="$miniforge_dir/bin:$PATH" +source $miniforge_dir/etc/profile.d/conda.sh +conda activate base + +# Verify conda and mamba are available +echo "Verifying conda installation..." +which conda +which mamba +conda --version +mamba --version + +# Install required packages +echo "Installing conda packages (this may take 5-10 minutes)..." +mamba install -y --override-channels conda-forge::python=3.12 conda-forge::nodejs conda-forge::poetry conda-forge::tmux + +# Verify installations +echo "Verifying package installations..." +which python +which node +which poetry + +# Clone OpenHands +if [ ! -d "$openhands_dir/.git" ]; then + echo "Cloning OpenHands..." + # Clean up any partial clone + rm -rf "$openhands_dir" + git clone $agent_framework_repo $openhands_dir +else + echo "OpenHands already cloned at $openhands_dir" +fi + +cd $openhands_dir +echo "Checking out $agent_framework_commit..." +git checkout $agent_framework_commit + +# Build OpenHands +echo "Building OpenHands (this may take 5-10 minutes)..." +export INSTALL_DOCKER=0 + + +# Remove any cached virtualenvs from previous runs +echo "Removing any cached poetry virtualenvs..." +rm -rf ~/.cache/pypoetry/virtualenvs/openhands-* || true + +# CRITICAL: Unset any active virtualenv from the host .venv +# This prevents poetry from getting confused about which venv to use +echo "Unsetting host virtualenv to avoid poetry confusion..." +unset VIRTUAL_ENV +unset PYTHONHOME +# Remove any venv paths from PATH to ensure clean environment +export PATH=$(echo "$PATH" | tr ':' '\n' | grep -v '\.venv' | tr '\n' ':' | sed 's/:$//') + +# Configure poetry to create virtualenv in the project directory (so it's mounted in container) +export POETRY_VIRTUALENVS_IN_PROJECT=true + +# Retry `make build` with a timeout guard on the first attempt +make build + +# Install Python dependencies with poetry +echo "Installing Python dependencies (creating .venv in OpenHands directory)..." +poetry install --no-interaction --no-root + +# Install datasets package +echo "Installing datasets package..." +poetry run python -m pip install datasets + +mkdir -p evaluation/oh +mkdir -p logs +mkdir -p .eval_sessions + +echo "Verifying .venv was created..." +if [ -d .venv ]; then + echo "✓ .venv created at $(pwd)/.venv" +else + echo "✗ ERROR: .venv was not created!" + exit 1 +fi + +echo "OpenHands setup complete!" diff --git a/responses_api_agents/swe_agents/setup_scripts/r2e_gym.sh b/responses_api_agents/swe_agents/setup_scripts/r2e_gym.sh new file mode 100755 index 000000000..a97eb7dfa --- /dev/null +++ b/responses_api_agents/swe_agents/setup_scripts/r2e_gym.sh @@ -0,0 +1,67 @@ +#!/bin/bash +set -e +set -x + +# Variables +setup_dir=$SETUP_DIR +uv_dir=$UV_DIR +python_dir=$PYTHON_DIR +r2e_gym_dir=$R2E_GYM_DIR +eval_harness_repo=$EVAL_HARNESS_REPO +eval_harness_commit=$EVAL_HARNESS_COMMIT + +cd $setup_dir + +export UV_INSTALL_DIR="$uv_dir" +export UV_PYTHON_INSTALL_DIR="$python_dir" +if [ ! -f "$uv_dir/bin/uv" ]; then + echo "Installing uv to $uv_dir..." + curl -LsSf https://astral.sh/uv/install.sh | sh +else + echo "uv already installed at $uv_dir" +fi + +export PATH="$uv_dir/bin:$PATH" +echo "Verifying uv installation..." +which uv +uv --version + +# Clone R2E-Gym +if [ ! -d "$r2e_gym_dir/.git" ]; then + echo "Cloning R2E-Gym..." + # Clean up any partial clone + rm -rf "$r2e_gym_dir" + git clone $eval_harness_repo $r2e_gym_dir +else + echo "R2E-Gym already cloned at $r2e_gym_dir" +fi + +cd $r2e_gym_dir +echo "Checking out $eval_harness_commit..." +git checkout $eval_harness_commit + +echo "Installing Python 3.12 to portable location..." +uv python install 3.12 + +echo "Python installations:" +uv python list + +echo "Creating virtual environment with uv..." +rm -rf venv +uv venv --python 3.12 venv + +echo "Installing R2E-Gym in editable mode..." +uv pip install -p $r2e_gym_dir/venv/bin/python -e . --no-cache + +echo "Verifying installation..." +$r2e_gym_dir/venv/bin/python -c "import r2egym; print('✓ r2egym installed successfully')" + +if [ -d venv ] && [ -f venv/bin/python ]; then + echo "✓ venv created at $(pwd)/venv" + echo "✓ Python version: $(venv/bin/python --version)" +else + echo "✗ ERROR: venv was not created properly!" + exit 1 +fi + +echo "R2E-Gym setup complete!" diff --git a/responses_api_agents/swe_agents/setup_scripts/swebench.sh b/responses_api_agents/swe_agents/setup_scripts/swebench.sh new file mode 100755 index 000000000..36c789170 --- /dev/null +++ b/responses_api_agents/swe_agents/setup_scripts/swebench.sh @@ -0,0 +1,64 @@ +#!/bin/bash +set -e +set -x + +# Variables +setup_dir=$SETUP_DIR +uv_dir=$UV_DIR +python_dir=$PYTHON_DIR +swebench_dir=$SWEBENCH_DIR +swebench_repo=$SWEBENCH_REPO +swebench_commit=$SWEBENCH_COMMIT + +cd $setup_dir + +export UV_INSTALL_DIR="$uv_dir" +export UV_PYTHON_INSTALL_DIR="$python_dir" +if [ ! -f "$uv_dir/bin/uv" ]; then + echo "Installing uv to $uv_dir..." + curl -LsSf https://astral.sh/uv/install.sh | sh +else + echo "uv already installed at $uv_dir" +fi + +export PATH="$uv_dir/bin:$PATH" +echo "Verifying uv installation..." +which uv +uv --version + +# Clone SWE-bench +if [ ! -d "$swebench_dir/.git" ]; then + echo "Cloning SWE-bench..." + # Clean up any partial clone + rm -rf "$swebench_dir" + git clone $swebench_repo $swebench_dir +else + echo "SWE-bench already cloned at $swebench_dir" +fi + +cd $swebench_dir +echo "Checking out $swebench_commit..." +git checkout $swebench_commit + +echo "Installing Python 3.12 to portable location..." +uv python install 3.12 + +echo "Python installations:" +uv python list + +echo "Creating virtual environment with uv..." +rm -rf venv +uv venv --python 3.12 venv + +echo "Installing SWE-bench..." +uv pip install -p $swebench_dir/venv/bin/python -e . + +if [ -d venv ] && [ -f venv/bin/python ]; then + echo "✓ venv created at $(pwd)/venv" + echo "✓ Python version: $(venv/bin/python --version)" +else + echo "✗ ERROR: venv was not created properly!" + exit 1 +fi + +echo "SWE-bench setup complete!" diff --git a/responses_api_agents/swe_agents/tests/test_app_2.py b/responses_api_agents/swe_agents/tests/test_app_2.py new file mode 100644 index 000000000..d4b26222a --- /dev/null +++ b/responses_api_agents/swe_agents/tests/test_app_2.py @@ -0,0 +1,57 @@ +from pathlib import Path +from unittest.mock import MagicMock + +from pytest import raises +from ray.exceptions import RayTaskError + +import responses_api_agents.swe_agents.app +from nemo_gym.config_types import ModelServerRef, OmegaConf +from nemo_gym.server_utils import ServerClient +from responses_api_agents.swe_agents.app import ( + BaseDatasetHarnessProcessor, + BaseRunRequest, + SWEBenchWrapper, + SWEBenchWrapperConfig, +) + + +class TestSWEBenchWrapper: + def _setup_wrapper(self, monkeypatch) -> SWEBenchWrapper: + monkeypatch.setattr( + responses_api_agents.swe_agents.app, "get_global_config_dict", MagicMock(return_value=OmegaConf.create({})) + ) + monkeypatch.setattr(BaseDatasetHarnessProcessor, "_run_setup_command", MagicMock(return_value=None)) + + config = SWEBenchWrapperConfig( + host="localhost", + port=9003, + name="test_swe_agent", + entrypoint="responses_api_agents/swe_agents", + agent_framework="swe_agent", + agent_config="custom/config", + agent_tools_file="tools.json", + agent_max_turns=50, + container_formatter=["docker://custom/{instance_id}"], + swebench_tests_timeout=900, + model_server=ModelServerRef( + type="responses_api_models", + name="test_model", + ), + ) + + wrapper = SWEBenchWrapper(config=config, server_client=MagicMock(spec=ServerClient)) + return wrapper + + def test_sanity(self, monkeypatch) -> None: + self._setup_wrapper(monkeypatch) + + async def test_sanity_run(self, monkeypatch) -> None: + wrapper = self._setup_wrapper(monkeypatch) + + monkeypatch.setattr(wrapper, "_find_container", MagicMock(return_value="test_container")) + + with (Path(__file__).parent / "../data/example.jsonl").open() as f: + lines = f.readlines() + + with raises(RayTaskError, match="Command failed with return code 1"): + await wrapper.run(body=BaseRunRequest.model_validate_json(lines[0])) diff --git a/responses_api_agents/swe_agents/utils.py b/responses_api_agents/swe_agents/utils.py deleted file mode 100644 index 5f08d3a7e..000000000 --- a/responses_api_agents/swe_agents/utils.py +++ /dev/null @@ -1,1234 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import copy -import fcntl -import json -import os -import shutil -import subprocess -import sys -from contextlib import contextmanager -from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple - -from openai.types.responses.function_tool import FunctionTool - -from nemo_gym.global_config import get_global_config_dict -from nemo_gym.openai_utils import ( - NeMoGymEasyInputMessage, - NeMoGymFunctionCallOutput, - NeMoGymMessage, - NeMoGymResponseCreateParamsNonStreaming, - NeMoGymResponseFunctionToolCall, - NeMoGymResponseOutputItem, - NeMoGymResponseOutputMessageForTraining, - NeMoGymResponseOutputText, -) -from nemo_gym.server_utils import get_first_server_config_dict -from responses_api_agents.swe_agents.run_openhands import ( - RunOpenHandsAgent, - SupportedAgentFrameworks, - SweBenchGenerationConfig, - SweBenchInferenceConfig, -) - - -### Trajectory Conversion Utils ### - - -def _extract_text_from_message(item) -> Optional[str]: - """Helper to extract text content from a message item.""" - if not (hasattr(item, "content") and item.content): - return None - - for content_item in item.content: - if isinstance(content_item, dict) and content_item.get("type") == "input_text": - return content_item.get("text", "") - - return None - - -def extract_input_messages_from_trajectory( - response_output: List, -) -> Tuple[List[NeMoGymEasyInputMessage], List]: - """Extract initial input messages from response output and return filtered output. - - These are the system/user messages that were actually sent to the agent, - which should be populated in the input field of responses_create_params. - - Args: - response_output: List of NeMoGymResponseOutputItem objects from the response - - Returns: - Tuple of (input_messages, filtered_output): - - input_messages: List of NeMoGymEasyInputMessage objects - - filtered_output: List with system/user/developer messages removed - """ - input_messages = [] - filtered_output = [] - - if not response_output: - return [], [] - - # Find where the assistant/function calls start - # TODO (sugam): check if we need the function call check. - for i, item in enumerate(response_output): - # Check if this is an assistant message or function call - is_assistant = hasattr(item, "role") and item.role == "assistant" - is_function = hasattr(item, "type") and item.type in [ - "function_call", - "function_call_output", - ] - - if is_assistant or is_function: - filtered_output.extend(response_output[i:]) - break - - # Process system/user/developer messages - if hasattr(item, "role") and item.role in ["system", "user", "developer"]: - # Try to extract text content - text_content = _extract_text_from_message(item) - if text_content: - input_messages.append( - NeMoGymEasyInputMessage( - role=item.role, - content=text_content, - type="message", - ) - ) - continue - - filtered_output.append(item) - - return input_messages, filtered_output - - -def convert_trajectory_to_output_items( - trajectory: List[Any], - agent_framework: str, -) -> List[NeMoGymResponseOutputItem]: - """Convert trajectory data to NeMoGym output items. - - Args: - trajectory: Raw trajectory data - problem_info: Problem information - agent_framework: Agent framework (swe_agent or openhands) - - Returns: - List of NeMoGym output items - """ - output_items = [] - - # For OpenHands, trajectory is already in OpenAI format - if agent_framework == "openhands" and isinstance(trajectory, list): - for item in trajectory: - if isinstance(item, dict): - role = item["role"] - - # Extract text content from content data - content_data = item.get("content", "") - text_content = "" - if isinstance(content_data, str): - text_content = content_data - elif isinstance(content_data, list): - # Handle list of content items - for c in content_data: - if isinstance(c, dict) and c.get("type") == "text": - text_content = c.get("text", "") - break # Take first text content - - if role in ["user", "system", "developer"]: - if text_content: - output_items.append( - NeMoGymMessage( - content=[{"type": "input_text", "text": text_content}], - role=role, - status="completed", - type="message", - ) - ) - - elif role == "assistant": - # Handle assistant messages with potential tool calls - tool_calls = item.get("tool_calls", []) - - # Add assistant message if there's content (even if there are also tool calls) - prompt_token_ids = item.get("prompt_token_ids", []) - generation_token_ids = item.get("generation_token_ids", []) - generation_log_probs = item.get("generation_log_probs", []) - - output_items.append( - NeMoGymResponseOutputMessageForTraining( - id=f"msg-{len(output_items)}", - content=[ - NeMoGymResponseOutputText( - type="output_text", - text=text_content, - annotations=[], - ) - ], - role="assistant", - status="completed", - type="message", - prompt_token_ids=prompt_token_ids, - generation_token_ids=generation_token_ids, - generation_log_probs=generation_log_probs, - ) - ) - - # Also add tool calls if present - if tool_calls: - # Create function call items - for tc in tool_calls: - if "function" in tc: - output_items.append( - NeMoGymResponseFunctionToolCall( - arguments=tc["function"].get("arguments", ""), - call_id=tc.get("id", ""), - name=tc["function"].get("name", ""), - type="function_call", - id=tc.get("id"), - status="completed", - ) - ) - - elif role == "tool": - # Tool response - content = item.get("content", "") - tool_call_id = item.get("tool_call_id") - if not tool_call_id and "tool_call_ids" in item: - tool_call_ids = item.get("tool_call_ids", []) - tool_call_id = tool_call_ids[0] if tool_call_ids else None - if tool_call_id: - output_items.append( - NeMoGymFunctionCallOutput( - call_id=tool_call_id, - output=text_content, - type="function_call_output", - status="completed", - ) - ) - - # For SWE-agent, trajectory format is similar to OpenAI but with additional fields - elif agent_framework == "swe_agent" and isinstance(trajectory, list): - for item in trajectory: - if isinstance(item, dict): - role = item.get("role", "") - content = item.get("content", "") - - if role in ["system", "user"]: - # Create input message - if content: - output_items.append( - NeMoGymMessage( - content=[{"type": "input_text", "text": content}], - role="system" if role == "system" else "user", - status="completed", - type="message", - ) - ) - - elif role == "assistant": - # Handle assistant messages which may have tool calls - tool_calls = item.get("tool_calls", []) - - prompt_token_ids = item.get("provider_specific_fields", {}).get("prompt_token_ids", []) - generation_token_ids = item.get("provider_specific_fields", {}).get("generation_token_ids", []) - generation_log_probs = item.get("provider_specific_fields", {}).get("generation_log_probs", []) - # Add assistant message if there's content (even if there are also tool calls) - if content: - output_items.append( - NeMoGymResponseOutputMessageForTraining( - id=f"msg-{len(output_items)}", - content=[ - NeMoGymResponseOutputText( - type="output_text", - text=content, - annotations=[], - logprobs=None, - ) - ], - role="assistant", - status="completed", - type="message", - prompt_token_ids=prompt_token_ids, - generation_token_ids=generation_token_ids, - generation_log_probs=generation_log_probs, - ) - ) - - # Also add tool calls if present - if tool_calls: - for tc in tool_calls: - if "function" in tc: - # Handle both dict and string formats for tc["function"] - func = tc["function"] - if isinstance(func, str): - # If it's a string, try to parse as JSON or use as name - try: - func = json.loads(func) - except (json.JSONDecodeError, TypeError): - # If not valid JSON, treat the string as the function name - func = {"name": func, "arguments": ""} - - output_items.append( - NeMoGymResponseFunctionToolCall( - arguments=func.get("arguments", ""), - call_id=tc.get("id", ""), - name=func.get("name", ""), - type="function_call", - id=tc.get("id"), - status="completed", - ) - ) - - elif role == "tool": - # Tool response - tool_call_ids = item.get("tool_call_ids", []) - if tool_call_ids and content: - output_items.append( - NeMoGymFunctionCallOutput( - call_id=tool_call_ids[0], # Use first ID - output=content if isinstance(content, str) else json.dumps(content), - type="function_call_output", - status="completed", - ) - ) - - return output_items - - -def get_trajectory_and_tools( - trajectories_dir: Path, - instance_id: str, - agent_framework: str, - agent_tools_file: Optional[str] = None, -) -> tuple: - """Get trajectory and tools from evaluation results. - - Args: - trajectories_dir: Directory containing trajectories - instance_id: Instance ID - agent_framework: Agent framework - agent_tools_file: Path to tools JSON file (for SWE-agent) - - Returns: - Tuple of (trajectory_data, tools) - """ - trajectory_data = None - tools = [] - - if agent_framework == "openhands": - trajectory_data, tools = get_openhands_trajectory_from_completions(trajectories_dir, instance_id) - # if trajectory_data: - # print( - # f"Loaded OpenHands trajectory from llm_completions ({len(trajectory_data)} messages)", - # flush=True, - # ) - # else: - # print(f"No trajectory files found in {trajectories_dir}", flush=True) - - elif agent_framework == "swe_agent": - # For SWE-agent, look for .traj files - if trajectories_dir.exists(): - traj_files = [f for f in trajectories_dir.glob("**/*.traj") if "demonstrations" not in str(f)] - - if traj_files: - # Read the first trajectory file found - try: - with open(traj_files[0], "r") as f: - traj_content = json.load(f) - history = traj_content["history"] - trajectory_steps = traj_content["trajectory"] - trajectory_data = extract_data_from_trajectory(trajectory_steps, history) - print(f"Found and loaded SWE-agent trajectory file: {traj_files[0]}", flush=True) - except Exception as e: - print(f"Failed to read trajectory file {traj_files[0]}: {e}", flush=True) - - # Load SWE-agent tools from the configured JSON file - if agent_tools_file: - tools_file = Path(__file__).parent / agent_tools_file - if tools_file.exists(): - with open(tools_file, "r") as f: - tools_data = json.load(f) - tools = tools_data.get("tools", []) - print(f"Loaded SWE-agent tools from {tools_file}", flush=True) - else: - print(f"SWE-agent tools file not found: {tools_file}", flush=True) - else: - print("No agent_tools_file configured for SWE-agent", flush=True) - else: - print(f"No trajectory files found in {trajectories_dir}", flush=True) - else: - print(f"Unsupported agent framework: {agent_framework}", flush=True) - - return trajectory_data, tools - - -def convert_tools_to_function_format(raw_tools: List[Dict]) -> List: - """Convert tools from ChatCompletion format to Response FunctionTool format. - - Args: - raw_tools: List of tools in ChatCompletion format - - Returns: - List of FunctionTool objects - """ - - tools = [] - for tool in raw_tools: - # Tools from SWE-agent are in ChatCompletion format with nested structure - # Convert to Response FunctionTool format which is flat - if tool.get("type") == "function" and "function" in tool: - func_def = tool["function"] - # Create FunctionTool object with flat structure - function_tool = FunctionTool( - type="function", - name=func_def.get("name", ""), - description=func_def.get("description"), - parameters=func_def.get("parameters"), - strict=func_def.get("strict"), # May be None - ) - tools.append(function_tool) - return tools - - -### SWE Agent Harness Utils ### - - -def extract_messages(trajectory_item) -> List[Dict]: - """ - Trajectory might have failed assistant messages, hence we take trajectory as ground truth instead of history. - Convert a trajectory item into assistant and tool messages. - Returns a list of messages. - """ - # Defensive check: if trajectory_item is not a dict, return empty list - if not isinstance(trajectory_item, dict): - print(f"trajectory_item is not a dict (type: {type(trajectory_item)}). Skipping.", flush=True) - return [] - - tool_calls = trajectory_item.get("tool_calls") - final_message = [] - - # Get extra_info safely - extra_info = trajectory_item.get("extra_info", {}) - if isinstance(extra_info, dict): - provider_specific_fields = extra_info.get("provider_specific_fields", {}) - else: - provider_specific_fields = {} - - # Create assistant message - assistant_msg = { - "role": "assistant", - "content": trajectory_item.get("response", ""), - "thought": trajectory_item.get("thought", ""), - "action": trajectory_item.get("action", ""), - "agent": "main", - "tool_calls": tool_calls, - "message_type": "action", - "thinking_blocks": [], - "provider_specific_fields": provider_specific_fields, - } - final_message.append(assistant_msg) - if tool_calls is not None: - # Create tool message - tool_msg = { - "role": "tool", - "content": trajectory_item.get("observation", ""), - "agent": "main", - "message_type": "observation", - "tool_call_ids": trajectory_item.get("tool_call_ids", [""]), - } - final_message.append(tool_msg) - - return final_message - - -def extract_data_from_trajectory( - trajectory_data: List[Dict], history: List[Dict] -) -> Tuple[List[Dict], Dict[int, Dict]]: - """ - Extract final trajectory from trajectory and history. - """ - final_trajectory = [] - history_copy = copy.deepcopy(history) - trajectories_copy = copy.deepcopy(trajectory_data) - - # Defensive checks for trajectory_data structure - if not trajectories_copy or len(trajectories_copy) == 0: - print("Empty trajectories_copy, returning empty trajectory", flush=True) - return [] - - # Check if last trajectory item is a dict - if not isinstance(trajectories_copy[-1], dict): - print( - f"Last trajectory item is not a dict (type: {type(trajectories_copy[-1])}), returning empty trajectory", - flush=True, - ) - return [] - - # Check if "query" key exists and is a list - if "query" not in trajectories_copy[-1] or not isinstance(trajectories_copy[-1]["query"], list): - print("'query' key missing or not a list in last trajectory item, returning empty trajectory", flush=True) - return [] - - if len(trajectories_copy[-1]["query"]) > 0 and len(trajectories_copy[-1]["query"][0]) == 0: # error case - if len(trajectories_copy) < 2: - print("Not enough trajectory items for error case, returning empty trajectory", flush=True) - return [] - if not isinstance(trajectories_copy[-2], dict) or "query" not in trajectories_copy[-2]: - print("Second-to-last trajectory item is malformed, returning empty trajectory", flush=True) - return [] - final_trajectory = trajectories_copy[-2]["query"].copy() - final_trajectory.extend(extract_messages(trajectories_copy[-2])) - if len(history_copy) >= 2: - user_message = history_copy.pop() - assistant_message = history_copy.pop() - if isinstance(user_message, dict) and isinstance(assistant_message, dict): - user_message["content"] = user_message.get("content", "") + "." + assistant_message.get("content", "") - final_trajectory.append(user_message) - else: - final_trajectory = trajectories_copy[-1]["query"].copy() - final_trajectory.extend(extract_messages(trajectories_copy[-1])) - - # Filter out any non-dict items that might have been added - final_trajectory = [item for item in final_trajectory if isinstance(item, dict)] - - return final_trajectory - - -### OpenHands Harness Utils ### - - -def get_openhands_trajectory_from_completions( - trajectories_dir: Path, - instance_id: str, -) -> tuple: - """Get trajectory from llm_completions directory for OpenHands. - - Args: - trajectories_dir: Trajectories directory - instance_id: Instance ID - - Returns: - Tuple of (messages, tools) - """ - messages = [] - tools = [] - completions_dir = trajectories_dir / instance_id / "llm_completions" / instance_id - - if not completions_dir.exists(): - print(f"No llm_completions directory found: {completions_dir}", flush=True) - return messages, tools - - completion_files = sorted(completions_dir.glob("*.json")) - - if not completion_files: - print(f"No completion files found in: {completions_dir}", flush=True) - return messages, tools - - last_file = completion_files[-1] - - try: - with open(last_file, "r") as f: - data = json.load(f) - - messages = data["messages"] - provider_specific_fields = data.get("provider_specific_fields", {}) - final_assistant_message = data["response"]["choices"][0]["message"] - - for key in ["prompt_token_ids", "generation_token_ids", "generation_log_probs"]: - if key in provider_specific_fields: - final_assistant_message[key] = provider_specific_fields[key] - - if final_assistant_message.get("content") or final_assistant_message.get("tool_calls"): - messages.append(final_assistant_message) - - tools = data.get("kwargs", {}).get("tools", []) - - # print( - # f"Loaded {len(messages)} messages from last completion file: {last_file}", - # flush=True, - # ) - - except Exception as e: - print(f"Failed to read completion file {last_file}: {e}", flush=True) - return [], [] - - for msg in messages: - if "content" in msg: - msg["content"] = msg["content"] or "" - if isinstance(msg["content"], list): - # Handle empty content lists (e.g., assistant messages with only tool calls) - if len(msg["content"]) == 0: - msg["content"] = "" - elif len(msg["content"]) == 1: - item = msg["content"][0] - if not isinstance(item, dict) or item.get("type") != "text" or "text" not in item: - raise ValueError(f"Expected content item to be {{type: 'text', text: '...'}}, got {item}") - msg["content"] = item["text"] - else: - raise ValueError(f"Expected 0 or 1 content items, got {len(msg['content'])}") - else: - raise ValueError(f"Expected content in message, got {msg}") - - return messages, tools - - -### Run SWE Harness Utils ### - - -def extract_problem_info( - body: NeMoGymResponseCreateParamsNonStreaming, - container_formatter: str | list[str], -) -> Dict: - # Get metadata - metadata = body.metadata - - # Build problem info - problem_info = { - "problem_statement": metadata["problem_statement"], - "instance_id": metadata["instance_id"], - "base_commit": metadata["base_commit"], - "dataset_name": metadata["dataset_name"], - "split": metadata["split"], - # TODO (sugam): refactor this to a cleaner approach - "instance_dict": metadata["instance_dict"], - "container_formatter": container_formatter, - } - - return problem_info - - -def get_model_endpoint(model_server_name: str) -> str: - global_config_dict = get_global_config_dict() - - model_server_config = get_first_server_config_dict( - global_config_dict, - model_server_name, - ) - - base_url = f"http://{model_server_config['host']}:{model_server_config['port']}/v1" - return base_url - - -async def run_swebench_evaluation( - problem_info: Dict, - model_endpoint: str, - body: NeMoGymResponseCreateParamsNonStreaming, - agent_framework: str, - agent_config: Optional[str], - agent_tools_file: Optional[str], - agent_max_turns: int, - swebench_tests_timeout: int, - swebench_agent_timeout: int, - persistent_dir: Path, - metrics_fpath: Path, - ng_global_config_dict_str: str, - model_server_name: str, - agent_framework_repo: Optional[str] = None, - agent_framework_commit: str = "HEAD", - openhands_setup_dir: Optional[Path] = None, - swebench_setup_dir: Optional[Path] = None, - r2e_gym_setup_dir: Optional[Path] = None, - dataset_path: Optional[str] = None, - ray_queue_time: Optional[float] = None, - ray_submit_time: Optional[float] = None, - openhands_should_log: bool = False, - debug: bool = False, - apptainer_memory_limit_mb: Optional[int] = None, - command_exec_timeout: Optional[int] = None, -) -> Dict: - instance_id = problem_info.get("instance_id", "unknown") - output_file = persistent_dir / "output.jsonl" - - inference_params = {} - - for param, key in [ - ("temperature", "temperature"), - ("top_p", "top_p"), - ("max_output_tokens", "tokens_to_generate"), - ]: - value = getattr(body, param, None) - if value is not None: - inference_params[key] = value - - inference_config = SweBenchInferenceConfig(**inference_params) - server = { - "model": body.model, - "base_url": model_endpoint, - } - - cfg = SweBenchGenerationConfig( - output_file=output_file, - agent_framework=SupportedAgentFrameworks.openhands, - agent_framework_repo=agent_framework_repo, - agent_framework_commit=agent_framework_commit, - agent_config=agent_config, - agent_max_turns=agent_max_turns, - swebench_tests_timeout=swebench_tests_timeout, - swebench_agent_timeout=swebench_agent_timeout, - apptainer_memory_limit_mb=apptainer_memory_limit_mb, - command_exec_timeout=command_exec_timeout, - inference=inference_config, - server=server, - ) - - run_oh = RunOpenHandsAgent( - cfg=cfg, - openhands_setup_dir=openhands_setup_dir, - swebench_setup_dir=swebench_setup_dir, - r2e_gym_setup_dir=r2e_gym_setup_dir, - dataset_path=dataset_path, - ng_global_config_dict_str=ng_global_config_dict_str, - openhands_should_log=openhands_should_log, - debug=debug, - model_server_name=model_server_name, - metrics_fpath=metrics_fpath, - ) - - result = await run_oh.process_single_datapoint(problem_info, persistent_dir) - print(f"Process completed for {instance_id}", flush=True) - - result["oh_time_metrics"]["ray_time_in_queue"] = ray_submit_time - ray_queue_time - - try: - with open(output_file, "w") as f: - json.dump(result, f) - except Exception as e: - print(f"Failed to write result to {output_file}: {e}", flush=True) - raise e - - # Read results - if not output_file.exists(): - raise RuntimeError(f"No output file generated: {output_file}") - - # Try to find and include trajectory file - trajectories_dir = persistent_dir / "trajectories" - trajectory_data, tools = get_trajectory_and_tools( - trajectories_dir, - instance_id, - agent_framework, - agent_tools_file if agent_framework == "swe_agent" else None, - ) - - result["tools"] = tools - result["trajectory"] = trajectory_data - - return result - - -### Harness and Evaluation Setup Utils ### - - -def _get_workspace_root() -> Path: - return Path(os.path.dirname(os.path.abspath(__file__))) - - -def _resolve_setup_directory(provided_dir: Optional[Path], default_subdir: str) -> Path: - base_dir = provided_dir or (_get_workspace_root() / default_subdir) - return base_dir.resolve() - - -@contextmanager -def _setup_directory_lock(setup_dir: Path, label: str): - """File-based lock to ensure only one process performs the setup.""" - lock_dir = setup_dir.parent - lock_dir.mkdir(parents=True, exist_ok=True) - lock_path = lock_dir / f".{setup_dir.name}.lock" - - with open(lock_path, "w") as lock_file: - print(f"Acquiring {label} setup lock at {lock_path}", flush=True) - fcntl.flock(lock_file, fcntl.LOCK_EX) - try: - yield - finally: - fcntl.flock(lock_file, fcntl.LOCK_UN) - - -def _run_setup_shell_script( - setup_dir: Path, - script_name: str, - script_content: str, - timeout_seconds: int, - label: str, - timeout_error_message: Optional[str] = None, - debug: bool = False, -) -> None: - script_path = setup_dir / script_name - - with open(script_path, "w") as f: - f.write(script_content) - script_path.chmod(0o755) - - print(f"Running {label} setup script...", flush=True) - print(f"Setup script: {script_path}", flush=True) - - process = None - try: - process = subprocess.Popen( - [str(script_path)], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - bufsize=1, - ) - - output_lines: List[str] = [] - if process.stdout is None: - raise RuntimeError("Failed to capture script output") - - target_file = sys.stderr if debug else sys.stdout - for line in process.stdout: - print(line, end="", file=target_file) - output_lines.append(line) - - process.wait(timeout=timeout_seconds) - - if process.returncode != 0: - full_output = "".join(output_lines) - raise RuntimeError(f"{label} setup failed with return code {process.returncode}:\n{full_output}") - - print(f"{label} setup completed successfully!", flush=True) - except subprocess.TimeoutExpired: - if process: - process.kill() - message = timeout_error_message or f"{label} setup timed out after {timeout_seconds} seconds" - raise RuntimeError(message) - except Exception as exc: - if isinstance(exc, RuntimeError): - raise - raise RuntimeError(f"{label} setup failed: {exc}") from exc - finally: - if process and process.stdout: - process.stdout.close() - - -def setup_swebench_environment( - swebench_repo: Optional[str] = "https://github.com/HeyyyyyyG/SWE-bench.git", - swebench_commit: str = "HEAD", - setup_dir: Optional[Path] = None, -) -> Path: - setup_dir = _resolve_setup_directory(setup_dir, "swe_swebench_setup") - - with _setup_directory_lock(setup_dir, "SWE-bench"): - swebench_dir = setup_dir / "SWE-bench" - uv_dir = setup_dir / "uv" - python_dir = setup_dir / "python" - - if swebench_dir.exists(): - print(f"SWE-bench already set up at {setup_dir}", flush=True) - print(f" - SWE-bench: {swebench_dir}", flush=True) - print(f" - venv: {swebench_dir / 'venv'}", flush=True) - print(f" - uv: {uv_dir}", flush=True) - print(f" - Python: {python_dir}", flush=True) - return setup_dir - - print(f"Setting up SWE-bench environment at {setup_dir}...", flush=True) - setup_dir.mkdir(parents=True, exist_ok=True) - - script_name = "setup_swebench.sh" - script_content = f"""#!/bin/bash -set -e -set -x - -cd {setup_dir} - -export UV_INSTALL_DIR="{uv_dir}" -export UV_PYTHON_INSTALL_DIR="{python_dir}" -if [ ! -f "{uv_dir}/bin/uv" ]; then - echo "Installing uv to {uv_dir}..." - curl -LsSf https://astral.sh/uv/install.sh | sh -else - echo "uv already installed at {uv_dir}" -fi - -export PATH="{uv_dir}/bin:$PATH" -echo "Verifying uv installation..." -which uv -uv --version - -# Clone SWE-bench -if [ ! -d "{swebench_dir}/.git" ]; then - echo "Cloning SWE-bench..." - # Clean up any partial clone - rm -rf "{swebench_dir}" - git clone {swebench_repo} {swebench_dir} -else - echo "SWE-bench already cloned at {swebench_dir}" -fi - -cd {swebench_dir} -echo "Checking out {swebench_commit}..." -git checkout {swebench_commit} - -echo "Installing Python 3.12 to portable location..." -uv python install 3.12 - -echo "Python installations:" -uv python list - -echo "Creating virtual environment with uv..." -rm -rf venv -uv venv --python 3.12 venv - -echo "Installing SWE-bench..." -uv pip install -p {swebench_dir}/venv/bin/python -e . - -if [ -d venv ] && [ -f venv/bin/python ]; then - echo "✓ venv created at $(pwd)/venv" - echo "✓ Python version: $(venv/bin/python --version)" -else - echo "✗ ERROR: venv was not created properly!" - exit 1 -fi - -echo "SWE-bench setup complete!" -""" - - _run_setup_shell_script( - setup_dir=setup_dir, - script_name=script_name, - script_content=script_content, - timeout_seconds=600, - label="SWE-bench", - timeout_error_message="SWE-bench setup timed out after 10 minutes", - ) - - print(f"Setup directory: {setup_dir}", flush=True) - print(f" - SWE-bench: {swebench_dir}", flush=True) - print(f" - venv: {swebench_dir / 'venv'}", flush=True) - print(f" - uv: {uv_dir}", flush=True) - print(f" - Python: {python_dir}", flush=True) - - return setup_dir - - -def setup_r2e_gym_environment( - eval_harness_repo: Optional[str] = None, - eval_harness_commit: str = "local-eval", - setup_dir: Optional[Path] = None, -) -> Path: - """Set up R2E-Gym environment once during initialization. - - This function builds R2E-Gym in a persistent location that can be mounted - into Apptainer containers, avoiding repeated setup for each request. - - Args: - eval_harness_repo: URL of the R2E-Gym repo (default: official repo) - eval_harness_commit: Commit/branch to use (default: local-eval) - setup_dir: Directory to set up R2E-Gym (default: workspace_root/swe_r2e_gym_setup) - - Returns: - Path to the built R2E-Gym directory - - Raises: - RuntimeError: If setup fails - """ - if eval_harness_repo is None: - eval_harness_repo = "https://github.com/ludwig-n/R2E-Gym.git" - - setup_dir = _resolve_setup_directory(setup_dir, "swe_r2e_gym_setup") - - with _setup_directory_lock(setup_dir, "R2E-Gym"): - r2e_gym_dir = setup_dir / "R2E-Gym" - uv_dir = setup_dir / "uv" - python_dir = setup_dir / "python" - - # Check if setup is complete by verifying venv and installed module - venv_dir = r2e_gym_dir / "venv" - if r2e_gym_dir.exists() and venv_dir.exists(): - # Verify r2egym module is actually installed - python_bin = venv_dir / "bin" / "python" - if python_bin.exists(): - import subprocess - - try: - result = subprocess.run([str(python_bin), "-c", "import r2egym"], capture_output=True, timeout=5) - if result.returncode == 0: - print(f"R2E-Gym already set up at {setup_dir}", flush=True) - print(f" - R2E-Gym: {r2e_gym_dir}", flush=True) - print(f" - venv: {venv_dir}", flush=True) - print(f" - uv: {uv_dir}", flush=True) - print(f" - Python: {python_dir}", flush=True) - return setup_dir - else: - print("R2E-Gym directory exists but module not properly installed, rebuilding...", flush=True) - except (subprocess.TimeoutExpired, Exception) as e: - print(f"R2E-Gym verification failed: {e}, rebuilding...", flush=True) - - print(f"Setting up R2E-Gym environment at {setup_dir}...", flush=True) - setup_dir.mkdir(parents=True, exist_ok=True) - - script_name = "setup_r2e_gym.sh" - script_content = f"""#!/bin/bash -set -e -set -x - -cd {setup_dir} - -export UV_INSTALL_DIR="{uv_dir}" -export UV_PYTHON_INSTALL_DIR="{python_dir}" -if [ ! -f "{uv_dir}/bin/uv" ]; then - echo "Installing uv to {uv_dir}..." - curl -LsSf https://astral.sh/uv/install.sh | sh -else - echo "uv already installed at {uv_dir}" -fi - -export PATH="{uv_dir}/bin:$PATH" -echo "Verifying uv installation..." -which uv -uv --version - -# Clone R2E-Gym -if [ ! -d "{r2e_gym_dir}/.git" ]; then - echo "Cloning R2E-Gym..." - # Clean up any partial clone - rm -rf "{r2e_gym_dir}" - git clone {eval_harness_repo} {r2e_gym_dir} -else - echo "R2E-Gym already cloned at {r2e_gym_dir}" -fi - -cd {r2e_gym_dir} -echo "Checking out {eval_harness_commit}..." -git checkout {eval_harness_commit} - -echo "Installing Python 3.12 to portable location..." -uv python install 3.12 - -echo "Python installations:" -uv python list - -echo "Creating virtual environment with uv..." -rm -rf venv -uv venv --python 3.12 venv - -echo "Installing R2E-Gym in editable mode..." -uv pip install -p {r2e_gym_dir}/venv/bin/python -e . --no-cache - -echo "Verifying installation..." -{r2e_gym_dir}/venv/bin/python -c "import r2egym; print('✓ r2egym installed successfully')" - -if [ -d venv ] && [ -f venv/bin/python ]; then - echo "✓ venv created at $(pwd)/venv" - echo "✓ Python version: $(venv/bin/python --version)" -else - echo "✗ ERROR: venv was not created properly!" - exit 1 -fi - -echo "R2E-Gym setup complete!" -""" - - _run_setup_shell_script( - setup_dir=setup_dir, - script_name=script_name, - script_content=script_content, - timeout_seconds=1200, - label="R2E-Gym", - timeout_error_message="R2E-Gym setup timed out after 20 minutes", - ) - - print(f"Setup directory: {setup_dir}", flush=True) - print(f" - R2E-Gym: {r2e_gym_dir}", flush=True) - print(f" - venv: {r2e_gym_dir / '.venv'}", flush=True) - print(f" - uv: {uv_dir}", flush=True) - print(f" - Python: {python_dir}", flush=True) - - return setup_dir - - -def setup_openhands_environment( - agent_framework_repo: Optional[str] = "https://github.com/sdevare-nv/nv-OpenHands.git", - agent_framework_commit: str = "gym", - setup_dir: Optional[Path] = None, - debug: bool = False, -) -> Path: - setup_dir = _resolve_setup_directory(setup_dir, "swe_openhands_setup") - - with _setup_directory_lock(setup_dir, "OpenHands"): - openhands_dir = setup_dir / "OpenHands" - miniforge_dir = setup_dir / "miniforge3" - - if openhands_dir.exists() and Path(openhands_dir / ".venv" / "bin" / "python").exists(): - print(f"OpenHands already set up at {setup_dir}", flush=True) - print(f" - Miniforge: {miniforge_dir}", flush=True) - print(f" - OpenHands: {openhands_dir}", flush=True) - return setup_dir - - print(f"Setting up OpenHands environment at {setup_dir}...", flush=True) - shutil.rmtree(setup_dir, ignore_errors=True) - setup_dir.mkdir(parents=True, exist_ok=True) - - script_name = "setup_openhands.sh" - script_content = f"""#!/bin/bash -set -e -set -x # Enable debug output - -cd {setup_dir} - -# Install miniforge if not properly installed -if [ ! -f "{miniforge_dir}/bin/conda" ] || [ ! -f "{miniforge_dir}/bin/mamba" ]; then - echo "Installing miniforge..." - # Clean up any partial installation - rm -rf "{miniforge_dir}" - rm -f Miniforge3-*.sh - - echo "Downloading miniforge..." - curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" - - echo "Running miniforge installer..." - bash Miniforge3-$(uname)-$(uname -m).sh -b -p {miniforge_dir} - - echo "Cleaning up installer..." - rm Miniforge3-$(uname)-$(uname -m).sh -else - echo "Miniforge already installed at {miniforge_dir}" -fi - -# Add conda to PATH and source conda setup -echo "Setting up conda environment..." -export PATH="{miniforge_dir}/bin:$PATH" -source {miniforge_dir}/etc/profile.d/conda.sh -conda activate base - -# Verify conda and mamba are available -echo "Verifying conda installation..." -which conda -which mamba -conda --version -mamba --version - -# Install required packages -echo "Installing conda packages (this may take 5-10 minutes)..." -mamba install -y --override-channels conda-forge::python=3.12 conda-forge::nodejs conda-forge::poetry conda-forge::tmux - -# Verify installations -echo "Verifying package installations..." -which python -which node -which poetry - -# Clone OpenHands -if [ ! -d "{openhands_dir}/.git" ]; then - echo "Cloning OpenHands..." - # Clean up any partial clone - rm -rf "{openhands_dir}" - git clone {agent_framework_repo} {openhands_dir} -else - echo "OpenHands already cloned at {openhands_dir}" -fi - -cd {openhands_dir} -echo "Checking out {agent_framework_commit}..." -git checkout {agent_framework_commit} - -# Build OpenHands -echo "Building OpenHands (this may take 5-10 minutes)..." -export INSTALL_DOCKER=0 - - -# Remove any cached virtualenvs from previous runs -echo "Removing any cached poetry virtualenvs..." -rm -rf ~/.cache/pypoetry/virtualenvs/openhands-* || true - -# CRITICAL: Unset any active virtualenv from the host .venv -# This prevents poetry from getting confused about which venv to use -echo "Unsetting host virtualenv to avoid poetry confusion..." -unset VIRTUAL_ENV -unset PYTHONHOME -# Remove any venv paths from PATH to ensure clean environment -export PATH=$(echo "$PATH" | tr ':' '\\n' | grep -v '\\.venv' | tr '\\n' ':' | sed 's/:$//') - -# Configure poetry to create virtualenv in the project directory (so it's mounted in container) -export POETRY_VIRTUALENVS_IN_PROJECT=true - -# Retry `make build` with a timeout guard on the first attempt -MAX_MAKE_BUILD_ATTEMPTS=2 -MAKE_BUILD_TIMEOUT_SECONDS=$((2 * 60)) -MAKE_BUILD_TIMEOUT_MINUTES=$((MAKE_BUILD_TIMEOUT_SECONDS / 60)) - -attempt=1 -while [ "$attempt" -le "$MAX_MAKE_BUILD_ATTEMPTS" ]; do - echo "Running make build (attempt $attempt/$MAX_MAKE_BUILD_ATTEMPTS)..." - - if [ "$attempt" -lt "$MAX_MAKE_BUILD_ATTEMPTS" ]; then - if timeout "$MAKE_BUILD_TIMEOUT_SECONDS" make build; then - echo "make build completed successfully." - break - fi - - exit_code=$? - if [ "$exit_code" -eq 124 ]; then - echo "make build timed out after $MAKE_BUILD_TIMEOUT_MINUTES minutes." - else - echo "make build failed with exit code $exit_code." - fi - - echo "Retrying make build after cleanup..." - make clean || true - attempt=$((attempt + 1)) - continue - fi - - if make build; then - echo "make build completed successfully." - break - fi - - exit_code=$? - echo "make build failed on the final attempt with exit code $exit_code." -done - - -# Install Python dependencies with poetry -echo "Installing Python dependencies (creating .venv in OpenHands directory)..." -poetry install --no-interaction --no-root - -# Install datasets package -echo "Installing datasets package..." -poetry run python -m pip install datasets - -mkdir -p evaluation/oh -mkdir -p logs -mkdir -p .eval_sessions - -echo "Verifying .venv was created..." -if [ -d .venv ]; then - echo "✓ .venv created at $(pwd)/.venv" -else - echo "✗ ERROR: .venv was not created!" - exit 1 -fi - -echo "OpenHands setup complete!" -""" - - _run_setup_shell_script( - setup_dir=setup_dir, - script_name=script_name, - script_content=script_content, - timeout_seconds=1800, - label="OpenHands", - timeout_error_message="OpenHands setup timed out after 30 minutes", - debug=debug, - ) - - print(f"Setup directory: {setup_dir}", flush=True) - print(f" - Miniforge: {miniforge_dir}", flush=True) - print(f" - OpenHands: {openhands_dir}", flush=True) - - return setup_dir diff --git a/responses_api_models/local_vllm_model/app.py b/responses_api_models/local_vllm_model/app.py index 147fd3f02..a1292149d 100644 --- a/responses_api_models/local_vllm_model/app.py +++ b/responses_api_models/local_vllm_model/app.py @@ -91,6 +91,7 @@ def __init__(self, server_args: Namespace, env_vars: Dict[str, str], server_name self._patch_signal_handler() self._patch_uvicorn_logger() self._maybe_patch_engine_stats() + self._patch_colocated_placement_group_logic() for k, v in self.env_vars.items(): environ[k] = v @@ -150,6 +151,51 @@ def _maybe_patch_engine_stats(self) -> None: ) metrics_logger.setLevel(ERROR) + def _patch_colocated_placement_group_logic(self) -> None: + """ + When running multiple local vLLM model instances on the same node, the placement group logic will error with the following since multiple placement groups are now on the same node. + + (LocalVLLMModelActor pid=504531) (APIServer pid=504531) File "responses_api_models/local_vllm_model/.venv/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 858, in launch_core_engines + (LocalVLLMModelActor pid=504531) (APIServer pid=504531) engine_actor_manager = CoreEngineActorManager( + (LocalVLLMModelActor pid=504531) (APIServer pid=504531) ^^^^^^^^^^^^^^^^^^^^^^^ + (LocalVLLMModelActor pid=504531) (APIServer pid=504531) File "responses_api_models/local_vllm_model/.venv/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 300, in __init__ + (LocalVLLMModelActor pid=504531) (APIServer pid=504531) CoreEngineActorManager.create_dp_placement_groups(vllm_config) + (LocalVLLMModelActor pid=504531) (APIServer pid=504531) File "responses_api_models/local_vllm_model/.venv/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 467, in create_dp_placement_groups + (LocalVLLMModelActor pid=504531) (APIServer pid=504531) assert len(node_ip_keys) == 1, ( + (LocalVLLMModelActor pid=504531) (APIServer pid=504531) ^^^^^^^^^^^^^^^^^^^^^^ + (LocalVLLMModelActor pid=504531) (APIServer pid=504531) AssertionError: Zero or multiple node IP keys found in node resources: ['node:10.65.9.15_group_a036a448bf98d155cd0d6a8991f902000000', 'node:10.65.9.15_group_1_8786b4bfb840f7ba7af007e7e41602000000', 'node:10.65.9.15', 'node:10.65.9.15_group_8786b4bfb840f7ba7af007e7e41602000000', 'node:10.65.9.15_group_1_a036a448bf98d155cd0d6a8991f902000000', 'node:10.65.9.15_group_0_8786b4bfb840f7ba7af007e7e41602000000', 'node:10.65.9.15_group_0_a036a448bf98d155cd0d6a8991f902000000'] + """ + from vllm.v1.engine.utils import CoreEngineActorManager + + original_create_dp_placement_groups = CoreEngineActorManager.create_dp_placement_groups + + def new_create_dp_placement_groups(*args, **kwargs): + from ray._private import state + + original_available_resources_per_node = state.available_resources_per_node + + def new_available_resources_per_node(*args, **kwargs): + result = original_available_resources_per_node(*args, **kwargs) + print(f"ORIGINAL available resources: {result}", file=sys.stderr) + + for node_hex_id, node_resources in list(result.items()): + result[node_hex_id] = { + resource_id: resource + for resource_id, resource in node_resources.items() + if "_group_" not in resource_id + } + print(f"MODIFIED available resources: {result}", file=sys.stderr) + + return result + + state.available_resources_per_node = new_available_resources_per_node + + result = original_create_dp_placement_groups(*args, **kwargs) + + return result + + CoreEngineActorManager.create_dp_placement_groups = new_create_dp_placement_groups + def base_url(self) -> str: return self._base_url diff --git a/responses_api_models/local_vllm_model/configs/multiple_models_at_once.yaml b/responses_api_models/local_vllm_model/configs/multiple_models_at_once.yaml new file mode 100644 index 000000000..f9de6e170 --- /dev/null +++ b/responses_api_models/local_vllm_model/configs/multiple_models_at_once.yaml @@ -0,0 +1,64 @@ +model1: + responses_api_models: + local_vllm_model: + entrypoint: app.py + model: Qwen/Qwen3-4B-Instruct-2507 + return_token_id_information: false + uses_reasoning_parser: true + debug: true + + # For example, if your model is downloaded at ~/.cache/huggingface/hub/models--Qwen--Qwen3-235B-A22B-Instruct-2507, your hf_home here would be `~/.cache/huggingface` + hf_home: null + + vllm_serve_env_vars: + # If your tp * pp is greater than one node or one model instance spans more than one node, set this to `span`! + VLLM_RAY_DP_PACK_STRATEGY: strict + + # If a hf_token is present in the env.yaml, it will be prepended to the hf download. + vllm_serve_kwargs: + # vLLM serving is a bit quirky at the moment. These parallelism configs are fairly sensitive and need to always be set properly! + data_parallel_size: 2 + data_parallel_size_local: 2 + tensor_parallel_size: 2 + pipeline_parallel_size: 1 + + trust_remote_code: true + gpu_memory_utilization: 0.9 + enable_auto_tool_choice: true + tool_call_parser: qwen3_coder + reasoning_parser: deepseek_r1 + model_loader_extra_config: + enable_multithread_load: true + num_threads: 16 +model2: + responses_api_models: + local_vllm_model: + entrypoint: app.py + model: Qwen/Qwen3-4B-Instruct-2507 + return_token_id_information: false + uses_reasoning_parser: true + debug: true + + # For example, if your model is downloaded at ~/.cache/huggingface/hub/models--Qwen--Qwen3-235B-A22B-Instruct-2507, your hf_home here would be `~/.cache/huggingface` + hf_home: null + + vllm_serve_env_vars: + # If your tp * pp is greater than one node or one model instance spans more than one node, set this to `span`! + VLLM_RAY_DP_PACK_STRATEGY: strict + + # If a hf_token is present in the env.yaml, it will be prepended to the hf download. + vllm_serve_kwargs: + # vLLM serving is a bit quirky at the moment. These parallelism configs are fairly sensitive and need to always be set properly! + data_parallel_size: 2 + data_parallel_size_local: 2 + tensor_parallel_size: 2 + pipeline_parallel_size: 1 + + trust_remote_code: true + gpu_memory_utilization: 0.9 + enable_auto_tool_choice: true + tool_call_parser: qwen3_coder + reasoning_parser: deepseek_r1 + model_loader_extra_config: + enable_multithread_load: true + num_threads: 16 diff --git a/responses_api_models/local_vllm_model/pyproject.toml b/responses_api_models/local_vllm_model/pyproject.toml index 1ab4ffd53..1d375a723 100644 --- a/responses_api_models/local_vllm_model/pyproject.toml +++ b/responses_api_models/local_vllm_model/pyproject.toml @@ -29,3 +29,6 @@ include = ["local_vllm_model"] [tool.uv.sources] nemo-gym = { path = "../..", editable = true } + +[tool.uv.pip] +torch-backend = "auto" diff --git a/responses_api_models/vllm_model/app.py b/responses_api_models/vllm_model/app.py index 46319303d..66fb80a40 100644 --- a/responses_api_models/vllm_model/app.py +++ b/responses_api_models/vllm_model/app.py @@ -44,6 +44,8 @@ NeMoGymChatCompletionToolParam, NeMoGymChatCompletionUserMessageParam, NeMoGymChoice, + NeMoGymEasyInputMessage, + NeMoGymFunctionCallOutput, NeMoGymFunctionDefinition, NeMoGymResponse, NeMoGymResponseCreateParamsNonStreaming, @@ -565,10 +567,12 @@ def _format_function_call( # ======================================================= def postprocess_chat_response(self, choice: NeMoGymChoice) -> List[NeMoGymResponseOutputItem]: - raw_message = choice.message.model_dump() + return self.postprocess_assistant_message_dict(choice.message.model_dump()) + + def postprocess_assistant_message_dict(self, message_dict: Dict[str, Any]) -> List[NeMoGymResponseOutputItem]: response_output = [] - content = raw_message.get("content") or "" + content = message_dict.get("content") or "" reasoning_matches, content = self._extract_reasoning_from_content(content) if reasoning_matches: reasoning_item = NeMoGymResponseReasoningItem( @@ -581,7 +585,7 @@ def postprocess_chat_response(self, choice: NeMoGymChoice) -> List[NeMoGymRespon ) response_output.append(reasoning_item) - tool_calls_raw = raw_message.get("tool_calls", []) or [] + tool_calls_raw = message_dict.get("tool_calls", []) or [] # We need to return at least one output item. When the model decides to just stop with no chat or tool calls # We just add an output item with empty or null content here. This is prevalent e.g. in the case of base models that may not be the most reliable since they have not been instruction tuned. has_empty_output = not (response_output or tool_calls_raw) @@ -590,7 +594,7 @@ def postprocess_chat_response(self, choice: NeMoGymChoice) -> List[NeMoGymRespon response_output.append( NeMoGymResponseOutputMessage( id=f"msg_{uuid4().hex}", - role=raw_message.get("role"), + role=message_dict.get("role"), content=[ NeMoGymResponseOutputText( type="output_text", @@ -618,14 +622,14 @@ def postprocess_chat_response(self, choice: NeMoGymChoice) -> List[NeMoGymRespon # `"prompt_token_ids" in raw_message`: sometimes the model endpoint may go out of context length, in which case we return an empty response # In these cases, there are no token id information provided. - if self.return_token_id_information and "prompt_token_ids" in raw_message: + if self.return_token_id_information and "prompt_token_ids" in message_dict: last_response_output_item = response_output[-1] train_cls = RESPONSES_TO_TRAIN[last_response_output_item.__class__] response_output[-1] = train_cls( **last_response_output_item.model_dump(), - prompt_token_ids=raw_message["prompt_token_ids"], - generation_token_ids=raw_message["generation_token_ids"], - generation_log_probs=raw_message["generation_log_probs"], + prompt_token_ids=message_dict["prompt_token_ids"], + generation_token_ids=message_dict["generation_token_ids"], + generation_log_probs=message_dict["generation_log_probs"], ) return response_output @@ -635,6 +639,43 @@ def _extract_reasoning_from_content(self, content: str) -> Tuple[List[str], str] # Maybe parameterize to support other model formats in the future. return self._parse_think_tags(content) + def chat_completions_messages_to_responses_items( + self, messages: List[Dict[str, Any]] + ) -> List[NeMoGymResponseOutputItem]: + output_items = [] + + for message in messages: + role = message["role"] + if role in ("user", "system", "developer"): + output_items.append(NeMoGymEasyInputMessage.model_validate(message)) + elif role == "assistant": + output_items.extend(self.postprocess_assistant_message_dict(message)) + elif role == "tool": + output_items.append( + NeMoGymFunctionCallOutput( + call_id=message["tool_call_id"], + output=message["content"], + status="completed", + ) + ) + else: + raise NotImplementedError(f"Unrecognized role: {role}!") + + return output_items + + +def split_responses_input_output_items( + items: List[NeMoGymResponseOutputItem], +) -> Tuple[List[NeMoGymResponseOutputItem], List[NeMoGymResponseOutputItem]]: + if not items: + return [], [] + + for i, item in enumerate(items): + if getattr(item, "role", None) == "assistant": + break + + return items[:i], items[i:] + if __name__ == "__main__": VLLMModel.run_webserver()