diff --git a/.github/actions/test-template/action.yml b/.github/actions/test-template/action.yml index 11469f92c4..edd2e943ed 100644 --- a/.github/actions/test-template/action.yml +++ b/.github/actions/test-template/action.yml @@ -29,6 +29,18 @@ inputs: test-data-path: description: Test data path selected by CI pre-flight required: true + model: + description: Optional model name exposed to the test script + required: false + default: "" + model-revision: + description: Optional immutable model revision exposed to the test script + required: false + default: "" + results-path: + description: Optional host directory mounted for test results + required: false + default: "" runs: using: composite @@ -38,11 +50,32 @@ runs: continue-on-error: true shell: bash -e -u -o pipefail {0} env: + CONTAINER_RESULTS_DIR: /opt/nemo-gym/results/gpu-e2e CONTAINER_IMAGE: ${{ inputs.container-image }} + MODEL: ${{ inputs.model }} + MODEL_REVISION: ${{ inputs.model-revision }} + RESULTS_PATH: ${{ inputs.results-path }} TEST_DATA_PATH: ${{ inputs.test-data-path }} TEST_SCRIPT: ${{ inputs.script }} TEST_TYPE: ${{ inputs.test-type }} run: | + prepare_mount_source() { + local label="$1" + local source="$2" + + if [[ "$source" != /* ]]; then + echo "$label must be an absolute path: $source" >&2 + return 1 + fi + mkdir -p "$source" + source="$(cd "$source" && pwd -P)" + if [[ "$source" == "/" ]]; then + echo "$label cannot resolve to the filesystem root." >&2 + return 1 + fi + printf '%s\n' "$source" + } + case "$TEST_TYPE" in cpu) gpu_args=() @@ -56,6 +89,26 @@ runs: ;; esac + TEST_DATA_PATH="$(prepare_mount_source test-data-path "$TEST_DATA_PATH")" + container_args=( + --env "TEST_DATA_PATH=/home/TestData" + --env "HF_HOME=/home/TestData/HF_HOME" + --volume "$TEST_DATA_PATH:/home/TestData" + ) + if [[ -n "$RESULTS_PATH" ]]; then + RESULTS_PATH="$(prepare_mount_source results-path "$RESULTS_PATH")" + container_args+=( + --env "RESULTS_DIR=$CONTAINER_RESULTS_DIR" + --volume "$RESULTS_PATH:$CONTAINER_RESULTS_DIR" + ) + fi + if [[ -n "$MODEL" ]]; then + container_args+=(--env "MODEL=$MODEL") + fi + if [[ -n "$MODEL_REVISION" ]]; then + container_args+=(--env "MODEL_REVISION=$MODEL_REVISION") + fi + echo -e "\033[1;34m┌─ launching test ─────────────────────────────────────────────────────────┐\033[0m" echo -e "\033[1;34m│ script : $TEST_SCRIPT\033[0m" echo -e "\033[1;34m│ type : $TEST_TYPE\033[0m" @@ -65,9 +118,8 @@ runs: docker pull "$CONTAINER_IMAGE" docker run --rm \ "${gpu_args[@]}" \ + "${container_args[@]}" \ --shm-size=64g \ - --env TEST_DATA_PATH="$TEST_DATA_PATH" \ - --volume "$TEST_DATA_PATH:$TEST_DATA_PATH" \ --entrypoint bash \ "$CONTAINER_IMAGE" \ -e -u -o pipefail "$TEST_SCRIPT" @@ -77,6 +129,8 @@ runs: shell: bash -e -u -o pipefail {0} env: CONTAINER_IMAGE: ${{ inputs.container-image }} + MODEL: ${{ inputs.model }} + RESULTS_PATH: ${{ inputs.results-path }} TEST_OUTCOME: ${{ steps.test.outcome }} TEST_SCRIPT: ${{ inputs.script }} TEST_TYPE: ${{ inputs.test-type }} @@ -108,6 +162,12 @@ runs: echo "|---|---|" echo "| Type | \`$TEST_TYPE\` |" echo "| Image | \`$CONTAINER_IMAGE\` |" + if [[ -n "$MODEL" ]]; then + echo "| Model | \`$MODEL\` |" + fi + if [[ -n "$RESULTS_PATH" ]]; then + echo "| Results | \`$RESULTS_PATH\` |" + fi } >> "$GITHUB_STEP_SUMMARY" if [[ "$TEST_OUTCOME" == "success" ]]; then diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index 493d43b3cc..09966fd5bf 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -120,6 +120,7 @@ jobs: NEMO_GYM_COMMIT=${{ github.sha }} NVIDIA_BUILD_ID=${{ github.run_id }} NVIDIA_BUILD_REF=${{ github.ref }} + NEMO_GYM_PREFETCH_CONFIGS=tests/e2e/gpu_e2e.yaml cache-from: type=registry,ref=${{ needs.pre-flight.outputs.registry }}/gym:${{ steps.image.outputs.cache-seed }}-buildcache cache-to: type=registry,ref=${{ needs.pre-flight.outputs.registry }}/gym:${{ steps.image.outputs.cache-key }}-buildcache,mode=max tags: | @@ -134,10 +135,14 @@ jobs: fail-fast: false matrix: include: - - name: GPU E2E - NVIDIA SMI + - name: GPU E2E - Qwen vLLM rollout script: ./tests/e2e/gpu_e2e_test.sh test_type: gpu + model: Qwen/Qwen2.5-0.5B-Instruct + model_revision: 7ae557604adf67be50417f59c2c2f167def9a775 # pragma: allowlist secret + artifact_name: gpu-e2e-qwen-vllm runs-on: ${{ needs.pre-flight.outputs.runner_prefix }} + timeout-minutes: 30 steps: - name: Checkout repository uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 @@ -149,6 +154,18 @@ jobs: test-type: ${{ matrix.test_type }} container-image: ${{ needs.container_build.outputs.image }} test-data-path: ${{ needs.pre-flight.outputs.test_data_path }} + model: ${{ matrix.model }} + model-revision: ${{ matrix.model_revision }} + results-path: ${{ runner.temp }}/nemo-gym-gpu-e2e/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.artifact_name }} + + - name: Upload GPU E2E artifacts + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ matrix.artifact_name }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/nemo-gym-gpu-e2e/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.artifact_name }} + if-no-files-found: warn + retention-days: 7 Nemo_CICD_Test: needs: [pre-flight, classify_changes, unit_tests, container_build, gpu_e2e_tests] diff --git a/tests/e2e/gpu_e2e.yaml b/tests/e2e/gpu_e2e.yaml new file mode 100644 index 0000000000..531a77a079 --- /dev/null +++ b/tests/e2e/gpu_e2e.yaml @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +config_paths: + - resources_servers/string_match/configs/string_match.yaml + - responses_api_models/vllm_model/configs/vllm_model.yaml + +policy_base_url: http://127.0.0.1:18000/v1 +policy_api_key: not-a-real-key # pragma: allowlist secret +policy_model_name: Qwen/Qwen2.5-0.5B-Instruct +skip_venv_if_present: true + +policy_model: + responses_api_models: + vllm_model: + uses_reasoning_parser: false + uses_interleaved_reasoning: false diff --git a/tests/e2e/gpu_e2e_test.sh b/tests/e2e/gpu_e2e_test.sh index 1bcfaf5bee..56ea986912 100644 --- a/tests/e2e/gpu_e2e_test.sh +++ b/tests/e2e/gpu_e2e_test.sh @@ -16,4 +16,183 @@ set -euo pipefail -nvidia-smi +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +E2E_DIR="${E2E_DIR:-${RUNNER_TEMP:-/tmp}/nemo-gym-gpu-e2e}" +RESULTS_DIR="${RESULTS_DIR:-$E2E_DIR/results}" +MODEL="${MODEL:-Qwen/Qwen2.5-0.5B-Instruct}" +MODEL_REVISION="${MODEL_REVISION:-7ae557604adf67be50417f59c2c2f167def9a775}" +EXPECTED_ANSWER="${EXPECTED_ANSWER:-Paris}" +GPU_DEVICE="${GPU_DEVICE:-0}" +VLLM_PORT="${VLLM_PORT:-18000}" +HEAD_PORT="${HEAD_PORT:-11000}" +MODEL_API_KEY="${MODEL_API_KEY:-not-a-real-key}" # pragma: allowlist secret +VLLM_STARTUP_TIMEOUT_SECONDS="${VLLM_STARTUP_TIMEOUT_SECONDS:-300}" +GYM_STARTUP_TIMEOUT_SECONDS="${GYM_STARTUP_TIMEOUT_SECONDS:-180}" +EVAL_TIMEOUT_SECONDS="${EVAL_TIMEOUT_SECONDS:-300}" +VLLM_PID="" +GYM_PID="" + +if [[ -z "${HF_HOME:-}" ]]; then + if [[ -n "${TEST_DATA_PATH:-}" ]]; then + HF_HOME="$TEST_DATA_PATH/HF_HOME" + else + HF_HOME="$HOME/.cache/huggingface" + fi +fi + +show_log_tail() { + local label="$1" + local log_path="$2" + + if [[ -f "$log_path" ]]; then + echo "===== Last 200 lines of $label =====" >&2 + tail -n 200 "$log_path" >&2 + fi +} + +stop_process() { + local pid="$1" + local signal="${2:-TERM}" + + if [[ -z "$pid" ]] || ! kill -0 "$pid" 2>/dev/null; then + return + fi + + kill "-$signal" "$pid" 2>/dev/null || true + for _ in $(seq 1 10); do + if ! kill -0 "$pid" 2>/dev/null; then + wait "$pid" 2>/dev/null || true + return + fi + sleep 1 + done + + kill -KILL "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true +} + +cleanup() { + local exit_code=$? + trap - EXIT + + stop_process "$GYM_PID" INT + stop_process "$VLLM_PID" + + if [[ "$exit_code" -ne 0 ]]; then + show_log_tail "Gym log" "$RESULTS_DIR/gym.log" + show_log_tail "vLLM log" "$RESULTS_DIR/vllm.log" + fi + + exit "$exit_code" +} +trap cleanup EXIT + +wait_for_url() { + local name="$1" + local url="$2" + local pid="$3" + local timeout_seconds="$4" + local log_path="$5" + local deadline=$((SECONDS + timeout_seconds)) + + echo "Waiting up to ${timeout_seconds}s for $name at $url ..." + until curl --connect-timeout 2 --max-time 5 --fail --silent "$url" >/dev/null; do + if ! kill -0 "$pid" 2>/dev/null; then + echo "$name exited before becoming ready." >&2 + show_log_tail "$name log" "$log_path" + return 1 + fi + if ((SECONDS >= deadline)); then + echo "$name did not become ready within ${timeout_seconds}s." >&2 + show_log_tail "$name log" "$log_path" + return 1 + fi + sleep 2 + done + echo "$name is ready." +} + +for command in curl gym nvidia-smi python python3 timeout uv vllm; do + if ! command -v "$command" >/dev/null; then + echo "Required command is not installed: $command" >&2 + exit 1 + fi +done + +for directory in "$E2E_DIR" "$RESULTS_DIR" "$HF_HOME"; do + if [[ "$directory" != /* ]]; then + echo "E2E_DIR, RESULTS_DIR, and HF_HOME must be absolute paths: $directory" >&2 + exit 1 + fi + mkdir -p "$directory" + if [[ "$(cd "$directory" && pwd -P)" == "/" ]]; then + echo "E2E_DIR, RESULTS_DIR, and HF_HOME cannot resolve to the filesystem root." >&2 + exit 1 + fi +done + +WORKSPACE_DIR="$(mktemp -d "$E2E_DIR/workspace.XXXXXX")" + +export CUDA_VISIBLE_DEVICES="$GPU_DEVICE" +export HF_HOME +export HF_HUB_DISABLE_IMPLICIT_TOKEN=1 +export NEMO_GYM_VLLM_TRANSPORT_LOG="$RESULTS_DIR/vllm-transport.jsonl" + +nvidia-smi | tee "$RESULTS_DIR/nvidia-smi.txt" +bash "$ROOT_DIR/docker/install_codec_deps.sh" + +vllm serve "$MODEL" \ + --revision "$MODEL_REVISION" \ + --tokenizer-revision "$MODEL_REVISION" \ + --served-model-name "$MODEL" \ + --host 127.0.0.1 \ + --port "$VLLM_PORT" \ + --dtype half \ + --enforce-eager \ + --gpu-memory-utilization 0.5 \ + --max-model-len 2048 \ + --tensor-parallel-size 1 \ + > "$RESULTS_DIR/vllm.log" 2>&1 & +VLLM_PID=$! + +wait_for_url \ + "vLLM" \ + "http://127.0.0.1:${VLLM_PORT}/v1/models" \ + "$VLLM_PID" \ + "$VLLM_STARTUP_TIMEOUT_SECONDS" \ + "$RESULTS_DIR/vllm.log" +curl --connect-timeout 2 --max-time 5 --fail --silent \ + "http://127.0.0.1:${VLLM_PORT}/v1/models" \ + > "$RESULTS_DIR/vllm-models.json" + +cd "$WORKSPACE_DIR" +# Bash starts asynchronous commands with SIGINT ignored. Reset it before exec so +# Gym can catch the cleanup interrupt and gracefully stop its child servers. +python3 -c \ + "import os, signal, sys; signal.signal(signal.SIGINT, signal.SIG_DFL); os.execvp(sys.argv[1], sys.argv[1:])" \ + gym env start \ + --config "$ROOT_DIR/tests/e2e/gpu_e2e.yaml" \ + --model-url "http://127.0.0.1:${VLLM_PORT}/v1" \ + --model-api-key "$MODEL_API_KEY" \ + --model "$MODEL" \ + "++head_server.host=127.0.0.1" \ + "++head_server.port=$HEAD_PORT" \ + "+nemo_gym_log_dir=$RESULTS_DIR/component-logs" \ + > "$RESULTS_DIR/gym.log" 2>&1 & +GYM_PID=$! +"$ROOT_DIR/scripts/wait_for_servers.sh" "$GYM_PID" "$HEAD_PORT" "$GYM_STARTUP_TIMEOUT_SECONDS" + +timeout --signal=INT --kill-after=30s "$EVAL_TIMEOUT_SECONDS" gym eval run \ + --no-serve \ + --agent string_match_simple_agent \ + --input "$ROOT_DIR/tests/e2e/gpu_smoke.jsonl" \ + --output "$RESULTS_DIR/rollouts.jsonl" \ + --limit 1 \ + --concurrency 1 \ + --temperature 0 \ + --max-output-tokens 64 + +python3 "$ROOT_DIR/tests/e2e/verify_gpu_rollout.py" \ + --rollouts "$RESULTS_DIR/rollouts.jsonl" \ + --expected-model "$MODEL" \ + --expected-answer "$EXPECTED_ANSWER" diff --git a/tests/e2e/gpu_smoke.jsonl b/tests/e2e/gpu_smoke.jsonl new file mode 100644 index 0000000000..dcca4e1aa2 --- /dev/null +++ b/tests/e2e/gpu_smoke.jsonl @@ -0,0 +1 @@ +{"responses_create_params":{"input":[{"role":"user","type":"message","content":[{"type":"input_text","text":"What is the capital of France? End your response with 'Final answer: '."}]}]},"expected_answer":"Paris","extraction_mode":"final_answer","case_sensitive":false} diff --git a/tests/e2e/verify_gpu_rollout.py b/tests/e2e/verify_gpu_rollout.py new file mode 100644 index 0000000000..209752d60f --- /dev/null +++ b/tests/e2e/verify_gpu_rollout.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +from pathlib import Path + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--rollouts", type=Path, required=True) + parser.add_argument("--expected-model", required=True) + parser.add_argument("--expected-answer", required=True) + args = parser.parse_args() + + with args.rollouts.open(encoding="utf-8") as rollouts_file: + rollouts = [json.loads(line) for line in rollouts_file if line.strip()] + + assert len(rollouts) == 1, f"expected one rollout, found {len(rollouts)}" + rollout = rollouts[0] + response = rollout["response"] + assert response["status"] == "completed" + assert response["error"] is None + assert response["model"] == args.expected_model + assert response["usage"]["input_tokens"] > 0 + assert response["usage"]["output_tokens"] > 0 + + messages = [item for item in response["output"] if item["type"] == "message"] + assert len(messages) == 1 + output_text = [content["text"] for content in messages[0]["content"] if content["type"] == "output_text"] + assert output_text and output_text[0].strip() + assert rollout["reward"] == 1.0 + assert rollout["expected_answer"] == args.expected_answer + assert isinstance(rollout["extracted_answer"], str) + assert rollout["extracted_answer"].strip() + assert rollout["agent_ref"] == {"name": "string_match_simple_agent"} + + +if __name__ == "__main__": + main() diff --git a/tests/unit_tests/test_ci_environment.py b/tests/unit_tests/test_ci_environment.py index 3127e9436c..942db3b191 100644 --- a/tests/unit_tests/test_ci_environment.py +++ b/tests/unit_tests/test_ci_environment.py @@ -1,9 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import json import os import shutil import subprocess +import sys from pathlib import Path import pytest @@ -14,6 +16,9 @@ CLASSIFY_CHANGES_ACTION = REPO_ROOT / ".github" / "actions" / "classify-changes" / "action.yml" FULL_TEST_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "full-test-suite.yml" GPU_E2E_SCRIPT = REPO_ROOT / "tests" / "e2e" / "gpu_e2e_test.sh" +GPU_E2E_CONFIG = REPO_ROOT / "tests" / "e2e" / "gpu_e2e.yaml" +GPU_E2E_DATASET = REPO_ROOT / "tests" / "e2e" / "gpu_smoke.jsonl" +GPU_E2E_VERIFIER = REPO_ROOT / "tests" / "e2e" / "verify_gpu_rollout.py" GITLAB_PIPELINE = REPO_ROOT / ".gitlab-ci.yml" IS_RETRYABLE_FULL_SUITE_FAILURE = REPO_ROOT / "scripts" / "ci" / "is_retryable_full_suite_failure.sh" RECLAIM_RUNNER_DISK = REPO_ROOT / "scripts" / "ci" / "reclaim_runner_disk.sh" @@ -23,6 +28,8 @@ SETUP_DEV = REPO_ROOT / "scripts" / "ci" / "setup_dev.sh" TEST_TEMPLATE_ACTION = REPO_ROOT / ".github" / "actions" / "test-template" / "action.yml" UNIT_TEST_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "unit-tests.yml" +QWEN_MODEL = "Qwen/Qwen2.5-0.5B-Instruct" +QWEN_REVISION = "7ae557604adf67be50417f59c2c2f167def9a775" # pragma: allowlist secret BEHAVIOR_CHANGING_ENV = { "GYM_CI_DEV_VENV_DIR": "/tmp/injected-driver-venv", @@ -182,6 +189,10 @@ def test_github_full_test_jobs_reclaim_disk_before_dependency_restore() -> None: def test_cicd_main_wires_preflight_cpu_and_gpu_workflows() -> None: workflow = CICD_MAIN_WORKFLOW.read_text() + results_path = ( + "${{ runner.temp }}/nemo-gym-gpu-e2e/" + "${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.artifact_name }}" + ) assert " - main\n" in workflow assert ' - "pull-request/[0-9]+"\n' in workflow @@ -206,6 +217,11 @@ def test_cicd_main_wires_preflight_cpu_and_gpu_workflows() -> None: assert "test-type: ${{ matrix.test_type }}" in workflow assert "test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}" in workflow assert "container-image: ${{ needs.container_build.outputs.image }}" in workflow + assert "model: ${{ matrix.model }}" in workflow + assert "model-revision: ${{ matrix.model_revision }}" in workflow + assert "hf-cache-path:" not in workflow + assert f"results-path: {results_path}" in workflow + assert f"path: {results_path}" in workflow def test_cicd_container_build_pushes_sha_image_after_unit_tests() -> None: @@ -221,6 +237,7 @@ def test_cicd_container_build_pushes_sha_image_after_unit_tests() -> None: assert 'echo "image=$REGISTRY/gym:$GITHUB_SHA"' in workflow assert "cache-from: type=registry" in workflow assert "cache-to: type=registry" in workflow + assert "NEMO_GYM_PREFETCH_CONFIGS=tests/e2e/gpu_e2e.yaml" in workflow def test_cicd_summary_accepts_only_expected_docs_only_skips() -> None: @@ -259,7 +276,13 @@ def test_test_template_runs_cpu_or_gpu_script_in_container() -> None: assert " gpu)" in action assert "gpu_args=(--runtime=nvidia --gpus all)" in action assert 'docker pull "$CONTAINER_IMAGE"' in action - assert '--volume "$TEST_DATA_PATH:$TEST_DATA_PATH"' in action + assert '--volume "$TEST_DATA_PATH:/home/TestData"' in action + assert '--env "TEST_DATA_PATH=/home/TestData"' in action + assert '--env "HF_HOME=/home/TestData/HF_HOME"' in action + assert '--volume "$RESULTS_PATH:$CONTAINER_RESULTS_DIR"' in action + assert '--env "MODEL=$MODEL"' in action + assert '--env "MODEL_REVISION=$MODEL_REVISION"' in action + assert 'TEST_DATA_PATH="$(prepare_mount_source test-data-path "$TEST_DATA_PATH")"' in action assert '-e -u -o pipefail "$TEST_SCRIPT"' in action assert "continue-on-error: true" in action assert " if: always()" in action @@ -273,14 +296,119 @@ def test_test_template_runs_cpu_or_gpu_script_in_container() -> None: assert action.count("required: true") == 4 -def test_gpu_e2e_matrix_uses_placeholder_script() -> None: +def test_gpu_e2e_matrix_runs_pinned_qwen_rollout_and_uploads_artifacts() -> None: workflow = CICD_MAIN_WORKFLOW.read_text() + script = GPU_E2E_SCRIPT.read_text() + config = GPU_E2E_CONFIG.read_text() + dataset = json.loads(GPU_E2E_DATASET.read_text()) assert "fail-fast: false" in workflow - assert "- name: GPU E2E - NVIDIA SMI" in workflow + assert "- name: GPU E2E - Qwen vLLM rollout" in workflow assert "script: ./tests/e2e/gpu_e2e_test.sh" in workflow assert "test_type: gpu" in workflow - assert GPU_E2E_SCRIPT.read_text().rstrip().endswith("nvidia-smi") + assert f"model: {QWEN_MODEL}" in workflow + assert f"model_revision: {QWEN_REVISION}" in workflow + assert "timeout-minutes: 30" in workflow + assert "if: always()" in workflow + assert "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" in workflow + assert "if-no-files-found: warn" in workflow + assert "retention-days: 7" in workflow + + for expected in ( + "vllm serve", + 'bash "$ROOT_DIR/docker/install_codec_deps.sh"', + '--tokenizer-revision "$MODEL_REVISION"', + "--tensor-parallel-size 1", + "gym env start", + "signal.SIG_DFL", + "gym eval run", + "--agent string_match_simple_agent", + "--temperature 0", + "verify_gpu_rollout.py", + ): + assert expected in script + assert "docker run" not in script + assert QWEN_REVISION in script + assert 'if [[ -z "${HF_HOME:-}" ]]' in script + assert 'HF_HOME="$TEST_DATA_PATH/HF_HOME"' in script + assert "HF_CACHE_DIR" not in script + + assert "resources_servers/string_match/configs/string_match.yaml" in config + assert "responses_api_models/vllm_model/configs/vllm_model.yaml" in config + assert "skip_venv_if_present: true" in config + assert QWEN_MODEL in config + assert dataset["expected_answer"] == "Paris" + assert dataset["extraction_mode"] == "final_answer" + assert dataset["case_sensitive"] is False + + +def _valid_gpu_rollout() -> dict: + return { + "response": { + "status": "completed", + "error": None, + "model": QWEN_MODEL, + "usage": {"input_tokens": 12, "output_tokens": 6}, + "output": [ + { + "type": "message", + "content": [{"type": "output_text", "text": "Final answer: Paris"}], + } + ], + }, + "reward": 1.0, + "expected_answer": "Paris", + "extracted_answer": "Paris", + "agent_ref": {"name": "string_match_simple_agent"}, + } + + +def _run_gpu_rollout_verifier(tmp_path: Path, rollout: dict) -> subprocess.CompletedProcess[str]: + rollouts_path = tmp_path / "rollouts.jsonl" + rollouts_path.write_text(json.dumps(rollout) + "\n") + return subprocess.run( + [ + sys.executable, + str(GPU_E2E_VERIFIER), + "--rollouts", + str(rollouts_path), + "--expected-model", + QWEN_MODEL, + "--expected-answer", + "Paris", + ], + capture_output=True, + text=True, + ) + + +def test_gpu_e2e_verifier_accepts_successful_qwen_rollout(tmp_path: Path) -> None: + result = _run_gpu_rollout_verifier(tmp_path, _valid_gpu_rollout()) + + assert result.returncode == 0, result.stderr + + +def test_gpu_e2e_verifier_accepts_case_insensitive_reward(tmp_path: Path) -> None: + rollout = _valid_gpu_rollout() + rollout["response"]["output"][0]["content"][0]["text"] = "Final answer: paris" + rollout["extracted_answer"] = "paris" + + result = _run_gpu_rollout_verifier(tmp_path, rollout) + + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize("failure", ["wrong-model", "zero-reward"]) +def test_gpu_e2e_verifier_rejects_vacuous_rollout(tmp_path: Path, failure: str) -> None: + rollout = _valid_gpu_rollout() + if failure == "wrong-model": + rollout["response"]["model"] = "unexpected/model" + else: + rollout["reward"] = 0.0 + + result = _run_gpu_rollout_verifier(tmp_path, rollout) + + assert result.returncode != 0 def test_runner_disk_reclamation_fails_fast_when_space_is_still_low(tmp_path: Path) -> None: