NVIDIA-NeMo · balasaajay · Aug 17, 2026 · Aug 21, 2026 · Aug 21, 2026
@@ -29,6 +29,18 @@ inputs:
   test-data-path:
     description: Test data path selected by CI pre-flight
     required: true
+  model:
+    description: Optional model name exposed to the test script
+    required: false
+    default: ""
+  model-revision:
+    description: Optional immutable model revision exposed to the test script
+    required: false
+    default: ""
+  results-path:
+    description: Optional host directory mounted for test results
+    required: false
+    default: ""
 
 runs:
   using: composite
@@ -38,11 +50,32 @@ runs:
       continue-on-error: true
       shell: bash -e -u -o pipefail {0}
       env:
+        CONTAINER_RESULTS_DIR: /opt/nemo-gym/results/gpu-e2e
         CONTAINER_IMAGE: ${{ inputs.container-image }}
+        MODEL: ${{ inputs.model }}
+        MODEL_REVISION: ${{ inputs.model-revision }}
+        RESULTS_PATH: ${{ inputs.results-path }}
         TEST_DATA_PATH: ${{ inputs.test-data-path }}
         TEST_SCRIPT: ${{ inputs.script }}
         TEST_TYPE: ${{ inputs.test-type }}
       run: |
+        prepare_mount_source() {
+          local label="$1"
+          local source="$2"
+
+          if [[ "$source" != /* ]]; then
+            echo "$label must be an absolute path: $source" >&2
+            return 1
+          fi
+          mkdir -p "$source"
+          source="$(cd "$source" && pwd -P)"
+          if [[ "$source" == "/" ]]; then
+            echo "$label cannot resolve to the filesystem root." >&2
+            return 1
+          fi
+          printf '%s\n' "$source"
+        }
+
         case "$TEST_TYPE" in
           cpu)
             gpu_args=()
@@ -56,6 +89,26 @@ runs:
             ;;
         esac
 
+        TEST_DATA_PATH="$(prepare_mount_source test-data-path "$TEST_DATA_PATH")"
+        container_args=(
+          --env "TEST_DATA_PATH=/home/TestData"
+          --env "HF_HOME=/home/TestData/HF_HOME"
+          --volume "$TEST_DATA_PATH:/home/TestData"
+        )
+        if [[ -n "$RESULTS_PATH" ]]; then
+          RESULTS_PATH="$(prepare_mount_source results-path "$RESULTS_PATH")"
+          container_args+=(
+            --env "RESULTS_DIR=$CONTAINER_RESULTS_DIR"
+            --volume "$RESULTS_PATH:$CONTAINER_RESULTS_DIR"
+          )
+        fi
+        if [[ -n "$MODEL" ]]; then
+          container_args+=(--env "MODEL=$MODEL")
+        fi
+        if [[ -n "$MODEL_REVISION" ]]; then
+          container_args+=(--env "MODEL_REVISION=$MODEL_REVISION")
+        fi
+
         echo -e "\033[1;34m┌─ launching test ─────────────────────────────────────────────────────────┐\033[0m"
         echo -e "\033[1;34m│  script    : $TEST_SCRIPT\033[0m"
         echo -e "\033[1;34m│  type      : $TEST_TYPE\033[0m"
@@ -65,9 +118,8 @@ runs:
         docker pull "$CONTAINER_IMAGE"
         docker run --rm \
           "${gpu_args[@]}" \
+          "${container_args[@]}" \
           --shm-size=64g \
-          --env TEST_DATA_PATH="$TEST_DATA_PATH" \
-          --volume "$TEST_DATA_PATH:$TEST_DATA_PATH" \
           --entrypoint bash \
           "$CONTAINER_IMAGE" \
           -e -u -o pipefail "$TEST_SCRIPT"
@@ -77,6 +129,8 @@ runs:
       shell: bash -e -u -o pipefail {0}
       env:
         CONTAINER_IMAGE: ${{ inputs.container-image }}
+        MODEL: ${{ inputs.model }}
+        RESULTS_PATH: ${{ inputs.results-path }}
         TEST_OUTCOME: ${{ steps.test.outcome }}
         TEST_SCRIPT: ${{ inputs.script }}
         TEST_TYPE: ${{ inputs.test-type }}
@@ -108,6 +162,12 @@ runs:
           echo "|---|---|"
           echo "| Type | \`$TEST_TYPE\` |"
           echo "| Image | \`$CONTAINER_IMAGE\` |"
+          if [[ -n "$MODEL" ]]; then
+            echo "| Model | \`$MODEL\` |"
+          fi
+          if [[ -n "$RESULTS_PATH" ]]; then
+            echo "| Results | \`$RESULTS_PATH\` |"
+          fi
         } >> "$GITHUB_STEP_SUMMARY"
 
         if [[ "$TEST_OUTCOME" == "success" ]]; then

@@ -120,6 +120,7 @@ jobs:
             NEMO_GYM_COMMIT=${{ github.sha }}
             NVIDIA_BUILD_ID=${{ github.run_id }}
             NVIDIA_BUILD_REF=${{ github.ref }}
+            NEMO_GYM_PREFETCH_CONFIGS=tests/e2e/gpu_e2e.yaml
           cache-from: type=registry,ref=${{ needs.pre-flight.outputs.registry }}/gym:${{ steps.image.outputs.cache-seed }}-buildcache
           cache-to: type=registry,ref=${{ needs.pre-flight.outputs.registry }}/gym:${{ steps.image.outputs.cache-key }}-buildcache,mode=max
           tags: |
@@ -134,10 +135,14 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: GPU E2E - NVIDIA SMI
+          - name: GPU E2E - Qwen vLLM rollout
             script: ./tests/e2e/gpu_e2e_test.sh
             test_type: gpu
+            model: Qwen/Qwen2.5-0.5B-Instruct
+            model_revision: 7ae557604adf67be50417f59c2c2f167def9a775 # pragma: allowlist secret
+            artifact_name: gpu-e2e-qwen-vllm
     runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}
+    timeout-minutes: 30
     steps:
       - name: Checkout repository
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
@@ -149,6 +154,18 @@ jobs:
           test-type: ${{ matrix.test_type }}
           container-image: ${{ needs.container_build.outputs.image }}
           test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}
+          model: ${{ matrix.model }}
+          model-revision: ${{ matrix.model_revision }}
+          results-path: ${{ runner.temp }}/nemo-gym-gpu-e2e/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.artifact_name }}
+
+      - name: Upload GPU E2E artifacts
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: ${{ matrix.artifact_name }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/nemo-gym-gpu-e2e/${{ github.run_id }}-${{ github.run_attempt }}/${{ matrix.artifact_name }}
+          if-no-files-found: warn
+          retention-days: 7
 
   Nemo_CICD_Test:
     needs: [pre-flight, classify_changes, unit_tests, container_build, gpu_e2e_tests]

diff --git a/tests/e2e/gpu_e2e.yaml b/tests/e2e/gpu_e2e.yaml
@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+config_paths:
+  - resources_servers/string_match/configs/string_match.yaml
+  - responses_api_models/vllm_model/configs/vllm_model.yaml
+
+policy_base_url: http://127.0.0.1:18000/v1
+policy_api_key: not-a-real-key # pragma: allowlist secret
+policy_model_name: Qwen/Qwen2.5-0.5B-Instruct
+skip_venv_if_present: true
+
+policy_model:
+  responses_api_models:
+    vllm_model:
+      uses_reasoning_parser: false
+      uses_interleaved_reasoning: false
diff --git a/tests/e2e/gpu_e2e_test.sh b/tests/e2e/gpu_e2e_test.sh
@@ -16,4 +16,183 @@
 
 set -euo pipefail
 
-nvidia-smi
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+E2E_DIR="${E2E_DIR:-${RUNNER_TEMP:-/tmp}/nemo-gym-gpu-e2e}"
+RESULTS_DIR="${RESULTS_DIR:-$E2E_DIR/results}"
+MODEL="${MODEL:-Qwen/Qwen2.5-0.5B-Instruct}"
+MODEL_REVISION="${MODEL_REVISION:-7ae557604adf67be50417f59c2c2f167def9a775}"
+EXPECTED_ANSWER="${EXPECTED_ANSWER:-Paris}"
+GPU_DEVICE="${GPU_DEVICE:-0}"
+VLLM_PORT="${VLLM_PORT:-18000}"
+HEAD_PORT="${HEAD_PORT:-11000}"
+MODEL_API_KEY="${MODEL_API_KEY:-not-a-real-key}" # pragma: allowlist secret
+VLLM_STARTUP_TIMEOUT_SECONDS="${VLLM_STARTUP_TIMEOUT_SECONDS:-300}"
+GYM_STARTUP_TIMEOUT_SECONDS="${GYM_STARTUP_TIMEOUT_SECONDS:-180}"
+EVAL_TIMEOUT_SECONDS="${EVAL_TIMEOUT_SECONDS:-300}"
+VLLM_PID=""
+GYM_PID=""
+
+if [[ -z "${HF_HOME:-}" ]]; then
+  if [[ -n "${TEST_DATA_PATH:-}" ]]; then
+    HF_HOME="$TEST_DATA_PATH/HF_HOME"
+  else
+    HF_HOME="$HOME/.cache/huggingface"
+  fi
+fi
+
+show_log_tail() {
+  local label="$1"
+  local log_path="$2"
+
+  if [[ -f "$log_path" ]]; then
+    echo "===== Last 200 lines of $label =====" >&2
+    tail -n 200 "$log_path" >&2
+  fi
+}
+
+stop_process() {
+  local pid="$1"
+  local signal="${2:-TERM}"
+
+  if [[ -z "$pid" ]] || ! kill -0 "$pid" 2>/dev/null; then
+    return
+  fi
+
+  kill "-$signal" "$pid" 2>/dev/null || true
+  for _ in $(seq 1 10); do
+    if ! kill -0 "$pid" 2>/dev/null; then
+      wait "$pid" 2>/dev/null || true
+      return
+    fi
+    sleep 1
+  done
+
+  kill -KILL "$pid" 2>/dev/null || true
+  wait "$pid" 2>/dev/null || true
+}
+
+cleanup() {
+  local exit_code=$?
+  trap - EXIT
+
+  stop_process "$GYM_PID" INT
+  stop_process "$VLLM_PID"
+
+  if [[ "$exit_code" -ne 0 ]]; then
+    show_log_tail "Gym log" "$RESULTS_DIR/gym.log"
+    show_log_tail "vLLM log" "$RESULTS_DIR/vllm.log"
+  fi
+
+  exit "$exit_code"
+}
+trap cleanup EXIT
+
+wait_for_url() {
+  local name="$1"
+  local url="$2"
+  local pid="$3"
+  local timeout_seconds="$4"
+  local log_path="$5"
+  local deadline=$((SECONDS + timeout_seconds))
+
+  echo "Waiting up to ${timeout_seconds}s for $name at $url ..."
+  until curl --connect-timeout 2 --max-time 5 --fail --silent "$url" >/dev/null; do
+    if ! kill -0 "$pid" 2>/dev/null; then
+      echo "$name exited before becoming ready." >&2
+      show_log_tail "$name log" "$log_path"
+      return 1
+    fi
+    if ((SECONDS >= deadline)); then
+      echo "$name did not become ready within ${timeout_seconds}s." >&2
+      show_log_tail "$name log" "$log_path"
+      return 1
+    fi
+    sleep 2
+  done
+  echo "$name is ready."
+}
+
+for command in curl gym nvidia-smi python python3 timeout uv vllm; do
+  if ! command -v "$command" >/dev/null; then
+    echo "Required command is not installed: $command" >&2
+    exit 1
+  fi
+done
+
+for directory in "$E2E_DIR" "$RESULTS_DIR" "$HF_HOME"; do
+  if [[ "$directory" != /* ]]; then
+    echo "E2E_DIR, RESULTS_DIR, and HF_HOME must be absolute paths: $directory" >&2
+    exit 1
+  fi
+  mkdir -p "$directory"
+  if [[ "$(cd "$directory" && pwd -P)" == "/" ]]; then
+    echo "E2E_DIR, RESULTS_DIR, and HF_HOME cannot resolve to the filesystem root." >&2
+    exit 1
+  fi
+done
+
+WORKSPACE_DIR="$(mktemp -d "$E2E_DIR/workspace.XXXXXX")"
+
+export CUDA_VISIBLE_DEVICES="$GPU_DEVICE"
+export HF_HOME
+export HF_HUB_DISABLE_IMPLICIT_TOKEN=1
+export NEMO_GYM_VLLM_TRANSPORT_LOG="$RESULTS_DIR/vllm-transport.jsonl"
+
+nvidia-smi | tee "$RESULTS_DIR/nvidia-smi.txt"
+bash "$ROOT_DIR/docker/install_codec_deps.sh"
+
+vllm serve "$MODEL" \
+  --revision "$MODEL_REVISION" \
+  --tokenizer-revision "$MODEL_REVISION" \
+  --served-model-name "$MODEL" \
+  --host 127.0.0.1 \
+  --port "$VLLM_PORT" \
+  --dtype half \
+  --enforce-eager \
+  --gpu-memory-utilization 0.5 \
+  --max-model-len 2048 \
+  --tensor-parallel-size 1 \
+  > "$RESULTS_DIR/vllm.log" 2>&1 &
+VLLM_PID=$!
+
+wait_for_url \
+  "vLLM" \
+  "http://127.0.0.1:${VLLM_PORT}/v1/models" \
+  "$VLLM_PID" \
+  "$VLLM_STARTUP_TIMEOUT_SECONDS" \
+  "$RESULTS_DIR/vllm.log"
+curl --connect-timeout 2 --max-time 5 --fail --silent \
+  "http://127.0.0.1:${VLLM_PORT}/v1/models" \
+  > "$RESULTS_DIR/vllm-models.json"
+
+cd "$WORKSPACE_DIR"
+# Bash starts asynchronous commands with SIGINT ignored. Reset it before exec so
+# Gym can catch the cleanup interrupt and gracefully stop its child servers.
+python3 -c \
+  "import os, signal, sys; signal.signal(signal.SIGINT, signal.SIG_DFL); os.execvp(sys.argv[1], sys.argv[1:])" \
+  gym env start \
+  --config "$ROOT_DIR/tests/e2e/gpu_e2e.yaml" \
+  --model-url "http://127.0.0.1:${VLLM_PORT}/v1" \
+  --model-api-key "$MODEL_API_KEY" \
+  --model "$MODEL" \
+  "++head_server.host=127.0.0.1" \
+  "++head_server.port=$HEAD_PORT" \
+  "+nemo_gym_log_dir=$RESULTS_DIR/component-logs" \
+  > "$RESULTS_DIR/gym.log" 2>&1 &
+GYM_PID=$!
+"$ROOT_DIR/scripts/wait_for_servers.sh" "$GYM_PID" "$HEAD_PORT" "$GYM_STARTUP_TIMEOUT_SECONDS"
+
+timeout --signal=INT --kill-after=30s "$EVAL_TIMEOUT_SECONDS" gym eval run \
+  --no-serve \
+  --agent string_match_simple_agent \
+  --input "$ROOT_DIR/tests/e2e/gpu_smoke.jsonl" \
+  --output "$RESULTS_DIR/rollouts.jsonl" \
+  --limit 1 \
+  --concurrency 1 \
+  --temperature 0 \
+  --max-output-tokens 64
+
+python3 "$ROOT_DIR/tests/e2e/verify_gpu_rollout.py" \
+  --rollouts "$RESULTS_DIR/rollouts.jsonl" \
+  --expected-model "$MODEL" \
+  --expected-answer "$EXPECTED_ANSWER"
diff --git a/tests/e2e/gpu_smoke.jsonl b/tests/e2e/gpu_smoke.jsonl
@@ -0,0 +1 @@
+{"responses_create_params":{"input":[{"role":"user","type":"message","content":[{"type":"input_text","text":"What is the capital of France? End your response with 'Final answer: <answer>'."}]}]},"expected_answer":"Paris","extraction_mode":"final_answer","case_sensitive":false}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"responses_create_params":{"input":[{"role":"user","type":"message","content":[{"type":"input_text","text":"What is the capital of France? End your response with 'Final answer: <answer>'."}]}]},"expected_answer":"Paris","extraction_mode":"final_answer","case_sensitive":false}