From 78424de6327c3e66ec25f81d13a7428845a97aa2 Mon Sep 17 00:00:00 2001 From: Charlie Truong Date: Wed, 19 Aug 2026 22:10:04 -0500 Subject: [PATCH 1/4] ci: add Gym GPU test workflow Signed-off-by: Charlie Truong --- .github/actions/classify-changes/action.yml | 88 +++++++++ .github/actions/test-template/action.yml | 53 +++++ .github/workflows/cicd-main.yml | 205 ++++++++++++++++++++ .github/workflows/unit-tests.yml | 90 ++------- tests/e2e/gpu_e2e_test.sh | 7 + tests/unit_tests/test_ci_environment.py | 98 ++++++++++ 6 files changed, 464 insertions(+), 77 deletions(-) create mode 100644 .github/actions/classify-changes/action.yml create mode 100644 .github/actions/test-template/action.yml create mode 100644 .github/workflows/cicd-main.yml create mode 100644 tests/e2e/gpu_e2e_test.sh diff --git a/.github/actions/classify-changes/action.yml b/.github/actions/classify-changes/action.yml new file mode 100644 index 0000000000..f166f7eaa6 --- /dev/null +++ b/.github/actions/classify-changes/action.yml @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Classify Changes +description: Classify Gym changes as full, server-only, or docs-only + +inputs: + base-ref: + description: Base commit or ref used to determine changed files + required: false + default: "" + force-run-all: + description: Run the full suite without inspecting changed files + required: false + default: "false" + +outputs: + run_full: + description: Whether to run the full test suite + value: ${{ steps.classify.outputs.run_full }} + run_servers: + description: Whether to run only changed server tests + value: ${{ steps.classify.outputs.run_servers }} + docs_only: + description: Whether all changed files are documentation-classified files + value: ${{ steps.classify.outputs.docs_only }} + server_all_changed_files: + description: Changed files belonging to server paths + value: ${{ steps.changed-files.outputs.server_all_changed_files }} + +runs: + using: composite + steps: + - name: Detect changed files + id: changed-files + if: inputs.force-run-all != 'true' + uses: step-security/changed-files@3dbe17c78367e7d60f00d78ae6781a35be47b4a1 # v45.0.1 + with: + base_sha: ${{ inputs.base-ref }} + # 'files' input enables global other_changed_files_count for uncategorized files. + files: | + **.md + fern/** + LICENSE + resources_servers/** + responses_api_agents/** + responses_api_models/** + benchmarks/** + files_yaml: | + doc: + - '**.md' + - fern/** + - LICENSE + - benchmarks/** + server: + - resources_servers/** + - responses_api_agents/** + - responses_api_models/** + + - name: Classify changes + id: classify + shell: bash -e -u -o pipefail {0} + env: + DOC_CHANGED: ${{ steps.changed-files.outputs.doc_any_changed }} + FORCE_RUN_ALL: ${{ inputs.force-run-all }} + OTHER_COUNT: ${{ steps.changed-files.outputs.other_changed_files_count }} + SERVER_CHANGED: ${{ steps.changed-files.outputs.server_any_changed }} + run: | + if [[ "$FORCE_RUN_ALL" == "true" || "${OTHER_COUNT:-0}" -gt 0 ]]; then + run_full=true + run_servers=false + docs_only=false + elif [[ "$SERVER_CHANGED" == "true" ]]; then + run_full=false + run_servers=true + docs_only=false + else + run_full=false + run_servers=false + docs_only=true + fi + + echo "run_full=$run_full" >> "$GITHUB_OUTPUT" + echo "run_servers=$run_servers" >> "$GITHUB_OUTPUT" + echo "docs_only=$docs_only" >> "$GITHUB_OUTPUT" + + echo "Changed files: docs=${DOC_CHANGED:-false}, servers=${SERVER_CHANGED:-false}, other=${OTHER_COUNT:-0}" + echo "Classification: run_full=$run_full, run_servers=$run_servers, docs_only=$docs_only" diff --git a/.github/actions/test-template/action.yml b/.github/actions/test-template/action.yml new file mode 100644 index 0000000000..1dd1e7a94d --- /dev/null +++ b/.github/actions/test-template/action.yml @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Test Template +description: Run a Gym CPU or GPU test script in the CI container + +inputs: + script: + description: Path to the test script inside the Gym repository + required: true + test-type: + description: Test hardware type (cpu or gpu) + required: true + container-image: + description: Gym container image to use for the test + required: true + test-data-path: + description: Test data path selected by CI pre-flight + required: true + +runs: + using: composite + steps: + - name: Run test + shell: bash -e -u -o pipefail {0} + env: + CONTAINER_IMAGE: ${{ inputs.container-image }} + TEST_DATA_PATH: ${{ inputs.test-data-path }} + TEST_SCRIPT: ${{ inputs.script }} + TEST_TYPE: ${{ inputs.test-type }} + run: | + case "$TEST_TYPE" in + cpu) + gpu_args=() + ;; + gpu) + gpu_args=(--runtime=nvidia --gpus all) + ;; + *) + echo "Unsupported test type: $TEST_TYPE (expected cpu or gpu)" + exit 1 + ;; + esac + + docker pull "$CONTAINER_IMAGE" + docker run --rm \ + "${gpu_args[@]}" \ + --shm-size=64g \ + --env TEST_DATA_PATH="$TEST_DATA_PATH" \ + --volume "$TEST_DATA_PATH:$TEST_DATA_PATH" \ + --entrypoint bash \ + "$CONTAINER_IMAGE" \ + -e -u -o pipefail "$TEST_SCRIPT" diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml new file mode 100644 index 0000000000..493d43b3cc --- /dev/null +++ b/.github/workflows/cicd-main.yml @@ -0,0 +1,205 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: CICD NeMo Gym + +on: + push: + branches: + - main + - "pull-request/[0-9]+" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: read + +jobs: + pre-flight: + uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@cf5acebace78c7c339bc1f804357a991336f1c4d # v1.8.10 + with: + default_runner_prefix: ${{ vars.DEFAULT_RUNNER_PREFIX }} + non_nvidia_runner_prefix: ${{ vars.NON_NVIDIA_RUNNER_PREFIX }} + default_test_data_path: ${{ vars.DEFAULT_TEST_DATA_PATH }} + non_nvidia_test_data_path: ${{ vars.NON_NVIDIA_TEST_DATA_PATH }} + default_registry: ${{ vars.DEFAULT_CONTAINER_REGISTRY }} + non_nvidia_registry: ${{ vars.NON_NVIDIA_CONTAINER_REGISTRY }} + sso_users_filename: ${{ vars.SSO_USERS_FILENAME }} + secrets: + NVIDIA_MANAGEMENT_ORG_PAT: ${{ secrets.NVIDIA_MANAGEMENT_ORG_PAT }} + + classify_changes: + name: Classify changes + needs: [pre-flight] + runs-on: ubuntu-latest + outputs: + docs_only: ${{ steps.changes.outputs.docs_only }} + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + fetch-depth: 0 + + - name: Classify changes + id: changes + uses: ./.github/actions/classify-changes + with: + base-ref: ${{ needs.pre-flight.outputs.base_ref }} + + unit_tests: + name: Unit tests + needs: [pre-flight, classify_changes] + if: needs.classify_changes.outputs.docs_only != 'true' + uses: ./.github/workflows/unit-tests.yml + with: + base-ref: ${{ needs.pre-flight.outputs.base_ref }} + + container_build: + name: Build Gym container + needs: [pre-flight, classify_changes, unit_tests] + if: needs.classify_changes.outputs.docs_only != 'true' + runs-on: ${{ needs.pre-flight.outputs.runner_prefix }} + environment: ${{ contains(needs.pre-flight.outputs.registry, 'azure') && 'nemo-ci' || '' }} + outputs: + image: ${{ steps.image.outputs.image }} + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Compute image and cache keys + id: image + shell: bash -e -u -o pipefail {0} + env: + REGISTRY: ${{ needs.pre-flight.outputs.registry }} + run: | + branch_key=$(echo "${GITHUB_REF_NAME}" | tr '/' '-' | tr -cd '[:alnum:]._-') + if [[ "${GITHUB_REF}" == "refs/heads/main" ]]; then + cache_key="main" + else + cache_key="$branch_key" + fi + + if [[ "$cache_key" != "main" ]] && \ + docker buildx imagetools inspect "$REGISTRY/gym:${cache_key}-buildcache" >/dev/null 2>&1; then + cache_seed="$cache_key" + else + cache_seed="main" + fi + + echo "cache-key=$cache_key" >> "$GITHUB_OUTPUT" + echo "cache-seed=$cache_seed" >> "$GITHUB_OUTPUT" + echo "image=$REGISTRY/gym:$GITHUB_SHA" >> "$GITHUB_OUTPUT" + + - name: Build and push + uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5 + with: + context: . + file: docker/Dockerfile + build-contexts: nemo-gym=. + target: release + platforms: linux/amd64 + push: true + build-args: | + NEMO_GYM_COMMIT=${{ github.sha }} + NVIDIA_BUILD_ID=${{ github.run_id }} + NVIDIA_BUILD_REF=${{ github.ref }} + cache-from: type=registry,ref=${{ needs.pre-flight.outputs.registry }}/gym:${{ steps.image.outputs.cache-seed }}-buildcache + cache-to: type=registry,ref=${{ needs.pre-flight.outputs.registry }}/gym:${{ steps.image.outputs.cache-key }}-buildcache,mode=max + tags: | + ${{ needs.pre-flight.outputs.registry }}/gym:${{ steps.image.outputs.cache-key }} + ${{ steps.image.outputs.image }} + + gpu_e2e_tests: + name: ${{ matrix.name }} + needs: [pre-flight, classify_changes, unit_tests, container_build] + if: needs.classify_changes.outputs.docs_only != 'true' + strategy: + fail-fast: false + matrix: + include: + - name: GPU E2E - NVIDIA SMI + script: ./tests/e2e/gpu_e2e_test.sh + test_type: gpu + runs-on: ${{ needs.pre-flight.outputs.runner_prefix }} + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Run ${{ matrix.name }} + uses: ./.github/actions/test-template + with: + script: ${{ matrix.script }} + test-type: ${{ matrix.test_type }} + container-image: ${{ needs.container_build.outputs.image }} + test-data-path: ${{ needs.pre-flight.outputs.test_data_path }} + + Nemo_CICD_Test: + needs: [pre-flight, classify_changes, unit_tests, container_build, gpu_e2e_tests] + if: always() && !cancelled() + runs-on: ubuntu-latest + steps: + - name: Check test results + shell: bash -e -u -o pipefail {0} + env: + CLASSIFY_RESULT: ${{ needs.classify_changes.result }} + DOCS_ONLY: ${{ needs.classify_changes.outputs.docs_only }} + PREFLIGHT_RESULT: ${{ needs.pre-flight.result }} + UNIT_TEST_RESULT: ${{ needs.unit_tests.result }} + CONTAINER_BUILD_RESULT: ${{ needs.container_build.result }} + GPU_E2E_TEST_RESULT: ${{ needs.gpu_e2e_tests.result }} + run: | + echo "Pre-flight: $PREFLIGHT_RESULT" + echo "Classify changes: $CLASSIFY_RESULT" + echo "Docs only: $DOCS_ONLY" + echo "Unit tests: $UNIT_TEST_RESULT" + echo "Container build: $CONTAINER_BUILD_RESULT" + echo "GPU E2E tests: $GPU_E2E_TEST_RESULT" + + if [[ "$PREFLIGHT_RESULT" != "success" ]]; then + echo "Pre-flight did not succeed." + exit 1 + fi + + if [[ "$CLASSIFY_RESULT" != "success" ]]; then + echo "Change classification did not succeed." + exit 1 + fi + + if [[ "$DOCS_ONLY" == "true" ]]; then + if [[ "$UNIT_TEST_RESULT" == "skipped" && \ + "$CONTAINER_BUILD_RESULT" == "skipped" && \ + "$GPU_E2E_TEST_RESULT" == "skipped" ]]; then + echo "Docs-only change: tests and container build were skipped as expected." + exit 0 + fi + + echo "Docs-only change had an unexpected test result." + exit 1 + fi + + if [[ "$UNIT_TEST_RESULT" == "success" && \ + "$CONTAINER_BUILD_RESULT" == "success" && \ + "$GPU_E2E_TEST_RESULT" == "success" ]]; then + echo "All relevant test jobs completed successfully." + exit 0 + fi + + echo "One or more relevant test jobs did not succeed." + exit 1 diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 433182c0fa..90df977c03 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -18,6 +18,12 @@ on: pull_request: types: [opened, synchronize, reopened, labeled, unlabeled] workflow_call: + inputs: + base-ref: + description: Base commit or ref used to classify changes + required: false + type: string + default: "" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -34,91 +40,21 @@ jobs: outputs: run_full: ${{ steps.changes.outputs.run_full }} run_servers: ${{ steps.changes.outputs.run_servers }} - server_all_changed_files: ${{ steps.changed-files.outputs.server_all_changed_files }} + docs_only: ${{ steps.changes.outputs.docs_only }} + server_all_changed_files: ${{ steps.changes.outputs.server_all_changed_files }} steps: - name: Checkout repository uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: submodules: 'recursive' - - - name: Detect changed files - id: changed-files - uses: step-security/changed-files@3dbe17c78367e7d60f00d78ae6781a35be47b4a1 # v45.0.1 - with: - # 'files' input enables global other_changed_files_count for uncategorized files - files: | - **.md - fern/** - LICENSE - resources_servers/** - responses_api_agents/** - responses_api_models/** - benchmarks/** - files_yaml: | - doc: - - '**.md' - - fern/** - - LICENSE - - benchmarks/** - server: - - resources_servers/** - - responses_api_agents/** - - responses_api_models/** + fetch-depth: 0 - name: Classify changes id: changes - run: | - # workflow_call or non-PR context: run everything - if [[ "${{ github.event_name }}" != "pull_request" ]]; then - echo "run_full=true" >> $GITHUB_OUTPUT - echo "run_servers=false" >> $GITHUB_OUTPUT - echo "No PR context - running full suite" - exit 0 - fi - - DOC_CHANGED="${{ steps.changed-files.outputs.doc_any_changed }}" - SERVER_CHANGED="${{ steps.changed-files.outputs.server_any_changed }}" - OTHER_COUNT="${{ steps.changed-files.outputs.other_changed_files_count }}" - - echo "============================================" - echo "File change categorization" - echo "============================================" - echo "" - echo "Categories:" - echo " doc (skip tests): **.md, fern/**, LICENSE, benchmarks/**" - echo " server (test changed): resources_servers/**, responses_api_agents/**, responses_api_models/**" - echo " other (full suite): everything else (core library, CI, scripts, etc.)" - echo " Priority: other > server > doc" - echo "" - echo "This PR:" - echo " doc changed: $DOC_CHANGED" - echo " server changed: $SERVER_CHANGED" - echo " other (uncategorized): ${OTHER_COUNT:-0} file(s)" - if [[ "${OTHER_COUNT:-0}" -gt 0 ]]; then - echo " other files: ${{ steps.changed-files.outputs.other_changed_files }}" - fi - echo "" - - # Files outside doc/server categories (CI, core library, config, etc.) → full suite - if [[ "${OTHER_COUNT:-0}" -gt 0 ]]; then - echo "Decision: FULL TEST SUITE" - echo "Reason: ${OTHER_COUNT} file(s) outside doc/server categories" - echo "run_full=true" >> $GITHUB_OUTPUT - echo "run_servers=false" >> $GITHUB_OUTPUT - # Only server files changed (possibly with doc changes) → test changed servers - elif [[ "$SERVER_CHANGED" == "true" ]]; then - echo "Decision: SERVER TESTS ONLY" - echo "Reason: only server files changed (+ possibly docs)" - echo "run_full=false" >> $GITHUB_OUTPUT - echo "run_servers=true" >> $GITHUB_OUTPUT - # Only doc files changed → skip tests - else - echo "Decision: SKIP TESTS" - echo "Reason: only doc/benchmark files changed" - echo "run_full=false" >> $GITHUB_OUTPUT - echo "run_servers=false" >> $GITHUB_OUTPUT - fi - echo "============================================" + uses: ./.github/actions/classify-changes + with: + base-ref: ${{ inputs.base-ref || github.event.pull_request.base.sha || '' }} + force-run-all: ${{ inputs.base-ref == '' && github.event_name != 'pull_request' }} # Core library unit tests (run once on a full run) + the changed-servers path. The full # server suite is sharded into the parallel `server-suite` matrix below. diff --git a/tests/e2e/gpu_e2e_test.sh b/tests/e2e/gpu_e2e_test.sh new file mode 100644 index 0000000000..e17a27c184 --- /dev/null +++ b/tests/e2e/gpu_e2e_test.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +nvidia-smi diff --git a/tests/unit_tests/test_ci_environment.py b/tests/unit_tests/test_ci_environment.py index 60637c1b57..ae5e9f209b 100644 --- a/tests/unit_tests/test_ci_environment.py +++ b/tests/unit_tests/test_ci_environment.py @@ -10,7 +10,10 @@ REPO_ROOT = Path(__file__).resolve().parents[2] +CICD_MAIN_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "cicd-main.yml" +CLASSIFY_CHANGES_ACTION = REPO_ROOT / ".github" / "actions" / "classify-changes" / "action.yml" FULL_TEST_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "full-test-suite.yml" +GPU_E2E_SCRIPT = REPO_ROOT / "tests" / "e2e" / "gpu_e2e_test.sh" GITLAB_PIPELINE = REPO_ROOT / ".gitlab-ci.yml" IS_RETRYABLE_FULL_SUITE_FAILURE = REPO_ROOT / "scripts" / "ci" / "is_retryable_full_suite_failure.sh" RECLAIM_RUNNER_DISK = REPO_ROOT / "scripts" / "ci" / "reclaim_runner_disk.sh" @@ -18,6 +21,7 @@ SANITIZER = REPO_ROOT / "scripts" / "ci" / "sanitize_env.sh" SERVER_TESTS = REPO_ROOT / "scripts" / "ci" / "server_tests.sh" SETUP_DEV = REPO_ROOT / "scripts" / "ci" / "setup_dev.sh" +TEST_TEMPLATE_ACTION = REPO_ROOT / ".github" / "actions" / "test-template" / "action.yml" UNIT_TEST_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "unit-tests.yml" BEHAVIOR_CHANGING_ENV = { @@ -176,6 +180,100 @@ def test_github_full_test_jobs_reclaim_disk_before_dependency_restore() -> None: assert section.index("reclaim_runner_disk.sh") < section.index("Cache uv dependencies") +def test_cicd_main_wires_preflight_cpu_and_gpu_workflows() -> None: + workflow = CICD_MAIN_WORKFLOW.read_text() + + assert " - main\n" in workflow + assert ' - "pull-request/[0-9]+"\n' in workflow + assert "deploy-release" not in workflow + assert "schedule:" not in workflow + assert "workflow_dispatch:" not in workflow + assert " contents: read\n" in workflow + assert " pull-requests: read\n" in workflow + assert "id-token:" not in workflow + assert "uses: ./.github/workflows/unit-tests.yml" in workflow + assert workflow.count("base-ref: ${{ needs.pre-flight.outputs.base_ref }}") == 2 + assert "uses: ./.github/actions/classify-changes" in workflow + assert "uses: ./.github/actions/test-template" in workflow + assert "base-ref: ${{ needs.pre-flight.outputs.base_ref }}" in workflow + assert "needs: [pre-flight, classify_changes, unit_tests]" in workflow + assert "needs: [pre-flight, classify_changes, unit_tests, container_build]" in workflow + assert workflow.count("if: needs.classify_changes.outputs.docs_only != 'true'") == 3 + assert "needs.pre-flight.outputs.docs_only" not in workflow + assert "runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}" in workflow + assert "matrix:" in workflow + assert "script: ${{ matrix.script }}" in workflow + assert "test-type: ${{ matrix.test_type }}" in workflow + assert "test-data-path: ${{ needs.pre-flight.outputs.test_data_path }}" in workflow + assert "container-image: ${{ needs.container_build.outputs.image }}" in workflow + + +def test_cicd_container_build_pushes_sha_image_after_unit_tests() -> None: + workflow = CICD_MAIN_WORKFLOW.read_text() + + assert "name: Build Gym container" in workflow + assert "runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}" in workflow + assert "uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f" in workflow + assert "uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25" in workflow + assert "build-contexts: nemo-gym=." in workflow + assert "target: release" in workflow + assert "push: true" in workflow + assert 'echo "image=$REGISTRY/gym:$GITHUB_SHA"' in workflow + assert "cache-from: type=registry" in workflow + assert "cache-to: type=registry" in workflow + + +def test_cicd_summary_accepts_only_expected_docs_only_skips() -> None: + workflow = CICD_MAIN_WORKFLOW.read_text() + + assert "needs: [pre-flight, classify_changes, unit_tests, container_build, gpu_e2e_tests]" in workflow + assert "if: always() && !cancelled()" in workflow + assert '"$PREFLIGHT_RESULT" != "success"' in workflow + assert '"$CLASSIFY_RESULT" != "success"' in workflow + assert '"$DOCS_ONLY" == "true"' in workflow + assert '"$CONTAINER_BUILD_RESULT" == "skipped"' in workflow + assert '"$CONTAINER_BUILD_RESULT" == "success"' in workflow + + +def test_shared_change_classifier_matches_gym_docs_and_server_paths() -> None: + action = CLASSIFY_CHANGES_ACTION.read_text() + unit_workflow = UNIT_TEST_WORKFLOW.read_text() + + for path in ("**.md", "fern/**", "LICENSE", "benchmarks/**"): + assert path in action + for path in ("resources_servers/**", "responses_api_agents/**", "responses_api_models/**"): + assert path in action + + assert "uses: ./.github/actions/classify-changes" in unit_workflow + assert "base-ref: ${{ inputs.base-ref || github.event.pull_request.base.sha || '' }}" in unit_workflow + assert "force-run-all: ${{ inputs.base-ref == '' && github.event_name != 'pull_request' }}" in unit_workflow + assert "gh pr view" not in action + assert "DOCS_ONLY_LABEL" not in action + + +def test_test_template_runs_cpu_or_gpu_script_in_container() -> None: + action = TEST_TEMPLATE_ACTION.read_text() + + assert ' case "$TEST_TYPE" in' in action + assert " cpu)" in action + assert " gpu)" in action + assert "gpu_args=(--runtime=nvidia --gpus all)" in action + assert 'docker pull "$CONTAINER_IMAGE"' in action + assert '--volume "$TEST_DATA_PATH:$TEST_DATA_PATH"' in action + assert '-e -u -o pipefail "$TEST_SCRIPT"' in action + assert action.count("required: true") == 4 + + +def test_gpu_e2e_matrix_uses_placeholder_script() -> None: + workflow = CICD_MAIN_WORKFLOW.read_text() + + assert "fail-fast: false" in workflow + assert "- name: GPU E2E - NVIDIA SMI" in workflow + assert "script: ./tests/e2e/gpu_e2e_test.sh" in workflow + assert "test_type: gpu" in workflow + assert GPU_E2E_SCRIPT.read_text().rstrip().endswith("nvidia-smi") + + def test_runner_disk_reclamation_fails_fast_when_space_is_still_low(tmp_path: Path) -> None: bin_dir = tmp_path / "bin" bin_dir.mkdir() From d9cfd1f59cb2ffbd6bdd50c291bb3783eb03337d Mon Sep 17 00:00:00 2001 From: Charlie Truong Date: Wed, 19 Aug 2026 22:18:15 -0500 Subject: [PATCH 2/4] ci: add full license headers Signed-off-by: Charlie Truong --- .github/actions/classify-changes/action.yml | 12 ++++++++++++ .github/actions/test-template/action.yml | 12 ++++++++++++ tests/e2e/gpu_e2e_test.sh | 12 ++++++++++++ 3 files changed, 36 insertions(+) diff --git a/.github/actions/classify-changes/action.yml b/.github/actions/classify-changes/action.yml index f166f7eaa6..bafc3c0fd2 100644 --- a/.github/actions/classify-changes/action.yml +++ b/.github/actions/classify-changes/action.yml @@ -1,5 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. name: Classify Changes description: Classify Gym changes as full, server-only, or docs-only diff --git a/.github/actions/test-template/action.yml b/.github/actions/test-template/action.yml index 1dd1e7a94d..a42411461f 100644 --- a/.github/actions/test-template/action.yml +++ b/.github/actions/test-template/action.yml @@ -1,5 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. name: Test Template description: Run a Gym CPU or GPU test script in the CI container diff --git a/tests/e2e/gpu_e2e_test.sh b/tests/e2e/gpu_e2e_test.sh index e17a27c184..1bcfaf5bee 100644 --- a/tests/e2e/gpu_e2e_test.sh +++ b/tests/e2e/gpu_e2e_test.sh @@ -1,6 +1,18 @@ #!/usr/bin/env bash # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. set -euo pipefail From 212eb7aba9996f73e7305716f08e0cf7954f4088 Mon Sep 17 00:00:00 2001 From: Charlie Truong Date: Wed, 19 Aug 2026 22:26:28 -0500 Subject: [PATCH 3/4] ci: add test result banners Signed-off-by: Charlie Truong --- .github/actions/test-template/action.yml | 48 ++++++++++++++++++++++++ tests/unit_tests/test_ci_environment.py | 6 +++ 2 files changed, 54 insertions(+) diff --git a/.github/actions/test-template/action.yml b/.github/actions/test-template/action.yml index a42411461f..3aa73b7e44 100644 --- a/.github/actions/test-template/action.yml +++ b/.github/actions/test-template/action.yml @@ -34,6 +34,8 @@ runs: using: composite steps: - name: Run test + id: test + continue-on-error: true shell: bash -e -u -o pipefail {0} env: CONTAINER_IMAGE: ${{ inputs.container-image }} @@ -54,6 +56,12 @@ runs: ;; esac + printf '\033[1;34m============================================================\n' + printf ' Running: %s\n' "$TEST_SCRIPT" + printf ' Type: %s\n' "$TEST_TYPE" + printf ' Image: %s\n' "$CONTAINER_IMAGE" + printf '============================================================\033[0m\n' + docker pull "$CONTAINER_IMAGE" docker run --rm \ "${gpu_args[@]}" \ @@ -63,3 +71,43 @@ runs: --entrypoint bash \ "$CONTAINER_IMAGE" \ -e -u -o pipefail "$TEST_SCRIPT" + + - name: Report result + if: always() + shell: bash -e -u -o pipefail {0} + env: + CONTAINER_IMAGE: ${{ inputs.container-image }} + TEST_OUTCOME: ${{ steps.test.outcome }} + TEST_SCRIPT: ${{ inputs.script }} + TEST_TYPE: ${{ inputs.test-type }} + run: | + if [[ "$TEST_OUTCOME" == "success" ]]; then + status="PASSED" + color='\033[1;32m' + symbol="✅" + else + status="FAILED" + color='\033[1;31m' + symbol="❌" + fi + + printf '%b============================================================\n' "$color" + printf ' %s: %s\n' "$status" "$TEST_SCRIPT" + printf '============================================================\033[0m\n' + + { + echo "### $symbol $TEST_SCRIPT — $status" + echo "" + echo "| Field | Value |" + echo "|---|---|" + echo "| Type | \`$TEST_TYPE\` |" + echo "| Image | \`$CONTAINER_IMAGE\` |" + } >> "$GITHUB_STEP_SUMMARY" + + if [[ "$TEST_OUTCOME" == "success" ]]; then + echo "::notice title=Test result::$TEST_SCRIPT — PASSED" + exit 0 + fi + + echo "::error title=Test result::$TEST_SCRIPT — FAILED" + exit 1 diff --git a/tests/unit_tests/test_ci_environment.py b/tests/unit_tests/test_ci_environment.py index ae5e9f209b..a8b77a8259 100644 --- a/tests/unit_tests/test_ci_environment.py +++ b/tests/unit_tests/test_ci_environment.py @@ -261,6 +261,12 @@ def test_test_template_runs_cpu_or_gpu_script_in_container() -> None: assert 'docker pull "$CONTAINER_IMAGE"' in action assert '--volume "$TEST_DATA_PATH:$TEST_DATA_PATH"' in action assert '-e -u -o pipefail "$TEST_SCRIPT"' in action + assert "continue-on-error: true" in action + assert " if: always()" in action + assert "TEST_OUTCOME: ${{ steps.test.outcome }}" in action + assert 'echo "::notice title=Test result::$TEST_SCRIPT — PASSED"' in action + assert 'echo "::error title=Test result::$TEST_SCRIPT — FAILED"' in action + assert '} >> "$GITHUB_STEP_SUMMARY"' in action assert action.count("required: true") == 4 From faa4248ffab14f146d8373257df64c2f42971bc7 Mon Sep 17 00:00:00 2001 From: Charlie Truong Date: Thu, 20 Aug 2026 00:56:28 -0500 Subject: [PATCH 4/4] ci: align test banners with Megatron Bridge Signed-off-by: Charlie Truong --- .github/actions/test-template/action.yml | 28 ++++++++++++++---------- tests/unit_tests/test_ci_environment.py | 3 +++ 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/.github/actions/test-template/action.yml b/.github/actions/test-template/action.yml index 3aa73b7e44..11469f92c4 100644 --- a/.github/actions/test-template/action.yml +++ b/.github/actions/test-template/action.yml @@ -56,11 +56,11 @@ runs: ;; esac - printf '\033[1;34m============================================================\n' - printf ' Running: %s\n' "$TEST_SCRIPT" - printf ' Type: %s\n' "$TEST_TYPE" - printf ' Image: %s\n' "$CONTAINER_IMAGE" - printf '============================================================\033[0m\n' + echo -e "\033[1;34m┌─ launching test ─────────────────────────────────────────────────────────┐\033[0m" + echo -e "\033[1;34m│ script : $TEST_SCRIPT\033[0m" + echo -e "\033[1;34m│ type : $TEST_TYPE\033[0m" + echo -e "\033[1;34m│ container : $CONTAINER_IMAGE\033[0m" + echo -e "\033[1;34m└──────────────────────────────────────────────────────────────────────────┘\033[0m" docker pull "$CONTAINER_IMAGE" docker run --rm \ @@ -83,18 +83,24 @@ runs: run: | if [[ "$TEST_OUTCOME" == "success" ]]; then status="PASSED" - color='\033[1;32m' symbol="✅" + echo -e "\033[1;32m╔══════════════════════════════════════════════════════════════════════════╗\033[0m" + echo -e "\033[1;32m║ ║\033[0m" + echo -e "\033[1;32m║ ✅ PASSED ║\033[0m" + echo -e "\033[1;32m║ $TEST_SCRIPT\033[0m" + echo -e "\033[1;32m║ ║\033[0m" + echo -e "\033[1;32m╚══════════════════════════════════════════════════════════════════════════╝\033[0m" else status="FAILED" - color='\033[1;31m' symbol="❌" + echo -e "\033[1;31m╔══════════════════════════════════════════════════════════════════════════╗\033[0m" + echo -e "\033[1;31m║ ║\033[0m" + echo -e "\033[1;31m║ ❌ FAILED ║\033[0m" + echo -e "\033[1;31m║ $TEST_SCRIPT\033[0m" + echo -e "\033[1;31m║ ║\033[0m" + echo -e "\033[1;31m╚══════════════════════════════════════════════════════════════════════════╝\033[0m" fi - printf '%b============================================================\n' "$color" - printf ' %s: %s\n' "$status" "$TEST_SCRIPT" - printf '============================================================\033[0m\n' - { echo "### $symbol $TEST_SCRIPT — $status" echo "" diff --git a/tests/unit_tests/test_ci_environment.py b/tests/unit_tests/test_ci_environment.py index a8b77a8259..3127e9436c 100644 --- a/tests/unit_tests/test_ci_environment.py +++ b/tests/unit_tests/test_ci_environment.py @@ -267,6 +267,9 @@ def test_test_template_runs_cpu_or_gpu_script_in_container() -> None: assert 'echo "::notice title=Test result::$TEST_SCRIPT — PASSED"' in action assert 'echo "::error title=Test result::$TEST_SCRIPT — FAILED"' in action assert '} >> "$GITHUB_STEP_SUMMARY"' in action + assert "┌─ launching test ─" in action + assert "║ ✅ PASSED" in action + assert "║ ❌ FAILED" in action assert action.count("required: true") == 4