integration-test.yml

name: Integration Tests

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: gpu-integration-${{ github.ref }}
  cancel-in-progress: true

jobs:
  integration:
    name: ${{ matrix.case }}
    runs-on: [self-hosted, linux, x64, h20, gpu]
    # Allow 120 minutes for image preparation plus the existing 65-minute test job budget.
    timeout-minutes: 185
    strategy:
      fail-fast: false
      matrix:
        include:
          - case: Qwen3-4B-4xgpu-async
            script: scripts/training/text/run-qwen3-4B-4xgpu-async.sh
            gpus: 4
          - case: Qwen3-VL-4B-2xgpu
            script: scripts/training/multimodal/run-qwen3-vl-4B-2xgpu.sh
            gpus: 2
    env:
      CI_CONTAINER_NAME: relax-integration-${{ github.run_id }}-${{ github.run_attempt }}-${{ strategy.job-index }}
      TRAINING_SCRIPT: ${{ matrix.script }}
      ARTIFACT_NAME: integration-${{ matrix.case }}-${{ github.run_id }}-${{ github.run_attempt }}
    steps:
      - name: Checkout
        uses: actions/checkout@v7
        with:
          persist-credentials: false

      - name: Prepare local test image
        id: image
        uses: ./.github/actions/prepare-image
        timeout-minutes: 120
        with:
          # Include patch targets as well as the symlinks under patch/latest.
          hash-key: ${{ hashFiles('docker/Dockerfile', '.dockerignore', 'requirements.txt', 'docker/patch/**', 'relax/backends/megatron/kernels/int4_qat/**', '.github/actions/prepare-image/action.yml') }}

      - name: Acquire test devices
        id: devices
        uses: ./.github/actions/acquire-devices
        timeout-minutes: 35
        with:
          backend: nvidia
          count: ${{ matrix.gpus }}
          timeout: 1800

      - name: Create test container
        env:
          CI_IMAGE: ${{ steps.image.outputs.image-id }}
          CI_DEVICES: ${{ steps.devices.outputs.devices }}
          NUM_GPUS: ${{ matrix.gpus }}
          UV_CACHE_DIR: /uv-cache
          UV_LINK_MODE: copy
          ARTIFACT_DIR: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
        run: |
          set -euo pipefail
          HOST_UV_CACHE_DIR="$HOME/.cache/uv"
          mkdir -p "$HOST_UV_CACHE_DIR" "$ARTIFACT_DIR"

          # Each runner provisions the cases' model and dataset files in ~/model and ~/data.
          # Keep Ray processes, network, IPC and /tmp private to this container.
          docker create --pull never --rm --init \
            --name "$CI_CONTAINER_NAME" \
            --gpus "\"device=$CI_DEVICES\"" \
            --shm-size=32g \
            -v "$GITHUB_WORKSPACE:/source:ro" \
            -v "$ARTIFACT_DIR:/artifacts" \
            -v "$HOST_UV_CACHE_DIR:$UV_CACHE_DIR" \
            --mount "type=bind,source=$HOME/model,target=/models,readonly" \
            --mount "type=bind,source=$HOME/data,target=/data,readonly" \
            --workdir /workspace/Relax \
            --env NUM_GPUS \
            --env NUM_ROLLOUT=4 \
            --env MODEL_DIR=/models \
            --env DATA_DIR=/data \
            --env EXP_DIR=/artifacts \
            --env CLEARML_OFFLINE_MODE=1 \
            --env UV_CACHE_DIR \
            --env UV_LINK_MODE \
            --env http_proxy \
            --env https_proxy \
            --env no_proxy \
            --entrypoint /bin/bash "$CI_IMAGE" -c 'sleep infinity'
          docker inspect --format 'Test image: {{.Config.Image}} ({{.Image}})' "$CI_CONTAINER_NAME"
          docker start "$CI_CONTAINER_NAME"

      - name: Prepare workspace and show GPUs
        run: |
          docker exec "$CI_CONTAINER_NAME" bash -euc '
            cp -a --no-preserve=ownership /source/. /workspace/Relax/
            nvidia-smi
          '

      - name: Install dependencies
        run: |
          docker exec "$CI_CONTAINER_NAME" uv pip install \
            --system --break-system-packages -r requirements.txt \
            --extra-index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple

      - name: Run integration test
        timeout-minutes: 10
        run: |
          docker exec --env TRAINING_SCRIPT "$CI_CONTAINER_NAME" bash -euo pipefail -c '
            unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY all_proxy ALL_PROXY
            unset RAY_NO_WAIT RAY_ADDRESS RELAX_ENTRYPOINT_MODE
            export no_proxy="*" NO_PROXY="*"
            bash "$TRAINING_SCRIPT" 2>&1 | tee /artifacts/training.log
          '

      - name: Collect Ray logs
        if: always()
        env:
          ARTIFACT_DIR: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
        run: |
          mkdir -p "$ARTIFACT_DIR/ray"
          docker cp "$CI_CONTAINER_NAME:/tmp/ray/session_latest/logs/." "$ARTIFACT_DIR/ray/" || true

      - name: Remove test container
        if: always()
        run: |
          if docker inspect "$CI_CONTAINER_NAME" >/dev/null 2>&1; then
            docker rm --force "$CI_CONTAINER_NAME"
          fi

      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: ${{ env.ARTIFACT_NAME }}
          path: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}/
          if-no-files-found: warn
          retention-days: 14

  npu-integration:
    name: Qwen3-4B-4xnpu-async
    runs-on: [self-hosted, linux, ARM64, 910c, npu]
    timeout-minutes: 185
    env:
      CI_CONTAINER_NAME: relax-npu-integration-${{ github.run_id }}-${{ github.run_attempt }}
      TRAINING_SCRIPT: scripts/training/text/run-qwen3-4B-4xgpu-async-npu.sh
      ARTIFACT_NAME: integration-Qwen3-4B-4xnpu-async-${{ github.run_id }}-${{ github.run_attempt }}
      CLEARML_TAGS: branch=${{ github.head_ref || github.ref_name }},commit=${{ github.sha }}
    steps:
      - name: Checkout
        uses: actions/checkout@v7
        with:
          persist-credentials: false

      - name: Prepare local test image
        id: image
        uses: ./.github/actions/prepare-image
        timeout-minutes: 120
        with:
          dockerfile: docker/Dockerfile.npu
          platform: linux/arm64
          build-args: |
            NO_PROXY=localhost,127.0.0.1,::1,mirrors.tuna.tsinghua.edu.cn
            SOC_VERSION=ascend910_9362
            SOC_VERSION_FOR_SGL_BUILD=Ascend910_9362
          hash-key: ${{ hashFiles('docker/Dockerfile.npu', '.dockerignore', 'requirements.txt', 'docker/npu_patch/**', '.github/actions/prepare-image/action.yml') }}

      - name: Acquire test devices
        id: devices
        uses: ./.github/actions/acquire-devices
        timeout-minutes: 35
        with:
          backend: ascend
          # Four compute chips; paired chips occupy two dual-chip 910C cards.
          count: 4
          timeout: 1800

      - name: Create test container
        env:
          CI_IMAGE: ${{ steps.image.outputs.image-id }}
          CI_DEVICES: ${{ steps.devices.outputs.devices }}
          ARTIFACT_DIR: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
        run: |
          set -euo pipefail
          mkdir -p "$ARTIFACT_DIR"

          # Same model/data layout as GPU runners; Ascend Runtime mounts the host driver.
          # Runtime selects host physical chips; CANN sees container-local IDs 0..3.
          docker create --pull never --rm --init \
            --name "$CI_CONTAINER_NAME" \
            --runtime ascend \
            --shm-size=32g \
            -v "$GITHUB_WORKSPACE:/source:ro" \
            -v "$ARTIFACT_DIR:/artifacts" \
            --mount "type=bind,source=$HOME/model,target=/models,readonly" \
            --mount "type=bind,source=$HOME/data,target=/data,readonly" \
            --workdir /workspace/Relax \
            --env "ASCEND_VISIBLE_DEVICES=$CI_DEVICES" \
            --env ASCEND_RT_VISIBLE_DEVICES=0,1,2,3 \
            --env CLEARML_TAGS \
            --env NUM_ROLLOUT=4 \
            --env MODEL_DIR=/models \
            --env DATA_DIR=/data \
            --env EXP_DIR=/artifacts \
            --env CLEARML_OFFLINE_MODE=1 \
            --entrypoint /bin/bash "$CI_IMAGE" -c 'sleep infinity'
          docker inspect --format 'Test image: {{.Config.Image}} ({{.Image}})' "$CI_CONTAINER_NAME"
          docker start "$CI_CONTAINER_NAME"

      - name: Prepare workspace
        run: |
          docker exec "$CI_CONTAINER_NAME" bash -euc '
            cp -a --no-preserve=ownership /source/. /workspace/Relax/
          '

      - name: Run integration test
        timeout-minutes: 15
        run: |
          docker exec --env TRAINING_SCRIPT "$CI_CONTAINER_NAME" bash -eo pipefail -c '
            source /usr/local/Ascend/ascend-toolkit/set_env.sh
            source /usr/local/Ascend/nnal/atb/set_env.sh
            source /usr/local/Ascend/cann-9.0.0/share/info/ascendnpu-ir/bin/set_env.sh
            export LD_LIBRARY_PATH=/usr/local/Ascend/driver/lib64/driver:/usr/local/Ascend/driver/lib64/common:${LD_LIBRARY_PATH:-}
            npu-smi info
            python3 -c "import torch, torch_npu; assert torch.npu.device_count() == 4, torch.npu.device_count()"
            unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY all_proxy ALL_PROXY
            unset RAY_NO_WAIT RAY_ADDRESS RELAX_ENTRYPOINT_MODE
            export no_proxy="*" NO_PROXY="*"
            bash "$TRAINING_SCRIPT" 2>&1 | tee /artifacts/training.log
          '

      - name: Collect Ray logs
        if: always()
        env:
          ARTIFACT_DIR: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}
        run: |
          mkdir -p "$ARTIFACT_DIR/ray"
          docker cp "$CI_CONTAINER_NAME:/tmp/ray/session_latest/logs/." "$ARTIFACT_DIR/ray/" || true

      - name: Remove test container
        if: always()
        run: |
          if docker inspect "$CI_CONTAINER_NAME" >/dev/null 2>&1; then
            docker rm --force "$CI_CONTAINER_NAME"
          fi

      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: ${{ env.ARTIFACT_NAME }}
          path: ${{ runner.temp }}/${{ env.ARTIFACT_NAME }}/
          if-no-files-found: warn
          retention-days: 14