ci.yml

name: CI

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  workflow_dispatch:

concurrency:
  group: ci-${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  pre-commit:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"
      - uses: pre-commit/action@v3.0.0
        with:
          extra_args: --all-files

  unit-tests:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Cache pip packages
        uses: actions/cache@v4
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-unit-tests
          restore-keys: |
            ${{ runner.os }}-pip-

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install pytest
          pip install torch --index-url https://download.pytorch.org/whl/cpu

      - name: Run golden unit tests
        run: pytest tests/golden -v

  detect-changes:
    runs-on: ubuntu-latest
    outputs:
      has_changes: ${{ steps.check.outputs.has_changes }}
      files: ${{ steps.check.outputs.files }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"
      - id: check
        run: |
          if [ "${{ github.event_name }}" != "pull_request" ]; then
            echo "has_changes=true" >> "$GITHUB_OUTPUT"
            echo "files=" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          # Map the raw diff to the runnable scripts CI must exercise:
          #  - a changed models/ source file selects every runnable file that
          #    (transitively) imports it, not just the file itself;
          #  - any change outside models/ (golden/, .github/, docs, examples, …)
          #    selects all runnable examples as a smoke test.
          # See .github/scripts/detect_changes.py for the rules.
          FILES=$(git diff --name-only \
            ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \
            | python .github/scripts/detect_changes.py)
          if [ -n "$FILES" ]; then
            echo "has_changes=true" >> "$GITHUB_OUTPUT"
          else
            echo "has_changes=false" >> "$GITHUB_OUTPUT"
          fi
          echo "files=$FILES" >> "$GITHUB_OUTPUT"
          echo "Selected files: $FILES"

  sim:
    needs: detect-changes
    if: github.event_name != 'pull_request' || needs.detect-changes.outputs.has_changes == 'true'
    # Simulator job: no device needed, so it runs on the CPU runner inside the
    # ci-py310 container (mirrors pypto's codegen-tests setup, registry on port
    # 5001). pypto/ptoas/pto-isa are installed exactly like the on-device a2a3
    # job — only the runner label and the lack of Ascend host env differ.
    runs-on: [self-hosted, linux, arm64, cpu]
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        platform: [a2a3sim, a5sim]
    env:
      PTOAS_ROOT: ${{ github.workspace }}/ptoas-bin
      PTO_ISA_ROOT: ${{ github.workspace }}/pto-isa
      CMAKE_BUILD_PARALLEL_LEVEL: 16
      CMAKE_C_COMPILER_LAUNCHER: ccache
      CMAKE_CXX_COMPILER_LAUNCHER: ccache
      CCACHE_DIR: /root/.cache/ccache
      CCACHE_MAXSIZE: 30G
      # Shared with the host-side a2a3 job (ci-runner) via the ci-cache mount;
      # 000 keeps every cache file group/world-writable so root (this container)
      # and ci-runner can both update and LRU-evict each other's entries.
      CCACHE_UMASK: '000'
      PIP_CACHE_DIR: /root/.cache/pip
    container:
      image: localhost:5001/ci-py310:latest
      # GitHub Actions containers do not inherit the host's large /dev/shm; they
      # use Docker's 64MB default. Distributed example kernels call torch
      # share_memory_() and overflow it. --shm-size gives each container its own
      # tmpfs (sized on demand, not pre-allocated), avoiding the --ipc=host name
      # clashes between the concurrent cpu-1..4 runners.
      options: >-
        --shm-size=128g
        -v /home/ci-runner/hw-native-sys-pypto-lib/ci-cache/pip-sim:/root/.cache/pip
        -v /home/ci-runner/hw-native-sys-pypto-lib/ci-cache/ccache:/root/.cache/ccache
        -v /home/ci-runner/hw-native-sys-pypto-lib/ci-cache/ptoas:/opt/ptoas-cache

    steps:
      - name: Checkout pypto-lib
        uses: actions/checkout@v4
        with:
          persist-credentials: false

      - name: Resolve toolchain + clone pypto
        # Clone the pypto we build against ONCE here (job-private /tmp — this CPU
        # host runs the a2a3sim/a5sim matrix concurrently, so a shared tree would
        # corrupt). The Build step below reuses this checkout, so the toolchain
        # pins and the installed pypto are always the same revision. pypto owns
        # ptoas (toolchain/versions.env) and pins pto-isa via its runtime submodule
        # (runtime/pto_isa.pin); read both.
        run: |
          rm -rf /tmp/pypto
          git clone --recurse-submodules --depth=1 https://github.com/hw-native-sys/pypto.git /tmp/pypto
          {
            grep -E '^PTOAS_VERSION=' /tmp/pypto/toolchain/versions.env
            echo "PTOAS_SHA256=$(sed -n 's/^PTOAS_SHA256_AARCH64=//p' /tmp/pypto/toolchain/versions.env)"
            echo "PTO_ISA_COMMIT=$(tr -d '[:space:]' < /tmp/pypto/runtime/pto_isa.pin)"
          } >> "$GITHUB_ENV"

      # Namespace the cache by pto-isa commit so a pto-isa bump can never serve
      # stale objects compiled against the old ISA headers (see issue #1139):
      # a different commit hashes to a different key, forcing a real recompile.
      - name: Scope ccache to pto-isa version
        run: echo "CCACHE_NAMESPACE=pto-isa-${PTO_ISA_COMMIT}" >> "$GITHUB_ENV"

      - name: Show ccache stats (before)
        run: ccache -s || true

      - name: Clone pto-isa repository (pinned)
        run: |
          rm -rf "$PTO_ISA_ROOT"
          timeout 60 git clone https://github.com/hw-native-sys/pto-isa.git "$PTO_ISA_ROOT" \
            || { rm -rf "$PTO_ISA_ROOT"; timeout 300 git clone https://gitcode.com/luohuan40/pto-isa.git "$PTO_ISA_ROOT"; }
          cd "$PTO_ISA_ROOT"
          git checkout ${{ env.PTO_ISA_COMMIT }}

      - name: Build and install pypto and simpler
        # Reuses /tmp/pypto cloned by the resolve step above (same revision the
        # toolchain pins were read from).
        run: |
          python -m pip install --upgrade pip
          pip install scikit-build-core nanobind cmake ninja
          pip install --no-build-isolation /tmp/pypto
          pip install --no-build-isolation /tmp/pypto/runtime

      - name: Install runner dependencies
        run: |
          pip install nanobind
          pip install torch

      - name: Show ccache stats (after)
        run: ccache -s || true

      - name: Install ptoas (local cache)
        env:
          PTOAS_CACHE_DIR: /opt/ptoas-cache
        run: |
          CACHE_ARCHIVE="$PTOAS_CACHE_DIR/ptoas-aarch64-${PTOAS_VERSION}-${PTOAS_SHA256}.tar.gz"
          download_ptoas() {
            echo "Downloading ptoas ${PTOAS_VERSION}"
            mkdir -p "$PTOAS_CACHE_DIR"
            curl --fail --location --retry 3 --retry-all-errors \
              https://github.com/hw-native-sys/PTOAS/releases/download/${PTOAS_VERSION}/ptoas-bin-aarch64.tar.gz \
              -o "$CACHE_ARCHIVE.tmp"
            echo "${PTOAS_SHA256}  $CACHE_ARCHIVE.tmp" | sha256sum -c -
            mv "$CACHE_ARCHIVE.tmp" "$CACHE_ARCHIVE"
          }
          if [ ! -f "$CACHE_ARCHIVE" ]; then
            echo "Cache miss"
            download_ptoas
          elif ! echo "${PTOAS_SHA256}  $CACHE_ARCHIVE" | sha256sum -c -; then
            echo "Cache corrupted — removing and re-downloading"
            rm -f "$CACHE_ARCHIVE"
            download_ptoas
          else
            echo "Cache hit — using $CACHE_ARCHIVE"
          fi
          mkdir -p "$PTOAS_ROOT"
          tar -xzf "$CACHE_ARCHIVE" -C "$PTOAS_ROOT"
          chmod +x "$PTOAS_ROOT/ptoas"
          chmod +x "$PTOAS_ROOT/bin/ptoas"

      - name: Run ${{ matrix.platform }} tests
        shell: bash
        env:
          PYTHONPATH: ${{ github.workspace }}
          PTO2_RING_DEP_POOL: 16384
          PTO2_RING_TASK_WINDOW: 16384
          PTO2_RING_HEAP: 1073741824
        run: |
          set +e
          if [ "${{ github.event_name }}" = "pull_request" ]; then
            FILES="${{ needs.detect-changes.outputs.files }}"
          else
            FILES=$(find examples -name '*.py' ! -name '*draft*' | sort)
          fi
          if [ -z "$FILES" ]; then
            echo "No runnable changed files; nothing to test."
            exit 0
          fi
          failed=()
          for f in $FILES; do
            echo "::group::$f"
            # Files marked `# ci: no-sim` (full multi-layer / multi-card
            # forwards) are device-only and would just argparse-reject *sim.
            if grep -qE '^#\s*ci:\s*no-sim' "$f"; then
              echo "skip (no-sim): $f"
              echo "::endgroup::"
              continue
            fi
            if ! python "$f" -p ${{ matrix.platform }}; then
              failed+=("$f")
              echo "::error file=${f}::FAIL"
            fi
            echo "::endgroup::"
          done
          if [ ${#failed[@]} -ne 0 ]; then
            printf 'FAILED: %s\n' "${failed[@]}"
            exit 1
          fi
          echo "All selected files passed."

      - name: Clean up build artifacts
        if: always()
        run: rm -rf build_output

  a2a3:
    needs: detect-changes
    if: github.event_name != 'pull_request' || needs.detect-changes.outputs.has_changes == 'true'
    # No container: multi-card kernels use HCCL, which needs a host-only env
    # (set_env.sh, HCCL plugin paths) — in docker the chip child silent-crashes
    # in comm_init. So this job runs directly on the host like pypto's
    # dist-system-tests, on a 2-device runner so it can also schedule the
    # multi-card files. Checkout / install happen under ./dist-checkout to avoid
    # colliding with any container-owned workspace left on the shared runner.
    runs-on: [self-hosted, linux, arm64, npu]
    timeout-minutes: 30
    defaults:
      run:
        working-directory: dist-checkout
    env:
      ASCEND_HOME_PATH: /usr/local/Ascend/cann-9.0.0
      PTOAS_ROOT: ${{ github.workspace }}/dist-checkout/ptoas-bin
      PTO_ISA_ROOT: ${{ github.workspace }}/dist-checkout/pto-isa
      CMAKE_BUILD_PARALLEL_LEVEL: 16
      CMAKE_C_COMPILER_LAUNCHER: ccache
      CMAKE_CXX_COMPILER_LAUNCHER: ccache
      CCACHE_DIR: /home/ci-runner/hw-native-sys-pypto-lib/ci-cache/ccache
      CCACHE_MAXSIZE: 30G
      # Shared with the container-side sim job (root) via the same ci-cache dir;
      # 000 keeps cache files mutually writable across root and ci-runner.
      CCACHE_UMASK: '000'
      # Per-job pip dir: unlike ccache, pip has no shared-umask knob, so a shared
      # pip cache would leave files owned by whichever job wrote first and block
      # the other. The sim job uses ci-cache/pip-sim (see its bind mount).
      PIP_CACHE_DIR: /home/ci-runner/hw-native-sys-pypto-lib/ci-cache/pip-a2a3

    steps:
      - name: Checkout pypto-lib
        uses: actions/checkout@v4
        with:
          path: dist-checkout
          persist-credentials: false

      - name: Resolve toolchain + sync pypto source
        # Sync the cached pypto checkout ONCE here; the Build step below reuses it,
        # so the toolchain pins and the installed pypto are always the same
        # revision. pypto owns ptoas (toolchain/versions.env) and pins pto-isa via
        # its runtime submodule (runtime/pto_isa.pin); read both.
        run: |
          PYPTO_SRC="/home/ci-runner/hw-native-sys-pypto-lib/ci-cache/pypto-src"
          if [ -d "$PYPTO_SRC/.git" ]; then
            echo "Cache hit — updating pypto"
            git -C "$PYPTO_SRC" fetch --depth=1 origin HEAD
            git -C "$PYPTO_SRC" reset --hard FETCH_HEAD
            git -C "$PYPTO_SRC" clean -ffdx
            git -C "$PYPTO_SRC" submodule foreach --recursive 'git reset --hard && git clean -ffdx' || true
          else
            echo "Cache miss — cloning pypto"
            git clone --recurse-submodules --depth=1 https://github.com/hw-native-sys/pypto.git "$PYPTO_SRC"
          fi
          git -C "$PYPTO_SRC" submodule update --init --recursive
          rm -rf "$PYPTO_SRC/build" "$PYPTO_SRC/_skbuild" "$PYPTO_SRC/runtime/build"
          {
            grep -E '^PTOAS_VERSION=' "$PYPTO_SRC/toolchain/versions.env"
            echo "PTOAS_SHA256=$(sed -n 's/^PTOAS_SHA256_AARCH64=//p' "$PYPTO_SRC/toolchain/versions.env")"
            echo "PTO_ISA_COMMIT=$(tr -d '[:space:]' < "$PYPTO_SRC/runtime/pto_isa.pin")"
          } >> "$GITHUB_ENV"

      - name: Check NPU
        working-directory: .
        run: npu-smi info

      # See the sim job: namespace the cache by pto-isa commit so an ISA bump
      # forces a real recompile instead of reusing stale objects (issue #1139).
      - name: Scope ccache to pto-isa version
        run: echo "CCACHE_NAMESPACE=pto-isa-${PTO_ISA_COMMIT}" >> "$GITHUB_ENV"

      - name: Clone pto-isa repository (pinned)
        run: |
          rm -rf "$PTO_ISA_ROOT"
          timeout 60 git clone https://github.com/hw-native-sys/pto-isa.git "$PTO_ISA_ROOT" \
            || { rm -rf "$PTO_ISA_ROOT"; timeout 300 git clone https://gitcode.com/luohuan40/pto-isa.git "$PTO_ISA_ROOT"; }
          cd "$PTO_ISA_ROOT"
          git checkout ${{ env.PTO_ISA_COMMIT }}

      - name: Bootstrap conda + per-job venv + write activate.sh
        # Per-job venv layered on conda py310-lib; activate.sh lets each later step
        # enter the env with a single `source activate.sh`. set_env.sh wires up
        # the Ascend/HCCL host environment; LD_LIBRARY_PATH is prepended with
        # $CONDA_PREFIX/lib because ptoas needs a newer GLIBCXX than the system.
        run: |
          source /home/ci-runner/miniconda3/etc/profile.d/conda.sh
          conda activate py310-lib
          python -m venv venv --system-site-packages
          cat > activate.sh <<'EOF'
          source /home/ci-runner/miniconda3/etc/profile.d/conda.sh
          conda activate py310-lib
          source "$GITHUB_WORKSPACE/dist-checkout/venv/bin/activate"
          source /usr/local/Ascend/cann-9.0.0/set_env.sh
          export LD_LIBRARY_PATH="$CONDA_PREFIX/lib:$LD_LIBRARY_PATH"
          EOF

      - name: Show ccache stats (before)
        run: ccache -s || true

      - name: Build and install pypto and simpler
        # Reuses the pypto source synced by the resolve step above (same revision
        # the toolchain pins were read from).
        run: |
          source activate.sh
          PYPTO_SRC="/home/ci-runner/hw-native-sys-pypto-lib/ci-cache/pypto-src"
          python -m pip install --upgrade pip
          pip install scikit-build-core nanobind cmake ninja
          pip install --no-build-isolation "$PYPTO_SRC"
          pip install --no-build-isolation "$PYPTO_SRC/runtime"

      - name: Install runner dependencies
        run: |
          source activate.sh
          pip install nanobind
          pip install torch

      - name: Show ccache stats (after)
        run: ccache -s || true

      - name: Install ptoas (local cache)
        env:
          PTOAS_CACHE_DIR: /home/ci-runner/hw-native-sys-pypto-lib/ci-cache/ptoas
        run: |
          CACHE_ARCHIVE="$PTOAS_CACHE_DIR/ptoas-aarch64-${PTOAS_VERSION}-${PTOAS_SHA256}.tar.gz"
          download_ptoas() {
            echo "Downloading ptoas ${PTOAS_VERSION}"
            mkdir -p "$PTOAS_CACHE_DIR"
            curl --fail --location --retry 3 --retry-all-errors \
              https://github.com/hw-native-sys/PTOAS/releases/download/${PTOAS_VERSION}/ptoas-bin-aarch64.tar.gz \
              -o "$CACHE_ARCHIVE.tmp"
            echo "${PTOAS_SHA256}  $CACHE_ARCHIVE.tmp" | sha256sum -c -
            mv "$CACHE_ARCHIVE.tmp" "$CACHE_ARCHIVE"
          }
          if [ ! -f "$CACHE_ARCHIVE" ]; then
            echo "Cache miss"
            download_ptoas
          elif ! echo "${PTOAS_SHA256}  $CACHE_ARCHIVE" | sha256sum -c -; then
            echo "Cache corrupted — removing and re-downloading"
            rm -f "$CACHE_ARCHIVE"
            download_ptoas
          else
            echo "Cache hit — using $CACHE_ARCHIVE"
          fi
          mkdir -p "$PTOAS_ROOT"
          tar -xzf "$CACHE_ARCHIVE" -C "$PTOAS_ROOT"
          chmod +x "$PTOAS_ROOT/ptoas"
          chmod +x "$PTOAS_ROOT/bin/ptoas"

      - name: Run a2a3 tests
        shell: bash
        env:
          PYTHONPATH: ${{ github.workspace }}/dist-checkout
          PTO2_RING_DEP_POOL: 16384
          PTO2_RING_TASK_WINDOW: 16384
          PTO2_RING_HEAP: 1073741824
        run: |
          source activate.sh
          set +e
          if [ "${{ github.event_name }}" = "pull_request" ]; then
            FILES="${{ needs.detect-changes.outputs.files }}"
          else
            FILES=$(find examples -name '*.py' ! -name '*draft*' | sort)
          fi
          if [ -z "$FILES" ]; then
            echo "No runnable changed files; nothing to test."
            exit 0
          fi
          failed=()
          for f in $FILES; do
            echo "::group::$f"
            # Files needing multiple NPUs declare `# ci: devices=N` (N>1) near
            # the top; ring sizes come from the step env: above.
            ndev=$(grep -oP '#\s*ci:\s*devices=\K[0-9]+' "$f" | head -1)
            ndev=${ndev:-1}
            if [ "$ndev" -gt 1 ]; then
              if [ -z "$DEVICE_RANGE" ]; then
                echo "::error file=${f}::needs $ndev devices but DEVICE_RANGE is unset"
                failed+=("$f")
                echo "::endgroup::"
                continue
              fi
              actual_ndev=$(tr ',' '\n' <<<"$DEVICE_RANGE" | sed '/^$/d' | wc -l)
              if [ "$actual_ndev" -lt "$ndev" ]; then
                echo "::error file=${f}::needs $ndev devices but DEVICE_RANGE only provides $actual_ndev ($DEVICE_RANGE)"
                failed+=("$f")
                echo "::endgroup::"
                continue
              fi
              python "$f" -p a2a3 -d "$(cut -d, -f1-"$ndev" <<<"$DEVICE_RANGE")"
            else
              python "$f" -p a2a3 -d "$DEVICE_ID"
            fi
            if [ $? -ne 0 ]; then
              failed+=("$f")
              echo "::error file=${f}::FAIL"
            fi
            echo "::endgroup::"
          done
          if [ ${#failed[@]} -ne 0 ]; then
            printf 'FAILED: %s\n' "${failed[@]}"
            exit 1
          fi
          echo "All selected files passed."

      - name: Clean up build artifacts
        if: always()
        run: rm -rf build_output