_build.yml

name: 15. _Build Distribution

on:
  workflow_call:
    inputs:
      os_json:
        description: 'JSON string of runner labels to build on (ubuntu-24.04=x86_64, ubuntu-24.04-arm=aarch64, macos-14=arm64, macos-15-intel=x86_64, windows-latest=x86_64)'
        required: false
        type: string
        default: '["ubuntu-24.04", "ubuntu-24.04-arm", "macos-14", "macos-15-intel", "windows-latest"]'
      python_json:
        description: 'JSON string of Python versions'
        required: false
        type: string
        default: '["3.10"]'
      build_sdist:
        description: 'Whether to build source distribution'
        required: false
        type: boolean
        default: true
      build_wheels:
        description: 'Whether to build wheel distribution'
        required: false
        type: boolean
        default: true
  workflow_dispatch:
    inputs:
      build_sdist:
        description: 'Whether to build source distribution'
        required: false
        type: boolean
        default: true
      build_wheels:
        description: 'Whether to build wheel distribution'
        required: false
        type: boolean
        default: true
      os_json:
        description: 'JSON string of runner labels to build on (ubuntu-24.04=x86_64, ubuntu-24.04-arm=aarch64, macos-14=arm64, macos-15-intel=x86_64, windows-latest=x86_64)'
        required: false
        default: '["ubuntu-24.04", "ubuntu-24.04-arm", "macos-14", "macos-15-intel", "windows-latest"]'
      python_json:
        description: 'JSON string of Python versions'
        required: false
        default: '["3.10"]'

jobs:
  build-sdist:
    name: Build source distribution py3.12
    if: inputs.build_sdist
    runs-on: ubuntu-24.04
    steps:
    - uses: actions/checkout@v6
      with:
        submodules: recursive
        fetch-depth: 0  # Required for setuptools_scm to detect version from git tags

    - name: Fetch all tags
      run: git fetch --force --tags

    - name: Set up Python
      uses: actions/setup-python@v6
      with:
        python-version: '3.12'

    - name: Install uv
      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true

    - name: Create venv
      run: uv venv

    - name: Install build dependencies
      run: uv pip install build setuptools_scm

    - name: Clean workspace (force ignore dirty)
      shell: bash
      run: |
        git reset --hard HEAD
        git clean -fd
        # For sdist, ensure local runtime binaries are not packaged even if present
        rm -rf openviking/bin openviking/lib third_party/agfs/bin || true
        rm -f openviking/storage/vectordb/*.so openviking/storage/vectordb/*.dylib openviking/storage/vectordb/*.dll openviking/storage/vectordb/*.exe || true
        rm -rf openviking/_version.py openviking.egg-info
        # Ignore uv.lock changes to avoid dirty state in setuptools_scm
        git update-index --assume-unchanged uv.lock || true

    - name: Debug Git and SCM
      shell: bash
      run: |
        echo "=== Git Describe ==="
        git describe --tags --long --dirty --always
        echo "=== Setuptools SCM Version ==="
        uv run --frozen python -m setuptools_scm
        echo "=== Git Status (Ignored included) ==="
        git status --ignored
        echo "=== Check openviking/_version.py ==="
        if [ -f openviking/_version.py ]; then cat openviking/_version.py; else echo "Not found"; fi

    - name: Build sdist
      run: uv build --sdist

    - name: Store the distribution packages
      uses: actions/upload-artifact@v7
      with:
        name: python-package-distributions-sdist
        path: dist/*.tar.gz

    - name: Display built sdist version
      continue-on-error: true
      run: |
        VERSION=$(ls dist/*.tar.gz | head -n 1 | xargs basename | sed -E 's/^[^-]+-(.+)\.tar\.gz$/\1/')
        echo "Build Version: $VERSION"
        echo "::notice::Build sdist Version: $VERSION"

  build-linux:
    name: Build distribution on Linux ${{ matrix.arch }} (glibc 2.31) py${{ matrix.python-version }}
    # Run if Linux runners are requested (explicit labels or generic 'linux')
    if: >-
      inputs.build_wheels &&
      (
        contains(inputs.os_json, 'linux') ||
        contains(inputs.os_json, '"ubuntu-24.04"') ||
        contains(inputs.os_json, 'ubuntu-24.04-arm')
      )
    runs-on: ${{ matrix.arch == 'aarch64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
    container: ubuntu:20.04
    env:
      DEBIAN_FRONTEND: noninteractive
      TZ: Etc/UTC
    strategy:
      fail-fast: false
      matrix:
        python-version: ${{ fromJson(inputs.python_json) }}
        arch: ${{ contains(inputs.os_json, 'linux') && fromJson('["x86_64","aarch64"]') || (contains(inputs.os_json, '"ubuntu-24.04"') && contains(inputs.os_json, 'ubuntu-24.04-arm')) && fromJson('["x86_64","aarch64"]') || contains(inputs.os_json, 'ubuntu-24.04-arm') && fromJson('["aarch64"]') || fromJson('["x86_64"]') }}
    
    steps:
    - name: Install system dependencies (Linux)
      run: |
        # Replace archive.ubuntu.com with azure.archive.ubuntu.com for better stability in GH Actions
        sed -i 's/http:\/\/archive.ubuntu.com\/ubuntu\//http:\/\/azure.archive.ubuntu.com\/ubuntu\//g' /etc/apt/sources.list
        # Retry apt-get update
        for i in 1 2 3 4 5; do apt-get update && break || sleep 5; done
        apt-get install -y \
          git ca-certificates cmake build-essential tzdata curl \
          libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev \
          libffi-dev liblzma-dev libgdbm-dev libnss3-dev libncurses5-dev \
          libncursesw5-dev tk-dev uuid-dev libexpat1-dev
        ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime
        dpkg-reconfigure -f noninteractive tzdata

    - uses: actions/checkout@v6
      with:
        submodules: recursive
        fetch-depth: 0  # Required for setuptools_scm to detect version from git tags

    - name: Fetch all tags
      run: |
        git config --global --add safe.directory "$GITHUB_WORKSPACE"
        git fetch --force --tags

    - name: Build CPython (Dynamic Selection)
      run: |
        # Map short version to full version for our specific build environment
        PYTHON_VERSION="${{ matrix.python-version }}"
        case "$PYTHON_VERSION" in
          "3.9") PYTHON_FULL="3.9.18" ;;
          "3.10") PYTHON_FULL="3.10.13" ;;
          "3.11") PYTHON_FULL="3.11.8" ;;
          "3.12") PYTHON_FULL="3.12.2" ;;
          "3.13") PYTHON_FULL="3.13.2" ;;
          "3.14") PYTHON_FULL="3.14.3" ;;
          *)
            echo "Error: Unknown python version $PYTHON_VERSION"
            exit 1
            ;;
        esac
        
        PYTHON_PREFIX="/opt/python/${PYTHON_FULL}"
        PYTHON_BIN="${PYTHON_PREFIX}/bin/python${{ matrix.python-version }}"
        if [ ! -x "$PYTHON_BIN" ]; then
          curl -fsSL -o /tmp/Python-${PYTHON_FULL}.tgz \
            https://www.python.org/ftp/python/${PYTHON_FULL}/Python-${PYTHON_FULL}.tgz
          tar -xzf /tmp/Python-${PYTHON_FULL}.tgz -C /tmp
          cd /tmp/Python-${PYTHON_FULL}
          CFLAGS="-fPIC" \
          ./configure --prefix="${PYTHON_PREFIX}" --with-ensurepip=install --enable-shared
          make -j"$(nproc)"
          make install
        fi
        echo "PYTHON_BIN=${PYTHON_BIN}" >> "$GITHUB_ENV"
        echo "LD_LIBRARY_PATH=${PYTHON_PREFIX}/lib:${LD_LIBRARY_PATH}" >> "$GITHUB_ENV"
        export LD_LIBRARY_PATH="${PYTHON_PREFIX}/lib:${LD_LIBRARY_PATH}"
        "$PYTHON_BIN" -V
    - name: Set up Go
      uses: actions/setup-go@v6
      with:
        go-version: '1.25.1'

    - name: Set up Rust
      uses: dtolnay/rust-toolchain@v1
      with:
        toolchain: stable
        targets: ${{ matrix.arch == 'aarch64' && 'aarch64-unknown-linux-gnu' || 'x86_64-unknown-linux-gnu' }}
    - name: Install uv
      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true

    - name: Create venv (Linux)
      run: uv venv --python "$PYTHON_BIN"

    - name: Seed pip (Linux)
      run: uv run python -m ensurepip --upgrade

    - name: Install dependencies
      run: uv sync --frozen

    - name: Install build dependencies
      run: uv pip install setuptools setuptools_scm cmake wheel build

    - name: Resolve OpenViking version for Rust CLI (Linux)
      shell: bash
      run: |
        OPENVIKING_VERSION=$(
          uv run --frozen python -c "from build_support.versioning import resolve_openviking_version; print(resolve_openviking_version())"
        )
        echo "OPENVIKING_VERSION=$OPENVIKING_VERSION" >> "$GITHUB_ENV"
        echo "Resolved OpenViking version: $OPENVIKING_VERSION"

    - name: Build Rust CLI (Linux)
      run: cargo build --release --target ${{ matrix.arch == 'aarch64' && 'aarch64-unknown-linux-gnu' || 'x86_64-unknown-linux-gnu' }} -p ov_cli

    - name: Copy Rust CLI binary (Linux)
      run: |
        mkdir -p openviking/bin
        cp target/${{ matrix.arch == 'aarch64' && 'aarch64-unknown-linux-gnu' || 'x86_64-unknown-linux-gnu' }}/release/ov openviking/bin/
        chmod +x openviking/bin/ov
    - name: Clean workspace (force ignore dirty)
      shell: bash
      run: |
        git reset --hard HEAD
        git clean -fd
        rm -rf openviking/_version.py openviking.egg-info
        # Ignore uv.lock changes to avoid dirty state in setuptools_scm
        git update-index --assume-unchanged uv.lock || true

    - name: Debug Git and SCM
      shell: bash
      run: |
        echo "=== Git Describe ==="
        git describe --tags --long --dirty --always
        echo "=== Setuptools SCM Version ==="
        uv run --frozen python -m setuptools_scm
        echo "=== Git Status (Ignored included) ==="
        git status --ignored
        echo "=== Check openviking/_version.py ==="
        if [ -f openviking/_version.py ]; then cat openviking/_version.py; else echo "Not found"; fi

    - name: Build package (Wheel Only)
      run: uv build --wheel

    - name: Install patchelf (Linux)
      run: |
        PATCHELF_VERSION=0.18.0
        curl -fsSL -o /tmp/patchelf-${PATCHELF_VERSION}.tar.gz \
          https://github.com/NixOS/patchelf/releases/download/${PATCHELF_VERSION}/patchelf-${PATCHELF_VERSION}.tar.gz
        tar -xzf /tmp/patchelf-${PATCHELF_VERSION}.tar.gz -C /tmp
        cd /tmp/patchelf-${PATCHELF_VERSION}
        ./configure
        make -j"$(nproc)"
        make install
        patchelf --version

    - name: Repair wheels (Linux)
      run: |
        uv pip install auditwheel
        # Repair wheels and output to a temporary directory
        uv run auditwheel repair dist/*.whl -w dist_fixed
        # Remove original non-compliant wheels
        rm dist/*.whl
        # Move repaired wheels back to dist
        mv dist_fixed/*.whl dist/
        rmdir dist_fixed

    - name: Store the distribution packages
      uses: actions/upload-artifact@v7
      with:
        name: python-package-distributions-linux-${{ matrix.arch }}-${{ matrix.python-version }}
        path: dist/

    - name: Display built wheel version
      continue-on-error: true
      run: |
        VERSION=$(ls dist/*.whl | head -n 1 | xargs basename | cut -d- -f2)
        echo "Build Version: $VERSION"
        echo "::notice::Build Wheel Version (Linux ${{ matrix.arch }} glibc 2.31 py${{ matrix.python-version }}): $VERSION"

  build-other:
    name: Build non-Linux distributions
    # Run only when non-Linux runners are explicitly requested
    if: inputs.build_wheels && (contains(inputs.os_json, 'macos') || contains(inputs.os_json, 'windows'))
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: ${{ fromJson(inputs.os_json) }}
        python-version: ${{ fromJson(inputs.python_json) }}
        # Exclude ubuntu-24.04 from this matrix if it was passed in inputs
        exclude:
          - os: linux
          - os: ubuntu-24.04
          - os: ubuntu-24.04-arm

    steps:
    - uses: actions/checkout@v6
      with:
        submodules: recursive
        fetch-depth: 0  # Required for setuptools_scm to detect version from git tags

    - name: Fetch all tags
      run: git fetch --force --tags

    - name: Set up Python
      uses: actions/setup-python@v6
      with:
        python-version: ${{ matrix.python-version }}

    - name: Configure macOS wheel architecture tag
      if: runner.os == 'macOS'
      shell: bash
      run: |
        if [[ "${{ matrix.os }}" == "macos-14" ]]; then
          TARGET_ARCH="arm64"
          MACOS_VERSION="14.0"
        elif [[ "${{ matrix.os }}" == "macos-15-intel" ]]; then
          TARGET_ARCH="x86_64"
          MACOS_VERSION="15.0"
        else
          echo "Unsupported macOS runner for release wheels: ${{ matrix.os }}"
          exit 1
        fi

        echo "ARCHFLAGS=-arch ${TARGET_ARCH}" >> "$GITHUB_ENV"
        echo "CMAKE_OSX_ARCHITECTURES=${TARGET_ARCH}" >> "$GITHUB_ENV"
        echo "_PYTHON_HOST_PLATFORM=macosx-${MACOS_VERSION}-${TARGET_ARCH}" >> "$GITHUB_ENV"
        echo "Configured macOS wheel platform: macosx-${MACOS_VERSION}-${TARGET_ARCH}"

    - name: Set up Go
      uses: actions/setup-go@v6
      with:
        go-version: '1.25.1'

    - name: Set up Rust
      uses: dtolnay/rust-toolchain@v1
      with:
        toolchain: stable
    - name: Install uv
      uses: astral-sh/setup-uv@v7
      with:
        enable-cache: true

    - name: Install system dependencies (macOS)
      if: runner.os == 'macOS'
      run: brew install cmake

    - name: Install system dependencies (Windows)
      if: runner.os == 'Windows'
      run: |
        choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System'
        choco install mingw

    - name: Install dependencies
      run: uv sync --frozen

    - name: Install build dependencies
      run: uv pip install setuptools setuptools_scm cmake wheel build

    - name: Resolve OpenViking version for Rust CLI (macOS/Windows)
      shell: bash
      run: |
        OPENVIKING_VERSION=$(
          uv run --frozen python -c "from build_support.versioning import resolve_openviking_version; print(resolve_openviking_version())"
        )
        echo "OPENVIKING_VERSION=$OPENVIKING_VERSION" >> "$GITHUB_ENV"
        echo "Resolved OpenViking version: $OPENVIKING_VERSION"

    - name: Build Rust CLI (macOS/Windows)
      shell: bash
      run: |
        if [[ "${{ matrix.os }}" == "windows-latest" ]]; then
          cargo build --release --target x86_64-pc-windows-msvc -p ov_cli
        else
          cargo build --release -p ov_cli
        fi

    - name: Copy Rust CLI binary (macOS/Windows)
      shell: bash
      run: |
        mkdir -p openviking/bin
        if [[ "${{ matrix.os }}" == "windows-latest" ]]; then
          cp target/x86_64-pc-windows-msvc/release/ov.exe openviking/bin/
        else
          cp target/release/ov openviking/bin/
          chmod +x openviking/bin/ov
        fi
    - name: Clean workspace (force ignore dirty)
      shell: bash
      run: |
        git reset --hard HEAD
        git clean -fd
        rm -rf openviking/_version.py openviking.egg-info
        # Ignore uv.lock changes to avoid dirty state in setuptools_scm
        git update-index --assume-unchanged uv.lock || true

    - name: Debug Git and SCM
      shell: bash
      run: |
        echo "=== Git Describe ==="
        git describe --tags --long --dirty --always
        echo "=== Setuptools SCM Version ==="
        uv run --frozen python -m setuptools_scm
        echo "=== Git Status (Ignored included) ==="
        git status --ignored
        echo "=== Check openviking/_version.py ==="
        if [ -f openviking/_version.py ]; then cat openviking/_version.py; else echo "Not found"; fi

    - name: Build package (Wheel Only)
      run: uv build --wheel

    - name: Smoke test built wheel (Windows)
      if: runner.os == 'Windows'
      shell: bash
      run: |
        python -m pip install --upgrade pip
        python -m pip install --force-reinstall dist/*.whl

        cd "$RUNNER_TEMP"
        python - <<'PY'
        import importlib.util

        import openviking.storage.vectordb.engine as engine

        print(f"Loaded runtime engine variant {engine.ENGINE_VARIANT}")
        print(f"Available engine variants {engine.AVAILABLE_ENGINE_VARIANTS}")

        module_name = f"openviking.storage.vectordb.engine._{engine.ENGINE_VARIANT}"
        backend_spec = importlib.util.find_spec(module_name)
        if backend_spec is None or backend_spec.origin is None:
            raise SystemExit(f"backend module {module_name} was not installed")

        print(f"Imported backend module {module_name}")
        print(f"Backend module origin {backend_spec.origin}")
        PY

    - name: Store the distribution packages
      uses: actions/upload-artifact@v7
      with:
        name: python-package-distributions-${{ matrix.os == 'macos-14' && 'macos-arm64' || matrix.os == 'macos-15-intel' && 'macos-x86_64' || matrix.os == 'windows-latest' && 'windows-x86_64' || matrix.os }}-${{ matrix.python-version }}
        path: dist/

    - name: Display built wheel version
      shell: bash
      continue-on-error: true
      run: |
        VERSION=$(ls dist/*.whl | head -n 1 | xargs basename | cut -d- -f2)
        echo "Build Version: $VERSION"
        echo "::notice::Build Wheel Version (${{ matrix.os == 'macos-14' && 'macOS arm64 (macos-14)' || matrix.os == 'macos-15-intel' && 'macOS x86_64 (macos-15-intel)' || matrix.os == 'windows-latest' && 'Windows x86_64 (windows-latest)' || matrix.os }} py${{ matrix.python-version }}): $VERSION"