diff --git a/.github/workflows/build_test_wheel.yml b/.github/workflows/build_test_wheel.yml index fd2b739b..65362db2 100644 --- a/.github/workflows/build_test_wheel.yml +++ b/.github/workflows/build_test_wheel.yml @@ -7,8 +7,8 @@ permissions: contents: read jobs: - build_wheels: - name: Build wheels on self-hosted manylinux_2_28 for TestPyPi + build_wheels_linux_x64: + name: Build wheels on self-hosted manylinux_2_28_x64 for TestPyPi runs-on: linux_x64 steps: @@ -54,3 +54,51 @@ jobs: # Run a simple smoke test python -c "import zvec; print('Import OK:', zvec.__version__)" shell: bash + + build_wheels_linux_arm64: + name: Build wheels on self-hosted manylinux_2_28_arm64 for TestPyPi + runs-on: linux_arm64 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set up Python (for cibuildwheel controller) + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install cibuildwheel + run: | + pip install --upgrade pip + pip install cibuildwheel==2.17.0 + - name: Build wheels using cibuildwheel + run: | + python -m cibuildwheel --output-dir wheelhouse + # Save list of built wheels for publishing + ls wheelhouse/*.whl | tee $GITHUB_STEP_SUMMARY + echo "wheels=$(ls wheelhouse/*.whl | tr '\n' ' ')" >> $GITHUB_ENV + - name: Publish to TestPyPI + if: success() && github.event_name == 'workflow_dispatch' + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} + TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/ + run: | + pip install twine + twine upload --skip-existing --verbose wheelhouse/*.whl + - name: (Optional) Install and test from TestPyPI + if: success() && github.event_name == 'workflow_dispatch' + run: | + # Create a clean venv + python -m venv test_env + source test_env/bin/activate + pip install --upgrade pip + # Install from TestPyPI (must allow pre-releases if version has dev/alpha) + pip install numpy + pip install --index-url https://test.pypi.org/simple/ zvec + # Run a simple smoke test + python -c "import zvec; print('Import OK:', zvec.__version__)" + shell: bash diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index db69d871..b56af990 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -7,10 +7,60 @@ permissions: contents: read jobs: - build_wheels: - name: Build wheels on self-hosted manylinux_2_28 + build_wheels_linux_x64: + name: Build wheels on self-hosted manylinux_2_28_x64 runs-on: linux_x64 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set up Python (for cibuildwheel controller) + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install cibuildwheel + run: | + pip install --upgrade pip + pip install cibuildwheel==2.17.0 + + - name: Build wheels using cibuildwheel + run: | + python -m cibuildwheel --output-dir wheelhouse + # Save list of built wheels for publishing + ls wheelhouse/*.whl | tee $GITHUB_STEP_SUMMARY + echo "wheels=$(ls wheelhouse/*.whl | tr '\n' ' ')" >> $GITHUB_ENV + + - name: Publish to PyPI + if: success() && github.event_name == 'workflow_dispatch' + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + TWINE_REPOSITORY: pypi + run: | + pip install twine + twine upload --skip-existing --verbose wheelhouse/*.whl + + - name: (Optional) Install and test from PyPI + if: success() && github.event_name == 'workflow_dispatch' + run: | + # Create a clean venv + python -m venv test_env + source test_env/bin/activate + pip install --upgrade pip + # Install from PyPI + pip install zvec + # Run a simple smoke test + python -c "import zvec; print('Import OK:', zvec.__version__)" + shell: bash + + build_wheels_linux_arm64: + name: Build wheels on self-hosted manylinux_2_28_arm64 + runs-on: linux_arm64 + steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/docker/Dockerfile.ubuntu18.10-glibc228 b/.github/workflows/docker/Dockerfile.linux_x64_glibc228 similarity index 100% rename from .github/workflows/docker/Dockerfile.ubuntu18.10-glibc228 rename to .github/workflows/docker/Dockerfile.linux_x64_glibc228 diff --git a/.github/workflows/linux_arm64_docker_ci.yml b/.github/workflows/linux_arm64_docker_ci.yml new file mode 100644 index 00000000..96a0f32d --- /dev/null +++ b/.github/workflows/linux_arm64_docker_ci.yml @@ -0,0 +1,146 @@ +name: Zvec LinuxARM64 CI + +on: + push: + branches: [ "main" ] + paths-ignore: + - '**.md' + merge_group: + pull_request: + branches: [ "main" ] + paths-ignore: + - '**.md' + workflow_dispatch: + +concurrency: + group: pr-${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + build: + name: Zvec LinuxARM64 CI + runs-on: linux_arm64 + + strategy: + matrix: + python-version: ['3.10'] + fail-fast: false + + container: + image: quay.io/pypa/manylinux_2_28_aarch64:2024-03-10-4935fcc + options: --user root + + steps: + - name: Set up Python path for manylinux + run: | + case "${{ matrix.python-version }}" in + "3.10") PY_PATH="/opt/python/cp310-cp310" ;; + "3.11") PY_PATH="/opt/python/cp311-cp311" ;; + "3.12") PY_PATH="/opt/python/cp312-cp312" ;; + *) echo "Unsupported Python version: ${{ matrix.python-version }}"; exit 1 ;; + esac + echo "PYTHON_BIN=$PY_PATH/bin/python" >> $GITHUB_ENV + echo "PIP_BIN=$PY_PATH/bin/pip" >> $GITHUB_ENV + echo "CLANG_FORMATTER_BIN=$PY_PATH/bin/clang-format" >> $GITHUB_ENV + $PY_PATH/bin/python --version + shell: bash + + - name: Prepare clean build directory + run: | + export CLEAN_WORKSPACE="/tmp/zvec" + mkdir -p "$CLEAN_WORKSPACE" + cd "$CLEAN_WORKSPACE" + + git config --global --add safe.directory "$CLEAN_WORKSPACE" + git clone --recursive "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" . + + if [ -n "${{ github.event.number }}" ]; then + git fetch origin "pull/${{ github.event.number }}/head" + git checkout FETCH_HEAD + else + git checkout "${{ github.sha }}" + fi + + echo "CLEAN_WORKSPACE=$CLEAN_WORKSPACE" >> $GITHUB_ENV + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash + + - name: Install Ruff + run: | + ${{ env.PIP_BIN }} install --upgrade pip ruff + shell: bash + + - name: Run Ruff Linter + run: | + cd "$CLEAN_WORKSPACE" + ${{ env.PYTHON_BIN }} -m ruff check . + shell: bash + + - name: Run Ruff Formatter Check + run: | + cd "$CLEAN_WORKSPACE" + ${{ env.PYTHON_BIN }} -m ruff format --check . + shell: bash + + - name: Run clang-format Check + run: | + ${{ env.PIP_BIN }} install clang-format==18.1.8 + cd "$CLEAN_WORKSPACE" + + + CPP_FILES=$(find . -type f \( -name "*.cpp" -o -name "*.h" -o -name "*.hpp" -o -name "*.cc" -o -name "*.cxx" \) \ + ! -path "./build/*" \ + ! -path "./tests/*" \ + ! -path "./scripts/*" \ + ! -path "./python/*" \ + ! -path "./thirdparty/*" \ + ! -path "./.git/*") + + if [ -z "$CPP_FILES" ]; then + echo "No C++ files found to check." + exit 0 + fi + + ${{ env.CLANG_FORMATTER_BIN }} --dry-run --Werror $CPP_FILES + shell: bash + + - name: Install Python dependencies and build package + run: | + cd "$CLEAN_WORKSPACE" + NPROC=$(nproc 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2) + + ${{ env.PIP_BIN }} install cmake ninja + + CMAKE_GENERATOR="Unix Makefiles" \ + CMAKE_BUILD_PARALLEL_LEVEL="$NPROC" \ + ${{ env.PIP_BIN }} install -v . --config-settings='cmake.define.BUILD_TOOLS="ON"' + shell: bash + + - name: Install test dependencies + run: | + ${{ env.PIP_BIN }} install pytest pytest-cov + shell: bash + + - name: Run Python Tests with Coverage + run: | + cd "$CLEAN_WORKSPACE" + ${{ env.PYTHON_BIN }} -m pytest python/tests/ --cov=zvec --cov-report=xml --no-cov-on-fail + shell: bash + + - name: Run Cpp Tests + run: | + ${{ env.PIP_BIN }} install pybind11==3.0 + cd "$CLEAN_WORKSPACE/build" + make unittest -j$(nproc) + shell: bash + + - name: Run Cpp Examples + run: | + cd "$CLEAN_WORKSPACE/examples/c++" + mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=Release + make -j $(nproc) && ./db-example && ./core-example && ./ailego-example + shell: bash diff --git a/.github/workflows/linux_x64_docker_ci.yml b/.github/workflows/linux_x64_docker_ci.yml index c839d963..2edd0995 100644 --- a/.github/workflows/linux_x64_docker_ci.yml +++ b/.github/workflows/linux_x64_docker_ci.yml @@ -30,29 +30,22 @@ jobs: fail-fast: false container: - image: zvec-registry.cn-hongkong.cr.aliyuncs.com/zvec/zvec:0.0.2 + image: quay.io/pypa/manylinux_2_28_x86_64:2024-03-10-4935fcc options: --user root steps: - - name: Activate Conda environment + - name: Set up Python path for manylinux run: | - # Map version to env name - if [[ "${{ matrix.python-version }}" == "3.10" ]]; then - ENV_NAME="py310" - elif [[ "${{ matrix.python-version }}" == "3.11" ]]; then - ENV_NAME="py311" - elif [[ "${{ matrix.python-version }}" == "3.12" ]]; then - ENV_NAME="py312" - else - echo "Unsupported Python version" - exit 1 - fi - echo "CONDA_ENV_NAME=$ENV_NAME" >> $GITHUB_ENV - source /opt/miniforge3/bin/activate $ENV_NAME - - pip install --upgrade pip pytest pytest-cov ruff - - python --version + case "${{ matrix.python-version }}" in + "3.10") PY_PATH="/opt/python/cp310-cp310" ;; + "3.11") PY_PATH="/opt/python/cp311-cp311" ;; + "3.12") PY_PATH="/opt/python/cp312-cp312" ;; + *) echo "Unsupported Python version: ${{ matrix.python-version }}"; exit 1 ;; + esac + echo "PYTHON_BIN=$PY_PATH/bin/python" >> $GITHUB_ENV + echo "PIP_BIN=$PY_PATH/bin/pip" >> $GITHUB_ENV + echo "CLANG_FORMATTER_BIN=$PY_PATH/bin/clang-format" >> $GITHUB_ENV + $PY_PATH/bin/python --version shell: bash - name: Prepare clean build directory @@ -76,25 +69,29 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash + - name: Install Ruff + run: | + ${{ env.PIP_BIN }} install --upgrade pip ruff + shell: bash + - name: Run Ruff Linter run: | - source /opt/miniforge3/bin/activate ${{ env.CONDA_ENV_NAME }} cd "$CLEAN_WORKSPACE" - ruff check . + ${{ env.PYTHON_BIN }} -m ruff check . shell: bash - - name: Run Ruff Formatter Check (ensure code is formatted) + - name: Run Ruff Formatter Check run: | - source /opt/miniforge3/bin/activate ${{ env.CONDA_ENV_NAME }} cd "$CLEAN_WORKSPACE" - ruff format --check . + ${{ env.PYTHON_BIN }} -m ruff format --check . shell: bash - name: Run clang-format Check run: | - source /opt/miniforge3/bin/activate ${{ env.CONDA_ENV_NAME }} + ${{ env.PIP_BIN }} install clang-format==18.1.8 cd "$CLEAN_WORKSPACE" + CPP_FILES=$(find . -type f \( -name "*.cpp" -o -name "*.h" -o -name "*.hpp" -o -name "*.cc" -o -name "*.cxx" \) \ ! -path "./build/*" \ ! -path "./tests/*" \ @@ -102,38 +99,41 @@ jobs: ! -path "./python/*" \ ! -path "./thirdparty/*" \ ! -path "./.git/*") - + if [ -z "$CPP_FILES" ]; then echo "No C++ files found to check." exit 0 fi - - clang-format-18 --dry-run --Werror $CPP_FILES + + ${{ env.CLANG_FORMATTER_BIN }} --dry-run --Werror $CPP_FILES shell: bash - name: Install Python dependencies and build package run: | - source /opt/miniforge3/bin/activate ${{ env.CONDA_ENV_NAME }} cd "$CLEAN_WORKSPACE" - - NPROC=$(nproc 2>/dev/null || echo $(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2)) - echo "ParallelGroup: Using $NPROC parallel jobs for build" - + NPROC=$(nproc 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2) + + ${{ env.PIP_BIN }} install cmake ninja + CMAKE_GENERATOR="Unix Makefiles" \ CMAKE_BUILD_PARALLEL_LEVEL="$NPROC" \ - pip install -v . \ - --config-settings='cmake.define.BUILD_TOOLS="ON"' + ${{ env.PIP_BIN }} install -v . --config-settings='cmake.define.BUILD_TOOLS="ON"' + shell: bash + + - name: Install test dependencies + run: | + ${{ env.PIP_BIN }} install pytest pytest-cov shell: bash - name: Run Python Tests with Coverage run: | - source /opt/miniforge3/bin/activate ${{ env.CONDA_ENV_NAME }} cd "$CLEAN_WORKSPACE" - python -m pytest python/tests/ --cov=zvec --cov-report=xml --no-cov-on-fail + ${{ env.PYTHON_BIN }} -m pytest python/tests/ --cov=zvec --cov-report=xml --no-cov-on-fail shell: bash - - name: Run Cpp Tests with Coverage + - name: Run Cpp Tests run: | + ${{ env.PIP_BIN }} install pybind11==3.0 cd "$CLEAN_WORKSPACE/build" make unittest -j$(nproc) shell: bash @@ -142,5 +142,5 @@ jobs: run: | cd "$CLEAN_WORKSPACE/examples/c++" mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=Release - make -j 16 && ./db-example && ./core-example && ./ailego-example + make -j $(nproc) && ./db-example && ./core-example && ./ailego-example shell: bash diff --git a/README.md b/README.md index 3a1f3b9b..4a9f9611 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,11 @@

Linux x64 CI + Linux ARM64 CI macOS ARM64 CI +
Code Coverage PyPI Release -
Python Versions License

@@ -46,7 +47,7 @@ pip install zvec - Python 3.10 - 3.12 - **Supported platforms**: - - Linux (x86_64) + - Linux (x86_64/ARM64) - macOS (ARM64) If you prefer to build Zvec from source, please check the [Building from Source](https://zvec.org/en/docs/build/) guide. diff --git a/pyproject.toml b/pyproject.toml index 7bf99fa3..dee6728d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,7 +89,7 @@ minimum-version = "0.11" metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" # CMake configuration -cmake.version = ">=3.15,<4.0" +cmake.version = ">=3.26,<4.0" ninja.version = ">=1.11" cmake.build-type = "Release" install.strip = true # Strip symbols in release builds to reduce wheel size @@ -161,9 +161,10 @@ test-requires = ["pytest", "numpy"] build-verbosity = 1 [tool.cibuildwheel.linux] -archs = ["x86_64"] +archs = ["auto"] test-command = "cd {project} && pytest python/tests -v --tb=short" manylinux-x86_64-image = "manylinux_2_28" +manylinux-aarch64-image = "manylinux_2_28" skip = "*musllinux*" [tool.cibuildwheel.macos]