Build System Rework

.github/workflows/build-variants.yml

@@ -0,0 +1,115 @@
+name: Build Variants
+
+on:
+  push:
+
+jobs:
+  # This job should mimic conda-feedstock build
+  libtiledbvcf:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Mamba environment
+        uses: mamba-org/setup-micromamba@v2
+        with:
+          environment-name: ci
+          create-args: -c tiledb tiledb=2.26 spdlog fmt cli11 htslib catch2=2.13.8
+          init-shell: >-
+            bash
+      - name: Configure CMake
+        shell: micromamba-shell {0}
+        run: |
+          cmake -S libtiledbvcf -B build \
+            -D CMAKE_BUILD_TYPE=Release \
+            -D BUILD_SHARED_LIBS=ON \
+            -D CMAKE_INSTALL_PREFIX=install \
+            -D TILEDBVCF_ENABLE_TESTS=OFF \
+            -D TILEDBVCF_FORCE_EXTERNAL_DEPENDENCIES=ON \
+            -D TILEDBVCF_INSTALL_TILEDB=OFF \
+            -D TILEDBVCF_SET_INSTALL_SUBPATH=OFF \
+            -D CMAKE_PREFIX_PATH=$CONDA_PREFIX
+      - name: Build using CMake
+        shell: micromamba-shell {0}
+        run: |
+          cmake --build build --config Release
+      - name: Install using CMake
+        shell: micromamba-shell {0}
+        run: |
+          cmake --install build --config Release
+
+      - name: Upload install folder
+        uses: actions/upload-artifact@v4
+        with:
+          name: libtiledbvcf
+          path: install/*
+
+      # Tests
+      - name: Run unit tests 
+        shell: micromamba-shell {0}
+        run: |
+          ctest --test-dir build
+      - name: Install cli test dependencies
+        run: |
+          sudo apt-get install -y bcftools
+      - name: Run CLI tests
+        shell: micromamba-shell {0}
+        run: |
+          libtiledbvcf/test/run-cli-tests.sh build libtiledbvcf/test/inputs
+
+
+  # This job should mimic conda-feedstock build
+  python-bindings-external-libtiledbvcf:
+    needs: libtiledbvcf
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Mamba environment
+        uses: mamba-org/setup-micromamba@v2
+        with:
+          environment-name: ci
+          create-args: -c tiledb htslib tiledb=2.26 tiledb-py python=3.11 pip pytest numpy pandas pyarrow
+          init-shell: >-
+            bash
+      - name: Download libtiledbvcf artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: libtiledbvcf
+          path: install
+      - name: Build Python bindings 
+        shell: micromamba-shell {0}
+        env:
+          tiledbvcf_DIR: "${{ github.workspace }}/install"
+        run: |
+          pip install -v --no-deps . \
+            --config-settings=cmake.define.TILEDBVCF_ONLY_PYTHON_BINDINGS=ON \
+            --config-settings=cmake.define.TILEDBVCF_SET_RPATH=OFF
+
+      - name: Install test dependencies
+        run: |
+          sudo apt-get install -y bcftools
+
+      - name: Run tests
+        shell: micromamba-shell {0}
+        run: |
+          export LD_LIBRARY_PATH="$CONDA_PREFIX/lib"
+          cp ${{ github.workspace }}/install/lib/libtiledbvcf* $LD_LIBRARY_PATH
+          ldd $CONDA_PREFIX/lib/python3.11/site-packages/tiledbvcf/libtiledbvcf.cpython-311-x86_64-linux-gnu.so
+          pytest apis/python
+
+
+  # This job should mimic wheel build
+  python-bindings-wheel:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install build dependencies
+        run: |
+          sudo apt-get install -y ninja-build bcftools
+      - name: Build and install using PiP
+        run: |
+          pip install -v .[test]
+
+      - name: Run tests
+        run: |
+          pytest apis/python
+

.github/workflows/build-wheels.yml

@@ -0,0 +1,168 @@
+name: Build and test wheels
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Version to use are release version
+        required: true
+        type: string
+      test_pypi:
+        description: Upload packages to test.pypi.org
+        required: false
+        type: boolean
+        default: false
+  push:
+    branches:
+      - "db/vcpkg"
+    tags:
+      - "*"
+
+env:
+  SETUPTOOLS_SCM_PRETEND_VERSION_FOR_TILEDB: ${{ inputs.version }}
+
+jobs:
+  build_wheels:
+    name: Wheel ${{ matrix.buildplat[0] }}-${{ matrix.buildplat[1] }}-${{ matrix.python }}
+    runs-on: ${{ matrix.buildplat[0] }}
+    strategy:
+      matrix:
+        buildplat:
+          - [ubuntu-22.04, manylinux_x86_64]
+          - [macos-13, macosx_x86_64]
+          - [macos-14, macosx_arm64]
+          # - [windows-2022, win_amd64]
+        python: ["cp39", "cp310", "cp311", "cp312"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: "Brew setup on macOS" # x-ref c8e49ba8f8b9ce
+        if: ${{ startsWith(matrix.buildplat[0], 'macos-') == true }}
+        run: |
+          set -e pipefail
+          brew update
+          brew install automake autoconf pkg-config bcftools libtool
+
+      - name: Build wheels
+        uses: pypa/[email protected]
+        env:
+          CIBW_BUILD_VERBOSITY: 3
+          CIBW_ENVIRONMENT_MACOS: >
+            CC=clang
+            CXX=clang++
+          CIBW_ARCHS: all
+          CIBW_PRERELEASE_PYTHONS: True
+          CIBW_BUILD: ${{ matrix.python }}-${{ matrix.buildplat[1] }}
+          CIBW_BEFORE_ALL_LINUX: yum install -y bzip2-devel xz-devel bcftools curl zip unzip tar
+          CIBW_TEST_COMMAND: pytest {package}/apis/python
+          CIBW_TEST_EXTRAS: test
+          MACOSX_DEPLOYMENT_TARGET: "13.0"
+        with:
+          output-dir: wheelhouse
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-${{ matrix.buildplat[0] }}-${{ matrix.buildplat[1] }}-${{ matrix.python }}
+          path: "./wheelhouse/*.whl"
+
+      - name: "Print log files (failed build only)"
+        run: |
+          cat /Users/runner/work/TileDB-VCF/TileDB-VCF/build/cp310-cp310-macosx_13_0_arm64/_deps/vcpkg-src/buildtrees/htscodecs/autoconf-arm64-osx-err.log
+        if: failure() # only run this job if the build step failed
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    outputs:
+      sdist_name: ${{ steps.get_sdist_name.outputs.sdist_name }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build sdist
+        run: pipx run build --sdist
+
+      - name: Get sdist package name
+        id: get_sdist_name
+        run: |
+          echo "sdist_name=$(ls dist/ | head -n 1)" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: sdist
+          path: dist/*.tar.gz
+
+  test_sdist:
+    name: Test source distribution package
+    needs: [build_sdist]
+    strategy:
+      matrix:
+        os:
+          - macos-13
+          - macos-14
+          # - windows-2022 Currently we do not know how to install bcftools on windows
+          - ubuntu-22.04
+        python: ["3.9", "3.10", "3.11", "3.12"]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Install bcftools MacOS
+        if: ${{ startsWith(matrix.os, 'macos-') == true }}
+        run: |
+          set -e pipefail
+          brew update
+          brew install automake pkg-config ninja llvm bzip2 xz autoconf zlib libdeflate bcftools libtool
+
+      - name: Install bcftools Ubuntu
+        if: ${{ startsWith(matrix.os, 'ubuntu-') == true }}
+        run: sudo apt install bcftools
+
+      - name: Download sdist artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: sdist
+          path: dist
+
+      - name: Install sdist artifact
+        run: pip install --verbose dist/${{ needs.build_sdist.outputs.sdist_name }}[test]
+
+      - uses: actions/checkout@v4
+
+      - name: Run tests
+        shell: bash
+        run: |
+          PROJECT_CWD=$PWD
+          cd ..
+          pytest -vv --showlocals $PROJECT_CWD/apis/python
+
+  upload_pypi:
+    needs: [build_wheels, test_sdist]
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    outputs:
+      package_version: ${{ steps.get_package_version.outputs.package_version }}
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: dist
+          merge-multiple: true
+
+      - id: get_package_version
+        run: |
+          echo "package_version=$(ls dist/ | head -n 1 | cut -d - -f 2)" >> "$GITHUB_OUTPUT"
+
+      - name: Upload to test-pypi
+        if: inputs.test_pypi
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+
+      # - name: Upload to pypi
+      #   if: startsWith(github.ref, 'refs/tags/')
+      #   uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/macos.yml

@@ -27,9 +27,12 @@ jobs:
         run: |
           cmake -S libtiledbvcf -B $(pwd)/libtiledbvcf/build \
             -D CMAKE_BUILD_TYPE=Debug \
+            -D BUILD_SHARED_LIBS=ON \
             -D CMAKE_INSTALL_PREFIX=$(pwd)/dist \
             -D OVERRIDE_INSTALL_PREFIX=OFF \
-            -D DOWNLOAD_TILEDB_PREBUILT=ON
+            -D DOWNLOAD_TILEDB_PREBUILT=ON \
+            -D TILEDBVCF_ENABLE_TESTS=ON \
+            -D TILEDBVCF_SET_INSTALL_SUBPATH=OFF
       - name: Build libtiledbvcf
         run: cmake --build $(pwd)/libtiledbvcf/build -j 2 --config Debug
       - name: Upload coredump as artifact if build failed
@@ -41,7 +44,7 @@ jobs:
           retention-days: 14
           if-no-files-found: error
       - name: Install libtiledbvcf
-        run: cmake --build $(pwd)/libtiledbvcf/build --config Debug --target install-libtiledbvcf
+        run: cmake --install $(pwd)/libtiledbvcf/build --config Debug
       - name: Upload libtiledbvcf as artifact
         uses: actions/upload-artifact@v4
         with:
@@ -57,8 +60,7 @@ jobs:
         run: brew install bcftools
       - name: Unit tests
         run: |
-          make -j 2 -C libtiledbvcf/build/libtiledbvcf tiledb_vcf_unit
-          ./libtiledbvcf/build/libtiledbvcf/test/tiledb_vcf_unit
+          ctest --test-dir libtiledbvcf/build
       - name: CLI tests (require bcftools)
         run: |
           # USAGE: run-cli-tests.sh <build-dir> <inputs-dir>
@@ -67,7 +69,7 @@ jobs:
     runs-on: macos-13
     needs: libtiledbvcf
     env:
-      DYLD_LIBRARY_PATH: "${{ github.workspace }}/dist/lib"
+      DYLD_LIBRARY_PATH: "${{ github.workspace }}/dist/tiledbvcf/lib"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -88,13 +90,10 @@ jobs:
           create-args: tiledb
       - name: Build tiledbvcf-py
         env:
-          LIBTILEDBVCF_PATH: "${{ github.workspace }}/dist"
+          tiledbvcf_DIR: "${{ github.workspace }}/dist"
           TileDB_DIR: "/Users/runner/micromamba/envs/ci/lib/cmake/TileDB"
         run: |
-          echo $DYLD_LIBRARY_PATH
-          echo $TileDB_DIR
-          cd apis/python
-          python -m pip install -v .[test]
+          python -m pip install -v --no-deps --config-settings=cmake.define.TILEDBVCF_ONLY_PYTHON_BINDINGS=ON .[test]
       - name: Confirm linking
         run: otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/libtiledbvcf.cpython-*-darwin.so
   java:
@@ -107,6 +106,11 @@ jobs:
         with:
           name: libtiledbvcf
           path: dist
+      - name: Debug
+        run: |
+          ls -R dist
+      - name: Build dependencies
+        run: brew install autoconf automake
       - name: Check format
         run: cd apis/java && ./gradlew checkFormat
       - name: Assemble
@@ -126,13 +130,11 @@ jobs:
         with:
           python-version: "3.11"
       - name: Build tiledbvcf-py
-        run: cd apis/python && python -m pip install -v .[test]
+        run: python -m pip install -v .[test]
       - name: Confirm linking
         run: |
           otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/libtiledbvcf.cpython-*-darwin.so
-          otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/libtiledbvcf.dylib
-          otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/libtiledb.dylib
-          otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/libhts.*.dylib
+          otool -L /Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/tiledbvcf/lib/libtiledb.dylib
       - name: Version
         run: python -c "import tiledbvcf; print(tiledbvcf.version)"
       - name: Install bcftools (for tests)

.github/workflows/nightly.yml

@@ -59,13 +59,13 @@ jobs:
       - name: Confirm linking (Linux)
         if: runner.os == 'Linux'
         run: |
-          ldd install/lib/libtiledb.so
-          ldd install/lib/libtiledbvcf.so
+          ldd install/tiledbvcf/lib/libtiledb.so*
+          ldd install/tiledbvcf/lib/libtiledbvcf.so*
       - name: Confirm linking (macOS)
         if: runner.os == 'macOS'
         run: |
-          otool -L install/lib/libtiledb.dylib
-          otool -L install/lib/libtiledbvcf.dylib
+          otool -L install/tiledbvcf/lib/libtiledb.dylib
+          otool -L install/tiledbvcf/lib/libtiledbvcf.dylib
       - name: Install bcftools (for tests) (Linux)
         if: runner.os == 'Linux'
         run: sudo apt-get install --yes bcftools