build: updates spun off mastiff PR (#2723)

(There are too many things happening in #2230, so I'll open a couple of PRs to simplify it) - Bring tox config as close as possible to `tox 4`, but keep using `tox 3` until [`tox-conda` is compatible with `tox 4`](tox-dev/tox-conda#156). Ended up adding an explicit `'tox>=3.27,<4'` to the conda dev env instructions to get the right version, and remove/replace deprecated options from `tox.ini` - Bump `MACOSX_DEPLOYMENT_TARGET` to `11.0` consistently for maturin and CI. - Remove Python installation for Rust tests in CI (not used) - Clean up `flake.nix` and avoid `naersk`, prefer regular `nixpkgs` tooling now that NixOS/nixpkgs#230596 is merged and we can keep them in sync - Move `.ci/install_cargo.sh` into `cibuildwheel` configuration, set up extra tools needed for `bindgen`/`cbindgen` (libclang and env vars)
sourmash-bio · Aug 19, 2023 · 0dd0e3b · 0dd0e3b
1 parent 9f6664e
commit 0dd0e3b
Show file tree

Hide file tree

Showing 9 changed files with 69 additions and 102 deletions.
diff --git a/.ci/install_cargo.sh b/.ci/install_cargo.sh
diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml
@@ -27,7 +27,7 @@ jobs:
           - build: macos-x86_64
             os: macos-latest
             arch: x86_64
-            macos_target: 'MACOSX_DEPLOYMENT_TARGET=10.11 CARGO_BUILD_TARGET=x86_64-apple-darwin'
+            macos_target: 'MACOSX_DEPLOYMENT_TARGET=11.0 CARGO_BUILD_TARGET=x86_64-apple-darwin'
           - build: macos-arm64
             os: macos-latest
             arch: arm64

diff --git a/.github/workflows/dev_envs.yml b/.github/workflows/dev_envs.yml
@@ -57,7 +57,7 @@ jobs:
 
     - name: install dependencies
       shell: bash -l {0}
-      run: mamba install tox-conda rust git compilers pandoc
+      run: mamba install 'tox>=3.27,<4' tox-conda rust git compilers pandoc
 
     - name: run tests for 3.9
       shell: bash -l {0}

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -63,17 +63,6 @@ jobs:
           toolchain: ${{ matrix.rust }}
           override: true
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.8"
-
-      - name: Install dependencies
-        continue-on-error: ${{ matrix.continue }}
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install -e .
-
       - name: Run tests
         uses: actions-rs/cargo@v1
         with:
@@ -90,16 +79,6 @@ jobs:
           toolchain: stable
           override: true
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.8"
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install -e .
-
       - uses: actions-rs/[email protected]
         with:
           crate: cargo-all-features

diff --git a/doc/developer.md b/doc/developer.md
@@ -25,7 +25,7 @@ and the [`conda-forge`](https://conda-forge.org/) channel by default).
 
 Once `mamba` is installed, run
 ```
-mamba create -n sourmash_dev tox-conda rust git compilers pandoc
+mamba create -n sourmash_dev 'tox>=3.27,<4' tox-conda rust git compilers pandoc
 ```
 to create an environment called `sourmash_dev` containing the programs needed
 for development.

diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -11,16 +11,9 @@
         flake-utils.follows = "utils";
       };
     };
-
-    naersk = {
-      url = "github:nix-community/naersk";
-      inputs = {
-        nixpkgs.follows = "nixpkgs";
-      };
-    };
   };
 
-  outputs = { self, nixpkgs, naersk, rust-overlay, utils }:
+  outputs = { self, nixpkgs, rust-overlay, utils }:
     utils.lib.eachDefaultSystem (system:
       let
         overlays = [ (import rust-overlay) ];
@@ -36,10 +29,6 @@
           cargo = rustVersion;
           rustc = rustVersion;
         };
-        naersk-lib = naersk.lib."${system}".override {
-          cargo = rustVersion;
-          rustc = rustVersion;
-        };
 
         python = pkgs.python311Packages;
 
@@ -49,15 +38,17 @@
       {
         packages = {
 
-          lib = naersk-lib.buildPackage {
-            pname = "libsourmash";
-            root = ./.;
+          lib = rustPlatform.buildRustPackage {
+            name = "libsourmash";
+            src = lib.cleanSource ./.;
             copyLibs = true;
+            cargoLock.lockFile = ./Cargo.lock;
+            nativeBuildInputs = with rustPlatform; [ bindgenHook ];
           };
 
           sourmash = python.buildPythonPackage rec {
             pname = "sourmash";
-            version = "4.8.3";
+            version = "4.8.4-dev";
             format = "pyproject";
 
             src = ./.;
@@ -66,7 +57,7 @@
               lockFile = ./Cargo.lock;
             };
 
-            nativeBuildInputs = with rustPlatform; [ cargoSetupHook maturinBuildHook ];
+            nativeBuildInputs = with rustPlatform; [ cargoSetupHook maturinBuildHook bindgenHook ];
 
             buildInputs = lib.optionals stdenv.isDarwin [ libiconv ];
             propagatedBuildInputs = with python; [ cffi deprecation cachetools bitstring numpy scipy matplotlib screed ];
@@ -93,9 +84,7 @@
         defaultPackage = self.packages.${system}.sourmash;
 
         devShell = mkShell {
-          nativeBuildInputs = [
-            clang_13
-          ];
+          nativeBuildInputs = with rustPlatform; [ bindgenHook ];
 
           buildInputs = [
             rustVersion
@@ -107,7 +96,6 @@
             (python311.withPackages (ps: with ps; [ virtualenv tox cffi ]))
             (python310.withPackages (ps: with ps; [ virtualenv ]))
             (python39.withPackages (ps: with ps; [ virtualenv ]))
-            (python38.withPackages (ps: with ps; [ virtualenv ]))
 
             rust-cbindgen
             maturin
@@ -122,15 +110,11 @@
             cargo-limit
             cargo-outdated
             cargo-udeps
+            cargo-deny
             nixpkgs-fmt
-
-            llvmPackages_13.libclang
-            llvmPackages_13.libcxxClang
           ];
 
-          BINDGEN_EXTRA_CLANG_ARGS = "-isystem ${llvmPackages_13.libclang.lib}/lib/clang/${lib.getVersion clang}/include";
-          LIBCLANG_PATH = "${llvmPackages_13.libclang.lib}/lib";
-          LD_LIBRARY_PATH = "${stdenv.cc.cc.lib}/lib64:$LD_LIBRARY_PATH";
+          LD_LIBRARY_PATH = lib.makeLibraryPath [ pkgs.stdenv.cc.cc.lib ];
 
           # workaround for https://github.com/NixOS/nixpkgs/blob/48dfc9fa97d762bce28cc8372a2dd3805d14c633/doc/languages-frameworks/python.section.md#python-setuppy-bdist_wheel-cannot-create-whl
           SOURCE_DATE_EPOCH = 315532800; # 1980

diff --git a/pyproject.toml b/pyproject.toml
@@ -148,6 +148,12 @@ features = ["maturin"]
 locked = true
 module-name = "sourmash._lowlevel"
 
+# macOS deployment target SDK version
+[tool.maturin.target.x86_64-apple-darwin]
+macos-deployment-target = "11.0"
+[tool.maturin.target.aarch64-apple-darwin]
+macos-deployment-target = "11.0"
+
 [tool.isort]
 known_third_party = ["deprecation", "hypothesis", "mmh3", "numpy", "pytest", "screed", "sourmash_tst_utils"]
 multi_line_output = 3 
@@ -159,10 +165,37 @@ known_first_party = ["sourmash"]
 [tool.cibuildwheel]
 build = "cp39-*"
 skip = "*-win32 *-manylinux_i686 *-musllinux_ppc64le *-musllinux_s390x"
-before-build = "source .ci/install_cargo.sh"
-environment = { PATH="$HOME/.cargo/bin:$PATH" }
+before-all = [
+  "curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain=stable",
+  "cargo update --dry-run",
+]
+macos.before-build = [
+  "rustup target add aarch64-apple-darwin",
+]
 build-verbosity = 3
 
+[tool.cibuildwheel.environment]
+CARGO_REGISTRIES_CRATES_IO_PROTOCOL="sparse"
+PATH="$HOME/.cargo/bin:$PATH"
+
+[tool.cibuildwheel.linux]
+before-all = [
+  "curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain=stable",
+  "cargo update --dry-run",
+  "if [ -f /etc/system-release ]; then yum -y install centos-release-scl; fi",
+  "if [ -f /etc/system-release ]; then yum -y install llvm-toolset-7.0; fi",
+]
+before-build = [
+  "if [ -f /etc/system-release ]; then source scl_source enable llvm-toolset-7.0; fi",
+  "if [ -f /etc/system-release ]; then source scl_source enable devtoolset-10; fi",
+]
+[tool.cibuildwheel.linux.environment]
+CARGO_REGISTRIES_CRATES_IO_PROTOCOL="sparse"
+PATH="$HOME/.cargo/bin:$PATH"
+LIBCLANG_PATH="/opt/rh/llvm-toolset-7.0/root/usr/lib64"
+LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/rh/llvm-toolset-7.0/root/usr/lib64"
+C_INCLUDE_PATH="/opt/rh/devtoolset-10/root/usr/lib/gcc/aarch64-redhat-linux/10/include:/opt/rh/devtoolset-10/root/usr/lib/gcc/x86_64-redhat-linux/10/include"
+
 [tool.pytest.ini_options]
 addopts = "--doctest-glob='doc/*.md' -n4"
 norecursedirs = [

diff --git a/tox.ini b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist =
+env_list =
   py310,
   py311,
   py39,
@@ -11,20 +11,20 @@ envlist =
   hypothesis,
   khmer,
   khmer_master
-minversion = 3.12
+min_version = 3.27
 isolated_build = true
 skip_missing_interpreters = true
 
 [testenv]
+description = run the tests with pytest under {basepython}
 package = wheel
 wheel_build_env = .pkg
-description = run the tests with pytest under {basepython}
-setenv =
+set_env =
     PIP_DISABLE_VERSION_CHECK = 1
     COVERAGE_FILE = {env:COVERAGE_FILE:{toxworkdir}/.coverage.{envname}}
     VIRTUALENV_NO_DOWNLOAD = 1
     PIP_EXTRA_INDEX_URL = https://antocuni.github.io/pypy-wheels/manylinux2010
-passenv =
+pass_env =
     TOXENV
     CURL_CA_BUNDLE
     http_proxy
@@ -36,6 +36,8 @@ passenv =
     PIP_CACHE_DIR
     CI
     PYTHONDEVMODE
+    LIBCLANG_PATH
+    BINDGEN_EXTRA_CLANG_ARGS
 deps =
     pip >= 19.3.1
 extras =
@@ -103,8 +105,8 @@ description = invoke sphinx-build to build the HTML docs
 basepython = python3.10
 extras = doc
 whitelist_externals = pandoc
-passenv = HOME
-changedir = {toxinidir}
+pass_env = HOME
+change_dir = {toxinidir}
 #commands = sphinx-build -d "{toxworkdir}/docs_doctree" doc "{toxworkdir}/docs_out" --color -W -bhtml {posargs}
 commands = sphinx-build -d "{toxworkdir}/docs_doctree" doc "{toxworkdir}/docs_out" --color -bhtml {posargs}
            python -c 'import pathlib; print("documentation available under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))'
@@ -117,15 +119,15 @@ deps = twine >= 1.12.1
        readme-renderer[md] >= 24.0
        pip >= 19.1
 skip_install = true
-changedir = {toxinidir}
+change_dir = {toxinidir}
 extras =
 commands = pip wheel -w {envtmpdir}/build --no-deps .
            twine check {envtmpdir}/build/*
 
 [testenv:mypy]
 description = run mypy checker
 basepython = python3.8
-passenv = {[testenv]passenv}
+pass_env = {[testenv]pass_env}
           # without PROGRAMDATA cloning using git for Windows will fail with an `error setting certificate verify locations` error
           PROGRAMDATA
 deps  = mypy
@@ -134,7 +136,7 @@ commands = mypy src/sourmash
 [testenv:fix_lint]
 description = format the code base to adhere to our styles, and complain about what we cannot do automatically
 basepython = python3.8
-passenv = {[testenv]passenv}
+pass_env = {[testenv]pass_env}
           # without PROGRAMDATA cloning using git for Windows will fail with an `error setting certificate verify locations` error
           PROGRAMDATA
           PRE_COMMIT_HOME
@@ -151,9 +153,9 @@ deps = {[testenv]deps}
        coverage >= 5.0.1
        diff_cover
 skip_install = True
-passenv = {[testenv]passenv}
+pass_env = {[testenv]pass_env}
           DIFF_AGAINST
-setenv = COVERAGE_FILE={toxworkdir}/.coverage
+set_env = COVERAGE_FILE={toxworkdir}/.coverage
 commands = coverage combine
            coverage report -i -m
            coverage xml -i -o {toxworkdir}/coverage.xml