diff --git a/.ci/install_cargo.sh b/.ci/install_cargo.sh deleted file mode 100755 index 8635836ef6..0000000000 --- a/.ci/install_cargo.sh +++ /dev/null @@ -1,10 +0,0 @@ -#! /bin/sh -curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain=stable -rustup show -export PATH="$HOME/.cargo/bin:$PATH" -rustc -V -rustup target add aarch64-apple-darwin - -# update crates.io index without updating Cargo.lock -export CARGO_NET_GIT_FETCH_WITH_CLI=true -cargo update --dry-run diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index c7b13ec599..6edeb79b1c 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -27,7 +27,7 @@ jobs: - build: macos-x86_64 os: macos-latest arch: x86_64 - macos_target: 'MACOSX_DEPLOYMENT_TARGET=10.11 CARGO_BUILD_TARGET=x86_64-apple-darwin' + macos_target: 'MACOSX_DEPLOYMENT_TARGET=11.0 CARGO_BUILD_TARGET=x86_64-apple-darwin' - build: macos-arm64 os: macos-latest arch: arm64 diff --git a/.github/workflows/dev_envs.yml b/.github/workflows/dev_envs.yml index c993e0a520..33b5069d00 100644 --- a/.github/workflows/dev_envs.yml +++ b/.github/workflows/dev_envs.yml @@ -57,7 +57,7 @@ jobs: - name: install dependencies shell: bash -l {0} - run: mamba install tox-conda rust git compilers pandoc + run: mamba install 'tox>=3.27,<4' tox-conda rust git compilers pandoc - name: run tests for 3.9 shell: bash -l {0} diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index fa16b58900..300169efa6 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -63,17 +63,6 @@ jobs: toolchain: ${{ matrix.rust }} override: true - - name: Set up Python 3.8 - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - - name: Install dependencies - continue-on-error: ${{ matrix.continue }} - run: | - python -m pip install --upgrade pip - python -m pip install -e . - - name: Run tests uses: actions-rs/cargo@v1 with: @@ -90,16 +79,6 @@ jobs: toolchain: stable override: true - - name: Set up Python 3.8 - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -e . - - uses: actions-rs/install@v0.1 with: crate: cargo-all-features diff --git a/doc/developer.md b/doc/developer.md index d3f83f7924..2368611e7a 100644 --- a/doc/developer.md +++ b/doc/developer.md @@ -25,7 +25,7 @@ and the [`conda-forge`](https://conda-forge.org/) channel by default). Once `mamba` is installed, run ``` -mamba create -n sourmash_dev tox-conda rust git compilers pandoc +mamba create -n sourmash_dev 'tox>=3.27,<4' tox-conda rust git compilers pandoc ``` to create an environment called `sourmash_dev` containing the programs needed for development. diff --git a/flake.lock b/flake.lock index f9fc0a31af..c39e6a9732 100644 --- a/flake.lock +++ b/flake.lock @@ -1,32 +1,12 @@ { "nodes": { - "naersk": { - "inputs": { - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1688534083, - "narHash": "sha256-/bI5vsioXscQTsx+Hk9X5HfweeNZz/6kVKsbdqfwW7g=", - "owner": "nix-community", - "repo": "naersk", - "rev": "abca1fb7a6cfdd355231fc220c3d0302dbb4369a", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "naersk", - "type": "github" - } - }, "nixpkgs": { "locked": { - "lastModified": 1689449371, - "narHash": "sha256-sK3Oi8uEFrFPL83wKPV6w0+96NrmwqIpw9YFffMifVg=", + "lastModified": 1692404241, + "narHash": "sha256-TRZlFHtrQI6Kh8RFqnjBF2uNNi/c66ldB4WuIcrwMzg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "29bcead8405cfe4c00085843eb372cc43837bb9d", + "rev": "2f9286912cb215969ece465147badf6d07aa43fe", "type": "github" }, "original": { @@ -38,7 +18,6 @@ }, "root": { "inputs": { - "naersk": "naersk", "nixpkgs": "nixpkgs", "rust-overlay": "rust-overlay", "utils": "utils" @@ -54,11 +33,11 @@ ] }, "locked": { - "lastModified": 1689475081, - "narHash": "sha256-lAyG+KKKjOAG1YxYnji1g1pV39WxzQQBHI3ZwoRzweM=", + "lastModified": 1692410823, + "narHash": "sha256-YM1QCenpghNqgleUmoCJUArTuMEBqScyQuhepA6JZaI=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "6e28f20574595b01e14f2bbb57d62b84393fdcc1", + "rev": "598b2f04ed252eb5808b108d7a10084c0c548753", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 21bee98f82..43ad3b8d78 100644 --- a/flake.nix +++ b/flake.nix @@ -11,16 +11,9 @@ flake-utils.follows = "utils"; }; }; - - naersk = { - url = "github:nix-community/naersk"; - inputs = { - nixpkgs.follows = "nixpkgs"; - }; - }; }; - outputs = { self, nixpkgs, naersk, rust-overlay, utils }: + outputs = { self, nixpkgs, rust-overlay, utils }: utils.lib.eachDefaultSystem (system: let overlays = [ (import rust-overlay) ]; @@ -36,10 +29,6 @@ cargo = rustVersion; rustc = rustVersion; }; - naersk-lib = naersk.lib."${system}".override { - cargo = rustVersion; - rustc = rustVersion; - }; python = pkgs.python311Packages; @@ -49,15 +38,17 @@ { packages = { - lib = naersk-lib.buildPackage { - pname = "libsourmash"; - root = ./.; + lib = rustPlatform.buildRustPackage { + name = "libsourmash"; + src = lib.cleanSource ./.; copyLibs = true; + cargoLock.lockFile = ./Cargo.lock; + nativeBuildInputs = with rustPlatform; [ bindgenHook ]; }; sourmash = python.buildPythonPackage rec { pname = "sourmash"; - version = "4.8.3"; + version = "4.8.4-dev"; format = "pyproject"; src = ./.; @@ -66,7 +57,7 @@ lockFile = ./Cargo.lock; }; - nativeBuildInputs = with rustPlatform; [ cargoSetupHook maturinBuildHook ]; + nativeBuildInputs = with rustPlatform; [ cargoSetupHook maturinBuildHook bindgenHook ]; buildInputs = lib.optionals stdenv.isDarwin [ libiconv ]; propagatedBuildInputs = with python; [ cffi deprecation cachetools bitstring numpy scipy matplotlib screed ]; @@ -93,9 +84,7 @@ defaultPackage = self.packages.${system}.sourmash; devShell = mkShell { - nativeBuildInputs = [ - clang_13 - ]; + nativeBuildInputs = with rustPlatform; [ bindgenHook ]; buildInputs = [ rustVersion @@ -107,7 +96,6 @@ (python311.withPackages (ps: with ps; [ virtualenv tox cffi ])) (python310.withPackages (ps: with ps; [ virtualenv ])) (python39.withPackages (ps: with ps; [ virtualenv ])) - (python38.withPackages (ps: with ps; [ virtualenv ])) rust-cbindgen maturin @@ -122,15 +110,11 @@ cargo-limit cargo-outdated cargo-udeps + cargo-deny nixpkgs-fmt - - llvmPackages_13.libclang - llvmPackages_13.libcxxClang ]; - BINDGEN_EXTRA_CLANG_ARGS = "-isystem ${llvmPackages_13.libclang.lib}/lib/clang/${lib.getVersion clang}/include"; - LIBCLANG_PATH = "${llvmPackages_13.libclang.lib}/lib"; - LD_LIBRARY_PATH = "${stdenv.cc.cc.lib}/lib64:$LD_LIBRARY_PATH"; + LD_LIBRARY_PATH = lib.makeLibraryPath [ pkgs.stdenv.cc.cc.lib ]; # workaround for https://github.com/NixOS/nixpkgs/blob/48dfc9fa97d762bce28cc8372a2dd3805d14c633/doc/languages-frameworks/python.section.md#python-setuppy-bdist_wheel-cannot-create-whl SOURCE_DATE_EPOCH = 315532800; # 1980 diff --git a/pyproject.toml b/pyproject.toml index 48d0152679..be7303d7fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -148,6 +148,12 @@ features = ["maturin"] locked = true module-name = "sourmash._lowlevel" +# macOS deployment target SDK version +[tool.maturin.target.x86_64-apple-darwin] +macos-deployment-target = "11.0" +[tool.maturin.target.aarch64-apple-darwin] +macos-deployment-target = "11.0" + [tool.isort] known_third_party = ["deprecation", "hypothesis", "mmh3", "numpy", "pytest", "screed", "sourmash_tst_utils"] multi_line_output = 3 @@ -159,10 +165,37 @@ known_first_party = ["sourmash"] [tool.cibuildwheel] build = "cp39-*" skip = "*-win32 *-manylinux_i686 *-musllinux_ppc64le *-musllinux_s390x" -before-build = "source .ci/install_cargo.sh" -environment = { PATH="$HOME/.cargo/bin:$PATH" } +before-all = [ + "curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain=stable", + "cargo update --dry-run", +] +macos.before-build = [ + "rustup target add aarch64-apple-darwin", +] build-verbosity = 3 +[tool.cibuildwheel.environment] +CARGO_REGISTRIES_CRATES_IO_PROTOCOL="sparse" +PATH="$HOME/.cargo/bin:$PATH" + +[tool.cibuildwheel.linux] +before-all = [ + "curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain=stable", + "cargo update --dry-run", + "if [ -f /etc/system-release ]; then yum -y install centos-release-scl; fi", + "if [ -f /etc/system-release ]; then yum -y install llvm-toolset-7.0; fi", +] +before-build = [ + "if [ -f /etc/system-release ]; then source scl_source enable llvm-toolset-7.0; fi", + "if [ -f /etc/system-release ]; then source scl_source enable devtoolset-10; fi", +] +[tool.cibuildwheel.linux.environment] +CARGO_REGISTRIES_CRATES_IO_PROTOCOL="sparse" +PATH="$HOME/.cargo/bin:$PATH" +LIBCLANG_PATH="/opt/rh/llvm-toolset-7.0/root/usr/lib64" +LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/rh/llvm-toolset-7.0/root/usr/lib64" +C_INCLUDE_PATH="/opt/rh/devtoolset-10/root/usr/lib/gcc/aarch64-redhat-linux/10/include:/opt/rh/devtoolset-10/root/usr/lib/gcc/x86_64-redhat-linux/10/include" + [tool.pytest.ini_options] addopts = "--doctest-glob='doc/*.md' -n4" norecursedirs = [ diff --git a/tox.ini b/tox.ini index 589d07b8cb..41734a6a3b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = +env_list = py310, py311, py39, @@ -11,20 +11,20 @@ envlist = hypothesis, khmer, khmer_master -minversion = 3.12 +min_version = 3.27 isolated_build = true skip_missing_interpreters = true [testenv] +description = run the tests with pytest under {basepython} package = wheel wheel_build_env = .pkg -description = run the tests with pytest under {basepython} -setenv = +set_env = PIP_DISABLE_VERSION_CHECK = 1 COVERAGE_FILE = {env:COVERAGE_FILE:{toxworkdir}/.coverage.{envname}} VIRTUALENV_NO_DOWNLOAD = 1 PIP_EXTRA_INDEX_URL = https://antocuni.github.io/pypy-wheels/manylinux2010 -passenv = +pass_env = TOXENV CURL_CA_BUNDLE http_proxy @@ -36,6 +36,8 @@ passenv = PIP_CACHE_DIR CI PYTHONDEVMODE + LIBCLANG_PATH + BINDGEN_EXTRA_CLANG_ARGS deps = pip >= 19.3.1 extras = @@ -103,8 +105,8 @@ description = invoke sphinx-build to build the HTML docs basepython = python3.10 extras = doc whitelist_externals = pandoc -passenv = HOME -changedir = {toxinidir} +pass_env = HOME +change_dir = {toxinidir} #commands = sphinx-build -d "{toxworkdir}/docs_doctree" doc "{toxworkdir}/docs_out" --color -W -bhtml {posargs} commands = sphinx-build -d "{toxworkdir}/docs_doctree" doc "{toxworkdir}/docs_out" --color -bhtml {posargs} python -c 'import pathlib; print("documentation available under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "docs_out" / "index.html"))' @@ -117,7 +119,7 @@ deps = twine >= 1.12.1 readme-renderer[md] >= 24.0 pip >= 19.1 skip_install = true -changedir = {toxinidir} +change_dir = {toxinidir} extras = commands = pip wheel -w {envtmpdir}/build --no-deps . twine check {envtmpdir}/build/* @@ -125,7 +127,7 @@ commands = pip wheel -w {envtmpdir}/build --no-deps . [testenv:mypy] description = run mypy checker basepython = python3.8 -passenv = {[testenv]passenv} +pass_env = {[testenv]pass_env} # without PROGRAMDATA cloning using git for Windows will fail with an `error setting certificate verify locations` error PROGRAMDATA deps = mypy @@ -134,7 +136,7 @@ commands = mypy src/sourmash [testenv:fix_lint] description = format the code base to adhere to our styles, and complain about what we cannot do automatically basepython = python3.8 -passenv = {[testenv]passenv} +pass_env = {[testenv]pass_env} # without PROGRAMDATA cloning using git for Windows will fail with an `error setting certificate verify locations` error PROGRAMDATA PRE_COMMIT_HOME @@ -151,9 +153,9 @@ deps = {[testenv]deps} coverage >= 5.0.1 diff_cover skip_install = True -passenv = {[testenv]passenv} +pass_env = {[testenv]pass_env} DIFF_AGAINST -setenv = COVERAGE_FILE={toxworkdir}/.coverage +set_env = COVERAGE_FILE={toxworkdir}/.coverage commands = coverage combine coverage report -i -m coverage xml -i -o {toxworkdir}/coverage.xml