Merge branch 'extend_edges'

tskit-dev · Aug 18, 2023 · b3de586 · b3de586
2 parents f18f3d1 + ebb79ac
commit b3de586
Show file tree

Hide file tree

Showing 96 changed files with 14,566 additions and 2,679 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,4 +1,6 @@
 version: 2.1
+orbs:
+  codecov: codecov/[email protected]
 
 commands:
   setup:
@@ -12,7 +14,6 @@ commands:
             sudo pip install meson
             pip install numpy==1.18.5
             pip install --user -r python/requirements/CI-complete/requirements.txt
-            pip install twine --user
             # Remove tskit installed by msprime
             pip uninstall tskit -y
             echo 'export PATH=/home/circleci/.local/bin:$PATH' >> $BASH_ENV
@@ -48,12 +49,14 @@ commands:
             ninja -C build-gcc test
 
       - run:
-          name: Run gcov & upload coverage.
+          name: Run gcov
           command: |
             cd build-gcc
             find ../c/tskit/*.c -type f -printf "%f\n" | xargs -i gcov -pb libtskit.a.p/tskit_{}.gcno ../c/tskit/{}
-            cd ..
-            bash <(curl -s https://codecov.io/bash) -X gcov -X coveragepy -F c-tests
+
+      - codecov/upload:
+          flags: c-tests
+          token: CODECOV_TOKEN
 
       - run:
           name: Valgrind for C tests.
@@ -116,14 +119,17 @@ commands:
             python -m pytest -n2
 
       - run:
-          name: Upload LWT coverage
+          name: Generate coverage
           command: |
             # Make sure the C coverage reports aren't lying around
             rm -fR build-gcc
             ls -R
             cd python/lwt_interface
             gcov -pb -o ./build/temp.linux*/*.gcno example_c_module.c
-            bash <(curl -s https://codecov.io/bash) -X gcov -F lwt-tests
+
+      - codecov/upload:
+          flags: lwt-tests
+          token: CODECOV_TOKEN
 
       - run:
           name: Run Python tests
@@ -132,30 +138,17 @@ commands:
             python -m pytest --cov=tskit  --cov-report=xml --cov-branch -n2 tests/test_lowlevel.py tests/test_tables.py tests/test_file_format.py
 
       - run:
-          name: Upload Python coverage
+          name: Generate Python coverage
           command: |
             # Make sure the C coverage reports aren't lying around
             rm -fR build-gcc
+            rm -f python/lwt_interface/*.gcov
             cd python
             gcov -pb -o ./build/temp.linux*/*.gcno _tskitmodule.c
-            bash <(curl -s https://codecov.io/bash) -X gcov -F python-c-tests
 
-      - run:
-          name: Build Python package
-          command: |
-            cd python
-            rm -fR build
-            python setup.py sdist
-            python setup.py check
-            python -m twine check dist/*.tar.gz
-            python -m venv venv
-            source venv/bin/activate
-            pip install --upgrade setuptools pip wheel
-            python setup.py build_ext
-            python setup.py egg_info
-            python setup.py bdist_wheel
-            pip install dist/*.tar.gz
-            tskit --help
+      - codecov/upload:
+          flags: python-c-tests
+          token: CODECOV_TOKEN
 
 jobs:
   build:
@@ -176,6 +169,28 @@ jobs:
           paths:
             - "/home/circleci/.local"
       - compile_and_test
+      - run:
+          name: Install dependencies for wheel test
+          command: |
+            ARGO_NET_GIT_FETCH_WITH_CLI=1 pip install twine --user
+            # Remove tskit installed by msprime
+            pip uninstall tskit -y
+      - run:
+          name: Build Python package
+          command: |
+            cd python
+            rm -fR build
+            python setup.py sdist
+            python setup.py check
+            python -m twine check dist/*.tar.gz
+            python -m venv venv
+            source venv/bin/activate
+            pip install --upgrade setuptools pip wheel
+            python setup.py build_ext
+            python setup.py egg_info
+            python setup.py bdist_wheel
+            pip install dist/*.tar.gz
+            tskit --help
 
   build-32:
     docker:
@@ -191,6 +206,8 @@ jobs:
           key: tskit-32-{{ .Branch }}-v7
           paths:
             - "/home/circleci/.local"
+      # We need to install curl for the codecov upload.
+      - run: sudo apt-get install -y curl
       - compile_and_test
 
 workflows:

diff --git a/.github/workflows/docker/shared.env b/.github/workflows/docker/shared.env
@@ -1,4 +1,5 @@
 PYTHON_VERSIONS=(
+    cp311-cp311
     cp310-cp310
     cp39-cp39
     cp38-cp38

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -19,42 +19,47 @@ env:
 jobs:
   build-deploy-docs:
     name: Docs
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
       - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.6.0
+        uses: styfle/cancel-workflow-action@0.11.0
         with:
           access_token: ${{ github.token }}
 
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
-          python-version: 3.8
-
-      - uses: actions/cache@v2
-        id: cache
-        with:
-          path: venv
-          key: docs-venv-v2-${{ hashFiles(env.REQUIREMENTS) }}
-
-      - name: Build virtualenv
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: python -m venv venv
-
-      - name: Install deps
-        run: venv/bin/activate && pip install -r ${{env.REQUIREMENTS}}
+          python-version: "3.10"
 
       - name: Install apt deps
         if: env.APTGET
         run: sudo apt-get install -y ${{env.APTGET}}
 
+      - uses: actions/cache@v3
+        id: venv-cache
+        with:
+          path: venv
+          key: docs-venv-v4-${{ hashFiles(env.REQUIREMENTS) }}
+
+      - name: Create venv and install deps (one by one to avoid conflict errors)
+        if: steps.venv-cache.outputs.cache-hit != 'true'
+        run: |
+          python -m venv venv
+          . venv/bin/activate
+          pip install --upgrade pip wheel
+          pip install -r ${{env.REQUIREMENTS}}          
+
       - name: Build C module
         if: env.MAKE_TARGET
-        run: venv/bin/activate && make $MAKE_TARGET
+        run: |
+          . venv/bin/activate
+          make $MAKE_TARGET
 
       - name: Build Docs
-        run: venv/bin/activate && make -C docs
+        run: |
+          . venv/bin/activate
+          make -C docs
 
       - name: Trigger docs site rebuild
         if: github.ref == 'refs/heads/main'

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -8,10 +8,10 @@ on:
 jobs:
   pre-commit:
     name: Lint
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
       - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.10.0
+        uses: styfle/cancel-workflow-action@0.11.0
         with:
           access_token: ${{ github.token }}
       - uses: actions/checkout@v3
@@ -21,9 +21,8 @@ jobs:
       - name: install clang-format
         if: steps.clang_format.outputs.cache-hit != 'true'
         run: |
-          sudo apt-get remove -y clang-6.0 libclang-common-6.0-dev libclang1-6.0 libllvm6.0
-          sudo apt-get autoremove
-          sudo apt-get install clang-format clang-format-6.0
+          sudo pip install clang-format==6.0.1
+          sudo ln -s /usr/local/bin/clang-format /usr/local/bin/clang-format-6.0
       - uses: pre-commit/[email protected]
 
   benchmark:
@@ -35,11 +34,11 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
           cache: 'pip'
-          cache-dependency-path: python/requirements/development.txt
+          cache-dependency-path: python/requirements/benchmark.txt
       - name: Install deps
-        run: pip install -r python/requirements/development.txt
+        run: pip install -r python/requirements/benchmark.txt
       - name: Build module
         run: |
           cd python
@@ -61,7 +60,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: [ 3.7, 3.9, "3.10" ]
+        python: [ 3.7, 3.9, "3.11" ]
         os:  [ macos-latest, ubuntu-latest, windows-latest ]
     defaults:
       run:
@@ -84,7 +83,7 @@ jobs:
             /usr/share/miniconda/envs/anaconda-client-env
             ~/osx-conda
             ~/.profile
-          key: ${{ runner.os }}-${{ matrix.python}}-conda-v11-${{ hashFiles('python/requirements/CI-tests-conda/requirements.txt') }}-${{ hashFiles('python/requirements/CI-tests-pip/requirements.txt') }}
+          key: ${{ runner.os }}-${{ matrix.python}}-conda-v12-${{ hashFiles('python/requirements/CI-tests-conda/requirements.txt') }}-${{ hashFiles('python/requirements/CI-tests-pip/requirements.txt') }}
 
       - name: Install Conda
         uses: conda-incubator/setup-miniconda@v2
@@ -143,6 +142,11 @@ jobs:
           conda activate anaconda-client-env
           python setup.py build_ext --inplace
 
+      - name: Remove py311 incompatible tests (lack of numba support for 3.11, needed for lshmm)
+        if: matrix.python == '3.11'
+        run: |
+          rm python/tests/test_genotype_matching_*
+
       - name: Run tests
         working-directory: python
         run: |
@@ -152,11 +156,11 @@ jobs:
           python -m pytest -x --cov=tskit --cov-report=xml --cov-branch -n2 tests
 
       - name: Upload coverage to Codecov
-        if: matrix.os == 'ubuntu-latest'
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v3
         with:
+          token: ${{ secrets.CODECOV_TOKEN }}
           working-directory: python
-          fail_ci_if_error: true
+          fail_ci_if_error: false
           flags: python-tests
           name: codecov-umbrella
           verbose: true

diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -15,12 +15,12 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        python: [3.7, 3.8, 3.9, "3.10"]
+        python: [3.7, 3.8, 3.9, "3.10", 3.11]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
       - name: Install deps
@@ -55,7 +55,7 @@ jobs:
     runs-on: windows-latest
     strategy:
       matrix:
-        python: [3.7, 3.8, 3.9, "3.10"]
+        python: [3.7, 3.8, 3.9, "3.10", 3.11]
         wordsize: [64]
     steps:
       - name: Checkout
@@ -108,7 +108,7 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: 3.8
 
@@ -127,7 +127,7 @@ jobs:
       - name: Build wheels in docker
         shell: bash
         run: |
-          docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2010_x86_64 bash .github/workflows/docker/buildwheel.sh
+          docker run --rm -v `pwd`:/project -w /project quay.io/pypa/manylinux2014_x86_64 bash .github/workflows/docker/buildwheel.sh
 
       - name: Upload Wheels
         uses: actions/upload-artifact@v2
@@ -140,14 +140,14 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        python: [3.7, 3.8, 3.9, "3.10"]
+        python: [3.7, 3.8, 3.9, "3.10", 3.11]
     steps:
       - name: Download wheels
         uses: actions/download-artifact@v2
         with:
           name: osx-wheel-${{ matrix.python }}
       - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
       - name: Install wheel and test
@@ -162,15 +162,15 @@ jobs:
     runs-on: windows-latest
     strategy:
       matrix:
-        python: [3.7, 3.8, 3.9, "3.10"]
+        python: [3.7, 3.8, 3.9, "3.10", 3.11]
         wordsize: [64]
     steps:
       - name: Download wheels
         uses: actions/download-artifact@v2
         with:
           name: win-wheel-${{ matrix.python }}-${{ matrix.wordsize }}
       - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
       - name: Install wheel and test
@@ -186,7 +186,7 @@ jobs:
     needs: ['manylinux']
     strategy:
       matrix:
-        python: [3.7, 3.8, 3.9, "3.10"]
+        python: [3.7, 3.8, 3.9, "3.10", 3.11]
         include:
           - python: 3.7
             wheel: cp37
@@ -196,13 +196,15 @@ jobs:
             wheel: cp39
           - python: "3.10"
             wheel: cp310
+          - python: 3.11
+            wheel: cp311
     steps:
       - name: Download wheels
         uses: actions/download-artifact@v2
         with:
           name: linux-wheels
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
       - name: Install wheel and test

diff --git a/.gitignore b/.gitignore
@@ -3,4 +3,6 @@ build-gcc
 python/benchmark/*.trees
 python/benchmark/*.json
 python/benchmark/*.html
+.venv
+.env