Merge branch 'main' into do_not_init_logging_on_import

.flake8

@@ -22,6 +22,8 @@ ignore =
     B028
     # do not compare types, for exact checks use `is` / `is not`, for instance checks use `isinstance()`
     E721
+    # multiple statements on one line; required for black compat
+    E701, E704
 
 per-file-ignores =
     **/tests/*:

.github/release-drafter.yml

@@ -0,0 +1,13 @@
+# These will be overridden by the publish workflow and set to the new tag
+name-template: 'Next Release'
+tag-template: 'next'
+
+change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
+change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks.
+
+template: |
+  ## Changes
+
+  $CHANGES  
+
+  See the [Changelog](https://docs.dask.org/en/stable/changelog.html) for more information.

.github/workflows/ci-pre-commit.yml

@@ -14,5 +14,5 @@ jobs:
       - uses: actions/[email protected]
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.12'
       - uses: pre-commit/[email protected]

.github/workflows/conda.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Set up Python
         uses: conda-incubator/[email protected]
         with:
-          miniforge-variant: Mambaforge
+          miniforge-version: latest
           use-mamba: true
           python-version: 3.9
           channel-priority: strict

.github/workflows/release-drafter.yml

@@ -0,0 +1,32 @@
+name: Release Drafter
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+jobs:
+  update_release_draft:
+    if: github.repository == 'dask/distributed'
+    permissions:
+      # Write permission is required to create a GitHub release
+      contents: write
+      pull-requests: read
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Required to get tag history
+      - name: Check if release commit
+        id: check_release_commit
+        run: git describe --exact-match --tags $(git rev-parse HEAD)
+        continue-on-error: true
+      - uses: release-drafter/release-drafter@v6
+        if: steps.check_release_commit.outcome != 'success'
+        with:
+          disable-autolabeler: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release-publish.yml

@@ -0,0 +1,33 @@
+name: Release Publisher
+
+on:
+  push:
+    tags:
+      - "*.*.*"
+
+permissions:
+  contents: read
+
+jobs:
+  publish_release:
+    if: github.repository == 'dask/distributed'
+    permissions:
+      # Write permission is required to publish a GitHub release
+      contents: write
+      pull-requests: read
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set version env
+        # Use a little bit of bash to extract the tag name from the GitHub ref
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+      - uses: release-drafter/release-drafter@v6
+        with:
+          disable-autolabeler: true
+          # Override the Release name/tag/version with the actual tag name
+          name: ${{ env.RELEASE_VERSION }}
+          tag: ${{ env.RELEASE_VERSION }}
+          version: ${{ env.RELEASE_VERSION }}
+          # Publish the Release
+          publish: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test-report.yaml

@@ -23,7 +23,6 @@ jobs:
       - name: Setup Conda Environment
         uses: conda-incubator/[email protected]
         with:
-          miniforge-variant: Mambaforge
           miniforge-version: latest
           condarc-file: continuous_integration/condarc
           use-mamba: true
@@ -54,7 +53,7 @@ jobs:
           mv test_report.html test_short_report.html deploy/
 
       - name: Deploy 🚀
-        uses: JamesIves/github-pages-deploy-action@v4.5.0
+        uses: JamesIves/github-pages-deploy-action@v4.6.8
         with:
           branch: gh-pages
           folder: deploy

.github/workflows/tests.yaml

@@ -26,7 +26,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        environment: [mindeps, "3.9", "3.10", "3.11", "3.12"]
+        environment: [mindeps, "3.10", "3.11", "3.12"]
         label: [default]
         extra_packages: [null]
         # Cherry-pick test modules to split the overall runtime roughly in half
@@ -36,8 +36,6 @@ jobs:
           # MacOS CI does not have any hosts available; run it on 3.12 only
           - os: macos-latest
             environment: mindeps
-          - os: macos-latest
-            environment: "3.9"
           - os: macos-latest
             environment: "3.10"
           - os: macos-latest
@@ -49,46 +47,46 @@ jobs:
         include:
           # Set distributed.scheduler.worker-saturation: .inf
           - os: ubuntu-latest
-            environment: "3.9"
+            environment: "3.10"
             label: no_queue
             partition: "ci1"
           - os: ubuntu-latest
-            environment: "3.9"
+            environment: "3.10"
             label: no_queue
             partition: "not ci1"
 
           # Set dataframe.query-planning: false
           - os: ubuntu-latest
-            environment: "3.9"
+            environment: "3.10"
             label: no_expr
             partition: "ci1"
           - os: ubuntu-latest
-            environment: "3.9"
+            environment: "3.10"
             label: no_expr
             partition: "not ci1"
 
           # dask.array P2P shuffle
           - os: ubuntu-latest
             environment: mindeps
             label: numpy
-            extra_packages: [numpy=1.21]
+            extra_packages: [numpy=1.24]
             partition: "ci1"
           - os: ubuntu-latest
             environment: mindeps
             label: numpy
-            extra_packages: [numpy=1.21]
+            extra_packages: [numpy=1.24]
             partition: "not ci1"
 
           # dask.dataframe P2P shuffle
           - os: ubuntu-latest
             environment: mindeps
             label: pandas
-            extra_packages: [numpy=1.21, pandas=1.3, pyarrow=7, pyarrow-hotfix]
+            extra_packages: [numpy=1.24, pandas=2.0, pyarrow=14.0.1]
             partition: "ci1"
           - os: ubuntu-latest
             environment: mindeps
             label: pandas
-            extra_packages: [numpy=1.21, pandas=1.3, pyarrow=7, pyarrow-hotfix]
+            extra_packages: [numpy=1.24, pandas=2.0, pyarrow=14.0.1]
             partition: "not ci1"
 
           - os: ubuntu-latest
@@ -127,7 +125,6 @@ jobs:
       - name: Setup Conda Environment
         uses: conda-incubator/[email protected]
         with:
-          miniforge-variant: Mambaforge
           miniforge-version: latest
           condarc-file: continuous_integration/condarc
           use-mamba: true
@@ -159,7 +156,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ env.CONDA }}/envs
-          key: conda-${{ matrix.os }}-${{ steps.get-date.outputs.today }}-${{ hashFiles(env.CONDA_FILE) }}-${{ env.CACHE_NUMBER }}
+          key: conda-${{ matrix.os }}-${{ matrix.environment }}-${{ matrix.label }}-${{ steps.get-date.outputs.today }}-${{ hashFiles(env.CONDA_FILE) }}-${{ env.CACHE_NUMBER }}
         env:
           # Increase this value to reset cache if
           # continuous_integration/environment-${{ matrix.environment }}.yaml has not

.github/workflows/update-gpuci.yaml

@@ -21,15 +21,15 @@ jobs:
 
       - name: Get latest cuDF nightly version
         id: cudf_latest
-        uses: jacobtomlinson/[email protected].3
+        uses: jacobtomlinson/[email protected].4
         with:
           org: "rapidsai-nightly"
           package: "cudf"
           version_system: "CalVer"
 
       - name: Get latest UCX-Py nightly version
         id: ucx_py_latest
-        uses: jacobtomlinson/[email protected].3
+        uses: jacobtomlinson/[email protected].4
         with:
           org: "rapidsai-nightly"
           package: "ucx-py"
@@ -58,7 +58,7 @@ jobs:
         if: ${{ env.UCX_PY_VER != env.NEW_UCX_PY_VER }}  # make sure new ucx-py nightlies are available
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          draft: true
+          draft: false
           commit-message: "Update gpuCI `RAPIDS_VER` to `${{ env.NEW_RAPIDS_VER }}`"
           title: "Update gpuCI `RAPIDS_VER` to `${{ env.NEW_RAPIDS_VER }}`"
           author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

.pre-commit-config.yaml

@@ -12,37 +12,37 @@ repos:
       - id: isort
         language_version: python3
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.0
+    rev: v3.17.0
     hooks:
       - id: pyupgrade
         args:
           - --py39-plus
   - repo: https://github.com/psf/black
-    rev: 23.12.1
+    rev: 24.8.0
     hooks:
       - id: black
         language_version: python3
         args:
           - --target-version=py39
   - repo: https://github.com/pycqa/flake8
-    rev: 7.0.0
+    rev: 7.1.1
     hooks:
       - id: flake8
         language_version: python3
         additional_dependencies:
         # NOTE: autoupdate does not pick up flake8-bugbear since it is a transitive
         #  dependency. Make sure to update flake8-bugbear manually on a regular basis.
-          - flake8-bugbear==23.12.2
+          - flake8-bugbear==24.8.19
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.6
+    rev: v2.3.0
     hooks:
       -   id: codespell
           additional_dependencies:
             - tomli
           types_or: [rst, markdown]
           files: docs
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.8.0
+    rev: v1.11.2
     hooks:
       - id: mypy
         # Override default --ignore-missing-imports

CODEOWNERS

@@ -12,3 +12,4 @@ distributed/widgets/*  @jacobtomlinson
 
 # GPU Support
 distributed/diagnostics/nvml.py  @jacobtomlinson @quasiben
+continuous_integration/gpuci/*  @jacobtomlinson @quasiben

continuous_integration/environment-3.10.yaml

@@ -11,6 +11,7 @@ dependencies:
   - click
   - cloudpickle
   - coverage
+  - cython  # Only tested here; also a dependency of crick
   - dask  # overridden by git tip below
   - filesystem-spec  # overridden by git tip below
   - gilknocker
@@ -21,33 +22,41 @@ dependencies:
   - jupyter-server-proxy
   - jupyterlab
   - locket
+  - lz4  # Only tested here
   - msgpack-python
   - netcdf4
   - paramiko
   - pre-commit
   - prometheus_client
   - psutil
   - pyarrow
+  - pynvml  # Only tested here
   - pytest
   - pytest-cov
   - pytest-faulthandler
   - pytest-repeat
   - pytest-rerunfailures
   - pytest-timeout
+  - python-snappy  # Only tested here
+  - pytorch  # Only tested here
   - requests
   - s3fs  # overridden by git tip below
   - scikit-learn
   - scipy
   - sortedcollections
   - tblib
   - toolz
+  - torchvision  # Only tested here
   - tornado
   - zict  # overridden by git tip below
   - zstandard
+  # Temporary fix for https://github.com/pypa/setuptools/issues/4496
+  - setuptools < 71
   - pip:
       - git+https://github.com/dask/dask
       - git+https://github.com/dask-contrib/dask-expr
       - git+https://github.com/dask/zict
+      - git+https://github.com/dask/crick  # Only tested here
       # Revert after https://github.com/dask/distributed/issues/8614 is fixed
       # - git+https://github.com/dask/s3fs
       # - git+https://github.com/fsspec/filesystem_spec

continuous_integration/environment-3.11.yaml

@@ -44,6 +44,8 @@ dependencies:
   - tornado
   - zict  # overridden by git tip below
   - zstandard
+  # Temporary fix for https://github.com/pypa/setuptools/issues/4496
+  - setuptools < 71
   - pip:
       - git+https://github.com/dask/dask
       - git+https://github.com/dask-contrib/dask-expr

continuous_integration/environment-3.12.yaml

@@ -44,6 +44,8 @@ dependencies:
   - tornado
   - zict  # overridden by git tip below
   - zstandard
+  # Temporary fix for https://github.com/pypa/setuptools/issues/4496
+  - setuptools < 71
   - pip:
       - git+https://github.com/dask/dask
       - git+https://github.com/dask-contrib/dask-expr