Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: dask/distributed
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 2024.11.1
Choose a base ref
...
head repository: dask/distributed
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: refs/heads/main
Choose a head ref
Loading
Showing with 218 additions and 995 deletions.
  1. +1 −1 .github/workflows/conda.yml
  2. +2 −2 .github/workflows/test-report.yaml
  3. +7 −20 .github/workflows/tests.yaml
  4. +0 −69 .github/workflows/update-gpuci.yaml
  5. +0 −1 CODEOWNERS
  6. +3 −0 continuous_integration/environment-3.10.yaml
  7. +3 −0 continuous_integration/environment-3.11.yaml
  8. +3 −0 continuous_integration/environment-3.12.yaml
  9. +58 −0 continuous_integration/environment-3.13.yaml
  10. +0 −14 continuous_integration/gpuci/axis.yaml
  11. +0 −62 continuous_integration/gpuci/build.sh
  12. +22 −10 distributed/client.py
  13. +1 −0 distributed/dashboard/tests/test_scheduler_bokeh.py
  14. +2 −1 distributed/diagnostics/nvml.py
  15. +3 −11 distributed/protocol/tests/test_highlevelgraph.py
  16. +22 −30 distributed/scheduler.py
  17. +2 −0 distributed/security.py
  18. +0 −6 distributed/shuffle/__init__.py
  19. +5 −14 distributed/shuffle/_core.py
  20. +4 −409 distributed/shuffle/_merge.py
  21. +6 −4 distributed/shuffle/_rechunk.py
  22. +2 −230 distributed/shuffle/_shuffle.py
  23. +0 −14 distributed/shuffle/tests/test_graph.py
  24. +3 −29 distributed/shuffle/tests/test_merge.py
  25. +1 −25 distributed/shuffle/tests/test_shuffle.py
  26. +1 −8 distributed/shuffle/tests/utils.py
  27. +5 −3 distributed/tests/test_active_memory_manager.py
  28. +8 −7 distributed/tests/test_as_completed.py
  29. +21 −4 distributed/tests/test_client.py
  30. +2 −2 distributed/tests/test_metrics.py
  31. +1 −1 distributed/tests/test_preload.py
  32. +15 −2 distributed/tests/test_scheduler.py
  33. +2 −2 distributed/tests/test_utils_test.py
  34. +6 −10 distributed/tests/test_worker.py
  35. +5 −3 distributed/utils_test.py
  36. +2 −1 pyproject.toml
2 changes: 1 addition & 1 deletion .github/workflows/conda.yml
Original file line number Diff line number Diff line change
@@ -30,7 +30,7 @@ jobs:
with:
fetch-depth: 0
- name: Set up Python
uses: conda-incubator/setup-miniconda@v3.0.3
uses: conda-incubator/setup-miniconda@v3.1.0
with:
miniforge-version: latest
use-mamba: true
4 changes: 2 additions & 2 deletions .github/workflows/test-report.yaml
Original file line number Diff line number Diff line change
@@ -21,7 +21,7 @@ jobs:
- uses: actions/checkout@v4.1.3

- name: Setup Conda Environment
uses: conda-incubator/setup-miniconda@v3.0.3
uses: conda-incubator/setup-miniconda@v3.1.0
with:
miniforge-version: latest
condarc-file: continuous_integration/condarc
@@ -53,7 +53,7 @@ jobs:
mv test_report.html test_short_report.html deploy/
- name: Deploy 🚀
uses: JamesIves/github-pages-deploy-action@v4.6.8
uses: JamesIves/github-pages-deploy-action@v4.7.2
with:
branch: gh-pages
folder: deploy
27 changes: 7 additions & 20 deletions .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
@@ -26,7 +26,7 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
environment: [mindeps, "3.10", "3.11", "3.12"]
environment: [mindeps, "3.10", "3.11", "3.12", "3.13"]
label: [default]
extra_packages: [null]
# Cherry-pick test modules to split the overall runtime roughly in half
@@ -40,6 +40,8 @@ jobs:
environment: "3.10"
- os: macos-latest
environment: "3.11"
- os: macos-latest
environment: "3.13"

- os: windows-latest
environment: mindeps
@@ -55,16 +57,6 @@ jobs:
label: no_queue
partition: "not ci1"

# Set dataframe.query-planning: false
- os: ubuntu-latest
environment: "3.10"
label: no_expr
partition: "ci1"
- os: ubuntu-latest
environment: "3.10"
label: no_expr
partition: "not ci1"

# dask.array P2P shuffle
- os: ubuntu-latest
environment: mindeps
@@ -81,12 +73,12 @@ jobs:
- os: ubuntu-latest
environment: mindeps
label: pandas
extra_packages: [numpy=1.24, pandas=2.0, pyarrow=14.0.1]
extra_packages: [numpy=1.24, pandas=2.0, pyarrow=14.0.1, dask-expr]
partition: "ci1"
- os: ubuntu-latest
environment: mindeps
label: pandas
extra_packages: [numpy=1.24, pandas=2.0, pyarrow=14.0.1]
extra_packages: [numpy=1.24, pandas=2.0, pyarrow=14.0.1, dask-expr]
partition: "not ci1"

- os: ubuntu-latest
@@ -123,7 +115,7 @@ jobs:
fetch-depth: 0

- name: Setup Conda Environment
uses: conda-incubator/setup-miniconda@v3.0.3
uses: conda-incubator/setup-miniconda@v3.1.0
with:
miniforge-version: latest
condarc-file: continuous_integration/condarc
@@ -219,11 +211,6 @@ jobs:
if: ${{ matrix.label == 'no_queue' }}
run: echo "DASK_DISTRIBUTED__SCHEDULER__WORKER_SATURATION=inf" >> $GITHUB_ENV

- name: Set up dask env to disable dask-expr
shell: bash -l {0}
if: ${{ matrix.label == 'no_expr' }}
run: echo "DASK_DATAFRAME__QUERY_PLANNING=False" >> $GITHUB_ENV

- name: Print host info
# host_info.py imports numpy, which isn't a direct dependency of distributed
if: matrix.environment != 'mindeps'
@@ -242,7 +229,7 @@ jobs:
set -o pipefail
mkdir reports
pytest distributed \
python -m pytest distributed \
-m "not avoid_ci and ${{ matrix.partition }}" --runslow \
--leaks=fds,processes,threads \
--junitxml reports/pytest.xml -o junit_suite_name=$TEST_ID \
69 changes: 0 additions & 69 deletions .github/workflows/update-gpuci.yaml

This file was deleted.

1 change: 0 additions & 1 deletion CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
@@ -12,4 +12,3 @@ distributed/widgets/* @jacobtomlinson

# GPU Support
distributed/diagnostics/nvml.py @jacobtomlinson @quasiben
continuous_integration/gpuci/* @jacobtomlinson @quasiben
3 changes: 3 additions & 0 deletions continuous_integration/environment-3.10.yaml
Original file line number Diff line number Diff line change
@@ -19,6 +19,7 @@ dependencies:
- ipykernel
- ipywidgets
- jinja2
- jupyter_events<0.11
- jupyter-server-proxy
- jupyterlab
- locket
@@ -52,6 +53,8 @@ dependencies:
- zstandard
# Temporary fix for https://github.com/pypa/setuptools/issues/4496
- setuptools < 71
# Temporary fix for https://github.com/jupyterlab/jupyterlab/issues/17012
- httpx<0.28.0
- pip:
- git+https://github.com/dask/dask
- git+https://github.com/dask/dask-expr
3 changes: 3 additions & 0 deletions continuous_integration/environment-3.11.yaml
Original file line number Diff line number Diff line change
@@ -18,6 +18,7 @@ dependencies:
- ipykernel
- ipywidgets
- jinja2
- jupyter_events<0.11
- jupyter-server-proxy
- jupyterlab
- locket
@@ -46,6 +47,8 @@ dependencies:
- zstandard
# Temporary fix for https://github.com/pypa/setuptools/issues/4496
- setuptools < 71
# Temporary fix for https://github.com/jupyterlab/jupyterlab/issues/17012
- httpx<0.28.0
- pip:
- git+https://github.com/dask/dask
- git+https://github.com/dask/dask-expr
3 changes: 3 additions & 0 deletions continuous_integration/environment-3.12.yaml
Original file line number Diff line number Diff line change
@@ -18,6 +18,7 @@ dependencies:
- ipykernel
- ipywidgets
- jinja2
- jupyter_events<0.11
- jupyter-server-proxy
- jupyterlab
- locket
@@ -46,6 +47,8 @@ dependencies:
- zstandard
# Temporary fix for https://github.com/pypa/setuptools/issues/4496
- setuptools < 71
# Temporary fix for https://github.com/jupyterlab/jupyterlab/issues/17012
- httpx<0.28.0
- pip:
- git+https://github.com/dask/dask
- git+https://github.com/dask/dask-expr
58 changes: 58 additions & 0 deletions continuous_integration/environment-3.13.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
name: dask-distributed-313
channels:
- conda-forge
dependencies:
- python=3.13
- packaging
- pip
- asyncssh
- bokeh>3
- click
- cloudpickle
- coverage
- dask # overridden by git tip below
- filesystem-spec # overridden by git tip below
- gilknocker
- h5py
- ipykernel
- ipywidgets
- jinja2
- jupyter_events<0.11
- jupyter-server-proxy
- jupyterlab
- locket
- msgpack-python
- netcdf4
- paramiko
- pre-commit
- prometheus_client
- psutil
- pyarrow
- pytest
- pytest-cov
- pytest-faulthandler
- pytest-repeat
- pytest-rerunfailures
- pytest-timeout
- requests
- s3fs # overridden by git tip below
- scikit-learn
- scipy
- sortedcollections
- tblib
- toolz
- tornado
- zict # overridden by git tip below
- zstandard
# Temporary fix for https://github.com/pypa/setuptools/issues/4496
- setuptools < 71
# Temporary fix for https://github.com/jupyterlab/jupyterlab/issues/17012
- httpx<0.28.0
- pip:
- git+https://github.com/dask/dask
- git+https://github.com/dask/dask-expr
- git+https://github.com/dask/zict
# Revert after https://github.com/dask/distributed/issues/8614 is fixed
# - git+https://github.com/dask/s3fs
# - git+https://github.com/fsspec/filesystem_spec
- keras
14 changes: 0 additions & 14 deletions continuous_integration/gpuci/axis.yaml

This file was deleted.

62 changes: 0 additions & 62 deletions continuous_integration/gpuci/build.sh

This file was deleted.

Loading