From a49e4b60b995fe91ec7aaa9fe201b781684330c3 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 2 Jul 2024 11:02:10 -0500 Subject: [PATCH 1/4] remove management of UCX system libraries in build scripts (#241) For https://github.com/rapidsai/build-planning/issues/57, #226 switched `ucxx` over to `libucx` wheels. To test that that was working, it added some code to building scripts to remove system installations of UCX libraries. That should no longer be necessary as of https://github.com/rapidsai/ci-imgs/pull/154. This proposes removing that code for managing system dependencies of UCX libraries, to simplify those build scripts a bit. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Peter Andreas Entschev (https://github.com/pentschev) - Mike Sarahan (https://github.com/msarahan) URL: https://github.com/rapidsai/ucxx/pull/241 --- ci/build_wheel.sh | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 9d1491da..762db3c5 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -6,19 +6,6 @@ set -euo pipefail package_name=$1 package_dir=$2 -# Clear out system ucx files to ensure that we're getting ucx from the wheel. -rm -rf /usr/lib64/ucx -rm -rf /usr/lib64/libucm.* -rm -rf /usr/lib64/libucp.* -rm -rf /usr/lib64/libucs.* -rm -rf /usr/lib64/libucs_signal.* -rm -rf /usr/lib64/libuct.* - -rm -rf /usr/include/ucm -rm -rf /usr/include/ucp -rm -rf /usr/include/ucs -rm -rf /usr/include/uct - source rapids-configure-sccache source rapids-date-string From 3985977756c0b21578a152e9f8d3afa17d3f8ea8 Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Tue, 2 Jul 2024 18:03:15 +0200 Subject: [PATCH 2/4] Fix tests on new Distributed releases (#242) Latest changes in Distributed have broken `test_nanny_closed_by_keyboard_interrupt` as it used a member attribute that is now supposed to be accessed via a method, the test is now updated to rely on the new behavior. Authors: - Peter Andreas Entschev (https://github.com/pentschev) Approvers: - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/ucxx/pull/242 --- python/distributed-ucxx/distributed_ucxx/tests/test_nanny.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/distributed-ucxx/distributed_ucxx/tests/test_nanny.py b/python/distributed-ucxx/distributed_ucxx/tests/test_nanny.py index 3867c25d..8e085796 100644 --- a/python/distributed-ucxx/distributed_ucxx/tests/test_nanny.py +++ b/python/distributed-ucxx/distributed_ucxx/tests/test_nanny.py @@ -21,4 +21,4 @@ async def test_nanny_closed_by_keyboard_interrupt(ucxx_loop): ) as n: await n.process.stopped.wait() # Check that the scheduler has been notified about the closed worker - assert "remove-worker" in str(s.events) + assert "remove-worker" in str(s.get_events()) From 4f44a48a4da41f82d2503215ffc7f7dc02b5efba Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 6 Jul 2024 14:33:24 -0500 Subject: [PATCH 3/4] skip CMake 3.30.0 (#244) Contributes to https://github.com/rapidsai/build-planning/issues/80 Adds constraints to avoid pulling in CMake 3.30.0, for the reasons described in that issue. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) - Peter Andreas Entschev (https://github.com/pentschev) URL: https://github.com/rapidsai/ucxx/pull/244 --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-122_arch-x86_64.yaml | 2 +- dependencies.yaml | 2 +- python/pyproject.toml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index dd9c34ec..f6eca017 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -11,7 +11,7 @@ dependencies: - automake - c-compiler - cloudpickle -- cmake>=3.26.4 +- cmake>=3.26.4,!=3.30.0 - cuda-version=11.8 - cudatoolkit - cudf==24.8.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index d5980711..21a00c3b 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -11,7 +11,7 @@ dependencies: - automake - c-compiler - cloudpickle -- cmake>=3.26.4 +- cmake>=3.26.4,!=3.30.0 - cuda-cudart-dev - cuda-version=12.2 - cudf==24.8.*,>=0.0.0a0 diff --git a/dependencies.yaml b/dependencies.yaml index 5da83c7a..992cfe29 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -130,7 +130,7 @@ dependencies: packages: - c-compiler - cxx-compiler - - &cmake_ver cmake>=3.26.4 + - &cmake_ver cmake>=3.26.4,!=3.30.0 - fmt>=10.1.1,<11 - librmm==24.8.*,>=0.0.0a0 - ninja diff --git a/python/pyproject.toml b/python/pyproject.toml index edb425c7..1f26f564 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -95,7 +95,7 @@ skip = [ build-backend = "scikit_build_core.build" dependencies-file = "../dependencies.yaml" requires = [ - "cmake>=3.26.4", + "cmake>=3.26.4,!=3.30.0", "cython>=3.0.0", "libucx==1.15.0", "ninja", From 36284cbe2848a296dba56b146c8650403fb27ae6 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 16 Jul 2024 13:59:08 -0400 Subject: [PATCH 4/4] Build and test with CUDA 12.5.1 (#245) This PR updates the latest CUDA build/test version 12.2.2 to 12.5.1. Contributes to https://github.com/rapidsai/build-planning/issues/73 Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/ucxx/pull/245 --- .github/workflows/build.yaml | 14 ++++++------ .github/workflows/pr.yaml | 22 +++++++++---------- .github/workflows/test.yaml | 10 ++++----- ..._64.yaml => all_cuda-125_arch-x86_64.yaml} | 4 ++-- dependencies.yaml | 6 ++++- 5 files changed, 30 insertions(+), 26 deletions(-) rename conda/environments/{all_cuda-122_arch-x86_64.yaml => all_cuda-125_arch-x86_64.yaml} (94%) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 631189e6..7a6e0f9d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: conda-cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -38,7 +38,7 @@ jobs: if: github.ref_type == 'branch' needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 with: arch: "amd64" branch: ${{ inputs.branch }} @@ -51,7 +51,7 @@ jobs: upload-conda: needs: [conda-cpp-build] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -59,7 +59,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-ucxx: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -69,7 +69,7 @@ jobs: wheel-publish-ucxx: needs: wheel-build-ucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -78,7 +78,7 @@ jobs: package-name: ucxx wheel-build-distributed-ucxx: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -88,7 +88,7 @@ jobs: wheel-publish-distributed-ucxx: needs: [wheel-build-ucxx, wheel-build-distributed-ucxx] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@cuda-12.5.1 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 96ffbc4a..97278868 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -23,22 +23,22 @@ jobs: - wheel-build-distributed-ucxx - wheel-tests-distributed-ucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@cuda-12.5.1 checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@cuda-12.5.1 with: enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@cuda-12.5.1 with: build_type: pull-request docs-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-12.5.1 with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -48,21 +48,21 @@ jobs: conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-12.5.1 with: build_type: pull-request container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" conda-python-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@cuda-12.5.1 with: build_type: pull-request container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" conda-python-distributed-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@cuda-12.5.1 with: build_type: pull-request script: "ci/test_python_distributed.sh" @@ -70,14 +70,14 @@ jobs: wheel-build-ucxx: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: build_type: pull-request script: ci/build_wheel_ucxx.sh wheel-tests-ucxx: needs: wheel-build-ucxx secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: build_type: pull-request container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" @@ -85,14 +85,14 @@ jobs: wheel-build-distributed-ucxx: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda-12.5.1 with: build_type: pull-request script: ci/build_wheel_distributed_ucxx.sh wheel-tests-distributed-ucxx: needs: [wheel-build-ucxx, wheel-build-distributed-ucxx] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: build_type: pull-request container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 33115354..a363bfec 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} @@ -25,7 +25,7 @@ jobs: container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" conda-python-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} @@ -34,7 +34,7 @@ jobs: container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" conda-python-distributed-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@cuda-12.5.1 with: build_type: nightly script: "ci/test_python_distributed.sh" @@ -44,7 +44,7 @@ jobs: container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000" wheel-tests-ucxx: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} @@ -54,7 +54,7 @@ jobs: script: ci/test_wheel_ucxx.sh wheel-tests-distributed-ucxx: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda-12.5.1 with: build_type: nightly branch: ${{ inputs.branch }} diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml similarity index 94% rename from conda/environments/all_cuda-122_arch-x86_64.yaml rename to conda/environments/all_cuda-125_arch-x86_64.yaml index 21a00c3b..b9d4bd01 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -13,7 +13,7 @@ dependencies: - cloudpickle - cmake>=3.26.4,!=3.30.0 - cuda-cudart-dev -- cuda-version=12.2 +- cuda-version=12.5 - cudf==24.8.*,>=0.0.0a0 - cupy>=12.0.0 - cxx-compiler @@ -42,4 +42,4 @@ dependencies: - setuptools>=64.0.0 - spdlog>=1.12.0,<1.13 - ucx>=1.15.0 -name: all_cuda-122_arch-x86_64 +name: all_cuda-125_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index 992cfe29..625f2bb1 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -3,7 +3,7 @@ files: all: output: conda matrix: - cuda: ["11.8", "12.2"] + cuda: ["11.8", "12.5"] arch: [x86_64] includes: - build_cpp @@ -192,6 +192,10 @@ dependencies: cuda: "12.2" packages: - cuda-version=12.2 + - matrix: + cuda: "12.5" + packages: + - cuda-version=12.5 cuda: specific: - output_types: conda