From d07e6044f4cd47b9910749a585fad8055541302c Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 1 Mar 2024 00:18:50 +0000 Subject: [PATCH 1/6] Serialize more runs to reduce overlapping failures --- .github/workflows/pr.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9e11993048f..241a33a0d1d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -39,7 +39,7 @@ jobs: with: enable_check_generated_files: false conda-cpp-build: - needs: checks + needs: devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.04 with: @@ -64,7 +64,7 @@ jobs: with: build_type: pull-request conda-python-cudf-tests: - needs: conda-python-build + needs: [conda-python-build, wheel-tests-cudf] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 with: @@ -72,7 +72,7 @@ jobs: test_script: "ci/test_python_cudf.sh" conda-python-other-tests: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism - needs: conda-python-build + needs: [conda-python-build, wheel-tests-dask-cudf] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 with: @@ -89,7 +89,7 @@ jobs: container_image: "rapidsai/ci-conda:latest" run_script: "ci/test_java.sh" conda-notebook-tests: - needs: conda-python-build + needs: [conda-python-build, wheel-tests-cudf] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 with: @@ -99,7 +99,7 @@ jobs: container_image: "rapidsai/ci-conda:latest" run_script: "ci/test_notebooks.sh" docs-build: - needs: conda-python-build + needs: [conda-python-build, wheel-tests-cudf] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 with: @@ -140,6 +140,7 @@ jobs: build_type: pull-request script: ci/test_wheel_dask_cudf.sh devcontainer: + needs: checks secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.04 with: From 1f3dc55bb851af49352f3290cb0892fea050b246 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 1 Mar 2024 00:22:06 +0000 Subject: [PATCH 2/6] Point at the new branch --- .github/workflows/pr.yaml | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 241a33a0d1d..680a41a883b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -32,41 +32,41 @@ jobs: #- pandas-tests-diff #- pandas-tests-diff-comment secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@feat/reduce_runs checks: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@feat/reduce_runs with: enable_check_generated_files: false conda-cpp-build: needs: devcontainer secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@feat/reduce_runs with: build_type: pull-request conda-cpp-checks: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@feat/reduce_runs with: build_type: pull-request enable_check_symbols: true conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@feat/reduce_runs with: build_type: pull-request conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@feat/reduce_runs with: build_type: pull-request conda-python-cudf-tests: needs: [conda-python-build, wheel-tests-cudf] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@feat/reduce_runs with: build_type: pull-request test_script: "ci/test_python_cudf.sh" @@ -74,14 +74,14 @@ jobs: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism needs: [conda-python-build, wheel-tests-dask-cudf] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@feat/reduce_runs with: build_type: pull-request test_script: "ci/test_python_other.sh" conda-java-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@feat/reduce_runs with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -91,7 +91,7 @@ jobs: conda-notebook-tests: needs: [conda-python-build, wheel-tests-cudf] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@feat/reduce_runs with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -101,7 +101,7 @@ jobs: docs-build: needs: [conda-python-build, wheel-tests-cudf] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@feat/reduce_runs with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -111,7 +111,7 @@ jobs: wheel-build-cudf: needs: checks secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@feat/reduce_runs with: build_type: pull-request build-2_28-wheels: "true" @@ -119,14 +119,14 @@ jobs: wheel-tests-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@feat/reduce_runs with: build_type: pull-request script: ci/test_wheel_cudf.sh wheel-build-dask-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@feat/reduce_runs with: matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.11" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) build_type: pull-request @@ -134,7 +134,7 @@ jobs: wheel-tests-dask-cudf: needs: wheel-build-dask-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@feat/reduce_runs with: matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.11" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) build_type: pull-request @@ -142,7 +142,7 @@ jobs: devcontainer: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.04 + uses: rapidsai/shared-action-workflows/.github/workflows/build-in-devcontainer.yaml@feat/reduce_runs with: build_command: | sccache -z; @@ -151,7 +151,7 @@ jobs: unit-tests-cudf-pandas: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@feat/reduce_runs with: matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.11" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.2.2"))) build_type: pull-request @@ -160,7 +160,7 @@ jobs: # # run the Pandas unit tests using PR branch # needs: wheel-build-cudf # secrets: inherit - # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.04 + # uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@feat/reduce_runs # with: # matrix_filter: map(select(.ARCH == "amd64")) | max_by(.CUDA_VER) | [.] # build_type: pull-request @@ -172,7 +172,7 @@ jobs: # needs: [pandas-tests-main, pandas-tests-pr] # secrets: inherit # # This branch exports a `job_output` output that the downstream job reads. - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.04 + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@feat/reduce_runs # with: # node_type: cpu4 # build_type: pull-request From cb9e57a0312e7de765c242f40618ffd37637c0aa Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 1 Mar 2024 00:24:21 +0000 Subject: [PATCH 3/6] Also wait for devcontainer to run wheel builds --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 680a41a883b..db0f77a6fa9 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -109,7 +109,7 @@ jobs: container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" wheel-build-cudf: - needs: checks + needs: devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@feat/reduce_runs with: From 0fafbe2602e95d2971fcf1d48a27f5db0a3147e0 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 1 Mar 2024 19:07:55 +0000 Subject: [PATCH 4/6] test From 23d05fffc00e3aa8455ac5fbf9297b5bbe3dd2c8 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 1 Mar 2024 19:12:43 +0000 Subject: [PATCH 5/6] test From b0391fcb07f8c02a68c627c4c8f527a519d5c0dd Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 1 Mar 2024 22:20:21 +0000 Subject: [PATCH 6/6] Parallelize again --- .github/workflows/pr.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index db0f77a6fa9..daa22c20fca 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -39,7 +39,7 @@ jobs: with: enable_check_generated_files: false conda-cpp-build: - needs: devcontainer + needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@feat/reduce_runs with: @@ -64,7 +64,7 @@ jobs: with: build_type: pull-request conda-python-cudf-tests: - needs: [conda-python-build, wheel-tests-cudf] + needs: conda-python-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@feat/reduce_runs with: @@ -72,7 +72,7 @@ jobs: test_script: "ci/test_python_cudf.sh" conda-python-other-tests: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism - needs: [conda-python-build, wheel-tests-dask-cudf] + needs: conda-python-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@feat/reduce_runs with: @@ -89,7 +89,7 @@ jobs: container_image: "rapidsai/ci-conda:latest" run_script: "ci/test_java.sh" conda-notebook-tests: - needs: [conda-python-build, wheel-tests-cudf] + needs: conda-python-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@feat/reduce_runs with: @@ -99,7 +99,7 @@ jobs: container_image: "rapidsai/ci-conda:latest" run_script: "ci/test_notebooks.sh" docs-build: - needs: [conda-python-build, wheel-tests-cudf] + needs: conda-python-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@feat/reduce_runs with: @@ -109,7 +109,7 @@ jobs: container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" wheel-build-cudf: - needs: devcontainer + needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@feat/reduce_runs with: