From adfd201e774078c00af8a4b209a7ae4584b1f325 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 15 Aug 2024 17:24:16 -0500 Subject: [PATCH 1/3] standardize and consolidate wheel installations in testing scripts --- ci/cudf_pandas_scripts/pandas-tests/run.sh | 7 +++++-- ci/cudf_pandas_scripts/run_tests.sh | 6 ++++-- ci/test_wheel_cudf_polars.sh | 11 +++++++---- ci/test_wheel_dask_cudf.sh | 7 ++++--- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index 48ee4a05628..38e75b13941 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -11,8 +11,11 @@ rapids-logger "Running Pandas tests using $PANDAS_TESTS_BRANCH branch and rapids rapids-logger "PR number: ${RAPIDS_REF_NAME:-"unknown"}" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep -python -m pip install $(ls ./local-cudf-dep/cudf*.whl)[test,pandas-tests] +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist + +# echo to expand wildcard before adding `[extra]` requires for pip +python -m pip install \ + "$(echo ./dist/cudf*.whl)[test,pandas-tests]" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index 1c3b99953fb..7a3ae3dbf5d 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -36,8 +36,10 @@ if [ "$no_cudf" = true ]; then echo "Skipping cudf install" else RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep - python -m pip install $(ls ./local-cudf-dep/cudf*.whl)[test,cudf-pandas-tests] + RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist + # echo to expand wildcard before adding `[extra]` requires for pip + python -m pip install \ + "$(echo ./dist/cudf*.whl)[test,cudf-pandas-tests]" fi python -m pytest -p cudf.pandas \ diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index cc9f5788685..3e9e2cef1db 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -21,11 +21,14 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist # Download the cudf built in the previous step -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep -python -m pip install ./local-cudf-dep/cudf*.whl +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -rapids-logger "Install cudf_polars" -python -m pip install $(echo ./dist/cudf_polars*.whl)[test] +rapids-logger "Install cudf_polars and its dependencies" + +# echo to expand wildcard before adding `[extra]` requires for pip +python -m pip install \ + "$(echo ./dist/cudf*.whl)" \ + "$(echo ./dist/cudf_polars*.whl)[test]" rapids-logger "Run cudf_polars tests" diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index c3800d3cc25..619feb6c834 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -7,11 +7,12 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist # Download the cudf built in the previous step -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep -python -m pip install ./local-cudf-dep/cudf*.whl +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/dask_cudf*.whl)[test] +python -m pip install \ + "$(echo ./dist/cudf*.whl)" \ + "$(echo ./dist/dask_cudf*.whl)[test]" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ From 22264d3a9f31b13c0c022e00e1e5a2f5cb764994 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 16 Aug 2024 09:17:00 -0500 Subject: [PATCH 2/3] add CUDA suffix --- ci/cudf_pandas_scripts/pandas-tests/run.sh | 2 +- ci/cudf_pandas_scripts/run_tests.sh | 2 +- ci/test_wheel_cudf.sh | 3 ++- ci/test_wheel_cudf_polars.sh | 4 ++-- ci/test_wheel_dask_cudf.sh | 4 ++-- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index 38e75b13941..a7c7e52e353 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -15,7 +15,7 @@ RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ - "$(echo ./dist/cudf*.whl)[test,pandas-tests]" + "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,pandas-tests]" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index 7a3ae3dbf5d..3cbcce06260 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -39,7 +39,7 @@ else RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ - "$(echo ./dist/cudf*.whl)[test,cudf-pandas-tests]" + "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,cudf-pandas-tests]" fi python -m pytest -p cudf.pandas \ diff --git a/ci/test_wheel_cudf.sh b/ci/test_wheel_cudf.sh index fdb61278d36..a1e6fe0a437 100755 --- a/ci/test_wheel_cudf.sh +++ b/ci/test_wheel_cudf.sh @@ -7,7 +7,8 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/cudf*.whl)[test] +python -m pip install \ + "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 3e9e2cef1db..6798b619ce2 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -27,8 +27,8 @@ rapids-logger "Install cudf_polars and its dependencies" # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ - "$(echo ./dist/cudf*.whl)" \ - "$(echo ./dist/cudf_polars*.whl)[test]" + "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ + "$(echo ./dist/cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" rapids-logger "Run cudf_polars tests" diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index 619feb6c834..db5bb963368 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -11,8 +11,8 @@ RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ - "$(echo ./dist/cudf*.whl)" \ - "$(echo ./dist/dask_cudf*.whl)[test]" + "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ + "$(echo ./dist/dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/ From b112f31ffd4ea89019923216f00956553d0446d6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 16 Aug 2024 15:14:49 -0500 Subject: [PATCH 3/3] Update ci/cudf_pandas_scripts/pandas-tests/run.sh Co-authored-by: Vyas Ramasubramani --- ci/cudf_pandas_scripts/pandas-tests/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index d2c027066ee..97c3139080f 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -19,7 +19,7 @@ RAPIDS_PY_WHEEL_NAME="pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install \ "$(echo ./dist/cudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,pandas-tests]" \ - "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test,pandas-tests]" + "$(echo ./dist/pylibcudf_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"} RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/