create conda ci test env in one step (#1387)

This is a port of rapidsai/cuml#5781 and part of rapidsai/build-planning#22 Authors: - Mike Sarahan (https://github.com/msarahan) - Paul Taylor (https://github.com/trxcllnt) Approvers: - Paul Taylor (https://github.com/trxcllnt) - https://github.com/jakirkham - Ray Douglass (https://github.com/raydouglass) URL: #1387
rapidsai · May 24, 2024 · eff6753 · eff6753
1 parent 61b8cc1
commit eff6753
Show file tree

Hide file tree

Showing 8 changed files with 53 additions and 41 deletions.
diff --git a/ci/build_docs.sh b/ci/build_docs.sh
@@ -6,27 +6,21 @@ set -euo pipefail
 rapids-logger "Create test conda environment"
 . /opt/conda/etc/profile.d/conda.sh
 
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL="$(rapids-download-conda-from-s3 cpp)"
+PYTHON_CHANNEL="$(rapids-download-conda-from-s3 python)"
+
 rapids-dependency-file-generator \
   --output conda \
   --file_key docs \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
+  --prepend-channel "${CPP_CHANNEL}" --prepend-channel "${PYTHON_CHANNEL}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n docs
 conda activate docs
 
 rapids-print-env
 
-rapids-logger "Downloading artifacts from previous jobs"
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
-
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${PYTHON_CHANNEL}" \
-  libcuspatial \
-  cuspatial \
-  cuproj
-
 export RAPIDS_VERSION="$(rapids-version)"
 export RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)"
 export RAPIDS_VERSION_NUMBER="24.06"

diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh
@@ -45,7 +45,10 @@ NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; prin
 DEPENDENCIES=(
   cudf
   cuml
+  cuspatial
   libcudf
+  libcuspatial
+  libcuspatial-tests
   librmm
   rmm
   cuspatial

diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh
@@ -5,11 +5,16 @@ set -euo pipefail
 
 . /opt/conda/etc/profile.d/conda.sh
 
+
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+
 rapids-logger "Generate C++ testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_cpp \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml
+  --file-key test_cpp \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" \
+  --prepend-channel "${CPP_CHANNEL}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test
 
@@ -18,7 +23,6 @@ set +u
 conda activate test
 set -u
 
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/
 mkdir -p "${RAPIDS_TESTS_DIR}"
 
@@ -27,10 +31,6 @@ export CUSPATIAL_HOME="${PWD}"
 
 rapids-print-env
 
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  libcuspatial libcuspatial-tests
-
 rapids-logger "Check GPU usage"
 nvidia-smi
 

diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh
@@ -5,11 +5,16 @@ set -euo pipefail
 
 . /opt/conda/etc/profile.d/conda.sh
 
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
+
 rapids-logger "Generate notebook testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_notebooks \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+  --file-key test_notebooks \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
+  --prepend-channel "${CPP_CHANNEL}" --prepend-channel "${PYTHON_CHANNEL}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test
 
@@ -20,15 +25,6 @@ set -u
 
 rapids-print-env
 
-rapids-logger "Downloading artifacts from previous jobs"
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
-
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${PYTHON_CHANNEL}" \
-  cuspatial libcuspatial cuproj
-
 NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")"
 pushd notebooks
 

diff --git a/ci/test_python.sh b/ci/test_python.sh
@@ -5,11 +5,16 @@ set -euo pipefail
 
 . /opt/conda/etc/profile.d/conda.sh
 
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
+
 rapids-logger "Generate Python testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_python \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+  --file-key test_python \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
+  --prepend-channel "${CPP_CHANNEL}" --prepend-channel "${PYTHON_CHANNEL}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test
 
@@ -18,10 +23,6 @@ set +u
 conda activate test
 set -u
 
-rapids-logger "Downloading artifacts from previous jobs"
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
-
 RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}
 RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"}
 mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}"
@@ -31,11 +32,6 @@ export CUSPATIAL_HOME="${PWD}"
 
 rapids-print-env
 
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${PYTHON_CHANNEL}" \
-  libcuspatial cuspatial cuproj
-
 rapids-logger "Check GPU usage"
 nvidia-smi
 

diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml
@@ -23,6 +23,8 @@ dependencies:
 - ipython
 - ipywidgets
 - libcudf==24.6.*
+- libcuspatial-tests==24.6.*
+- libcuspatial==24.6.*
 - librmm==24.6.*
 - myst-parser
 - nbsphinx

diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml
@@ -26,6 +26,8 @@ dependencies:
 - ipython
 - ipywidgets
 - libcudf==24.6.*
+- libcuspatial-tests==24.6.*
+- libcuspatial==24.6.*
 - librmm==24.6.*
 - myst-parser
 - nbsphinx

diff --git a/dependencies.yaml b/dependencies.yaml
@@ -18,27 +18,31 @@ files:
       - depends_on_cuml
       - depends_on_cupy
       - run_python_cuspatial
+      - test_libcuspatial
       - test_python_cuspatial
       - test_python_cuproj
       - notebooks
   test_cpp:
     output: none
     includes:
       - cuda_version
+      - test_libcuspatial
   test_python:
     output: none
     includes:
       - cuda_version
       - py_version
       - test_python_cuspatial
       - test_python_cuproj
+      - test_cuspatial
   test_notebooks:
     output: none
     includes:
       - cuda_version
       - depends_on_cuml
       - notebooks
       - py_version
+      - test_cuspatial
   checks:
     output: none
     includes:
@@ -50,6 +54,7 @@ files:
       - cuda_version
       - docs
       - py_version
+      - test_cuspatial
   py_build_cuspatial:
     output: [pyproject]
     pyproject_dir: python/cuspatial
@@ -105,6 +110,7 @@ files:
     includes:
       - test_python_cuproj
       - depends_on_cuspatial
+      - test_cuspatial
 
 channels:
   - rapidsai
@@ -451,3 +457,16 @@ dependencies:
             packages:
               - cupy-cuda11x>=12.0.0
           - {matrix: null, packages: [cupy-cuda11x>=12.0.0]}
+  test_libcuspatial:
+    common:
+      - output_types: conda
+        packages:
+          - libcuspatial==24.6.*
+          - libcuspatial-tests==24.6.*
+  test_cuspatial:
+    common:
+      - output_types: conda
+        packages:
+          - libcuspatial==24.6.*
+          - cuspatial==24.6.*
+          - cuproj==24.6.*