From eff6753c9d6b1e18e965b19cd2730f69bf5c6925 Mon Sep 17 00:00:00 2001
From: Mike Sarahan <msarahan@gmail.com>
Date: Thu, 23 May 2024 19:53:07 -0500
Subject: [PATCH] create conda ci test env in one step (#1387)

This is a port of https://github.com/rapidsai/cuml/pull/5781 and part of https://github.com/rapidsai/build-planning/issues/22

Authors:
  - Mike Sarahan (https://github.com/msarahan)
  - Paul Taylor (https://github.com/trxcllnt)

Approvers:
  - Paul Taylor (https://github.com/trxcllnt)
  - https://github.com/jakirkham
  - Ray Douglass (https://github.com/raydouglass)

URL: https://github.com/rapidsai/cuspatial/pull/1387
---
 ci/build_docs.sh                              | 18 ++++++------------
 ci/release/update-version.sh                  |  3 +++
 ci/test_cpp.sh                                | 14 +++++++-------
 ci/test_notebooks.sh                          | 18 +++++++-----------
 ci/test_python.sh                             | 18 +++++++-----------
 .../all_cuda-118_arch-x86_64.yaml             |  2 ++
 .../all_cuda-122_arch-x86_64.yaml             |  2 ++
 dependencies.yaml                             | 19 +++++++++++++++++++
 8 files changed, 53 insertions(+), 41 deletions(-)

diff --git a/ci/build_docs.sh b/ci/build_docs.sh
index 15eb639a0..0d4e97488 100755
--- a/ci/build_docs.sh
+++ b/ci/build_docs.sh
@@ -6,27 +6,21 @@ set -euo pipefail
 rapids-logger "Create test conda environment"
 . /opt/conda/etc/profile.d/conda.sh
 
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL="$(rapids-download-conda-from-s3 cpp)"
+PYTHON_CHANNEL="$(rapids-download-conda-from-s3 python)"
+
 rapids-dependency-file-generator \
   --output conda \
   --file_key docs \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
+  --prepend-channel "${CPP_CHANNEL}" --prepend-channel "${PYTHON_CHANNEL}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n docs
 conda activate docs
 
 rapids-print-env
 
-rapids-logger "Downloading artifacts from previous jobs"
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
-
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${PYTHON_CHANNEL}" \
-  libcuspatial \
-  cuspatial \
-  cuproj
-
 export RAPIDS_VERSION="$(rapids-version)"
 export RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)"
 export RAPIDS_VERSION_NUMBER="24.06"
diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh
index 5392183dc..ae13d7629 100755
--- a/ci/release/update-version.sh
+++ b/ci/release/update-version.sh
@@ -45,7 +45,10 @@ NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; prin
 DEPENDENCIES=(
   cudf
   cuml
+  cuspatial
   libcudf
+  libcuspatial
+  libcuspatial-tests
   librmm
   rmm
   cuspatial
diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh
index 35a981e93..0f0a09ef1 100755
--- a/ci/test_cpp.sh
+++ b/ci/test_cpp.sh
@@ -5,11 +5,16 @@ set -euo pipefail
 
 . /opt/conda/etc/profile.d/conda.sh
 
+
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+
 rapids-logger "Generate C++ testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_cpp \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml
+  --file-key test_cpp \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" \
+  --prepend-channel "${CPP_CHANNEL}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test
 
@@ -18,7 +23,6 @@ set +u
 conda activate test
 set -u
 
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/
 mkdir -p "${RAPIDS_TESTS_DIR}"
 
@@ -27,10 +31,6 @@ export CUSPATIAL_HOME="${PWD}"
 
 rapids-print-env
 
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  libcuspatial libcuspatial-tests
-
 rapids-logger "Check GPU usage"
 nvidia-smi
 
diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh
index a4ae9c21c..61ae4f194 100755
--- a/ci/test_notebooks.sh
+++ b/ci/test_notebooks.sh
@@ -5,11 +5,16 @@ set -euo pipefail
 
 . /opt/conda/etc/profile.d/conda.sh
 
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
+
 rapids-logger "Generate notebook testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_notebooks \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+  --file-key test_notebooks \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
+  --prepend-channel "${CPP_CHANNEL}" --prepend-channel "${PYTHON_CHANNEL}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test
 
@@ -20,15 +25,6 @@ set -u
 
 rapids-print-env
 
-rapids-logger "Downloading artifacts from previous jobs"
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
-
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${PYTHON_CHANNEL}" \
-  cuspatial libcuspatial cuproj
-
 NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")"
 pushd notebooks
 
diff --git a/ci/test_python.sh b/ci/test_python.sh
index c6b80ed45..9a7cada37 100755
--- a/ci/test_python.sh
+++ b/ci/test_python.sh
@@ -5,11 +5,16 @@ set -euo pipefail
 
 . /opt/conda/etc/profile.d/conda.sh
 
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
+
 rapids-logger "Generate Python testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_python \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+  --file-key test_python \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
+  --prepend-channel "${CPP_CHANNEL}" --prepend-channel "${PYTHON_CHANNEL}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test
 
@@ -18,10 +23,6 @@ set +u
 conda activate test
 set -u
 
-rapids-logger "Downloading artifacts from previous jobs"
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
-
 RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}
 RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"}
 mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}"
@@ -31,11 +32,6 @@ export CUSPATIAL_HOME="${PWD}"
 
 rapids-print-env
 
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${PYTHON_CHANNEL}" \
-  libcuspatial cuspatial cuproj
-
 rapids-logger "Check GPU usage"
 nvidia-smi
 
diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml
index 1ec5a27e3..ae933a67c 100644
--- a/conda/environments/all_cuda-118_arch-x86_64.yaml
+++ b/conda/environments/all_cuda-118_arch-x86_64.yaml
@@ -23,6 +23,8 @@ dependencies:
 - ipython
 - ipywidgets
 - libcudf==24.6.*
+- libcuspatial-tests==24.6.*
+- libcuspatial==24.6.*
 - librmm==24.6.*
 - myst-parser
 - nbsphinx
diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml
index 9619b19fb..4f48db253 100644
--- a/conda/environments/all_cuda-122_arch-x86_64.yaml
+++ b/conda/environments/all_cuda-122_arch-x86_64.yaml
@@ -26,6 +26,8 @@ dependencies:
 - ipython
 - ipywidgets
 - libcudf==24.6.*
+- libcuspatial-tests==24.6.*
+- libcuspatial==24.6.*
 - librmm==24.6.*
 - myst-parser
 - nbsphinx
diff --git a/dependencies.yaml b/dependencies.yaml
index 97af06191..bd5d25e11 100644
--- a/dependencies.yaml
+++ b/dependencies.yaml
@@ -18,6 +18,7 @@ files:
       - depends_on_cuml
       - depends_on_cupy
       - run_python_cuspatial
+      - test_libcuspatial
       - test_python_cuspatial
       - test_python_cuproj
       - notebooks
@@ -25,6 +26,7 @@ files:
     output: none
     includes:
       - cuda_version
+      - test_libcuspatial
   test_python:
     output: none
     includes:
@@ -32,6 +34,7 @@ files:
       - py_version
       - test_python_cuspatial
       - test_python_cuproj
+      - test_cuspatial
   test_notebooks:
     output: none
     includes:
@@ -39,6 +42,7 @@ files:
       - depends_on_cuml
       - notebooks
       - py_version
+      - test_cuspatial
   checks:
     output: none
     includes:
@@ -50,6 +54,7 @@ files:
       - cuda_version
       - docs
       - py_version
+      - test_cuspatial
   py_build_cuspatial:
     output: [pyproject]
     pyproject_dir: python/cuspatial
@@ -105,6 +110,7 @@ files:
     includes:
       - test_python_cuproj
       - depends_on_cuspatial
+      - test_cuspatial
 
 channels:
   - rapidsai
@@ -451,3 +457,16 @@ dependencies:
             packages:
               - cupy-cuda11x>=12.0.0
           - {matrix: null, packages: [cupy-cuda11x>=12.0.0]}
+  test_libcuspatial:
+    common:
+      - output_types: conda
+        packages:
+          - libcuspatial==24.6.*
+          - libcuspatial-tests==24.6.*
+  test_cuspatial:
+    common:
+      - output_types: conda
+        packages:
+          - libcuspatial==24.6.*
+          - cuspatial==24.6.*
+          - cuproj==24.6.*