Merge branch 'add-ci' of https://github.com/alexbarghi-nv/cugraph-gnn …

…into add-ci
rapidsai · Oct 11, 2024 · 78c4db4 · 78c4db4
2 parents 66bcde6 + 5db48ab
commit 78c4db4
Show file tree

Hide file tree

Showing 9 changed files with 62 additions and 68 deletions.
diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh
@@ -5,6 +5,8 @@ set -Eeuo pipefail
 
 . /opt/conda/etc/profile.d/conda.sh
 
+RAPIDS_VERSION="$(rapids-version)"
+
 rapids-logger "Generate notebook testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
@@ -27,7 +29,9 @@ PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
 rapids-mamba-retry install \
   --channel "${CPP_CHANNEL}" \
   --channel "${PYTHON_CHANNEL}" \
-  libcugraph pylibcugraph cugraph
+  "libcugraph=${RAPIDS_VERSION}" \
+  "pylibcugraph=${RAPIDS_VERSION}" \
+  "cugraph=${RAPIDS_VERSION}"
 
 NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")"
 NOTEBOOK_LIST="$(realpath "$(dirname "$0")/notebook_list.py")"

diff --git a/ci/test_python.sh b/ci/test_python.sh
@@ -8,6 +8,8 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
 
 . /opt/conda/etc/profile.d/conda.sh
 
+RAPIDS_VERSION="$(rapids-version)"
+
 rapids-logger "Generate Python testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
@@ -59,10 +61,10 @@ if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then
     --channel conda-forge \
     --channel dglteam/label/th23_cu118 \
     --channel nvidia \
-    "pylibwholegraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-    "pylibcugraphops=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-    "cugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-    "cugraph-dgl=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
+    "pylibwholegraph=${RAPIDS_VERSION}.*" \
+    "pylibcugraphops=${RAPIDS_VERSION}.*" \
+    "cugraph=${RAPIDS_VERSION}.*" \
+    "cugraph-dgl=${RAPIDS_VERSION}.*" \
     'pytorch::pytorch>=2.3,<2.4' \
     'cuda-version=11.8' \
     "ogb"
@@ -111,10 +113,10 @@ if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then
     --channel "${CPP_CHANNEL}" \
     --channel "${PYTHON_CHANNEL}" \
     --channel pytorch \
-    "pylibwholegraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-    "pylibcugraphops=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-    "cugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-    "cugraph-pyg=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
+    "pylibwholegraph=${RAPIDS_VERSION}.*" \
+    "pylibcugraphops=${RAPIDS_VERSION}.*" \
+    "cugraph=${RAPIDS_VERSION}.*" \
+    "cugraph-pyg=${RAPIDS_VERSION}.*" \
     "pytorch::pytorch>=2.3,<2.4" \
     "ogb"
 
@@ -160,7 +162,7 @@ if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then
     --channel "${CPP_CHANNEL}" \
     --channel "${PYTHON_CHANNEL}" \
     --channel pytorch \
-    "pylibwholegraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
+    "pylibwholegraph=${RAPIDS_VERSION}.*" \
     "pytorch::pytorch>=2.3,<2.4" \
     "ogb"
 

diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh
@@ -4,7 +4,6 @@
 set -eoxu pipefail
 
 package_name=$1
-package_dir=$2
 
 python_package_name=$(echo ${package_name}|sed 's/-/_/g')
 

diff --git a/ci/test_wheel_cugraph-dgl.sh b/ci/test_wheel_cugraph-dgl.sh
@@ -4,24 +4,16 @@
 set -eoxu pipefail
 
 package_name="cugraph-dgl"
-package_dir="python/cugraph-dgl"
-
-python_package_name=$(echo ${package_name}|sed 's/-/_/g')
 
 mkdir -p ./dist
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-# Download wheels built during this job.
+# Download the pylibcugraph, cugraph, and cugraph-dgl built in the previous step
 RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
 RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
-python -m pip install ./local-deps/*.whl
-
-# use 'ls' to expand wildcard before adding `[extra]` requires for pip
 RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist
-# pip creates wheels using python package names
-python -m pip install $(ls ./dist/${python_package_name}*.whl)[test]
-
 
+# determine pytorch and DGL sources
 PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')"
 PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2}
 if [[ "${PKG_CUDA_VER_MAJOR}" == "12" ]]; then
@@ -30,20 +22,17 @@ else
   PYTORCH_CUDA_VER=$PKG_CUDA_VER
 fi
 PYTORCH_URL="https://download.pytorch.org/whl/cu${PYTORCH_CUDA_VER}"
-DGL_URL="https://data.dgl.ai/wheels/cu${PYTORCH_CUDA_VER}/repo.html"
-
-# Starting from 2.2, PyTorch wheels depend on nvidia-nccl-cuxx>=2.19 wheel and
-# dynamically link to NCCL. RAPIDS CUDA 11 CI images have an older NCCL version that
-# might shadow the newer NCCL required by PyTorch during import (when importing
-# `cupy` before `torch`).
-if [[ "${NCCL_VERSION}" < "2.19" ]]; then
-  PYTORCH_VER="2.1.0"
-else
-  PYTORCH_VER="2.3.0"
-fi
-
-rapids-logger "Installing PyTorch and DGL"
-rapids-retry python -m pip install "torch==${PYTORCH_VER}" --index-url ${PYTORCH_URL}
-rapids-retry python -m pip install dgl==2.0.0 --find-links ${DGL_URL}
+DGL_URL="https://data.dgl.ai/wheels/torch-2.3/cu${PYTORCH_CUDA_VER}/repo.html"
+
+# echo to expand wildcard before adding `[extra]` requires for pip
+python -m pip install \
+    -v \
+    --extra-index-url "${PYTORCH_URL}" \
+    --find-links "${DGL_URL}" \
+    "$(echo ./local-deps/pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
+    "$(echo ./local-deps/cugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
+    "$(echo ./dist/cugraph_dgl_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \
+    'dgl==2.4.0' \
+    'torch>=2.3.0,<2.4'
 
 python -m pytest python/cugraph-dgl/tests
diff --git a/ci/test_wheel_cugraph-pyg.sh b/ci/test_wheel_cugraph-pyg.sh
@@ -4,46 +4,44 @@
 set -eoxu pipefail
 
 package_name="cugraph-pyg"
-package_dir="python/cugraph-pyg"
-
-python_package_name=$(echo ${package_name}|sed 's/-/_/g')
 
 mkdir -p ./dist
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-# Download wheels built during this job.
+# Download the pylibcugraph, cugraph, and cugraph-pyg built in the previous step
 RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
 RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
-python -m pip install ./local-deps/*.whl
-
-# use 'ls' to expand wildcard before adding `[extra]` requires for pip
 RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist
-# pip creates wheels using python package names
-python -m pip install $(ls ./dist/${python_package_name}*.whl)[test]
-
-# RAPIDS_DATASET_ROOT_DIR is used by test scripts
-export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)"
-
-# Used to skip certain examples in CI due to memory limitations
-export CI_RUN=1
 
+# determine pytorch and pyg sources
 if [[ "${CUDA_VERSION}" == "11.8.0" ]]; then
   PYTORCH_URL="https://download.pytorch.org/whl/cu118"
-  PYG_URL="https://data.pyg.org/whl/torch-2.1.0+cu118.html"
+  PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu118.html"
 else
   PYTORCH_URL="https://download.pytorch.org/whl/cu121"
-  PYG_URL="https://data.pyg.org/whl/torch-2.1.0+cu121.html"
+  PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu121.html"
 fi
-rapids-logger "Installing PyTorch and PyG dependencies"
-rapids-retry python -m pip install torch==2.1.0 --index-url ${PYTORCH_URL}
-rapids-retry python -m pip install "torch-geometric>=2.5,<2.6"
-rapids-retry python -m pip install \
-  ogb \
-  pyg_lib \
-  torch_scatter \
-  torch_sparse \
-  tensordict \
-  -f ${PYG_URL}
+
+# echo to expand wildcard before adding `[extra]` requires for pip
+python -m pip install \
+    -v \
+    --extra-index-url "${PYTORCH_URL}" \
+    --find-links "${PYG_URL}" \
+    "$(echo ./local-deps/pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
+    "$(echo ./local-deps/cugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
+    "$(echo ./dist/cugraph_pyg_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \
+    'ogb' \
+    'pyg_lib' \
+    'torch>=2.3.0,<2.4' \
+    'torch-geometric>=2.5,<2.6' \
+    'torch_scatter' \
+    'torch_sparse'
+
+# RAPIDS_DATASET_ROOT_DIR is used by test scripts
+export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)"
+
+# Used to skip certain examples in CI due to memory limitations
+export CI_RUN=1
 
 rapids-logger "pytest cugraph-pyg (single GPU)"
 pushd python/cugraph-pyg/cugraph_pyg

diff --git a/conda/recipes/cugraph-dgl/meta.yaml b/conda/recipes/cugraph-dgl/meta.yaml
@@ -23,13 +23,13 @@ requirements:
     - python
   run:
     - cugraph ={{ version }}
-    - dgl >=1.1.0.cu*
+    - dgl >=2.4.0.th23.cu*
     - numba >=0.57
     - numpy >=1.23,<3.0a0
     - pylibcugraphops ={{ minor_version }}
     - tensordict >=0.1.2
     - python
-    - pytorch >=2.0
+    - pytorch >=2.3,<2.4.0a0
     - cupy >=12.0.0
 
 tests:

diff --git a/conda/recipes/cugraph-pyg/meta.yaml b/conda/recipes/cugraph-pyg/meta.yaml
@@ -30,12 +30,12 @@ requirements:
     - numba >=0.57
     - numpy >=1.23,<3.0a0
     - python
-    - pytorch >=2.0
+    - pytorch >=2.3,<2.4.0a0
     - cupy >=12.0.0
     - cugraph ={{ version }}
     - pylibcugraphops ={{ minor_version }}
     - tensordict >=0.1.2
-    - pyg >=2.5,<2.6
+    - pytorch_geometric >=2.5,<2.6
 
 tests:
   imports:

diff --git a/conda/recipes/libwholegraph/conda_build_config.yaml b/conda/recipes/libwholegraph/conda_build_config.yaml
@@ -17,7 +17,7 @@ doxygen_version:
   - ">=1.8.11"
 
 nccl_version:
-  - ">=2.9.9"
+  - ">=2.19"
 
 c_stdlib:
   - sysroot

diff --git a/conda/recipes/pylibwholegraph/meta.yaml b/conda/recipes/pylibwholegraph/meta.yaml
@@ -75,4 +75,6 @@ requirements:
 
 about:
   home: https://rapids.ai/
+  license: Apache-2.0
+  license_file: ../../../LICENSE
   summary: pylibwholegraph library