From b49ff8dac434893cf38844777430553438d3c18f Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Wed, 20 Dec 2023 22:18:32 -0600 Subject: [PATCH 1/2] Use dependencies.yaml as source-of-truth for environment files. (#1441) Adds rapids-dependency-file-generator pre-commit checks for local and CI, and updates scripts to use the generated environment files. Contributes to #1389 ## By Submitting this PR I confirm: - I am familiar with the [Contributing Guidelines](https://github.com/nv-morpheus/Morpheus/blob/main/docs/source/developer_guide/contributing.md). - When the PR is ready for review, new or existing tests cover these changes. - When the PR is ready for review, the documentation is up to date with these changes. Authors: - Christopher Harris (https://github.com/cwharris) Approvers: - Michael Demoret (https://github.com/mdemoret-nv) URL: https://github.com/nv-morpheus/Morpheus/pull/1441 --- .../opt/morpheus/bin/post-attach-command.sh | 4 +- .github/workflows/ci_pipe.yml | 3 - .../workflows/{pull_request.yml => pr.yaml} | 18 +- .gitmodules | 2 +- .pre-commit-config.yaml | 24 +++ ci/check_style.sh | 23 +++ ci/conda/recipes/python-dbg/meta.yaml | 2 +- ci/release/update-version.sh | 4 +- ci/scripts/copyright.py | 3 +- ci/scripts/github/build.sh | 7 +- ci/scripts/github/checks.sh | 7 +- ci/scripts/github/common.sh | 32 +-- ci/scripts/github/docs.sh | 7 +- ci/scripts/github/test.sh | 11 +- ci/scripts/run_ci_local.sh | 12 +- .../all_cuda-118_arch-x86_64.yaml | 116 +++++++++++ .../dev_cuda-118_arch-x86_64.yaml | 92 +++++++++ dependencies.yaml | 193 +++++++++++++++++- 18 files changed, 499 insertions(+), 61 deletions(-) rename .github/workflows/{pull_request.yml => pr.yaml} (83%) create mode 100644 .pre-commit-config.yaml create mode 100755 ci/check_style.sh create mode 100644 conda/environments/all_cuda-118_arch-x86_64.yaml create mode 100644 conda/environments/dev_cuda-118_arch-x86_64.yaml diff --git a/.devcontainer/opt/morpheus/bin/post-attach-command.sh b/.devcontainer/opt/morpheus/bin/post-attach-command.sh index c1384cba6d..43729d7367 100755 --- a/.devcontainer/opt/morpheus/bin/post-attach-command.sh +++ b/.devcontainer/opt/morpheus/bin/post-attach-command.sh @@ -27,6 +27,6 @@ ENV_NAME=${ENV_NAME:-morpheus} sed -ri "s/conda activate base/conda activate $ENV_NAME/g" ~/.bashrc; if conda_env_find "${ENV_NAME}" ; \ -then mamba env update --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.8_dev.yml --prune; \ -else mamba env create --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.8_dev.yml; \ +then mamba env update --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/conda/environments/all_cuda-118_arch-x86_64.yaml --prune; \ +else mamba env create --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/conda/environments/all_cuda-118_arch-x86_64.yaml; \ fi diff --git a/.github/workflows/ci_pipe.yml b/.github/workflows/ci_pipe.yml index c1a92cdbd5..4989172fb3 100644 --- a/.github/workflows/ci_pipe.yml +++ b/.github/workflows/ci_pipe.yml @@ -132,7 +132,6 @@ jobs: env: NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} PARALLEL_LEVEL: '10' - MERGE_EXAMPLES_YAML: '1' strategy: fail-fast: true @@ -164,8 +163,6 @@ jobs: username: '$oauthtoken' password: ${{ secrets.NGC_API_KEY }} image: ${{ inputs.container }} - env: - MERGE_DOCS_YAML: '1' strategy: fail-fast: true diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pr.yaml similarity index 83% rename from .github/workflows/pull_request.yml rename to .github/workflows/pr.yaml index b0a799047b..0109a5eec7 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pr.yaml @@ -42,6 +42,15 @@ permissions: statuses: none jobs: + + pr-builder: + needs: + - prepare + - checks + - ci_pipe + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.02 + prepare: # Executes the get-pr-info action to determine if the PR has the skip-ci label, if the action fails we assume the # PR does not have the label @@ -57,11 +66,18 @@ jobs: outputs: has_skip_ci_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'skip-ci') || false }} + checks: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.02 + # Only run the CI pipeline if the PR does not have the skip-ci label + if: ${{ ! fromJSON(needs.prepare.outputs.has_skip_ci_label) }} + with: + enable_check_generated_files: false + ci_pipe: name: CI Pipeline needs: [prepare] uses: ./.github/workflows/ci_pipe.yml - # Only run the CI pipeline if the PR does not have the skip-ci label if: ${{ ! fromJSON(needs.prepare.outputs.has_skip_ci_label) }} with: diff --git a/.gitmodules b/.gitmodules index 4cb2d7fd83..a92915a355 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,7 +2,7 @@ path = external/morpheus-visualizations url = https://github.com/nv-morpheus/morpheus-visualizations.git branch = branch-24.03 -[submodule "morpheus_utils"] +[submodule "external/utilities"] path = external/utilities url = https://github.com/nv-morpheus/utilities.git branch = branch-24.03 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..956031a526 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +repos: + - repo: https://github.com/rapidsai/dependency-file-generator + rev: v1.7.1 + hooks: + - id: rapids-dependency-file-generator + args: ["--clean"] + +default_language_version: + python: python3 diff --git a/ci/check_style.sh b/ci/check_style.sh new file mode 100755 index 0000000000..82b6ab825d --- /dev/null +++ b/ci/check_style.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +rapids-logger "Create checks conda environment" +. /opt/conda/etc/profile.d/conda.sh + +rapids-dependency-file-generator \ + --output conda \ + --file_key checks \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + +rapids-mamba-retry env create --force -f env.yaml -n checks +conda activate checks + +# Run pre-commit checks +pre-commit run --all-files --show-diff-on-failure diff --git a/ci/conda/recipes/python-dbg/meta.yaml b/ci/conda/recipes/python-dbg/meta.yaml index 3bf036a532..fe56414f5d 100644 --- a/ci/conda/recipes/python-dbg/meta.yaml +++ b/ci/conda/recipes/python-dbg/meta.yaml @@ -16,7 +16,7 @@ {% set version = environ.get('MORPHEUS_PYTHON_VER') %} package: - name: python + name: morpheus-python-dbg version: {{ version }} source: diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index d072bbe2c9..dd8023344f 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -77,8 +77,8 @@ fi # Root CMakeLists.txt sed_runner 's/'"VERSION ${CURRENT_FULL_VERSION}.*"'/'"VERSION ${NEXT_FULL_VERSION}"'/g' CMakeLists.txt -# Conda environment file -sed_runner "s/mrc=${CURRENT_SHORT_TAG}/mrc=${NEXT_SHORT_TAG}/g" docker/conda/environments/cuda11.8_dev.yml +# Depedencies file +sed_runner "s/mrc=${CURRENT_SHORT_TAG}/mrc=${NEXT_SHORT_TAG}/g" dependencies.yaml # examples/digital_fingerprinting sed_runner "s/v${CURRENT_FULL_VERSION}-runtime/v${NEXT_FULL_VERSION}-runtime/g" \ diff --git a/ci/scripts/copyright.py b/ci/scripts/copyright.py index 3eab43e8c9..102afbb456 100755 --- a/ci/scripts/copyright.py +++ b/ci/scripts/copyright.py @@ -48,7 +48,8 @@ re.compile(r"^[^ \/\n]*build[^ \/\n]*\/.*$"), # Ignore any build*/ folder re.compile(r"^external\/.*$"), # Ignore external re.compile(r"[^ \/\n]*docs/source/(_lib|_modules|_templates)/.*$"), - re.compile(r"PULL_REQUEST_TEMPLATE.md"), # Ignore the PR template + re.compile(r"PULL_REQUEST_TEMPLATE.md"), # Ignore the PR template, + re.compile(r"[^ \/\n]*conda/environments/.*\.yaml$"), # Ignore generated environment files ] # this will break starting at year 10000, which is probably OK :) diff --git a/ci/scripts/github/build.sh b/ci/scripts/github/build.sh index c03cc60d84..35fcd2d9f8 100755 --- a/ci/scripts/github/build.sh +++ b/ci/scripts/github/build.sh @@ -18,7 +18,12 @@ set -e source ${WORKSPACE}/ci/scripts/github/common.sh -update_conda_env +rapids-dependency-file-generator \ + --output conda \ + --file_key build \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + +update_conda_env env.yaml log_toolchain diff --git a/ci/scripts/github/checks.sh b/ci/scripts/github/checks.sh index 7426754590..4abfacd157 100755 --- a/ci/scripts/github/checks.sh +++ b/ci/scripts/github/checks.sh @@ -18,7 +18,12 @@ set -e source ${WORKSPACE}/ci/scripts/github/common.sh -update_conda_env +rapids-dependency-file-generator \ + --output conda \ + --file_key build \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + +update_conda_env env.yaml log_toolchain diff --git a/ci/scripts/github/common.sh b/ci/scripts/github/common.sh index 8756ea09bd..685ba7282a 100644 --- a/ci/scripts/github/common.sh +++ b/ci/scripts/github/common.sh @@ -61,10 +61,6 @@ export SCCACHE_REGION="us-east-2" export SCCACHE_IDLE_TIMEOUT=32768 #export SCCACHE_LOG=debug -export CONDA_ENV_YML=${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_dev.yml -export CONDA_EXAMPLES_YML=${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER}_examples.yml -export CONDA_DOCS_YML=${MORPHEUS_ROOT}/docs/conda_docs.yml - export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DMORPHEUS_CUDA_ARCHITECTURES=60;70;75;80 -DMORPHEUS_BUILD_BENCHMARKS=ON -DMORPHEUS_BUILD_EXAMPLES=ON -DMORPHEUS_BUILD_TESTS=ON -DMORPHEUS_USE_CONDA=ON -DMORPHEUS_PYTHON_INPLACE_BUILD=OFF -DMORPHEUS_PYTHON_BUILD_STUBS=ON -DMORPHEUS_USE_CCACHE=ON" export FETCH_STATUS=0 @@ -75,31 +71,10 @@ function update_conda_env() { # Deactivate the environment first before updating conda deactivate - ENV_YAML=${CONDA_ENV_YML} - if [[ "${MERGE_EXAMPLES_YAML}" == "1" || "${MERGE_DOCS_YAML}" == "1" ]]; then - # Merge the dev, docs and examples envs, otherwise --prune will remove the examples packages - ENV_YAML=${WORKSPACE_TMP}/merged_env.yml - YAMLS="${CONDA_ENV_YML}" - if [[ "${MERGE_EXAMPLES_YAML}" == "1" ]]; then - YAMLS="${YAMLS} ${CONDA_EXAMPLES_YML}" - fi - if [[ "${MERGE_DOCS_YAML}" == "1" ]]; then - YAMLS="${YAMLS} ${CONDA_DOCS_YML}" - fi - - rapids-logger "Merging conda envs: ${YAMLS}" - conda run -n morpheus --live-stream conda-merge ${YAMLS} > ${ENV_YAML} - fi - - if [[ "${SKIP_CONDA_ENV_UPDATE}" == "" ]]; then - rapids-logger "Checking for updates to conda env" + rapids-logger "Checking for updates to conda env" - # Remove default/conflicting channels from base image - rm /opt/conda/.condarc - - # Update the packages - rapids-mamba-retry env update -n morpheus --prune -q --file ${ENV_YAML} - fi + # Update the packages + rapids-mamba-retry env update -n morpheus --prune -q --file "$1" # Finally, reactivate conda activate morpheus @@ -132,6 +107,7 @@ function fetch_base_branch_gh_api() { function fetch_base_branch_local() { rapids-logger "Retrieving base branch from git" + git remote remove upstream git remote add upstream ${GIT_UPSTREAM_URL} git fetch upstream --tags source ${MORPHEUS_ROOT}/ci/scripts/common.sh diff --git a/ci/scripts/github/docs.sh b/ci/scripts/github/docs.sh index 4c2a5703f2..d03f79ebb0 100755 --- a/ci/scripts/github/docs.sh +++ b/ci/scripts/github/docs.sh @@ -18,7 +18,12 @@ set -e source ${WORKSPACE}/ci/scripts/github/common.sh -update_conda_env +rapids-dependency-file-generator \ + --output conda \ + --file_key docs \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + +update_conda_env env.yaml download_artifact "wheel.tar.bz" diff --git a/ci/scripts/github/test.sh b/ci/scripts/github/test.sh index d96c2ed888..cb45189733 100755 --- a/ci/scripts/github/test.sh +++ b/ci/scripts/github/test.sh @@ -19,7 +19,12 @@ set -e source ${WORKSPACE}/ci/scripts/github/common.sh /usr/bin/nvidia-smi -update_conda_env +rapids-dependency-file-generator \ + --output conda \ + --file_key test \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + +update_conda_env env.yaml rapids-logger "Check versions" python3 --version @@ -69,8 +74,8 @@ rapids-logger "Pulling LFS assets" git lfs install ${MORPHEUS_ROOT}/scripts/fetch_data.py fetch tests validation -# List missing files -rapids-logger "Listing missing files" +# Listing LFS-known files +rapids-logger "Listing LFS-known files" git lfs ls-files REPORTS_DIR="${WORKSPACE_TMP}/reports" diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh index 4b13ef4f7f..96248fcc35 100755 --- a/ci/scripts/run_ci_local.sh +++ b/ci/scripts/run_ci_local.sh @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -exo pipefail + case "$1" in "" ) STAGES=("bash") @@ -72,16 +74,12 @@ for STAGE in "${STAGES[@]}"; do DOCKER_RUN_ARGS="--rm -ti --net=host -v "${LOCAL_CI_TMP}":/ci_tmp ${ENV_LIST} --env STAGE=${STAGE}" if [[ "${STAGE}" == "test" || "${USE_GPU}" == "1" ]]; then CONTAINER="${TEST_CONTAINER}" - DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=nvidia --gpus all" - if [[ "${STAGE}" == "test" ]]; then - DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --env MERGE_EXAMPLES_YAML=1 --cap-add=sys_nice" - fi + DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=nvidia" + DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --gpus all" + DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --cap-add=sys_nice" else CONTAINER="${BUILD_CONTAINER}" DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --runtime=runc" - if [[ "${STAGE}" == "docs" ]]; then - DOCKER_RUN_ARGS="${DOCKER_RUN_ARGS} --env MERGE_DOCS_YAML=1" - fi fi if [[ "${STAGE}" == "bash" ]]; then diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml new file mode 100644 index 0000000000..d2557e605b --- /dev/null +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -0,0 +1,116 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- conda-forge +- huggingface +- rapidsai +- nvidia/label/cuda-11.8.0 +- nvidia +- rapidsai-nightly +- nvidia/label/dev +- pytorch +- dglteam/label/cu118 +dependencies: +- appdirs +- arxiv=1.4 +- benchmark=1.6.0 +- boost-cpp=1.82 +- boto3 +- breathe=4.34.0 +- clangdev=16 +- click >=8 +- click>=8 +- cmake=3.25 +- cuda-cudart-dev=11.8 +- cuda-nvcc +- cuda-nvml-dev=11.8 +- cuda-nvrtc-dev=11.8 +- cuda-python=11.8.2 +- cuda-tools=11.8 +- cuda-version=11.8 +- cudf=23.06 +- cuml=23.06 +- cxx-compiler +- cython=0.29.24 +- dask>=2023.1.1 +- datacompy=0.8 +- dgl=1.0.2 +- dill +- dill=0.3.6 +- distributed>=2023.1.1 +- doxygen=1.9.2 +- elasticsearch==8.9.0 +- environs=9.5 +- exhale=0.3.6 +- feedparser=6.0.10 +- flake8 +- gcc_linux-64=11.2 +- git-lfs +- gputil +- grpcio +- gxx_linux-64=11.2 +- huggingface_hub=0.10.1 +- include-what-you-use=0.20 +- ipython +- isort +- langchain=0.0.190 +- librdkafka=1.9.2 +- libwebp>=1.3.2 +- minio=7.1 +- mlflow>=2.2.1,<3 +- mrc=24.03 +- myst-parser=0.17.2 +- nb_conda_kernels +- nbsphinx +- newspaper3k=0.2 +- ninja=1.10 +- nlohmann_json=3.9 +- nodejs=18.* +- numpydoc=1.4 +- nvtabular=23.06 +- openai=0.28 +- papermill=2.3.4 +- pip +- pluggy=1.0 +- pre-commit +- protobuf=4.21.* +- pybind11-stubgen=0.10 +- pylint>=2.17.4,<2.18 +- pypdf=3.16 +- pytest +- pytest-asyncio +- pytest-benchmark>=4.0 +- pytest-cov +- python-confluent-kafka=1.9.2 +- python-dotenv=1.0 +- python-graphviz +- pytorch-cuda +- pytorch=2.0.1 +- rapidjson=1.1.0 +- requests-cache=1.1 +- s3fs>=2023.6 +- scikit-build=0.17.1 +- scikit-learn=1.2.2 +- sentence-transformers +- sphinx +- sphinx_rtd_theme +- sqlalchemy<2.0 +- tqdm=4 +- transformers=4.30.2 +- tritonclient=2.26 +- typing_utils=0.1 +- ucx=1.14 +- ujson=5.8 +- watchdog=2.1 +- websockets +- yapf=0.40.1 +- pip: + - databricks-connect + - google-search-results==2.4 + - grpcio-status==1.58 + - milvus==2.3.2 + - nemollm + - pyarrow_hotfix + - pymilvus==2.3.2 + - pytest-kafka==0.6.0 +name: all_cuda-118_arch-x86_64 diff --git a/conda/environments/dev_cuda-118_arch-x86_64.yaml b/conda/environments/dev_cuda-118_arch-x86_64.yaml new file mode 100644 index 0000000000..c0d127a07f --- /dev/null +++ b/conda/environments/dev_cuda-118_arch-x86_64.yaml @@ -0,0 +1,92 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- conda-forge +- huggingface +- rapidsai +- nvidia/label/cuda-11.8.0 +- nvidia +- rapidsai-nightly +- nvidia/label/dev +- pytorch +- dglteam/label/cu118 +dependencies: +- appdirs +- benchmark=1.6.0 +- boost-cpp=1.82 +- breathe=4.34.0 +- clangdev=16 +- click >=8 +- click>=8 +- cmake=3.25 +- cuda-cudart-dev=11.8 +- cuda-nvcc +- cuda-nvml-dev=11.8 +- cuda-nvrtc-dev=11.8 +- cuda-python=11.8.2 +- cuda-tools=11.8 +- cuda-version=11.8 +- cudf=23.06 +- cxx-compiler +- cython=0.29.24 +- datacompy=0.8 +- dill +- doxygen=1.9.2 +- elasticsearch==8.9.0 +- exhale=0.3.6 +- feedparser=6.0.10 +- flake8 +- gcc_linux-64=11.2 +- git-lfs +- gputil +- grpcio +- gxx_linux-64=11.2 +- include-what-you-use=0.20 +- ipython +- isort +- librdkafka=1.9.2 +- mlflow>=2.2.1,<3 +- mrc=24.03 +- myst-parser=0.17.2 +- nb_conda_kernels +- nbsphinx +- ninja=1.10 +- nlohmann_json=3.9 +- nodejs=18.* +- numpydoc=1.4 +- nvtabular=23.06 +- pip +- pluggy=1.0 +- pre-commit +- protobuf=4.21.* +- pybind11-stubgen=0.10 +- pylint>=2.17.4,<2.18 +- pytest +- pytest-asyncio +- pytest-benchmark>=4.0 +- pytest-cov +- python-confluent-kafka=1.9.2 +- python-graphviz +- pytorch-cuda +- pytorch=2.0.1 +- rapidjson=1.1.0 +- requests-cache=1.1 +- scikit-build=0.17.1 +- scikit-learn=1.2.2 +- sphinx +- sphinx_rtd_theme +- sqlalchemy<2.0 +- tqdm=4 +- tritonclient=2.26 +- typing_utils=0.1 +- ucx=1.14 +- watchdog=2.1 +- websockets +- yapf=0.40.1 +- pip: + - databricks-connect + - milvus==2.3.2 + - pyarrow_hotfix + - pymilvus==2.3.2 + - pytest-kafka==0.6.0 +name: dev_cuda-118_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index 4cec9440ca..fd035b733d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -21,20 +21,101 @@ files: cuda: ["11.8"] arch: [x86_64] includes: + - data_retrieval - build_cpp - - development_cpp - - run_morpheus + - development + - benchmark_cpp + - examples + - runtime - test_python_morpheus + - docs - cudatoolkit + - runtime + - checks + + dev: + output: conda + matrix: + cuda: ["11.8"] + arch: [x86_64] + includes: + - data_retrieval + - build_cpp + - development + - benchmark_cpp + - runtime + - test_python_morpheus + - docs + - cudatoolkit + - runtime + - checks + + build: + output: none + matrix: + cuda: ["11.8"] + arch: [x86_64] + includes: + - build_cpp + - development + - benchmark_cpp + - runtime + - test_python_morpheus + - cudatoolkit + + test: + output: none + matrix: + cuda: ["11.8"] + arch: [x86_64] + includes: + - data_retrieval + - build_cpp + - development + - benchmark_cpp + - runtime + - test_python_morpheus + - examples + - cudatoolkit + + docs: + output: none + matrix: + cuda: ["11.8"] + arch: [x86_64] + includes: + - data_retrieval + - build_cpp + - benchmark_cpp + - cudatoolkit + - docs + - runtime + - examples + + runtime: + output: none + matrix: + cuda: ["11.8"] + arch: [x86_64] + includes: + - runtime + + checks: + output: none + includes: + - checks + channels: + - conda-forge + - huggingface - rapidsai - nvidia/label/cuda-11.8.0 - nvidia - rapidsai-nightly - nvidia/label/dev - pytorch - - conda-forge + - dglteam/label/cu118 dependencies: @@ -42,12 +123,15 @@ dependencies: common: - output_types: [conda] packages: + - mrc=24.03 # should this be in build, or somewhere else? + - gcc_linux-64=11.2 + - gxx_linux-64=11.2 + - cmake=3.25 - boost-cpp=1.82 - cuda-nvcc - cudf=23.06 - cxx-compiler - cython=0.29.24 - - libhwloc=2.9.2 - librdkafka=1.9.2 - ninja=1.10 - nlohmann_json=3.9 @@ -57,15 +141,91 @@ dependencies: - scikit-build=0.17.1 - tritonclient=2.26 # Required by NvTabular, force the version, so we get protobufs compatible with 4.21 - ucx=1.14 - development_cpp: + + checks: common: - output_types: [conda] packages: - - clangdev=14 - - include-what-you-use=0.18 + - pre-commit + + data_retrieval: + common: + - output_types: [conda] + packages: + - git-lfs + + development: + common: + - output_types: [conda] + packages: + - clangdev=16 + - flake8 + - include-what-you-use=0.20 - isort + - pylint>=2.17.4,<2.18 # 2.17.4 contains a fix for toml support - yapf=0.40.1 - run_morpheus: + + docs: + common: + - output_types: [conda] + packages: + - breathe=4.34.0 + - doxygen=1.9.2 + - exhale=0.3.6 + - ipython + - myst-parser=0.17.2 + - nbsphinx + - python-graphviz + - sphinx + - sphinx_rtd_theme + - numpydoc=1.4 + - pluggy=1.0 + - &click click >=8 + + benchmark_cpp: + common: + - output_types: [conda] + packages: + - benchmark=1.6.0 + + examples: + common: + - output_types: [conda] + packages: + - arxiv=1.4 + - boto3 + - cuml=23.06 + - dask>=2023.1.1 + - dgl=1.0.2 + - dill=0.3.6 + - distributed>=2023.1.1 + - huggingface_hub=0.10.1 # work-around for https://github.com/UKPLab/sentence-transformers/issues/1762 + - langchain=0.0.190 + - libwebp>=1.3.2 # Required for CVE mitigation: https://nvd.nist.gov/vuln/detail/CVE-2023-4863 + - mlflow>=2.2.1,<3 + - newspaper3k=0.2 + - openai=0.28 + - papermill=2.3.4 + - pypdf=3.16 + - s3fs>=2023.6 + - sentence-transformers + - transformers=4.30.2 # newer versions are incompatible with our pinned version of huggingface_hub + ####### Pip Transitive Dependencies (keep sorted!) ####### + # These are dependencies that are available on conda, but are required by the pip packages listed below. Its much + # better to install them with conda than pip to allow for better dependency resolution. + - environs=9.5 + - minio=7.1 + - python-dotenv=1.0 + - ujson=5.8 + ####### Pip Dependencies (keep sorted!) ####### + - pip + - pip: + - google-search-results==2.4 + - grpcio-status==1.58 # To keep in sync with 1.58 grpcio which is installed for Morpheus + - nemollm + + + runtime: common: - output_types: [conda] packages: @@ -73,28 +233,42 @@ dependencies: - click>=8 - datacompy=0.8 - dill + - elasticsearch==8.9.0 + - feedparser=6.0.10 - gputil - grpcio - mlflow>=2.2.1,<3 + - nb_conda_kernels - numpydoc=1.4 - nvtabular=23.06 - python-confluent-kafka=1.9.2 - python-graphviz - pytorch-cuda - pytorch=2.0.1 + - requests-cache=1.1 - scikit-learn=1.2.2 - sqlalchemy<2.0 # 2.0 is incompatible with pandas=1.3 - tqdm=4 - typing_utils=0.1 - watchdog=2.1 + - websockets + - pip + - pip: + - databricks-connect + - milvus==2.3.2 + - pyarrow_hotfix + - pymilvus==2.3.2 + test_python_morpheus: common: - output_types: [conda] packages: - - pip + - nodejs=18.* - pytest + - pytest-asyncio - pytest-benchmark>=4.0 - pytest-cov + - pip - pip: - pytest-kafka==0.6.0 @@ -108,5 +282,6 @@ dependencies: - cuda-cudart-dev=11.8 - cuda-nvml-dev=11.8 - cuda-nvrtc-dev=11.8 + - cuda-python=11.8.2 - cuda-tools=11.8 - cuda-version=11.8 From 36f1c190770f609984612345eef58a10d2513c35 Mon Sep 17 00:00:00 2001 From: Michael Demoret <42954918+mdemoret-nv@users.noreply.github.com> Date: Fri, 22 Dec 2023 11:22:48 -0700 Subject: [PATCH 2/2] Updating the workspace settings to remove deprecated python options (#1440) Removes the deprecated python settings in favor of the options recommended in https://github.com/microsoft/vscode-python/wiki/Migration-to-Python-Tools-Extensions Authors: - Michael Demoret (https://github.com/mdemoret-nv) Approvers: - Devin Robison (https://github.com/drobison00) URL: https://github.com/nv-morpheus/Morpheus/pull/1440 --- morpheus.code-workspace | 72 +++++++++++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 24 deletions(-) diff --git a/morpheus.code-workspace b/morpheus.code-workspace index 273057e0d5..dd9ed51763 100644 --- a/morpheus.code-workspace +++ b/morpheus.code-workspace @@ -1,4 +1,25 @@ { + "extensions": { + // See https://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations. + // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp + // List of extensions which should be recommended for users of this workspace. + "recommendations": [ + "eeyore.yapf", + "josetr.cmake-language-support-vscode", + "llvm-vs-code-extensions.vscode-clangd", + "matepek.vscode-catch2-test-adapter", + "ms-python.flake8", + "ms-python.isort", + "ms-python.pylint", + "ms-vscode.cmake-tools", + "stkb.rewrap", + "twxs.cmake" + ], + // List of extensions recommended by VS Code that should not be recommended for users of this workspace. + "unwantedRecommendations": [ + "austin.code-gnu-global" + ] + }, "folders": [ { "path": "." @@ -565,52 +586,54 @@ "type": "cppdbg" }, { - "name": "Python: GNN DGL inference", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/examples/gnn_fraud_detection_pipeline/run.py", - "console": "integratedTerminal", - "cwd": "${workspaceFolder}/examples/gnn_fraud_detection_pipeline", - "justMyCode": false, "args": [ "--input_file=validation.csv", "--training_file=training.csv", "--output_file=result.csv", "--model_dir=model" - ] - }, - { - "name": "Python: GNN model training", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/models/training-tuning-scripts/fraud-detection-models/training.py", + ], "console": "integratedTerminal", - "cwd": "${workspaceFolder}/models/training-tuning-scripts/fraud-detection-models", + "cwd": "${workspaceFolder}/examples/gnn_fraud_detection_pipeline", "justMyCode": false, + "name": "Python: GNN DGL inference", + "program": "${workspaceFolder}/examples/gnn_fraud_detection_pipeline/run.py", + "request": "launch", + "type": "python" + }, + { "args": [ "--training-data=${workspaceFolder}/examples/gnn_fraud_detection_pipeline/training.csv", "--validation-data=${workspaceFolder}/examples/gnn_fraud_detection_pipeline/validation.csv", "--output-file=result.csv", "--model-dir=models", "--epochs=5" - ] + ], + "console": "integratedTerminal", + "cwd": "${workspaceFolder}/models/training-tuning-scripts/fraud-detection-models", + "justMyCode": false, + "name": "Python: GNN model training", + "program": "${workspaceFolder}/models/training-tuning-scripts/fraud-detection-models/training.py", + "request": "launch", + "type": "python" } ] }, "settings": { "[cpp]": { + "editor.defaultFormatter": "llvm-vs-code-extensions.vscode-clangd", "editor.detectIndentation": false, "editor.formatOnSave": true, "editor.semanticHighlighting.enabled": true, "editor.suggest.insertMode": "replace", "editor.tabSize": 4, - "editor.wordBasedSuggestions": false, + "editor.wordBasedSuggestions": "off", "editor.wordWrapColumn": 120 }, "[python]": { "editor.codeActionsOnSave": { - "source.organizeImports": true + "source.organizeImports": "explicit" }, + "editor.defaultFormatter": "eeyore.yapf", "editor.formatOnSave": true, "editor.tabSize": 4 }, @@ -628,6 +651,9 @@ "files.insertFinalNewline": true, "files.trimFinalNewlines": true, "files.trimTrailingWhitespace": true, + "flake8.args": [ + "--style=${workspaceFolder}/setup.cfg" + ], "pylint.args": [ "--rcfile=${workspaceFolder}/pyproject.toml", "--init-hook=import sys; sys.path.append(\"${workspaceFolder}\")" @@ -636,17 +662,15 @@ "python.analysis.extraPaths": [ "./examples/digital_fingerprinting/production/morpheus" ], - "python.formatting.provider": "yapf", - "python.formatting.yapfArgs": [ - "--style=${workspaceFolder}/setup.cfg" - ], - "python.linting.flake8Enabled": true, "python.testing.pytestArgs": [ "-s", "tests" ], "python.testing.pytestEnabled": true, "python.testing.unittestEnabled": false, - "rewrap.wrappingColumn": 120 + "rewrap.wrappingColumn": 120, + "yapf.args": [ + "--style=${workspaceFolder}/setup.cfg" + ] } }