[RELEASE] MRC v24.03.00

Merge pull request #457 from nv-morpheus/branch-24.03

.clang-tidy

@@ -20,7 +20,11 @@ Checks: >
   -modernize-macro-to-enum,
   google-readability-todo
 
-#WarningsAsErrors: '*'
+WarningsAsErrors: >
+  *,
+  -clang-diagnostic-unused-command-line-argument,
+  -clang-diagnostic-ignored-optimization-argument
+
 HeaderFilterRegex: '.*\/include\/mrc\/.*'
 AnalyzeTemporaryDtors: false
 FormatStyle: file

.devcontainer/conda/Dockerfile

@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM rapidsai/devcontainers:23.04-cuda11.8-mambaforge-ubuntu22.04 AS base
+FROM rapidsai/devcontainers:23.04-cuda12.1-mambaforge-ubuntu22.04 AS base
 
 ENV PATH="${PATH}:/workspaces/mrc/.devcontainer/bin"

.devcontainer/opt/mrc/bin/post-attach-command.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -28,6 +28,6 @@ sed -ri "s/conda activate base/conda activate $ENV_NAME/g" ~/.bashrc;
 
 if conda_env_find "${ENV_NAME}" ; \
 
-then mamba env update --name ${ENV_NAME} -f ${MRC_ROOT}/ci/conda/environments/dev_env.yml --prune; \
-else mamba env create --name ${ENV_NAME} -f ${MRC_ROOT}/ci/conda/environments/dev_env.yml; \
+then mamba env update --name ${ENV_NAME} -f ${MRC_ROOT}/conda/environments/all_cuda-121_arch-x86_64.yaml --prune; \
+else mamba env create --name ${ENV_NAME} -f ${MRC_ROOT}/conda/environments/all_cuda-121_arch-x86_64.yaml; \
 fi

.devcontainer/opt/mrc/conda/Dockerfile

@@ -13,6 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM rapidsai/devcontainers:23.04-cuda11.8-mambaforge-ubuntu22.04 AS base
+FROM rapidsai/devcontainers:23.04-cuda12.1-mambaforge-ubuntu22.04 AS base
 
 ENV PATH="${PATH}:/workspaces/mrc/.devcontainer/bin"

.github/ops-bot.yaml

@@ -5,3 +5,4 @@ auto_merger: true
 branch_checker: true
 label_checker: true
 release_drafter: true
+forward_merger: true

.github/workflows/ci_pipe.yml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -61,6 +61,7 @@ env:
   GH_TOKEN: "${{ github.token }}"
   GIT_COMMIT: "${{ github.sha }}"
   MRC_ROOT: "${{ github.workspace }}/mrc"
+  RAPIDS_CONDA_RETRY_MAX: "5"
   WORKSPACE: "${{ github.workspace }}/mrc"
   WORKSPACE_TMP: "${{ github.workspace }}/tmp"
 

.github/workflows/pull_request.yml → .github/workflows/pr.yaml

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -43,6 +43,13 @@ permissions:
   statuses: none
 
 jobs:
+  pr-builder:
+    needs:
+      - checks
+      - prepare
+      - ci_pipe
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
   prepare:
     name: Prepare
     runs-on: ubuntu-latest
@@ -58,11 +65,22 @@ jobs:
       is_main_branch: ${{ github.ref_name == 'main' }}
       is_dev_branch: ${{ startsWith(github.ref_name, 'branch-') }}
       has_conda_build_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'conda-build') || false }}
+      has_skip_ci_label: ${{ steps.get-pr-info.outcome == 'success' && contains(fromJSON(steps.get-pr-info.outputs.pr-info).labels.*.name, 'skip-ci') || false }}
       pr_info: ${{ steps.get-pr-info.outcome == 'success' && steps.get-pr-info.outputs.pr-info || '' }}
+
+  checks:
+    needs: [prepare]
+    if: ${{ !fromJSON(needs.prepare.outputs.has_skip_ci_label) && fromJSON(needs.prepare.outputs.is_pr )}}
+    secrets: inherit
+    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
+    with:
+      enable_check_generated_files: false
+
   ci_pipe:
     name: CI Pipeline
     needs: [prepare]
     uses: ./.github/workflows/ci_pipe.yml
+    if: ${{ ! fromJSON(needs.prepare.outputs.has_skip_ci_label) }}
     with:
       # Run checks for any PR branch
       run_check: ${{ fromJSON(needs.prepare.outputs.is_pr) }}
@@ -71,9 +89,9 @@ jobs:
       # Update conda package only for non PR branches. Use 'main' for main branch and 'dev' for all other branches
       conda_upload_label: ${{ !fromJSON(needs.prepare.outputs.is_pr) && (fromJSON(needs.prepare.outputs.is_main_branch) && 'main' || 'dev') || '' }}
       # Build container
-      container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-build-230920
+      container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-build-240214
       # Test container
-      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-test-230920
+      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:mrc-ci-test-240214
       # Info about the PR. Empty for non PR branches. Useful for extracting PR number, title, etc.
       pr_info: ${{ needs.prepare.outputs.pr_info }}
     secrets:

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "morpheus_utils"]
 	path = external/utilities
 	url = https://github.com/nv-morpheus/utilities.git
-	branch = branch-23.11
+	branch = branch-24.03

.pre-commit-config.yaml

@@ -0,0 +1,29 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+repos:
+      - repo: https://github.com/rapidsai/dependency-file-generator
+        rev: v1.7.1
+        hooks:
+            - id: rapids-dependency-file-generator
+              args: ["--clean"]
+      - repo: https://github.com/PyCQA/isort
+        rev: 5.12.0
+        hooks:
+              - id: isort
+                args: ["--settings-file=./python/setup.cfg"]
+                files: ^python/
+      - repo: https://github.com/PyCQA/flake8
+        rev: 6.1.0
+        hooks:
+              - id: flake8
+                args: ["--config=./python/setup.cfg"]
+                files: ^python/
+      - repo: https://github.com/google/yapf
+        rev: v0.40.2
+        hooks:
+              - id: yapf
+                args: ["--style", "./python/setup.cfg"]
+                files: ^python/
+
+default_language_version:
+      python: python3

CHANGELOG.md

@@ -1,3 +1,41 @@
+# MRC 24.03.00 (7 Apr 2024)
+
+## 🚨 Breaking Changes
+
+- Update cast_from_pyobject to throw on unsupported types rather than returning null ([#451](https://github.com/nv-morpheus/MRC/pull/451)) [@dagardner-nv](https://github.com/dagardner-nv)
+- RAPIDS 24.02 Upgrade ([#433](https://github.com/nv-morpheus/MRC/pull/433)) [@cwharris](https://github.com/cwharris)
+
+## 🐛 Bug Fixes
+
+- Update CR year ([#460](https://github.com/nv-morpheus/MRC/pull/460)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Removing the INFO log when creating an AsyncioRunnable ([#456](https://github.com/nv-morpheus/MRC/pull/456)) [@mdemoret-nv](https://github.com/mdemoret-nv)
+- Update cast_from_pyobject to throw on unsupported types rather than returning null ([#451](https://github.com/nv-morpheus/MRC/pull/451)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Adopt updated builds of CI runners ([#442](https://github.com/nv-morpheus/MRC/pull/442)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Update Conda channels to prioritize `conda-forge` over `nvidia` ([#436](https://github.com/nv-morpheus/MRC/pull/436)) [@cwharris](https://github.com/cwharris)
+- Remove redundant copy of libmrc_pymrc.so ([#429](https://github.com/nv-morpheus/MRC/pull/429)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Unifying cmake exports name across all Morpheus repos ([#427](https://github.com/nv-morpheus/MRC/pull/427)) [@mdemoret-nv](https://github.com/mdemoret-nv)
+- Updating the workspace settings to remove deprecated python options ([#425](https://github.com/nv-morpheus/MRC/pull/425)) [@mdemoret-nv](https://github.com/mdemoret-nv)
+- Use `dependencies.yaml` to generate environment files ([#416](https://github.com/nv-morpheus/MRC/pull/416)) [@cwharris](https://github.com/cwharris)
+
+## 📖 Documentation
+
+- Update minimum requirements ([#467](https://github.com/nv-morpheus/MRC/pull/467)) [@dagardner-nv](https://github.com/dagardner-nv)
+
+## 🚀 New Features
+
+- Add maximum simultaneous tasks support to `TaskContainer` ([#464](https://github.com/nv-morpheus/MRC/pull/464)) [@cwharris](https://github.com/cwharris)
+- Add `TestScheduler` to support testing time-based coroutines without waiting for timeouts ([#453](https://github.com/nv-morpheus/MRC/pull/453)) [@cwharris](https://github.com/cwharris)
+- Adding RoundRobinRouter node type for distributing values to downstream nodes ([#449](https://github.com/nv-morpheus/MRC/pull/449)) [@mdemoret-nv](https://github.com/mdemoret-nv)
+- Add IoScheduler to enable epoll-based Task scheduling ([#448](https://github.com/nv-morpheus/MRC/pull/448)) [@cwharris](https://github.com/cwharris)
+- Update ops-bot.yaml ([#446](https://github.com/nv-morpheus/MRC/pull/446)) [@AyodeAwe](https://github.com/AyodeAwe)
+- RAPIDS 24.02 Upgrade ([#433](https://github.com/nv-morpheus/MRC/pull/433)) [@cwharris](https://github.com/cwharris)
+
+## 🛠️ Improvements
+
+- Update MRC to use CCCL instead of libcudacxx ([#444](https://github.com/nv-morpheus/MRC/pull/444)) [@cwharris](https://github.com/cwharris)
+- Optionally skip the CI pipeline if the PR contains the skip-ci label ([#426](https://github.com/nv-morpheus/MRC/pull/426)) [@dagardner-nv](https://github.com/dagardner-nv)
+- Add flake8, yapf, and isort pre-commit hooks. ([#420](https://github.com/nv-morpheus/MRC/pull/420)) [@cwharris](https://github.com/cwharris)
+
 # MRC 23.11.00 (30 Nov 2023)
 
 ## 🐛 Bug Fixes

CMakeLists.txt

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -37,7 +37,7 @@ option(MRC_USE_CONDA "Enables finding dependencies via conda. All dependencies m
   environment" ON)
 option(MRC_USE_IWYU "Enable running include-what-you-use as part of the build process" OFF)
 
-set(MRC_RAPIDS_VERSION "23.06" CACHE STRING "Which version of RAPIDS to build for. Sets default versions for RAPIDS CMake and RMM.")
+set(MRC_RAPIDS_VERSION "24.02" CACHE STRING "Which version of RAPIDS to build for. Sets default versions for RAPIDS CMake and RMM.")
 
 set(MRC_CACHE_DIR "${CMAKE_SOURCE_DIR}/.cache" CACHE PATH "Directory to contain all CPM and CCache data")
 mark_as_advanced(MRC_CACHE_DIR)
@@ -79,7 +79,7 @@ morpheus_utils_initialize_package_manager(
 morpheus_utils_initialize_cuda_arch(mrc)
 
 project(mrc
-  VERSION 23.11.00
+  VERSION 24.03.00
   LANGUAGES C CXX
 )
 

CONTRIBUTING.md

@@ -85,7 +85,7 @@ cd $MRC_ROOT
 #### Create MRC Conda environment
 ```bash
 # note: `mamba` may be used in place of `conda` for better performance.
-conda env create -n mrc --file $MRC_ROOT/ci/conda/environments/dev_env.yml
+conda env create -n mrc --file $MRC_ROOT/conda/environments/all_cuda-121_arch-x86_64.yaml
 conda activate mrc
 ```
 #### Build MRC

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.3
 
-# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,38 +17,48 @@
 
 
 ARG FROM_IMAGE="rapidsai/ci-conda"
-ARG CUDA_VER=11.8.0
+ARG CUDA_VER=12.1.1
 ARG LINUX_DISTRO=ubuntu
-ARG LINUX_VER=20.04
+ARG LINUX_VER=22.04
 ARG PYTHON_VER=3.10
 
 # ============= base ===================
-FROM ${FROM_IMAGE}:cuda11.8.0-ubuntu20.04-py3.10 AS base
+FROM ${FROM_IMAGE}:cuda${CUDA_VER}-${LINUX_DISTRO}${LINUX_VER}-py${PYTHON_VER} AS base
 
 ARG PROJ_NAME=mrc
+ARG USERNAME=morpheus
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
 
 SHELL ["/bin/bash",  "-c"]
 
 RUN --mount=type=cache,target=/var/cache/apt \
     apt update &&\
     apt install --no-install-recommends -y \
-    libnuma1 && \
+    libnuma1 \
+    sudo && \
     rm -rf /var/lib/apt/lists/*
 
-COPY ./ci/conda/environments/* /opt/mrc/conda/environments/
+# create a user inside the container
+RUN useradd --uid $USER_UID --gid $USER_GID -m $USERNAME && \
+    usermod --shell /bin/bash $USERNAME && \
+    echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME && \
+    chmod 0440 /etc/sudoers.d/$USERNAME
+
+COPY ./conda/environments/all_cuda-121_arch-x86_64.yaml /opt/mrc/conda/environments/all_cuda-121_arch-x86_64.yaml
 
 RUN --mount=type=cache,target=/opt/conda/pkgs,sharing=locked \
     echo "create env: ${PROJ_NAME}" && \
+    sudo -g conda -u $USERNAME \
     CONDA_ALWAYS_YES=true \
-    /opt/conda/bin/mamba env create -q -n ${PROJ_NAME} --file /opt/mrc/conda/environments/dev_env.yml && \
-    /opt/conda/bin/mamba env update -q -n ${PROJ_NAME} --file /opt/mrc/conda/environments/clang_env.yml && \
-    /opt/conda/bin/mamba env update -q -n ${PROJ_NAME} --file /opt/mrc/conda/environments/ci_env.yml && \
+    /opt/conda/bin/mamba env create -q -n ${PROJ_NAME} --file /opt/mrc/conda/environments/all_cuda-121_arch-x86_64.yaml && \
     chmod -R a+rwX /opt/conda && \
     rm -rf /tmp/conda
 
 RUN /opt/conda/bin/conda init --system &&\
     sed -i 's/xterm-color)/xterm-color|*-256color)/g' ~/.bashrc &&\
-    echo "conda activate ${PROJ_NAME}" >> ~/.bashrc
+    echo "conda activate ${PROJ_NAME}" >> ~/.bashrc && \
+    cp /root/.bashrc /home/$USERNAME/.bashrc
 
 # disable sscache wrappers around compilers
 ENV CMAKE_CUDA_COMPILER_LAUNCHER=
@@ -78,7 +88,6 @@ RUN --mount=type=cache,target=/var/cache/apt \
     less \
     openssh-client \
     psmisc \
-    sudo \
     vim-tiny \
     && \
     rm -rf /var/lib/apt/lists/*
@@ -93,17 +102,6 @@ RUN --mount=type=cache,target=/var/cache/apt \
     apt-get install --no-install-recommends -y dotnet-sdk-6.0 &&\
     rm -rf /var/lib/apt/lists/*
 
-# create a user inside the container
-ARG USERNAME=morpheus
-ARG USER_UID=1000
-ARG USER_GID=$USER_UID
-
-RUN useradd --uid $USER_UID --gid $USER_GID -m $USERNAME && \
-    usermod --shell /bin/bash $USERNAME && \
-    echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME && \
-    chmod 0440 /etc/sudoers.d/$USERNAME && \
-    cp /root/.bashrc /home/$USERNAME/.bashrc
-
 USER $USERNAME
 
 # default working directory

README.md

@@ -38,8 +38,8 @@ MRC includes both Python and C++ bindings and supports installation via [conda](
 
 ### Prerequisites
 
-- Pascal architecture (Compute capability 6.0) or better
-- NVIDIA driver `450.80.02` or higher
+- Volta architecture (Compute capability 7.0) or better
+- [CUDA 12.1](https://developer.nvidia.com/cuda-12-1-0-download-archive)
 - [conda or miniconda](https://conda.io/projects/conda/en/latest/user-guide/install/linux.html)
 - If using Docker:
   - [Docker](https://docs.docker.com/get-docker/)
@@ -118,7 +118,7 @@ cd $MRC_ROOT
 #### Create MRC Conda Environment
 ```bash
 # note: `mamba` may be used in place of `conda` for better performance.
-conda env create -n mrc-dev --file $MRC_ROOT/ci/conda/environments/dev_env.yml
+conda env create -n mrc-dev --file $MRC_ROOT/conda/environments/all_cuda-121_arch-x86_64.yaml
 conda activate mrc-dev
 ```
 <!-- omit in toc -->

ci/conda/environments/ci_env.yml → ci/check_style.sh

@@ -1,4 +1,5 @@
-# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,10 +14,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Additional dependencies only needed during a CI build
-name: mrc
-channels:
-    - conda-forge
-dependencies:
-    - codecov=2.1
-    - conda-merge>=0.2
+set -euo pipefail
+
+rapids-logger "Create checks conda environment"
+. /opt/conda/etc/profile.d/conda.sh
+
+rapids-dependency-file-generator \
+  --output conda \
+  --file_key checks \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+
+rapids-mamba-retry env create --force -f env.yaml -n checks
+conda activate checks
+
+# Run pre-commit checks
+pre-commit run --all-files --show-diff-on-failure