diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index d835aab82..000000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -name: Bug report -about: Create a bug report to help us improve SRF -title: "[BUG]" -labels: "? - Needs Triage, bug" -assignees: '' - ---- - -**Describe the bug** -A clear and concise description of what the bug is. - -**Steps/Code to reproduce bug** -Follow this guide http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports to craft a minimal bug report. This helps us reproduce the issue you're having and resolve the issue more quickly. - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Environment overview (please complete the following information)** - - Environment location: [Bare-metal, Docker, Cloud(specify cloud provider)] - - Method of SRF install: [conda, Docker, or from source] - - If method of install is [Docker], provide `docker pull` & `docker run` commands used - -**Environment details** -Please run and paste the output of the `./scripts/print_env.sh` script here, to gather any other relevant environment details - -**Additional context** -Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 000000000..e11bfa046 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Bug Report +description: File a bug report +title: "[BUG]: " +labels: ["bug"] + +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + + - type: input + id: version + attributes: + label: Version + description: What version of SRF are you running? + placeholder: "example: 22.09" + validations: + required: true + + - type: dropdown + id: installation-method + attributes: + label: Which installation method(s) does this occur on? + multiple: true + options: + - Docker + - Conda + - Source + + - type: textarea + id: description + attributes: + label: Describe the bug. + description: Also tell us, what did you expect to happen? + placeholder: XYZ occured, I expected QRS results + validations: + required: true + + - type: textarea + id: mvr + attributes: + label: Minimum reproducible example + description: Please supply a [minimum reproducible code example](https://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports) here + render: shell + + - type: textarea + id: logs + attributes: + label: Relevant log output + description: Please paste relevant error and log output here + render: shell + + - type: textarea + id: env-printout + attributes: + label: Full env printout + description: Please run and paste the output of the `./scripts/print_env.sh` script here, to gather any other relevant environment details + render: shell + + - type: textarea + id: misc + attributes: + label: Other/Misc. + description: Please enter any other helpful information here. + + - type: checkboxes + id: terms + attributes: + label: Code of Conduct + description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/nv-morpheus/SRF/blob/main/CODE_OF_CONDUCT.md) + options: + - label: I agree to follow SRF's Code of Conduct + required: true + - label: I have searched the [open bugs](https://github.com/nv-morpheus/SRF/issues?q=is%3Aopen+is%3Aissue+label%3Abug) and have found no duplicates for this bug report + required: true diff --git a/.github/ISSUE_TEMPLATE/documentation-request.md b/.github/ISSUE_TEMPLATE/documentation-request.md deleted file mode 100644 index 89a026f34..000000000 --- a/.github/ISSUE_TEMPLATE/documentation-request.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -name: Documentation request -about: Report incorrect or needed documentation -title: "[DOC]" -labels: "? - Needs Triage, doc" -assignees: '' - ---- - -## Report incorrect documentation - -**Location of incorrect documentation** -Provide links and line numbers if applicable. - -**Describe the problems or issues found in the documentation** -A clear and concise description of what you found to be incorrect. - -**Steps taken to verify documentation is incorrect** -List any steps you have taken: - -**Suggested fix for documentation** -Detail proposed changes to fix the documentation if you have any. - ---- - -## Report needed documentation - -**Report needed documentation** -A clear and concise description of what documentation you believe it is needed and why. - -**Describe the documentation you'd like** -A clear and concise description of what you want to happen. - -**Steps taken to search for needed documentation** -List any steps you have taken: diff --git a/.github/ISSUE_TEMPLATE/documentation_request_correction.yml b/.github/ISSUE_TEMPLATE/documentation_request_correction.yml new file mode 100644 index 000000000..ad67aadec --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation_request_correction.yml @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Documentation - Correction/Update Request +description: Request corrections or updates to existing documentation +title: "[DOC]: " +labels: ["doc"] + +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to improve our documentation! + + - type: dropdown + id: criticality + attributes: + label: How would you describe the priority of this documentation request + options: + - Critical (currently preventing usage) + - High + - Medium + - Low (would be nice) + validations: + required: true + + - type: input + id: correction_location + attributes: + label: Please provide a link or source to the relevant docs + placeholder: "ex: https://github.com/nv-morpheus/SRF/blob/main/README.md" + validations: + required: true + + - type: textarea + id: problem + attributes: + label: Describe the problems in the documentation + placeholder: The documents say to use foo.func(args) however an AttributeError is thrown + validations: + required: true + + - type: textarea + id: correction + attributes: + label: (Optional) Propose a correction + placeholder: foo.func() was deprecated, replace documentation with foo.new_func() + + - type: checkboxes + id: terms + attributes: + label: Code of Conduct + description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/nv-morpheus/SRF/blob/main/CODE_OF_CONDUCT.md) + options: + - label: I agree to follow SRF's Code of Conduct + required: true + - label: I have searched the [open documentation issues](https://github.com/nv-morpheus/SRF/issues?q=is%3Aopen+is%3Aissue+label%3Adoc) and have found no duplicates for this bug report + required: true diff --git a/.github/ISSUE_TEMPLATE/documentation_request_new.yml b/.github/ISSUE_TEMPLATE/documentation_request_new.yml new file mode 100644 index 000000000..a61aae8ed --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation_request_new.yml @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Documentation - New Documentation Request +description: Request additions to Morpheus documentation +title: "[DOC]: " +labels: ["doc"] + +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to improve our documentation! + + - type: dropdown + id: criticality + attributes: + label: How would you describe the priority of this documentation request + options: + - Critical (currently preventing usage) + - High + - Medium + - Low (would be nice) + validations: + required: true + + - type: textarea + id: problem + attributes: + label: Describe the future/missing documentation + placeholder: A code snippet mentions function foo(args) but I cannot find any documentation on it. + validations: + required: true + + - type: textarea + id: search_locs + attributes: + label: Where have you looked? + placeholder: | + https://github.com/nv-morpheus/SRF/blob/main/docs/quickstart/README.md and + https://github.com/nv-morpheus/SRF/blob/main/README.md + + - type: checkboxes + id: terms + attributes: + label: Code of Conduct + description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/nv-morpheus/SRF/blob/main/CODE_OF_CONDUCT.md) + options: + - label: I agree to follow SRF's Code of Conduct + required: true + - label: I have searched the [open documentation issues](https://github.com/nv-morpheus/SRF/issues?q=is%3Aopen+is%3Aissue+label%3Adoc) and have found no duplicates for this bug report + required: true diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index 9d883cfb9..000000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for SRF -title: "[FEA]" -labels: "? - Needs Triage, feature request" -assignees: '' - ---- - -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I wish I could use SRF to do [...] - -**Describe the solution you'd like** -A clear and concise description of what you want to happen. - -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. - -**Additional context** -Add any other context, code examples, or references to existing implementations about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 000000000..a864631f9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Feature Request Form +description: Request new or improved functionality or changes to existing functionality +title: "[FEA]: " +labels: ["feature request"] + +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this feature request! + + - type: dropdown + id: new_or_improvement + attributes: + label: Is this a new feature, an improvement, or a change to existing functionality? + options: + - New Feature + - Improvement + - Change + validations: + required: true + + - type: dropdown + id: criticality + attributes: + label: How would you describe the priority of this feature request + options: + - Critical (currently preventing usage) + - High + - Medium + - Low (would be nice) + validations: + required: true + + - type: textarea + id: problem + attributes: + label: Please provide a clear description of problem this feature solves + description: Real usage examples are especially helpful, non-code. + placeholder: I want SRF to do _____, because I need to _____. + validations: + required: true + + - type: textarea + id: solution + attributes: + label: Describe your ideal solution + description: Please describe the functionality you would like added. + placeholder: > + A new function that takes in the information in this form, and triages the issue + + def feature_request(form_info): + parse(form_info) + return triage_outcome + validations: + required: true + + - type: textarea + id: alternatives + attributes: + label: Describe any alternatives you have considered + description: List any other libraries, or approaches you have looked at or tried. + placeholder: I have looked at library xyz and qrs, but they do not offer GPU accleration + + - type: textarea + id: misc + attributes: + label: Additional context + description: Add any other context, code examples, or references to existing implementations about the feature request here. + + - type: checkboxes + id: terms + attributes: + label: Code of Conduct + description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/nv-morpheus/SRF/blob/main/CODE_OF_CONDUCT.md) + options: + - label: I agree to follow SRF's Code of Conduct + required: true + - label: I have searched the [open feature requests](https://github.com/nv-morpheus/SRF/issues?q=is%3Aopen+is%3Aissue+label%3A%22feature+request%22%2Cimprovement%2Cenhancement) and have found no duplicates for this feature request + required: true diff --git a/.github/ISSUE_TEMPLATE/submit-question.md b/.github/ISSUE_TEMPLATE/submit-question.md deleted file mode 100644 index 4a5d33f19..000000000 --- a/.github/ISSUE_TEMPLATE/submit-question.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -name: Submit question -about: Ask a general question about SRF -title: "[QST]" -labels: "? - Needs Triage, question" -assignees: '' - ---- - -**What is your question?** diff --git a/.github/ISSUE_TEMPLATE/submit_question.yml b/.github/ISSUE_TEMPLATE/submit_question.yml new file mode 100644 index 000000000..6a02f9fc4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/submit_question.yml @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Submit Question +description: Ask a general question about SRF +title: "[QST]: " +labels: ["question"] + +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to ask us a question! + + - type: textarea + id: question + attributes: + label: What is your question? + description: Please be specific and we will answer your question as soon as possible. + placeholder: Does SRF integrate with XYZ software? + validations: + required: true diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 000000000..adc83a43f --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,34 @@ +#Configuration File for CodeCov +coverage: + status: + project: + default: + informational: true + target: auto + threshold: 0% + base: auto + paths: + - "include" + - "python" + - "src" + - "tools" + patch: + default: + informational: true + target: auto + threshold: 0% + base: auto + paths: + - "include" + - "python" + - "src" + - "tools" +comment: + behavior: new + +# Suggested workaround to fix "missing base report" issue when using Squash and +# Merge Strategy in GitHub. See this comment from CodeCov support about this +# undocumented option: +# https://community.codecov.io/t/unable-to-determine-a-parent-commit-to-compare-against-in-base-branch-after-squash-and-merge/2480/15 +codecov: + allow_coverage_offsets: true diff --git a/.github/workflows/add_to_project.yml b/.github/workflows/add_to_project.yml index 00f0ac7fa..572804724 100644 --- a/.github/workflows/add_to_project.yml +++ b/.github/workflows/add_to_project.yml @@ -5,7 +5,7 @@ on: types: - opened - pull_request: + pull_request_target: types: - opened @@ -17,4 +17,4 @@ jobs: - uses: actions/add-to-project@v0.3.0 with: project-url: https://github.com/orgs/nv-morpheus/projects/2 - github-token: ${{ secrets.ACTIONS_ACCESS_TOKEN }} + github-token: ${{ github.token}} diff --git a/.github/workflows/ci_pipe.yml b/.github/workflows/ci_pipe.yml new file mode 100644 index 000000000..4157f1a54 --- /dev/null +++ b/.github/workflows/ci_pipe.yml @@ -0,0 +1,230 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: CI Pipeline +run-name: CI Pipeline + +on: + workflow_call: + inputs: + aws_region: + default: 'us-west-2' + type: string + run_check: + required: true + type: boolean + run_package_conda: + required: true + type: boolean + container: + required: true + type: string + test_container: + required: true + type: string + secrets: + CODECOV_TOKEN: + required: true + CONDA_TOKEN: + required: true + GHA_AWS_ACCESS_KEY_ID: + required: true + GHA_AWS_SECRET_ACCESS_KEY: + required: true + NGC_API_KEY: + required: true + +env: + AWS_DEFAULT_REGION: ${{ inputs.aws_region }} + AWS_ACCESS_KEY_ID: "${{ secrets.GHA_AWS_ACCESS_KEY_ID }}" + AWS_SECRET_ACCESS_KEY: "${{ secrets.GHA_AWS_SECRET_ACCESS_KEY }}" + BUILD_CC: "gcc" + CHANGE_TARGET: "${{ github.base_ref }}" + GH_TOKEN: "${{ github.token }}" + GIT_COMMIT: "${{ github.sha }}" + SRF_ROOT: "${{ github.workspace }}/srf" + WORKSPACE: "${{ github.workspace }}/srf" + WORKSPACE_TMP: "${{ github.workspace }}/tmp" + + +jobs: + check: + if: ${{ inputs.run_check }} + name: Check + runs-on: [self-hosted, linux, amd64, cpu4] + timeout-minutes: 60 + container: + credentials: + username: '$oauthtoken' + password: ${{ secrets.NGC_API_KEY }} + image: ${{ inputs.container }} + strategy: + fail-fast: true + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + lfs: false + path: 'srf' + fetch-depth: 0 + + - name: Check + shell: bash + run: ./srf/ci/scripts/github/checks.sh + + build: + name: Build + runs-on: [self-hosted, linux, amd64, cpu16] + timeout-minutes: 60 + container: + credentials: + username: '$oauthtoken' + password: ${{ secrets.NGC_API_KEY }} + image: ${{ inputs.container }} + strategy: + fail-fast: true + matrix: + build_cc: ["gcc", "gcc-coverage", "clang"] + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + lfs: false + path: 'srf' + + - name: Build:linux:x86_64 + shell: bash + env: + BUILD_CC: ${{ matrix.build_cc }} + run: ./srf/ci/scripts/github/build.sh + + test: + name: Test + needs: [build] + runs-on: [self-hosted, linux, amd64, gpu-v100-495-1] + timeout-minutes: 60 + container: + credentials: + username: '$oauthtoken' + password: ${{ secrets.NGC_API_KEY }} + image: ${{ inputs.test_container }} + options: --cap-add=sys_nice + env: + NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} + PARALLEL_LEVEL: '10' + strategy: + fail-fast: true + matrix: + build_cc: ["gcc", "gcc-coverage"] + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + lfs: false + path: 'srf' + + - name: Test:linux:x86_64 + shell: bash + env: + BUILD_CC: ${{ matrix.build_cc }} + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + run: ./srf/ci/scripts/github/test.sh + + documentation: + name: Documentation + needs: [build] + runs-on: [self-hosted, linux, amd64, cpu4] + timeout-minutes: 60 + container: + credentials: + username: '$oauthtoken' + password: ${{ secrets.NGC_API_KEY }} + image: ${{ inputs.container }} + strategy: + fail-fast: true + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + lfs: false + path: 'srf' + + - name: build_docs + shell: bash + run: ./srf/ci/scripts/github/docs.sh + + benchmark: + name: Benchmark + needs: [build] + runs-on: [self-hosted, linux, amd64, cpu4] + timeout-minutes: 60 + container: + credentials: + username: '$oauthtoken' + password: ${{ secrets.NGC_API_KEY }} + image: ${{ inputs.container }} + options: --cap-add=sys_nice + strategy: + fail-fast: true + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + lfs: false + path: 'srf' + + - name: pre_benchmark + shell: bash + run: ./srf/ci/scripts/github/pre_benchmark.sh + - name: benchmark + shell: bash + run: ./srf/ci/scripts/github/benchmark.sh + - name: post_benchmark + shell: bash + run: ./srf/ci/scripts/github/benchmark.sh + + + package: + name: Package + if: ${{ inputs.run_package_conda }} + needs: [benchmark, documentation, test] + runs-on: [self-hosted, linux, amd64, cpu16] + timeout-minutes: 60 + container: + credentials: + username: '$oauthtoken' + password: ${{ secrets.NGC_API_KEY }} + image: ${{ inputs.container }} + strategy: + fail-fast: true + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + lfs: false + path: 'srf' + fetch-depth: 0 + + - name: conda + shell: bash + env: + CONDA_TOKEN: "${{ secrets.CONDA_TOKEN }}" + run: ./srf/ci/scripts/github/conda.sh diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml new file mode 100644 index 000000000..b583c0c05 --- /dev/null +++ b/.github/workflows/pull_request.yml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Build pull request + +on: + push: + branches: + - 'pull-request/**' + - "branch-*" + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true + +jobs: + ci_pipe: + uses: ./.github/workflows/ci_pipe.yml + with: + run_check: ${{ startsWith(github.ref_name, 'pull-request/') }} + run_package_conda: ${{ startsWith(github.ref_name, 'branch-') }} + container: nvcr.io/ea-nvidia-morpheus/morpheus:srf-ci-driver-221103 + test_container: nvcr.io/ea-nvidia-morpheus/morpheus:srf-ci-base-221103 + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + CONDA_TOKEN: ${{ secrets.CONDA_TOKEN }} + GHA_AWS_ACCESS_KEY_ID: ${{ secrets.GHA_AWS_ACCESS_KEY_ID }} + GHA_AWS_SECRET_ACCESS_KEY: ${{ secrets.GHA_AWS_SECRET_ACCESS_KEY }} + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} diff --git a/.gitignore b/.gitignore index 470b577a5..4e6f0fd66 100755 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,9 @@ docs/quickstart/**/*.pyi python/**/*.pyi +# Ignore version header file +include/srf/version.hpp + # Ignore all vscode options that are in the workspace file .vscode/extensions.json .vscode/launch.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 59509759c..d7d045ee2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,41 @@ +# SRF 22.11.00 (18 Nov 2022) + +## 🚨 Breaking Changes + +- update abseil, grpc, and ucx versions for cuml compatibility ([#177](https://github.com/nv-morpheus/SRF/pull/177)) [@cwharris](https://github.com/cwharris) + +## 🐛 Bug Fixes + +- Fix throwing errors from `Executor.join_async()` ([#208](https://github.com/nv-morpheus/SRF/pull/208)) [@mdemoret-nv](https://github.com/mdemoret-nv) +- Fix help string for SRF_BUILD_DOCS ([#202](https://github.com/nv-morpheus/SRF/pull/202)) [@dagardner-nv](https://github.com/dagardner-nv) +- change pull_request to pull_request_target ([#201](https://github.com/nv-morpheus/SRF/pull/201)) [@jarmak-nv](https://github.com/jarmak-nv) +- Registered memory should be released before the UCX Context is destroyed ([#193](https://github.com/nv-morpheus/SRF/pull/193)) [@ryanolson](https://github.com/ryanolson) +- Fix tests so that the proper upstream build is used for the coverage test ([#192](https://github.com/nv-morpheus/SRF/pull/192)) [@dagardner-nv](https://github.com/dagardner-nv) +- Updating SRF versions from 22.09 to 22.11 ([#191](https://github.com/nv-morpheus/SRF/pull/191)) [@mdemoret-nv](https://github.com/mdemoret-nv) +- Fixes "Add new issue/PR to project" action ([#189](https://github.com/nv-morpheus/SRF/pull/189)) [@dagardner-nv](https://github.com/dagardner-nv) +- Fetch history and tags for package step ([#188](https://github.com/nv-morpheus/SRF/pull/188)) [@dagardner-nv](https://github.com/dagardner-nv) +- Fix CI deps ([#187](https://github.com/nv-morpheus/SRF/pull/187)) [@dagardner-nv](https://github.com/dagardner-nv) +- Emit the value before incrementing the iterator fixes ([#180](https://github.com/nv-morpheus/SRF/pull/180)) [@dagardner-nv](https://github.com/dagardner-nv) +- Fix returning of thread_binding attr ([#179](https://github.com/nv-morpheus/SRF/pull/179)) [@dagardner-nv](https://github.com/dagardner-nv) +- update abseil, grpc, and ucx versions for cuml compatibility ([#177](https://github.com/nv-morpheus/SRF/pull/177)) [@cwharris](https://github.com/cwharris) + +## 📖 Documentation + +- Add documentation on how to build the doxygen docs ([#183](https://github.com/nv-morpheus/SRF/pull/183)) [@dagardner-nv](https://github.com/dagardner-nv) + +## 🚀 New Features + +- : Replacing SRF markdown templates with yml forms ([#200](https://github.com/nv-morpheus/SRF/pull/200)) [@jarmak-nv](https://github.com/jarmak-nv) + +## 🛠️ Improvements + +- Improve NVML + MIG Behavior ([#206](https://github.com/nv-morpheus/SRF/pull/206)) [@ryanolson](https://github.com/ryanolson) +- Add dockerfile for CI runners ([#199](https://github.com/nv-morpheus/SRF/pull/199)) [@dagardner-nv](https://github.com/dagardner-nv) +- Add codecov upload ([#197](https://github.com/nv-morpheus/SRF/pull/197)) [@dagardner-nv](https://github.com/dagardner-nv) +- SRF Modules and Module Registry Implementation ([#196](https://github.com/nv-morpheus/SRF/pull/196)) [@drobison00](https://github.com/drobison00) +- Allow building build without GPU and without a driver ([#195](https://github.com/nv-morpheus/SRF/pull/195)) [@dagardner-nv](https://github.com/dagardner-nv) +- Switch to github actions ([#182](https://github.com/nv-morpheus/SRF/pull/182)) [@dagardner-nv](https://github.com/dagardner-nv) + # SRF 22.09.00 (30 Sep 2022) ## 📖 Documentation diff --git a/CMakeLists.txt b/CMakeLists.txt index 74c1c1c81..e306fecd2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,7 @@ list(APPEND CMAKE_MESSAGE_CONTEXT "srf") # Options of the form: SRF_USE_XXX, enable linting or alter the environment and are OFF by default option(BUILD_SHARED_LIBS "Default value for whether or not to build shared or static libraries" ON) option(SRF_BUILD_BENCHMARKS "Whether or not to build SRF benchmarks" OFF) -option(SRF_BUILD_DOCS "Enable building the python bindings for Srf" OFF) +option(SRF_BUILD_DOCS "Enable building of API documentation" OFF) option(SRF_BUILD_LIBRARY "Whether the entire SRF library should be built. If set to OFF, only the pieces needed for a target will be built. Set to ON if installing the library" ON) option(SRF_BUILD_PYTHON "Enable building the python bindings for Srf" ON) option(SRF_BUILD_TESTS "Whether or not to build SRF tests" ON) @@ -58,10 +58,12 @@ endif() rapids_cuda_init_architectures(srf) project(srf - VERSION 22.09.00 + VERSION 22.11.00 LANGUAGES C CXX ) +rapids_cmake_write_version_file(${CMAKE_BINARY_DIR}/autogenerated/include/srf/version.hpp) + # Delay enabling CUDA until after we have determined our CXX compiler if(NOT DEFINED CMAKE_CUDA_HOST_COMPILER) message(STATUS "Setting CUDA host compiler to match CXX compiler: ${CMAKE_CXX_COMPILER}") @@ -149,8 +151,8 @@ include(cmake/setup_iwyu.cmake) add_library(libsrf src/internal/data_plane/callbacks.cpp src/internal/data_plane/client.cpp - src/internal/data_plane/resources.cpp src/internal/data_plane/request.cpp + src/internal/data_plane/resources.cpp src/internal/data_plane/server.cpp src/internal/executor/executor.cpp src/internal/executor/iexecutor.cpp @@ -166,10 +168,10 @@ add_library(libsrf src/internal/pipeline/port_graph.cpp src/internal/pipeline/resources.cpp src/internal/resources/manager.cpp - src/internal/resources/partition_resources.cpp src/internal/resources/partition_resources_base.cpp - src/internal/runnable/engine_factory.cpp + src/internal/resources/partition_resources.cpp src/internal/runnable/engine.cpp + src/internal/runnable/engine_factory.cpp src/internal/runnable/engines.cpp src/internal/runnable/fiber_engine.cpp src/internal/runnable/fiber_engines.cpp @@ -200,8 +202,8 @@ add_library(libsrf src/internal/system/system.cpp src/internal/system/system_provider.cpp src/internal/system/thread.cpp - src/internal/system/thread_pool.cpp src/internal/system/thread.cpp + src/internal/system/thread_pool.cpp src/internal/system/topology.cpp src/internal/ucx/context.cpp src/internal/ucx/endpoint.cpp @@ -214,6 +216,8 @@ add_library(libsrf src/internal/utils/parse_config.cpp src/internal/utils/parse_ints.cpp src/internal/utils/shared_resource_bit_map.cpp + src/public/benchmarking/tracer.cpp + src/public/benchmarking/trace_statistics.cpp src/public/benchmarking/util.cpp src/public/channel/channel.cpp src/public/codable/encoded_object.cpp @@ -228,6 +232,10 @@ add_library(libsrf src/public/memory/buffer_view.cpp src/public/metrics/counter.cpp src/public/metrics/registry.cpp + src/public/modules/module_registry.cpp + src/public/modules/plugins.cpp + src/public/modules/sample_modules.cpp + src/public/modules/segment_modules.cpp src/public/node/edge_adapter_registry.cpp src/public/node/edge_builder.cpp src/public/node/edge_registry.cpp @@ -246,8 +254,6 @@ add_library(libsrf src/public/runnable/types.cpp src/public/segment/builder.cpp src/public/segment/definition.cpp - src/public/benchmarking/trace_statistics.cpp - src/public/benchmarking/tracer.cpp src/public/utils/bytes_to_string.cpp src/public/utils/thread_utils.cpp src/public/utils/type_utils.cpp @@ -285,10 +291,11 @@ target_link_libraries(libsrf target_include_directories(libsrf PUBLIC - $ - $ + $ + $ + $ PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/src + ${CMAKE_CURRENT_SOURCE_DIR}/src ) target_compile_definitions(libsrf diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aa995fd62..9b396db5a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,7 +27,7 @@ More information can be found at: [Contributor Code of Conduct](CODE_OF_CONDUCT. 1. Find an issue to work on. The best way is to look for issues with the [good first issue](https://github.com/NVIDIA/SRF/issues) label. 2. Comment on the issue stating that you are going to work on it. -3. Code! Make sure to update unit tests and confirm that test coverage has not decreased (see below)! Ensure the +3. Code! Make sure to update unit tests and confirm that test coverage has not decreased (see below)! Ensure the [license headers are set properly](#Licensing). 4. When done, [create your pull request](https://github.com/NVIDIA/SRF/compare). 5. Wait for other developers to review your code and update code as needed. @@ -37,7 +37,7 @@ Remember, if you are unsure about anything, don't hesitate to comment on issues ## Unit testing and Code Coverage Prior to submitting a pull request, you should ensure that all your contributed code is covered by unit tests, and that -unit test coverage percentages have not decreased (even better if they've increased). To test, from the SRF root +unit test coverage percentages have not decreased (even better if they've increased). To test, from the SRF root directory: 1. Generate a code coverage report and ensure your additions are covered. @@ -114,6 +114,13 @@ pip install -e $SRF_HOME/build/python pytest $SRF_HOME/python ``` +### Building API Documentation +From the root of the SRF repo, configure CMake with `SRF_BUILD_DOCS=ON` then build the `srf_docs` target. Once built the documentation will be located in the `build/docs/html` directory. +```bash +cmake -B build -DSRF_BUILD_DOCS=ON . +cmake --build build --target srf_docs +``` + ## Licensing SRF is licensed under the Apache v2.0 license. All new source files including CMake and other build scripts should contain the Apache v2.0 license header. Any edits to existing source code should update the date range of the copyright to the current year. The format for the license header is: diff --git a/Dockerfile b/Dockerfile index 89512e255..5f2bf6d99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,7 @@ RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/ RUN conda config --set ssl_verify false &&\ conda config --add pkgs_dirs /opt/conda/pkgs &&\ conda config --env --add channels conda-forge &&\ - /opt/conda/bin/conda install -y -n base -c conda-forge "mamba >=0.22" "boa >=0.10" python=${PYTHON_VER} + /opt/conda/bin/conda install -y -n base -c conda-forge "mamba >=0.22" "boa >=0.12" python=${PYTHON_VER} # conda clean -afy # All code will be under /work diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile deleted file mode 100644 index 877e50b92..000000000 --- a/ci/Jenkinsfile +++ /dev/null @@ -1,318 +0,0 @@ -@Library('jenkins_shared_lib') _ - -pipeline { - agent any - options { - disableConcurrentBuilds(abortPrevious: true) - } - environment { - BUILD_CC = 'gcc' - BUILD_TYPE = sh(returnStdout: true, script: 'rapids-build-type') - } - stages { - stage('Checks') { - when { environment name: 'BUILD_TYPE', value: 'pull-request' } - options { - timeout(time: 1, unit: 'HOURS') - } - environment { - PARALLEL_LEVEL = '4' - HOME = "${WORKSPACE}" - GH_TOKEN = credentials('gputester-github-token') - } - agent { - docker { - image 'gpuci/rapidsai-driver:21.10-cuda11.4-devel-ubuntu20.04-py3.8' - label 'cpu' - } - } - steps { - cleanWs( - deleteDirs: true, - externalDelete: 'sudo rm -rf %s' - ) - checkout scm - sh "${WORKSPACE}/ci/scripts/jenkins/checks.sh" - } - } - stage('Builds') { - failFast true - parallel { - stage('Build:linux:x86_64:gcc:release') { - options { - timeout(time: 1, unit: 'HOURS') - } - environment { - PARALLEL_LEVEL = '16' - HOME = "${WORKSPACE}" - } - agent { - docker { - image 'gpuci/rapidsai-driver:21.10-cuda11.4-devel-ubuntu20.04-py3.8' - label 'cpu4' - } - } - steps { - cleanWs( - deleteDirs: true, - externalDelete: 'sudo rm -rf %s' - ) - checkout scm - withCredentials([[ - $class: 'AmazonWebServicesCredentialsBinding', - credentialsId: "aws-s3-gpuci", - accessKeyVariable: 'AWS_ACCESS_KEY_ID', - secretKeyVariable: 'AWS_SECRET_ACCESS_KEY' - ]]) - { - sh "${WORKSPACE}/ci/scripts/jenkins/build.sh" - } - } - } - stage('Build:linux:x86_64:gcc:debug') { - options { - timeout(time: 1, unit: 'HOURS') - } - environment { - BUILD_CC= "gcc-coverage" - PARALLEL_LEVEL = '16' - HOME = "${WORKSPACE}" - } - agent { - docker { - image 'gpuci/rapidsai-driver:21.10-cuda11.4-devel-ubuntu20.04-py3.8' - label 'cpu4' - } - } - steps { - cleanWs( - deleteDirs: true, - externalDelete: 'sudo rm -rf %s' - ) - checkout scm - withCredentials([[ - $class: 'AmazonWebServicesCredentialsBinding', - credentialsId: "aws-s3-gpuci", - accessKeyVariable: 'AWS_ACCESS_KEY_ID', - secretKeyVariable: 'AWS_SECRET_ACCESS_KEY' - ]]) - { - sh "${WORKSPACE}/ci/scripts/jenkins/build.sh" - } - } - } - stage('Build:linux:x86_64:clang') { - options { - timeout(time: 1, unit: 'HOURS') - } - environment { - PARALLEL_LEVEL = '16' - BUILD_CC = 'clang' - HOME = "${WORKSPACE}" - } - agent { - docker { - image 'gpuci/rapidsai-driver:21.10-cuda11.4-devel-ubuntu20.04-py3.8' - label 'cpu4' - } - } - steps { - cleanWs( - deleteDirs: true, - externalDelete: 'sudo rm -rf %s' - ) - checkout scm - withCredentials([[ - $class: 'AmazonWebServicesCredentialsBinding', - credentialsId: "aws-s3-gpuci", - accessKeyVariable: 'AWS_ACCESS_KEY_ID', - secretKeyVariable: 'AWS_SECRET_ACCESS_KEY' - ]]) - { - sh "${WORKSPACE}/ci/scripts/jenkins/build.sh" - } - } - } - stage('Build:Documentation') { - options { - timeout(time: 1, unit: 'HOURS') - } - environment { - PARALLEL_LEVEL = '4' - HOME = "${WORKSPACE}" - } - agent { - docker { - image 'gpuci/rapidsai-driver:21.10-cuda11.4-devel-ubuntu20.04-py3.8' - label 'cpu' - } - } - steps { - cleanWs( - deleteDirs: true, - externalDelete: 'sudo rm -rf %s' - ) - checkout scm - withCredentials([[ - $class: 'AmazonWebServicesCredentialsBinding', - credentialsId: "aws-s3-gpuci", - accessKeyVariable: 'AWS_ACCESS_KEY_ID', - secretKeyVariable: 'AWS_SECRET_ACCESS_KEY' - ]]) - { - sh "${WORKSPACE}/ci/scripts/jenkins/docs.sh" - } - } - } - } - } - stage('Tests') { - failFast true - parallel { - stage('TestDebug') { - options { - timeout(time: 1, unit: 'HOURS') - } - agent { - docker { - image 'gpuci/rapidsai:21.10-cuda11.4-devel-ubuntu20.04-py3.8' - label 'driver-495' - args '--cap-add=sys_nice --runtime "nvidia" -e "NVIDIA_VISIBLE_DEVICES=$EXECUTOR_NUMBER"' - } - } - environment { - BUILD_TYPE = "Debug" - HOME = "${WORKSPACE}" - } - steps { - cleanWs( - deleteDirs: true, - externalDelete: 'sudo rm -rf %s' - ) - checkout scm - withCredentials([[ - $class: 'AmazonWebServicesCredentialsBinding', - credentialsId: "aws-s3-gpuci", - accessKeyVariable: 'AWS_ACCESS_KEY_ID', - secretKeyVariable: 'AWS_SECRET_ACCESS_KEY' - ]]) - { - sh "${WORKSPACE}/ci/scripts/jenkins/test.sh" - } - } - } - stage('TestRelease') { - options { - timeout(time: 1, unit: 'HOURS') - } - agent { - docker { - image 'gpuci/rapidsai:21.10-cuda11.4-devel-ubuntu20.04-py3.8' - label 'driver-495' - args '--cap-add=sys_nice --runtime "nvidia" -e "NVIDIA_VISIBLE_DEVICES=$EXECUTOR_NUMBER"' - } - } - environment { - BUILD_TYPE = "Release" - HOME = "${WORKSPACE}" - } - steps { - cleanWs( - deleteDirs: true, - externalDelete: 'sudo rm -rf %s' - ) - checkout scm - withCredentials([[ - $class: 'AmazonWebServicesCredentialsBinding', - credentialsId: "aws-s3-gpuci", - accessKeyVariable: 'AWS_ACCESS_KEY_ID', - secretKeyVariable: 'AWS_SECRET_ACCESS_KEY' - ]]) - { - sh "${WORKSPACE}/ci/scripts/jenkins/test.sh" - } - } - } - stage('Benchmark') { - options { - timeout(time: 1, unit: 'HOURS') - } - agent { - docker { - image 'gpuci/rapidsai-driver:21.10-cuda11.4-devel-ubuntu20.04-py3.8' - label 'cpu' - args '--cap-add=sys_nice' - } - } - environment { - HOME = "${WORKSPACE}" - } - steps { - cleanWs( - deleteDirs: true, - externalDelete: 'sudo rm -rf %s' - ) - checkout scm - withCredentials([[ - $class: 'AmazonWebServicesCredentialsBinding', - credentialsId: "aws-s3-gpuci", - accessKeyVariable: 'AWS_ACCESS_KEY_ID', - secretKeyVariable: 'AWS_SECRET_ACCESS_KEY' - ]]) - { - sh "${WORKSPACE}/ci/scripts/jenkins/pre_benchmark.sh" - sh "${WORKSPACE}/ci/scripts/jenkins/benchmark.sh" - sh "${WORKSPACE}/ci/scripts/jenkins/post_benchmark.sh" - } - } - } - } - } - stage('package:conda') { - when { - anyOf { - environment name: 'BUILD_TYPE', value: 'branch' - environment name: 'BUILD_TYPE', value: 'nightly' - } - } - options { - timeout(time: 1, unit: 'HOURS') - } - agent { - docker { - image 'gpuci/rapidsai-driver:21.10-cuda11.4-devel-ubuntu20.04-py3.8' - label 'cpu4' - } - } - environment { - HOME = "${WORKSPACE}" - PARALLEL_LEVEL = '16' - CONDA_PKG_LABEL = 'dev-ci' - } - steps { - cleanWs( - deleteDirs: true, - externalDelete: 'sudo rm -rf %s' - ) - checkout scm - withCredentials([[ - $class: 'AmazonWebServicesCredentialsBinding', - credentialsId: "aws-s3-gpuci", - accessKeyVariable: 'AWS_ACCESS_KEY_ID', - secretKeyVariable: 'AWS_SECRET_ACCESS_KEY' - ]]) - { - sh "${WORKSPACE}/ci/scripts/jenkins/conda.sh" - } - } - } - } - post { - always { - cleanWs( - deleteDirs: true, - externalDelete: 'sudo rm -rf %s' - ) - } - } -} diff --git a/ci/conda/environments/ci_env.yml b/ci/conda/environments/ci_env.yml index 993824428..9df4969bb 100644 --- a/ci/conda/environments/ci_env.yml +++ b/ci/conda/environments/ci_env.yml @@ -18,6 +18,4 @@ name: srf channels: - conda-forge dependencies: - - boa>=0.1 - - conda-pack=0.7 - - sccache=0.3 + - codecov=2.1 diff --git a/ci/conda/environments/dev_env.yml b/ci/conda/environments/dev_env.yml index b6ccd973e..879ae15c8 100644 --- a/ci/conda/environments/dev_env.yml +++ b/ci/conda/environments/dev_env.yml @@ -21,7 +21,7 @@ dependencies: - glog=0.6 - gmock=1.10 - graphviz=3.0 - - grpc-cpp=1.45 + - grpc-cpp=1.46 - gtest=1.10 - gxx_linux-64=9.4 - jinja2=3.0 @@ -42,7 +42,7 @@ dependencies: - scikit-build>=0.12 - spdlog=1.8.5 - sysroot_linux-64=2.17 - - ucx=1.12 + - ucx=1.13 - pip: - cython - flake8 diff --git a/ci/conda/environments/dev_env_nogcc.yml b/ci/conda/environments/dev_env_nogcc.yml index f1b8ec610..5be1b1455 100644 --- a/ci/conda/environments/dev_env_nogcc.yml +++ b/ci/conda/environments/dev_env_nogcc.yml @@ -17,7 +17,7 @@ dependencies: - glog=0.6 - gmock=1.10 - graphviz=3.0 - - grpc-cpp=1.45 + - grpc-cpp=1.46 - gtest=1.10 - libhwloc=2.5 - libprotobuf=3.20 @@ -28,7 +28,7 @@ dependencies: - python=3.8 - scikit-build>=0.12 - spdlog=1.8.5 - - ucx=1.12 + - ucx=1.13 - pip: - cython - flake8 diff --git a/ci/conda/recipes/libsrf/build.sh b/ci/conda/recipes/libsrf/build.sh index 711277d0d..11a4254a0 100644 --- a/ci/conda/recipes/libsrf/build.sh +++ b/ci/conda/recipes/libsrf/build.sh @@ -62,6 +62,7 @@ CMAKE_ARGS="-DSRF_BUILD_PYTHON=ON ${CMAKE_ARGS}" CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"ALL"} ${CMAKE_ARGS}" CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" CMAKE_ARGS="-DSRF_RAPIDS_VERSION=${rapids_version} ${CMAKE_ARGS}" +CMAKE_ARGS="-DUCX_VERSION=${ucx} ${CMAKE_ARGS}" echo "CC : ${CC}" echo "CXX : ${CXX}" diff --git a/ci/conda/recipes/libsrf/conda_build_config.yaml b/ci/conda/recipes/libsrf/conda_build_config.yaml index d11d84f59..84db3f5eb 100644 --- a/ci/conda/recipes/libsrf/conda_build_config.yaml +++ b/ci/conda/recipes/libsrf/conda_build_config.yaml @@ -30,19 +30,36 @@ python: - 3.9 # Setup the dependencies to build with multiple versions of RAPIDS -rapids_version: - - 22.04 # Keep around compatibility with current version -1 +rapids_version: # Keep around compatibility with current version -1 - 22.06 - 22.08 + - 22.08 +# Multiple versions of abseil are required to satisfy the solver for some +# environments. RAPIDS 22.06 only works with gRPC 1.45 and 22.08 only works with +# 1.46. For each version of gRPC, support 2 abseil versions. Zip all of the keys +# together to avoid impossible combinations abseil_cpp: - - 20211102.0 - - 20210324.2 - 20210324.2 + - 20211102.0 + - 20220623.0 + +grpc_cpp: + - 1.45 + - 1.46 + - 1.46 + +# UCX 1.12 is required for RAPIDS 22.06 +ucx: + - 1.12 + - 1.13 + - 1.13 zip_keys: - rapids_version - abseil_cpp + - grpc_cpp + - ucx # The following mimic what is available in the pinning feedstock: # https://github.com/conda-forge/conda-forge-pinning-feedstock/blob/main/recipe/conda_build_config.yaml @@ -54,12 +71,8 @@ gflags: - 2.2 glog: - 0.6 -grpc_cpp: - - 1.45 libprotobuf: - 3.20 -ucx: - - 1.12 pin_run_as_build: diff --git a/ci/conda/recipes/libsrf/meta.yaml b/ci/conda/recipes/libsrf/meta.yaml index e39a5ee8a..2eb2ff035 100644 --- a/ci/conda/recipes/libsrf/meta.yaml +++ b/ci/conda/recipes/libsrf/meta.yaml @@ -127,15 +127,16 @@ outputs: host: # Only should need libsrf and python. Keep sorted! - {{ pin_subpackage('libsrf', exact=True) }} + - abseil-cpp # srf does not require abseil at build time. See https://github.com/conda-forge/arrow-cpp-feedstock/issues/814 - python {{ python }} run: - {{ pin_subpackage('libsrf', exact=True) }} - - rmm {{ rapids_version }} # This is not necessary but required until this is fixed: https://github.com/mamba-org/boa/issues/232 + - rmm {{ rapids_version }}.* # This is not necessary but required until this is fixed: https://github.com/mamba-org/boa/issues/232 - python test: imports: - srf - script: test_libsrf.sh + script: test_srf.sh source_files: # Copy the pytest source files - python/pytest.ini @@ -145,6 +146,7 @@ outputs: - numpy - nvtx - pytest + - cuml {{ rapids_version }}.* # Ensure we can install cuml. This can cause issues solving abseil-cpp about: home: https://www.nvidia.com/ diff --git a/ci/runner/Dockerfile b/ci/runner/Dockerfile new file mode 100644 index 000000000..c0d510927 --- /dev/null +++ b/ci/runner/Dockerfile @@ -0,0 +1,72 @@ +# syntax=docker/dockerfile:1.3 + +# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Args used in FROM commands must come first +ARG FROM_IMAGE="rapidsai/ci" +ARG CUDA_VER=11.4.1 +ARG LINUX_DISTRO=ubuntu +ARG LINUX_VER=20.04 +ARG PYTHON_VER=3.8 + +# Configure the base docker img +FROM ${FROM_IMAGE}:cuda${CUDA_VER}-${LINUX_DISTRO}${LINUX_VER}-py${PYTHON_VER} AS base + +ARG PROJ_NAME=srf +ARG CUDA_SHORT_VER=11.4 +ARG CLANG_VER=12 + +SHELL ["/bin/bash", "-c"] + +# OS deps +RUN apt-get update &&\ + apt-get install --no-install-recommends -y \ + libnuma1 && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* + +# Create conda environment +COPY ./ci/conda/environments/* /tmp/conda/ + +RUN CONDA_ALWAYS_YES=true /opt/conda/bin/mamba env create -n ${PROJ_NAME} -q --file /tmp/conda/dev_env.yml && \ + /opt/conda/bin/mamba env update -q -n ${PROJ_NAME} --file /tmp/conda/clang_env.yml && \ + /opt/conda/bin/mamba env update -q -n ${PROJ_NAME} --file /tmp/conda/ci_env.yml && \ + sed -i "s/conda activate base/conda activate ${PROJ_NAME}/g" ~/.bashrc && \ + conda clean -afy && \ + rm -rf /tmp/conda + + +# Install IWYU (remove once we are on clang-14) +RUN git clone https://github.com/include-what-you-use/include-what-you-use.git /tmp/iwyu &&\ + cd /tmp/iwyu &&\ + git checkout clang_${CLANG_VER} &&\ + source activate ${PROJ_NAME} && \ + cmake -G Ninja \ + -DCMAKE_C_COMPILER=$(which clang) \ + -DCMAKE_CXX_COMPILER=$(which clang++) \ + -DCMAKE_INSTALL_PREFIX=${CONDA_PREFIX} . && \ + cmake --build . --parallel $(nproc) --target install && \ + cd - && \ + rm -rf /tmp/iwyu + +# ============ driver ================== +FROM base as driver + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC \ + apt install --no-install-recommends -y libnvidia-compute-495 && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/ci/runner/README.md b/ci/runner/README.md new file mode 100644 index 000000000..cf4ab727e --- /dev/null +++ b/ci/runner/README.md @@ -0,0 +1,43 @@ + + +The `Dockerfile` in this directory defines the images used by the CI runner not for SRF itself. + +# Building CI images +The `Dockerfile` defines two targets: `base` and `driver`. The `driver` target includes the Nvidia driver needed to build SRF on a machine without access to a GPU. + +To build the images from the root of the SRF repo run: +```bash +SKIP_PUSH=1 ci/runner/build_and_push.sh +``` + +# Build and push CI images +This will require being a member of the `Morpheus Early Access CI` group in [NGC](https://catalog.ngc.nvidia.com) and logging into the `nvcr.io` registry prior to running. + +From the root of the SRF repo run: +```bash +ci/runner/build_and_push.sh +``` + +If the images are already built, the build step can be skipped by setting `SKIP_BUILD=1`. + +# Updating CI to use the new images +Update `.github/workflows/pull_request.yml` changing these two lines with the new image names: +```yaml + container: nvcr.io/ea-nvidia-morpheus/morpheus:srf-ci-driver-221102 + test_container: nvcr.io/ea-nvidia-morpheus/morpheus:srf-ci-base-221102 +``` diff --git a/ci/runner/build_and_push.sh b/ci/runner/build_and_push.sh new file mode 100755 index 000000000..cc4c4b1e1 --- /dev/null +++ b/ci/runner/build_and_push.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DOCKER_TARGET=${DOCKER_TARGET:-"base" "driver"} +DOCKER_BUILDKIT=${DOCKER_BUILDKIT:-1} +DOCKER_REGISTRY_SERVER=${DOCKER_REGISTRY_SERVER:-"nvcr.io"} +DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH:-"/ea-nvidia-morpheus/morpheus"} +DOCKER_TAG_PREFIX=${DOCKER_TAG_PREFIX:-"srf-ci"} +DOCKER_TAG_POSTFIX=${DOCKER_TAG_POSTFIX:-"$(date +'%y%m%d')"} +DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""} + +SKIP_BUILD=${SKIP_BUILD:-""} +SKIP_PUSH=${SKIP_PUSH:-""} + +set -e + +function get_image_full_name() { + echo "${DOCKER_REGISTRY_SERVER}${DOCKER_REGISTRY_PATH}:${DOCKER_TAG_PREFIX}-${build_target}-${DOCKER_TAG_POSTFIX}" +} + +if [[ "${SKIP_BUILD}" == "" ]]; then + for build_target in ${DOCKER_TARGET[@]}; do + FULL_NAME=$(get_image_full_name) + echo "Building target \"${build_target}\" as ${FULL_NAME}"; + docker build --network=host ${DOCKER_EXTRA_ARGS} --target ${build_target} -t ${FULL_NAME} -f ci/runner/Dockerfile . + done +fi + +if [[ "${SKIP_PUSH}" == "" ]]; then + for build_target in ${DOCKER_TARGET[@]}; do + FULL_NAME=$(get_image_full_name) + echo "Pushing ${FULL_NAME}"; + docker push ${FULL_NAME} + done +fi diff --git a/ci/scripts/fix_all.sh b/ci/scripts/fix_all.sh index 14e249d29..79fd1ac2f 100755 --- a/ci/scripts/fix_all.sh +++ b/ci/scripts/fix_all.sh @@ -59,7 +59,9 @@ fi # Run include-what-you-use if [[ "${SKIP_IWYU}" == "" ]]; then - IWYU_TOOL=$(find_iwyu_tool) + if [[ "${IWYU_TOOL}" == "" ]]; then + IWYU_TOOL=$(find_iwyu_tool) + fi if [[ -x "${IWYU_TOOL}" ]]; then echo "Running include-what-you-use from '${IWYU_TOOL}'..." diff --git a/ci/scripts/jenkins/benchmark.sh b/ci/scripts/github/benchmark.sh similarity index 82% rename from ci/scripts/jenkins/benchmark.sh rename to ci/scripts/github/benchmark.sh index a4195df4e..4d00deb70 100755 --- a/ci/scripts/jenkins/benchmark.sh +++ b/ci/scripts/github/benchmark.sh @@ -16,24 +16,22 @@ set -e -source ${WORKSPACE}/ci/scripts/jenkins/common.sh +source ${WORKSPACE}/ci/scripts/github/common.sh REPORTS_DIR="${WORKSPACE_TMP}/reports" -conda activate srf - BENCHMARKS=($(find ${SRF_ROOT}/build/benchmarks -name "*.x")) -gpuci_logger "Running Benchmarks..." +rapids-logger "Running Benchmarks..." BENCH_RESULTS=0 for benchmark in "${BENCHMARKS[@]}"; do bench_name=$(basename ${benchmark}) - gpuci_logger "Running ${bench_name}" + rapids-logger "Running ${bench_name}" set +e taskset -c 0 ${benchmark} --benchmark_out_format=json --benchmark_out="${REPORTS_DIR}/${bench_name}.json" BENCH_RESULT=$? - gpuci_logger "${bench_name} completed with an exit status of ${BENCH_RESULT}" + rapids-logger "${bench_name} completed with an exit status of ${BENCH_RESULT}" BENCH_RESULTS=$(($BENCH_RESULTS+$BENCH_RESULT)) set -e @@ -42,4 +40,4 @@ done # We want the archive step to run, even if we failed, save our exit status, which the next step can use echo ${BENCH_RESULTS} > ${WORKSPACE_TMP}/exit_status -gpuci_logger "Running Benchmarks... Done" +rapids-logger "Running Benchmarks... Done" diff --git a/ci/scripts/github/build.sh b/ci/scripts/github/build.sh new file mode 100755 index 000000000..96d5d870f --- /dev/null +++ b/ci/scripts/github/build.sh @@ -0,0 +1,68 @@ +#!/usr/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +source ${WORKSPACE}/ci/scripts/github/common.sh + +update_conda_env + +CMAKE_CACHE_FLAGS="-DCCACHE_PROGRAM_PATH=$(which sccache) -DSRF_USE_CCACHE=ON" + +rapids-logger "Check versions" +python3 --version +cmake --version +ninja --version + +if [[ "${BUILD_CC}" == "gcc" ]]; then + rapids-logger "Building with GCC" + gcc --version + g++ --version + CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES} ${CMAKE_CACHE_FLAGS}" +elif [[ "${BUILD_CC}" == "gcc-coverage" ]]; then + rapids-logger "Building with GCC with gcov profile '-g -fprofile-arcs -ftest-coverage" + gcc --version + g++ --version + CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES} ${CMAKE_BUILD_WITH_CODECOV} ${CMAKE_CACHE_FLAGS}" +else + rapids-logger "Building with Clang" + clang --version + clang++ --version + CMAKE_CLANG_OPTIONS="-DCMAKE_C_COMPILER:FILEPATH=$(which clang) -DCMAKE_CXX_COMPILER:FILEPATH=$(which clang++) -DCMAKE_CUDA_COMPILER:FILEPATH=$(which nvcc)" + CMAKE_FLAGS="${CMAKE_CLANG_OPTIONS} ${CMAKE_BUILD_ALL_FEATURES} ${CMAKE_CACHE_FLAGS}" +fi + +show_conda_info + +rapids-logger "Configuring for build and test" +cmake -B build -G Ninja ${CMAKE_FLAGS} . + +rapids-logger "Building SRF" +cmake --build build --parallel ${PARALLEL_LEVEL} + +rapids-logger "sccache usage for SRF build:" +sccache --show-stats + +rapids-logger "Archiving results" +tar cfj "${WORKSPACE_TMP}/dot_cache.tar.bz" .cache +tar cfj "${WORKSPACE_TMP}/build.tar.bz" build +ls -lh ${WORKSPACE_TMP}/ + +rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/" +aws s3 cp --no-progress "${WORKSPACE_TMP}/build.tar.bz" "${ARTIFACT_URL}/build.tar.bz" +aws s3 cp --no-progress "${WORKSPACE_TMP}/dot_cache.tar.bz" "${ARTIFACT_URL}/dot_cache.tar.bz" + +rapids-logger "Success" diff --git a/ci/scripts/jenkins/conda.sh b/ci/scripts/github/checks.sh similarity index 52% rename from ci/scripts/jenkins/conda.sh rename to ci/scripts/github/checks.sh index 2a0ab8c14..cc4c4fa2c 100755 --- a/ci/scripts/jenkins/conda.sh +++ b/ci/scripts/github/checks.sh @@ -16,23 +16,23 @@ set -e -source ${WORKSPACE}/ci/scripts/jenkins/common.sh +source ${WORKSPACE}/ci/scripts/github/common.sh +fetch_base_branch -restore_conda_env +update_conda_env -gpuci_logger "Building Conda Package" -CONDA_BLD_OUTPUT="${WORKSPACE_TMP}/conda-bld" -mkdir -p ${CONDA_BLD_OUTPUT} +rapids-logger "Configuring CMake" +cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} . -CONDA_ARGS=() -CONDA_ARGS+=("--output-folder=${CONDA_BLD_OUTPUT}") -CONDA_ARGS+=("--label" "${CONDA_PKG_LABEL}") -CONDA_ARGS="${CONDA_ARGS[@]}" ${SRF_ROOT}/ci/conda/recipes/run_conda_build.sh +rapids-logger "Building targets that generate source code" +cmake --build build --target srf_style_checks --parallel ${PARALLEL_LEVEL} -gpuci_logger "Archiving Conda Package" -cd $(dirname ${CONDA_BLD_OUTPUT}) -tar cfj ${WORKSPACE_TMP}/conda_pkg.tar.bz $(basename ${CONDA_BLD_OUTPUT}) +rapids-logger "Running C++ style checks" +${SRF_ROOT}/ci/scripts/cpp_checks.sh -gpuci_logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/" -aws s3 cp ${WORKSPACE_TMP}/conda_pkg.tar.bz "${ARTIFACT_URL}/conda_pkg.tar.bz" +rapids-logger "Runing Python style checks" +${SRF_ROOT}/ci/scripts/python_checks.sh + +rapids-logger "Checking copyright headers" +python ${SRF_ROOT}/ci/scripts/copyright.py --verify-apache-v2 --git-diff-commits ${CHANGE_TARGET} ${GIT_COMMIT} diff --git a/ci/scripts/jenkins/common.sh b/ci/scripts/github/common.sh similarity index 68% rename from ci/scripts/jenkins/common.sh rename to ci/scripts/github/common.sh index d01bd32b6..3dd9ee948 100644 --- a/ci/scripts/jenkins/common.sh +++ b/ci/scripts/github/common.sh @@ -14,15 +14,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -gpuci_logger "Env Setup" +rapids-logger "Env Setup" source /opt/conda/etc/profile.d/conda.sh export SRF_ROOT=${SRF_ROOT:-$(git rev-parse --show-toplevel)} -gpuci_logger "Procs: $(nproc)" -gpuci_logger "Memory" +cd ${SRF_ROOT} +# For non-gpu hosts nproc will correctly report the number of cores we are able to use +# On a GPU host however nproc will report the total number of cores and PARALLEL_LEVEL +# will be defined specifying the subset we are allowed to use. +NUM_CORES=$(nproc) +export PARALLEL_LEVEL=${PARALLEL_LEVEL:-${NUM_CORES}} +rapids-logger "Procs: ${NUM_CORES}" +rapids-logger "Memory" /usr/bin/free -g -gpuci_logger "user info" +rapids-logger "user info" id # NUM_PROC is used by some of the other scripts @@ -30,16 +36,17 @@ export NUM_PROC=${PARALLEL_LEVEL:-$(nproc)} export CONDA_ENV_YML="${SRF_ROOT}/ci/conda/environments/dev_env.yml" -export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DSRF_BUILD_BENCHMARKS=ON -DSRF_BUILD_EXAMPLES=ON -DSRF_BUILD_PYTHON=ON -DSRF_BUILD_TESTS=ON -DSRF_USE_CONDA=ON" +export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DSRF_BUILD_BENCHMARKS=ON -DSRF_BUILD_EXAMPLES=ON -DSRF_BUILD_PYTHON=ON -DSRF_BUILD_TESTS=ON -DSRF_USE_CONDA=ON -DSRF_PYTHON_BUILD_STUBS=ON" export CMAKE_BUILD_WITH_CODECOV="-DCMAKE_BUILD_TYPE=Debug -DSRF_ENABLE_CODECOV=ON" # Set the depth to allow git describe to work export GIT_DEPTH=1000 # For PRs, $GIT_BRANCH is like: pull-request/989 -REPO_NAME=$(basename "${GIT_URL}" .git) -ORG_NAME=$(basename "$(dirname "${GIT_URL}")") -PR_NUM="${GIT_BRANCH##*/}" +REPO_NAME=$(basename "${GITHUB_REPOSITORY}") +ORG_NAME="${GITHUB_REPOSITORY_OWNER}" +PR_NUM="${GITHUB_REF_NAME##*/}" + # S3 vars export S3_URL="s3://rapids-downloads/ci/srf" @@ -51,21 +58,34 @@ export DISPLAY_ARTIFACT_URL="${DISPLAY_URL}${ARTIFACT_ENDPOINT}" # Set sccache env vars export SCCACHE_S3_KEY_PREFIX=srf-${NVARCH}-${BUILD_CC} export SCCACHE_BUCKET=rapids-sccache -export SCCACHE_REGION=us-west-2 +export SCCACHE_REGION="${AWS_DEFAULT_REGION}" export SCCACHE_IDLE_TIMEOUT=32768 #export SCCACHE_LOG=debug -gpuci_logger "Environ:" -env | sort +mkdir -p ${WORKSPACE_TMP} + +function print_env_vars() { + rapids-logger "Environ:" + env | grep -v -E "AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|TOKEN" | sort +} + +function update_conda_env() { + rapids-logger "Checking for updates to conda env" + rapids-mamba-retry env update -n srf -q --file ${CONDA_ENV_YML} + conda deactivate + conda activate srf +} + +print_env_vars function fetch_base_branch() { - gpuci_logger "Retrieving base branch from GitHub API" + rapids-logger "Retrieving base branch from GitHub API" [[ -n "$GH_TOKEN" ]] && CURL_HEADERS=('-H' "Authorization: token ${GH_TOKEN}") RESP=$( curl -s \ -H "Accept: application/vnd.github.v3+json" \ "${CURL_HEADERS[@]}" \ - "https://api.github.com/repos/${ORG_NAME}/${REPO_NAME}/pulls/${PR_NUM}" + "${GITHUB_API_URL}/repos/${ORG_NAME}/${REPO_NAME}/pulls/${PR_NUM}" ) BASE_BRANCH=$(echo "${RESP}" | jq -r '.base.ref') @@ -73,7 +93,7 @@ function fetch_base_branch() { # Change target is the branch name we are merging into but due to the weird way jenkins does # the checkout it isn't recognized by git without the origin/ prefix export CHANGE_TARGET="origin/${BASE_BRANCH}" - gpuci_logger "Base branch: ${BASE_BRANCH}" + rapids-logger "Base branch: ${BASE_BRANCH}" } function fetch_s3() { @@ -90,27 +110,8 @@ function fetch_s3() { function show_conda_info() { - gpuci_logger "Check Conda info" + rapids-logger "Check Conda info" conda info conda config --show-sources conda list --show-channel-urls } - -function restore_conda_env() { - - gpuci_logger "Downloading build artifacts from ${DISPLAY_ARTIFACT_URL}/" - fetch_s3 "${ARTIFACT_ENDPOINT}/conda_env.tar.gz" "${WORKSPACE_TMP}/conda_env.tar.gz" - - gpuci_logger "Extracting" - mkdir -p /opt/conda/envs/srf - - # We are using the --no-same-owner flag since user id & group id's are inconsistent between nodes in our CI pool - tar xf "${WORKSPACE_TMP}/conda_env.tar.gz" --no-same-owner --directory /opt/conda/envs/srf - - gpuci_logger "Setting conda env" - conda activate srf - conda-unpack - - show_conda_info -} - diff --git a/ci/scripts/github/conda.sh b/ci/scripts/github/conda.sh new file mode 100755 index 000000000..dc183e627 --- /dev/null +++ b/ci/scripts/github/conda.sh @@ -0,0 +1,25 @@ +#!/usr/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +source ${WORKSPACE}/ci/scripts/github/common.sh + +update_conda_env + +rapids-logger "Building Conda Package" + +${SRF_ROOT}/ci/conda/recipes/run_conda_build.sh upload diff --git a/ci/scripts/jenkins/docs.sh b/ci/scripts/github/docs.sh similarity index 71% rename from ci/scripts/jenkins/docs.sh rename to ci/scripts/github/docs.sh index 6b8d91a77..ba492c2db 100755 --- a/ci/scripts/jenkins/docs.sh +++ b/ci/scripts/github/docs.sh @@ -16,16 +16,11 @@ set -e -source ${WORKSPACE}/ci/scripts/jenkins/common.sh +source ${WORKSPACE}/ci/scripts/github/common.sh -rm -rf ${SRF_ROOT}/.cache/ ${SRF_ROOT}/build/ +update_conda_env -gpuci_logger "Creating conda env" -mamba env create -n srf -q --file ${CONDA_ENV_YML} -conda deactivate -conda activate srf - -gpuci_logger "Check versions" +rapids-logger "Check versions" python3 --version cmake --version ninja --version @@ -33,17 +28,17 @@ doxygen --version show_conda_info -gpuci_logger "Configuring for docs" +rapids-logger "Configuring for docs" cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} -DSRF_BUILD_DOCS=ON . -gpuci_logger "Building docs" +rapids-logger "Building docs" cmake --build build --target srf_docs -gpuci_logger "Tarring the docs" +rapids-logger "Tarring the docs" tar cfj "${WORKSPACE_TMP}/docs.tar.bz" build/docs/html -gpuci_logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/" +rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/" aws s3 cp --no-progress "${WORKSPACE_TMP}/docs.tar.bz" "${ARTIFACT_URL}/docs.tar.bz" -gpuci_logger "Success" +rapids-logger "Success" diff --git a/ci/scripts/jenkins/post_benchmark.sh b/ci/scripts/github/post_benchmark.sh similarity index 86% rename from ci/scripts/jenkins/post_benchmark.sh rename to ci/scripts/github/post_benchmark.sh index 342e69666..d08bce2b4 100755 --- a/ci/scripts/jenkins/post_benchmark.sh +++ b/ci/scripts/github/post_benchmark.sh @@ -16,15 +16,15 @@ set -e -source ${WORKSPACE}/ci/scripts/jenkins/common.sh +source ${WORKSPACE}/ci/scripts/github/common.sh REPORTS_DIR="${WORKSPACE_TMP}/reports" -gpuci_logger "Archiving benchmark reports" +rapids-logger "Archiving benchmark reports" cd $(dirname ${REPORTS_DIR}) tar cfj ${WORKSPACE_TMP}/benchmark_reports.tar.bz $(basename ${REPORTS_DIR}) -gpuci_logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/" +rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/" aws s3 cp ${WORKSPACE_TMP}/benchmark_reports.tar.bz "${ARTIFACT_URL}/benchmark_reports.tar.bz" exit $(cat ${WORKSPACE_TMP}/exit_status) diff --git a/ci/scripts/jenkins/pre_benchmark.sh b/ci/scripts/github/pre_benchmark.sh similarity index 66% rename from ci/scripts/jenkins/pre_benchmark.sh rename to ci/scripts/github/pre_benchmark.sh index 987e441db..419df25c2 100755 --- a/ci/scripts/jenkins/pre_benchmark.sh +++ b/ci/scripts/github/pre_benchmark.sh @@ -16,15 +16,13 @@ set -e -source ${WORKSPACE}/ci/scripts/jenkins/common.sh +source ${WORKSPACE}/ci/scripts/github/common.sh -restore_conda_env +update_conda_env -gpuci_logger "Fetching Build artifacts from ${DISPLAY_ARTIFACT_URL}/" -fetch_s3 "${ARTIFACT_ENDPOINT}/cpp_tests.tar.bz" "${WORKSPACE_TMP}/cpp_tests.tar.bz" -fetch_s3 "${ARTIFACT_ENDPOINT}/dsos.tar.bz" "${WORKSPACE_TMP}/dsos.tar.bz" +rapids-logger "Fetching Build artifacts from ${DISPLAY_ARTIFACT_URL}/" +fetch_s3 "${ARTIFACT_ENDPOINT}/build.tar.bz" "${WORKSPACE_TMP}/build.tar.bz" -tar xf "${WORKSPACE_TMP}/cpp_tests.tar.bz" -tar xf "${WORKSPACE_TMP}/dsos.tar.bz" +tar xf "${WORKSPACE_TMP}/build.tar.bz" mkdir -p ${WORKSPACE_TMP}/reports diff --git a/ci/scripts/jenkins/test.sh b/ci/scripts/github/test.sh similarity index 59% rename from ci/scripts/jenkins/test.sh rename to ci/scripts/github/test.sh index 133927f4d..6511c2be3 100755 --- a/ci/scripts/jenkins/test.sh +++ b/ci/scripts/github/test.sh @@ -16,37 +16,38 @@ set -e -source ${WORKSPACE}/ci/scripts/jenkins/common.sh +source ${WORKSPACE}/ci/scripts/github/common.sh /usr/bin/nvidia-smi -restore_conda_env +update_conda_env -gpuci_logger "Fetching Build artifacts from ${DISPLAY_ARTIFACT_URL}/" -fetch_s3 "${ARTIFACT_ENDPOINT}/cpp_tests.tar.bz" "${WORKSPACE_TMP}/cpp_tests.tar.bz" -fetch_s3 "${ARTIFACT_ENDPOINT}/dsos.tar.bz" "${WORKSPACE_TMP}/dsos.tar.bz" -fetch_s3 "${ARTIFACT_ENDPOINT}/python_build.tar.bz" "${WORKSPACE_TMP}/python_build.tar.bz" +rapids-logger "Fetching Build artifacts from ${DISPLAY_ARTIFACT_URL}/" +fetch_s3 "${ARTIFACT_ENDPOINT}/dot_cache.tar.bz" "${WORKSPACE_TMP}/dot_cache.tar.bz" +fetch_s3 "${ARTIFACT_ENDPOINT}/build.tar.bz" "${WORKSPACE_TMP}/build.tar.bz" -tar xf "${WORKSPACE_TMP}/cpp_tests.tar.bz" -tar xf "${WORKSPACE_TMP}/dsos.tar.bz" -tar xf "${WORKSPACE_TMP}/python_build.tar.bz" +tar xf "${WORKSPACE_TMP}/dot_cache.tar.bz" +tar xf "${WORKSPACE_TMP}/build.tar.bz" REPORTS_DIR="${WORKSPACE_TMP}/reports" mkdir -p ${WORKSPACE_TMP}/reports -# ctest requires cmake to be configured in order to locate tests +rapids-logger "Installing SRF" +cmake -P ${SRF_ROOT}/build/cmake_install.cmake +pip install ${SRF_ROOT}/build/python -if [[ "${BUILD_TYPE}" == "Debug" ]]; then - cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} ${CMAKE_BUILD_WITH_CODECOV} . +if [[ "${BUILD_CC}" == "gcc-coverage" ]]; then + CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES} ${CMAKE_BUILD_WITH_CODECOV}" else - cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} . + CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES}" fi -gpuci_logger "Running C++ Tests" +cmake -B build -G Ninja ${CMAKE_FLAGS} . + +rapids-logger "Running C++ Tests" cd ${SRF_ROOT}/build set +e # Tests known to be failing # Issues: -# * test_srf_benchmarking - https://github.com/nv-morpheus/SRF/issues/32 # * test_srf_private - https://github.com/nv-morpheus/SRF/issues/33 # * nvrpc - https://github.com/nv-morpheus/SRF/issues/34 ctest --output-on-failure \ @@ -57,28 +58,31 @@ CTEST_RESULTS=$? set -e cd ${SRF_ROOT} -gpuci_logger "Running Python Tests" +rapids-logger "Running Python Tests" cd ${SRF_ROOT}/build/python set +e pytest -v --junit-xml=${WORKSPACE_TMP}/report_pytest.xml PYTEST_RESULTS=$? set -e -if [[ "${BUILD_TYPE}" == "Debug" ]]; then - gpuci_logger "Generating codecov report" +if [[ "${BUILD_CC}" == "gcc-coverage" ]]; then + rapids-logger "Generating codecov report" cd ${SRF_ROOT} - cmake --build build --target gcovr-html-report + cmake --build build --target gcovr-html-report gcovr-xml-report - gpuci_logger "Archiving codecov report" + rapids-logger "Archiving codecov report" tar cfj ${WORKSPACE_TMP}/coverage_reports.tar.bz ${SRF_ROOT}/build/gcovr-html-report aws s3 cp ${WORKSPACE_TMP}/coverage_reports.tar.bz "${ARTIFACT_URL}/coverage_reports.tar.bz" + + gpuci_logger "Upload codecov report" + codecov --root ${SRF_ROOT} -f ${SRF_ROOT}/build/gcovr-xml-report.xml fi -gpuci_logger "Archiving test reports" +rapids-logger "Archiving test reports" cd $(dirname ${REPORTS_DIR}) tar cfj ${WORKSPACE_TMP}/test_reports.tar.bz $(basename ${REPORTS_DIR}) -gpuci_logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/" +rapids-logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/" aws s3 cp ${WORKSPACE_TMP}/test_reports.tar.bz "${ARTIFACT_URL}/test_reports.tar.bz" TEST_RESULTS=$(($CTEST_RESULTS+$PYTEST_RESULTS)) diff --git a/ci/scripts/jenkins/build.sh b/ci/scripts/jenkins/build.sh deleted file mode 100755 index b8e1bef0d..000000000 --- a/ci/scripts/jenkins/build.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -source ${WORKSPACE}/ci/scripts/jenkins/common.sh - -rm -rf ${SRF_ROOT}/.cache/ ${SRF_ROOT}/build/ - -gpuci_logger "Creating conda env" -mamba env create -n srf -q --file ${CONDA_ENV_YML} -conda deactivate -conda activate srf - -mamba env update -q -n srf --file ${SRF_ROOT}/ci/conda/environments/ci_env.yml - -CMAKE_CACHE_FLAGS="-DCCACHE_PROGRAM_PATH=$(which sccache) -DSRF_USE_CCACHE=ON" - -gpuci_logger "Check versions" -python3 --version -cmake --version -ninja --version - -if [[ "${BUILD_CC}" == "gcc" ]]; then - gpuci_logger "Building with GCC" - gcc --version - g++ --version - CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES} ${CMAKE_CACHE_FLAGS}" -elif [[ "${BUILD_CC}" == "gcc-coverage" ]]; then - gpuci_logger "Building with GCC with gcov profile '-g -fprofile-arcs -ftest-coverage" - gcc --version - g++ --version - CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES} ${CMAKE_BUILD_WITH_CODECOV} ${CMAKE_CACHE_FLAGS}" -else - gpuci_logger "Installing Clang" - mamba env update -q -n srf --file ${SRF_ROOT}/ci/conda/environments/clang_env.yml - gpuci_logger "Building with Clang" - clang --version - clang++ --version - CMAKE_CLANG_OPTIONS="-DCMAKE_C_COMPILER:FILEPATH=$(which clang) -DCMAKE_CXX_COMPILER:FILEPATH=$(which clang++) -DCMAKE_CUDA_COMPILER:FILEPATH=$(which nvcc)" - CMAKE_FLAGS="${CMAKE_CLANG_OPTIONS} ${CMAKE_BUILD_ALL_FEATURES} ${CMAKE_CACHE_FLAGS}" -fi - -show_conda_info - -gpuci_logger "Configuring for build and test" -cmake -B build -G Ninja ${CMAKE_FLAGS} . - -gpuci_logger "Building SRF" -cmake --build build --parallel ${PARALLEL_LEVEL} - -gpuci_logger "sccache usage for SRF build:" -sccache --show-stats - -gpuci_logger "Installing SRF" -cmake -P ${SRF_ROOT}/build/cmake_install.cmake -pip install ${SRF_ROOT}/build/python - -gpuci_logger "Archiving results" -mamba pack --quiet --force --ignore-missing-files --n-threads ${PARALLEL_LEVEL} -n srf -o ${WORKSPACE_TMP}/conda_env.tar.gz -tar cfj "${WORKSPACE_TMP}/cpp_tests.tar.bz" $(find build/ -name "*.x") -tar cfj "${WORKSPACE_TMP}/dsos.tar.bz" $(find build/ -name "*.so") -tar cfj "${WORKSPACE_TMP}/python_build.tar.bz" build/python -ls -lh ${WORKSPACE_TMP}/ - -gpuci_logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/" -aws s3 cp --no-progress "${WORKSPACE_TMP}/conda_env.tar.gz" "${ARTIFACT_URL}/conda_env.tar.gz" -aws s3 cp --no-progress "${WORKSPACE_TMP}/cpp_tests.tar.bz" "${ARTIFACT_URL}/cpp_tests.tar.bz" -aws s3 cp --no-progress "${WORKSPACE_TMP}/dsos.tar.bz" "${ARTIFACT_URL}/dsos.tar.bz" -aws s3 cp --no-progress "${WORKSPACE_TMP}/python_build.tar.bz" "${ARTIFACT_URL}/python_build.tar.bz" - -gpuci_logger "Success" diff --git a/ci/scripts/jenkins/checks.sh b/ci/scripts/jenkins/checks.sh deleted file mode 100755 index bd354bedb..000000000 --- a/ci/scripts/jenkins/checks.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -source ${WORKSPACE}/ci/scripts/jenkins/common.sh -export IWYU_DIR="${WORKSPACE_TMP}/iwyu" - -rm -rf ${SRF_ROOT}/.cache/ ${SRF_ROOT}/build/ ${IWYU_DIR} - -fetch_base_branch - -gpuci_logger "Creating conda env" -mamba env create -n srf -q --file ${CONDA_ENV_YML} - -gpuci_logger "Installing Clang" -mamba env update -q -n srf --file ${SRF_ROOT}/ci/conda/environments/clang_env.yml - -conda deactivate -conda activate srf - -show_conda_info - -gpuci_logger "Installing IWYU" -git clone https://github.com/include-what-you-use/include-what-you-use.git ${IWYU_DIR} -pushd ${IWYU_DIR} -git checkout clang_12 -cmake -G Ninja \ - -DCMAKE_PREFIX_PATH=$(llvm-config --cmakedir) \ - -DCMAKE_C_COMPILER=$(which clang) \ - -DCMAKE_CXX_COMPILER=$(which clang++) \ - -DCMAKE_INSTALL_PREFIX=${CONDA_PREFIX} \ - . - -cmake --build . --parallel ${PARALLEL_LEVEL} --target install - -popd - -gpuci_logger "Configuring CMake" -cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} . - -gpuci_logger "Building targets that generate source code" -cmake --build build --target srf_style_checks --parallel ${PARALLEL_LEVEL} - -gpuci_logger "Running C++ style checks" -${SRF_ROOT}/ci/scripts/cpp_checks.sh - -gpuci_logger "Runing Python style checks" -${SRF_ROOT}/ci/scripts/python_checks.sh - -gpuci_logger "Checking copyright headers" -python ${SRF_ROOT}/ci/scripts/copyright.py --verify-apache-v2 --git-diff-commits ${CHANGE_TARGET} ${GIT_COMMIT} diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index 72262cca5..dc162a29c 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -64,7 +64,7 @@ include(deps/Configure_boost) # UCX # === -set(UCX_VERSION "1.12" CACHE STRING "Version of ucx to use") +set(UCX_VERSION "1.13" CACHE STRING "Version of ucx to use") include(deps/Configure_ucx) # hwloc diff --git a/cmake/deps/Configure_gRPC.cmake b/cmake/deps/Configure_gRPC.cmake deleted file mode 100644 index fddae0dd5..000000000 --- a/cmake/deps/Configure_gRPC.cmake +++ /dev/null @@ -1,41 +0,0 @@ -#============================================================================= -# SPDX-FileCopyrightText: Copyright (c) 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -function(find_and_configure_gRPC VERSION) - - list(APPEND CMAKE_MESSAGE_CONTEXT "gRPC") - - rapids_cpm_find(gRPC ${GRPC_VERSION} - GLOBAL_TARGETS - gRPC::grpc++_unsecure gRPC::grpc_cpp_plugin gRPC::gpr - BUILD_EXPORT_SET - ${PROJECT_NAME}-core-exports - INSTALL_EXPORT_SET - ${PROJECT_NAME}-core-exports - CPM_ARGS - GIT_REPOSITORY https://github.com/grpc/grpc.git - GIT_TAG v${GRPC_VERSION} - GIT_SHALLOW TRUE - OPTIONS "BUILD_TESTS OFF" - "BUILD_BENCHMARKS OFF" - "CUDA_STATIC_RUNTIME ON" - "DISABLE_DEPRECATION_WARNING ${DISABLE_DEPRECATION_WARNINGS}" - ) - -endfunction() - -find_and_configure_gRPC(${GRPC_VERSION}) diff --git a/cmake/deps/Configure_ucx.cmake b/cmake/deps/Configure_ucx.cmake index 2bcd5da9f..550999d30 100644 --- a/cmake/deps/Configure_ucx.cmake +++ b/cmake/deps/Configure_ucx.cmake @@ -20,7 +20,7 @@ function(find_and_configure_ucx version) list(APPEND CMAKE_MESSAGE_CONTEXT "ucx") # Try to find UCX and download from source if not found - rapids_cpm_find(ucx 1.12 + rapids_cpm_find(ucx ${version} GLOBAL_TARGETS ucx ucx::ucp ucx::uct ucx_ucx ucx::ucp ucx::uct ucx::ucx BUILD_EXPORT_SET diff --git a/docs/quickstart/CMakeLists.txt b/docs/quickstart/CMakeLists.txt index 0f692e350..b3db84fea 100644 --- a/docs/quickstart/CMakeLists.txt +++ b/docs/quickstart/CMakeLists.txt @@ -25,7 +25,7 @@ list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake") include(import_rapids_cmake) project(srf-quickstart - VERSION 22.09.00 + VERSION 22.11.00 LANGUAGES C CXX ) diff --git a/docs/quickstart/environment_cpp.yml b/docs/quickstart/environment_cpp.yml index 3d8464080..992c2e6ae 100644 --- a/docs/quickstart/environment_cpp.yml +++ b/docs/quickstart/environment_cpp.yml @@ -30,7 +30,7 @@ dependencies: - python=3.8 - scikit-build>=0.12 - spdlog=1.8.5 - - srf=22.09 + - srf=22.11 - sysroot_linux-64=2.17 - pip: - cython diff --git a/include/srf/modules/module_registry.hpp b/include/srf/modules/module_registry.hpp new file mode 100644 index 000000000..effafb3f2 --- /dev/null +++ b/include/srf/modules/module_registry.hpp @@ -0,0 +1,123 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +namespace srf::modules { + +class SegmentModule; + +/** + * Simple, thread safe, global module registry. + */ +class ModuleRegistry +{ + public: + using module_constructor_t = + std::function(std::string module_name, nlohmann::json config)>; + + using module_registry_map_t = std::map; + using module_namespace_map_t = std::map; + using module_name_map_t = std::map>; + using registry_version_t = std::vector; + + ModuleRegistry() = delete; + + /** + * Return true if the registry contains 'name' else false + * @param name Name of the module + * @return boolean indicating if the registry contains the required module. + */ + static bool contains(const std::string& name, const std::string& registry_namespace = "default"); + + /** + * Return true if the registry contains the namespace, else false + * @param registry_namespace Namespace name + * @return boolean indicating if the registry contains the required namespace. + */ + static bool contains_namespace(const std::string& registry_namespace); + + /** + * Attempt to retrieve the module constructor for a given module name; throws an error + * if the given module does not exist. + * @param name Name of the module + * @return Module constructor + */ + static module_constructor_t get_module_constructor(const std::string& name, + const std::string& registry_namespace = "default"); + + /** + * Retrieve a map of namespace -> registered module name vectors + * @return Map of namespace -> registered module vector pairs + */ + static const module_name_map_t& registered_modules(); + + /** + * Simple register call, places the module into the default namespace + * @param name Name of the module + * @param fn_constructor Module constructor + */ + static void register_module(std::string name, + const std::vector& release_version, + module_constructor_t fn_constructor); + + /** + * Attempt to register the provided module constructor for the given name; throws an error + * if the module already exists. + * @param name Name of the module + * @param registry_namespace Namespace where the module `name` should be registered. + * @param fn_constructor Module constructor + */ + static void register_module(std::string name, + std::string registry_namespace, + const registry_version_t& release_version, + module_constructor_t fn_constructor); + + /** + * Unregister an existing module + * @param name Name of the module to un-register + * @param registry_namespace Namespace where module `name` should reside. + * @param optional If true, then it is not an error if the module does not exist. + */ + static void unregister_module(const std::string& name, const std::string& registry_namespace, bool optional = true); + + /** + * @param release_version vector of unsigned integers corresponding to the version string to check against the + * registry version. + * @return true if release version is compatible with registry version, false otherwise. + */ + static bool is_version_compatible(const registry_version_t& release_version); + + private: + static const unsigned int VersionElements; + static const registry_version_t Version; + + static module_name_map_t s_module_name_map; + static module_namespace_map_t s_module_namespace_registry; + static std::recursive_mutex s_mutex; +}; + +} // namespace srf::modules diff --git a/include/srf/modules/module_registry_util.hpp b/include/srf/modules/module_registry_util.hpp new file mode 100644 index 000000000..23fd34918 --- /dev/null +++ b/include/srf/modules/module_registry_util.hpp @@ -0,0 +1,53 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "srf/modules/module_registry.hpp" +#include "srf/modules/segment_modules.hpp" + +#include +#include + +namespace srf::modules { +struct ModelRegistryUtil +{ + /** + * Helper function for registering a new module; automatically check that the type of the object is a segment + * module, and build the constructor boiler plate. + * @tparam ModuleTypeT Module type, must have modules::SegmentModule as a base class + * @param name Name of the Module + * @param registry_namespace Namespace where `name` should be registered. + */ + template + static void create_registered_module(std::string name, + std::string registry_namespace, + const std::vector& release_version) + { + static_assert(std::is_base_of_v, + "ModuleTypeT must derive from SegmentModule"); + + ModuleRegistry::register_module(std::move(name), + std::move(registry_namespace), + release_version, + [](std::string module_name, nlohmann::json config) { + return std::make_shared(std::move(module_name), + std::move(config)); + }); + } +}; +} // namespace srf::modules diff --git a/include/srf/modules/plugins.hpp b/include/srf/modules/plugins.hpp new file mode 100644 index 000000000..a6884cf20 --- /dev/null +++ b/include/srf/modules/plugins.hpp @@ -0,0 +1,111 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace srf::modules { + +class PluginModule +{ + using module_plugin_map_t = std::map>; + + public: + PluginModule() = delete; + PluginModule(PluginModule&&) = delete; + PluginModule(const PluginModule&) = delete; + + ~PluginModule() = default; + + void operator=(const PluginModule&) = delete; + + /** + * Prevent duplicate versions of a plugin library from existing + * @param plugin_library_name Path to the library file + * @return A shared pointer to an existing or newly created PluginModule + */ + static std::shared_ptr create_or_acquire(const std::string& plugin_library_name); + + // Configuration so that dependent libraries will be searched for in + // 'path' during OpenLibraryHandle. + void set_library_directory(std::string path); + + // Reset any configuration done by SetLibraryDirectory. + void reset_library_directory(); + + /** + * Load plugin module -- will load the plugin library and call its loader entrypoint to register + * any modules it contains. + * @param throw_on_error Flag indicating if failure to load a library is an error; true by default. + * @return true if the library was successfully loaded, false if throw_on_error is false and load failed + */ + bool load(bool throw_on_error = true); + + /** + * Unload the plugin module -- this will call the unload entrypoint of the plugin, which will then + * unload any registered models. + * @param throw_on_error Flag indicating if failure to load a library is an error; true by default. + * @return true if the library was successfully unloaded, false if throw_on_error is false and unload failed + */ + bool unload(bool throw_on_error = true); + + /** + * Unload and re-load the given module + */ + void reload(); + + /** + * Return a list of modules published by the plugin + */ + std::vector list_modules(); + + private: + explicit PluginModule(std::string plugin_library_name); + + static std::recursive_mutex s_mutex; + static module_plugin_map_t s_plugin_map; + + static const std::string PluginEntrypointLoad; + static const std::string PluginEntrypointUnload; + static const std::string PluginEntrypointList; + + const std::string m_plugin_library_name; + std::string m_plugin_library_dir{}; + + void* m_plugin_handle{nullptr}; + bool m_loaded{false}; + + bool (*m_plugin_load)(); + bool (*m_plugin_unload)(); + unsigned int (*m_plugin_list)(const char***); + + bool try_load_plugin(bool throw_on_error = true); + bool try_unload_plugin(bool throw_on_error = true); + bool try_build_plugin_interface(bool throw_on_error = true); + bool try_open_library_handle(bool throw_on_error = true); + bool try_close_library_handle(bool throw_on_error = true); + + void get_entrypoint(const std::string& entrypoint_name, void** entrypoint); + void clear_plugin_interface(); +}; + +} // namespace srf::modules diff --git a/include/srf/modules/sample_modules.hpp b/include/srf/modules/sample_modules.hpp new file mode 100644 index 000000000..cdd22c8b6 --- /dev/null +++ b/include/srf/modules/sample_modules.hpp @@ -0,0 +1,260 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "srf/core/utils.hpp" +#include "srf/modules/segment_modules.hpp" +#include "srf/segment/builder.hpp" + +#include +#include + +#include +#include + +namespace srf::modules { + +/** + * Create a 2 input 2 output SegmentModule + * Inputs: input1:bool, input2:bool + * Outputs: output1:std::string, output2:std::string + */ +class SimpleModule : public SegmentModule +{ + using type_t = SimpleModule; + + public: + SimpleModule(std::string module_name); + SimpleModule(std::string module_name, nlohmann::json config); + + bool m_was_configured{false}; + + protected: + void initialize(segment::Builder& builder) override; + std::string module_type_name() const override; + + private: + bool m_initialized{false}; +}; + +/** + * Create a 1 input 1 output module that sets 'm_was_configured' variable if 'config_key_1' is found in the config. + * Inputs: configurable_input_a:bool + * Outputs: configureable_output_x:std::string + */ +class ConfigurableModule : public SegmentModule +{ + using type_t = ConfigurableModule; + + public: + ConfigurableModule(std::string module_name); + ConfigurableModule(std::string module_name, nlohmann::json config); + + bool m_was_configured{false}; + + protected: + void initialize(segment::Builder& builder) override; + std::string module_type_name() const override; + + private: + bool m_initialized; +}; + +/** + * Create a module that acts as a data source with one output + * By default emits a single value, configurable by passing 'source_count' in the config. + * Outputs: source:bool + */ +class SourceModule : public SegmentModule +{ + using type_t = SourceModule; + + public: + SourceModule(std::string module_name); + SourceModule(std::string module_name, nlohmann::json config); + + protected: + void initialize(segment::Builder& builder) override; + std::string module_type_name() const override; +}; + +/** + * Create a module that acts as a data sink with one input + * Inputs: sink:bool + */ +class SinkModule : public SegmentModule +{ + using type_t = SinkModule; + + public: + SinkModule(std::string module_name); + SinkModule(std::string module_name, nlohmann::json config); + + unsigned int m_packet_count{0}; + + protected: + void initialize(segment::Builder& builder) override; + std::string module_type_name() const override; +}; + +/** + * Creates a single output module that: + * - Creates a nested ConfigurableModule + * - Creates a nested SourceModule + * - Creates an edge between the SourceModule's output and the ConfigurableModule's input + * - Publishes the ConfigurableModule's 'configurable_output_x' as NestedModule's 'nested_module_output' + * Outputs: nested_module_output:bool + */ +class NestedModule : public SegmentModule +{ + using type_t = NestedModule; + + public: + NestedModule(std::string module_name); + NestedModule(std::string module_name, nlohmann::json config); + + protected: + void initialize(segment::Builder& builder) override; + std::string module_type_name() const override; +}; + +/** + * Creates a data source that emits OutputTypeT data elements + * @tparam OutputTypeT Type of data to emit + * Outputs: source:OutputTypeT + */ +template +class TemplateModule : public SegmentModule +{ + using type_t = TemplateModule; + + public: + TemplateModule(std::string module_name); + TemplateModule(std::string module_name, nlohmann::json config); + + void initialize(segment::Builder& builder) override; + std::string module_type_name() const override; +}; + +template +TemplateModule::TemplateModule(std::string module_name) : SegmentModule(std::move(module_name)) +{} + +template +TemplateModule::TemplateModule(std::string module_name, nlohmann::json config) : + SegmentModule(std::move(module_name), std::move(config)) +{} + +template +void TemplateModule::initialize(segment::Builder& builder) +{ + unsigned int count{1}; + + if (config().contains("source_count")) + { + count = config()["source_count"]; + } + + auto source = builder.make_source("source", [count](rxcpp::subscriber& sub) { + if (sub.is_subscribed()) + { + for (unsigned int i = 0; i < count; ++i) + { + sub.on_next(std::move(OutputTypeT())); + } + } + + sub.on_completed(); + }); + + // Register the submodules output as one of this module's outputs + register_output_port("source", source); +} + +template +std::string TemplateModule::module_type_name() const +{ + return std::string(::srf::type_name()); +} + +/** + * Creates a data source that emits OutputTypeT data elements, and takes a lambda function used to initialize the + * emitted data element. + * @tparam OutputTypeT Type of data to emit + * @tparam Initializer Lambda function taking no inputs and returning a object of OutputTypeT + * Outputs: source:OutputTypeT + */ +template +class TemplateWithInitModule : public SegmentModule +{ + using type_t = TemplateWithInitModule; + + public: + TemplateWithInitModule(std::string module_name); + TemplateWithInitModule(std::string module_name, nlohmann::json config); + + void initialize(segment::Builder& builder) override; + std::string module_type_name() const override; +}; + +template +TemplateWithInitModule::TemplateWithInitModule(std::string module_name) : + SegmentModule(std::move(module_name)) +{} + +template +TemplateWithInitModule::TemplateWithInitModule(std::string module_name, + nlohmann::json config) : + SegmentModule(std::move(module_name), std::move(config)) +{} + +template +void TemplateWithInitModule::initialize(segment::Builder& builder) +{ + unsigned int count{1}; + + if (config().contains("source_count")) + { + count = config()["source_count"]; + } + + auto source = builder.make_source("source", [count](rxcpp::subscriber& sub) { + if (sub.is_subscribed()) + { + for (unsigned int i = 0; i < count; ++i) + { + auto data = Initializer(); + + sub.on_next(std::move(data)); + } + } + + sub.on_completed(); + }); + + // Register the submodules output as one of this module's outputs + register_output_port("source", source); +} + +template +std::string TemplateWithInitModule::module_type_name() const +{ + return std::string(::srf::type_name()); +} + +} // namespace srf::modules diff --git a/include/srf/modules/segment_modules.hpp b/include/srf/modules/segment_modules.hpp new file mode 100644 index 000000000..2175e893f --- /dev/null +++ b/include/srf/modules/segment_modules.hpp @@ -0,0 +1,169 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "srf/segment/forward.hpp" +#include "srf/segment/object.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace srf::modules { + +class SegmentModule +{ + friend srf::segment::Builder; + + public: + using segment_module_port_map_t = std::map>; + using segment_module_port_t = std::shared_ptr; + using segment_module_typeindex_map_t = std::map; + + virtual ~SegmentModule() = default; + + SegmentModule() = delete; + + SegmentModule(std::string module_name); + SegmentModule(std::string module_name, nlohmann::json config); + + std::string component_prefix() const; + const nlohmann::json& config() const; + const std::string& name() const; + + /** + * Return vector of input ids -- these are only understood by the SegmentModule + * @return std::vector + */ + const std::vector& input_ids() const; + + /** + * Return a vector of output ids -- these are only understood by the SegmentModule class + * @return std::vector + */ + const std::vector& output_ids() const; + + /** + * Return a set of ObjectProperties for module input_ids + * @return ObjectProperties + */ + const segment_module_port_map_t& input_ports() const; + + /** + * Return the ObjectProperties object corresponding to input_name + * @param input_name Name of the module port + * @return ObjectProperties + */ + segment_module_port_t input_port(const std::string& input_name); + + /** + * Return a map of module port id : type indices + * @return std::map + */ + const segment_module_typeindex_map_t& input_port_type_ids() const; + + /** + * Return the type index of a given input name + * @return Type index + */ + std::type_index input_port_type_id(const std::string& input_name); + + /** + * Return a set of ObjectProperties for module input_ids + * @return ObjectProperties + */ + const segment_module_port_map_t& output_ports() const; + + /** + * Return an ObjectProperties for the module port corresponding to output name + * @param output_name Name of the module port to return + * @return ObjectProperties + */ + segment_module_port_t output_port(const std::string& output_name); + + /** + * Return a map of module port id : type indices + * @return std::map + */ + const segment_module_typeindex_map_t& output_port_type_ids() const; + + /** + * Return the type index of a given input name + * @param input_name Name of the module port to return a type index for + * @return Type index + */ + std::type_index output_port_type_id(const std::string& output_name); + + /** + * Functional entrypoint for module constructor during build -- this lets us act like a std::function + * @param builder + */ + void operator()(segment::Builder& builder); + + /** + * Retrieve the class name for the module, defaults to 'segment_module' + * @return + */ + virtual std::string module_type_name() const = 0; + + protected: + // Derived class interface functions + /* Virtual Functions */ + /** + * Entrypoint for module constructor during build + * @param builder + */ + virtual void initialize(segment::Builder& builder) = 0; + + /* Interface Functions */ + /** + * Register an input port that should be exposed for the module + * @param input_name Port name + * @param object ObjectProperties object associated with the port + */ + void register_input_port(std::string input_name, std::shared_ptr object); + + /** + * Register an output port that should be exposed for the module + * @param input_name Port name + * @param object ObjectProperties object assocaited with the port + */ + void register_output_port(std::string output_name, std::shared_ptr object); + + private: + const std::string m_module_instance_name; + + std::string m_module_instance_registered_namespace{}; + + std::vector m_input_port_ids{}; + std::vector m_output_port_ids{}; + + segment_module_typeindex_map_t m_input_port_type_indices{}; + segment_module_typeindex_map_t m_output_port_type_indices{}; + + segment_module_port_map_t m_input_ports{}; + segment_module_port_map_t m_output_ports{}; + + const nlohmann::json m_config; +}; + +} // namespace srf::modules diff --git a/include/srf/segment/builder.hpp b/include/srf/segment/builder.hpp index 1e63380f8..1e26bcd64 100644 --- a/include/srf/segment/builder.hpp +++ b/include/srf/segment/builder.hpp @@ -21,6 +21,7 @@ #include "srf/core/watcher.hpp" #include "srf/engine/segment/ibuilder.hpp" #include "srf/exceptions/runtime_error.hpp" +#include "srf/modules/segment_modules.hpp" #include "srf/node/edge_builder.hpp" #include "srf/node/rx_node.hpp" #include "srf/node/rx_sink.hpp" @@ -40,6 +41,7 @@ #include // IWYU pragma: keep #include +#include #include #include @@ -50,6 +52,7 @@ #include #include #include +#include // IWYU pragma: no_include // IWYU pragma: no_include @@ -118,12 +121,13 @@ class Builder final template std::shared_ptr> construct_object(std::string name, ArgsT&&... args) { - auto uptr = std::make_unique(std::forward(args)...); + auto ns_name = m_namespace_prefix.empty() ? name : m_namespace_prefix + "/" + name; + auto uptr = std::make_unique(std::forward(args)...); - ::add_stats_watcher_if_rx_source(*uptr, name); - ::add_stats_watcher_if_rx_sink(*uptr, name); + ::add_stats_watcher_if_rx_source(*uptr, ns_name); + ::add_stats_watcher_if_rx_sink(*uptr, ns_name); - return make_object(std::move(name), std::move(uptr)); + return make_object(std::move(ns_name), std::move(uptr)); } template >(name, std::forward(ops)...); } + /** + * Instantiate a segment module of `ModuleTypeT`, intialize it, and return it to the caller + * @tparam ModuleTypeT Type of module to create + * @param module_name Unique name of this instance of the module + * @param config Configuration to pass to the module + * @return Return a shared pointer to the new module, which is a derived class of SegmentModule + */ + template + std::shared_ptr make_module(std::string module_name, nlohmann::json config = {}) + { + static_assert(std::is_base_of_v); + + auto module = std::make_shared(std::move(module_name), std::move(config)); + init_module(module); + + return std::move(module); + } + + /** + * Initialize a SegmentModule that was instantiated outside of the builder. + * @param module Module to initialize + */ + void init_module(std::shared_ptr module); + + /** + * Register an input port on the given module -- note: this in generally only necessary for dynamically + * created modules that use an alternate initializer function independent of the derived class. + * See: PythonSegmentModule + * @param input_name Unique name of the input port + * @param object shared pointer to type erased Object associated with 'input_name' on this module instance. + */ + void register_module_input(std::string input_name, std::shared_ptr object); + + /** + * Register an output port on the given module -- note: this in generally only necessary for dynamically + * created modules that use an alternate initializer function independent of the derived class. + * See: PythonSegmentModule + * @param output_name Unique name of the output port + * @param object shared pointer to type erased Object associated with 'output_name' on this module instance. + */ + void register_module_output(std::string output_name, std::shared_ptr object); + + std::shared_ptr load_module_from_registry(const std::string& module_id, + const std::string& registry_namespace, + std::string module_name, + nlohmann::json config = {}); + template void make_edge(std::shared_ptr> source, std::shared_ptr> sink) { @@ -189,6 +240,70 @@ class Builder final node::make_edge(source, sink); } + /** + * Given a typed source and a typeless sink, attempt to construct an edge between them -- assumes that source and + * sink types are convertible. + * + * @tparam InputT + * @param source + * @param sink + */ + template + void make_edge(node::SourceProperties& source, std::shared_ptr sink) + { + DVLOG(10) << "forming segment edge between two node objects"; + node::make_edge(source, sink->template sink_typed()); + } + + /** + * Partial dynamic edge construction: + * + * Create edge using a fully constructed Object and a type erased Object + * We extract the underlying node object (Likely an RxNode) and call make_edge with it and the type erased + * object. This works via a cascaded type extraction process. + * @tparam SourceNodeTypeT + * @param source Fully typed, wrapped, object + * @param sink Type erased object -- assumed to be convertible to source type + */ + template + void make_edge(std::shared_ptr>& source, std::shared_ptr sink) + { + DVLOG(10) << "forming segment edge between a segment source and typeless Object"; + this->make_edge(source->object(), sink); + } + + /** + * Given a typeless source and a typed sink, attempt to construct an edge between them -- assumes that + * source and sink type's are convertible. + * + * @tparam OutputT + * @param source + * @param sink + */ + template + void make_edge(std::shared_ptr source, node::SinkProperties& sink) + { + DVLOG(10) << "forming segment edge between two node objects"; + node::make_edge(source->template source_typed(), sink); + } + + /** + * Partial dynamic edge construction: + * + * Create edge using a fully constructed Object and a type erased Object + * We extract the underlying node object (Likely an RxNode) and call make_edge with it and the type erased + * object. This works via a cascaded type extraction process. + * @tparam SinkNodeTypeT + * @param source Fully typed, wrapped, object + * @param sink Fully typed, wrapped, object + */ + template + void make_edge(std::shared_ptr source, std::shared_ptr>& sink) + { + DVLOG(10) << "forming segment edge between a typeless object and a segment sink"; + this->make_edge(source, sink->object()); + } + template void make_dynamic_edge(const std::string& source_name, const std::string& sink_name) { @@ -229,14 +344,25 @@ class Builder final } private: + using sp_segment_module_t = std::shared_ptr; + using sp_obj_prop_t = std::shared_ptr; + + std::string m_namespace_prefix; + std::vector m_namespace_stack{}; + std::vector m_module_stack{}; + internal::segment::IBuilder& m_backend; + void ns_push(sp_segment_module_t module); + void ns_pop(); + friend Definition; }; template std::shared_ptr> Builder::make_object(std::string name, std::unique_ptr node) { + // Note: name should have any prefix modifications done prior to getting here. if (m_backend.has_object(name)) { LOG(ERROR) << "A Object named " << name << " is already registered"; diff --git a/python/srf/__init__.py b/python/srf/__init__.py index a816b4a7f..37cf6e61f 100644 --- a/python/srf/__init__.py +++ b/python/srf/__init__.py @@ -15,13 +15,17 @@ from .core import logging from .core import operators +from .core.common import __version__ from .core.executor import Executor from .core.executor import Future from .core.node import SegmentObject from .core.options import Config from .core.options import Options from .core.pipeline import Pipeline +from .core.plugins import PluginModule from .core.segment import Builder +from .core.segment import ModuleRegistry +from .core.segment import SegmentModule from .core.subscriber import Observable from .core.subscriber import Observer from .core.subscriber import Subscriber diff --git a/python/srf/_pysrf/CMakeLists.txt b/python/srf/_pysrf/CMakeLists.txt index cee6b3c88..206f81caf 100644 --- a/python/srf/_pysrf/CMakeLists.txt +++ b/python/srf/_pysrf/CMakeLists.txt @@ -18,42 +18,46 @@ find_package(prometheus-cpp REQUIRED) # Keep all source files sorted!!! add_library(pysrf - src/executor.cpp - src/logging.cpp - src/module_wrappers/pickle.cpp - src/module_wrappers/shared_memory.cpp - src/node.cpp - src/operators.cpp - src/options.cpp - src/pipeline.cpp - src/segment.cpp - src/subscriber.cpp - src/system.cpp - src/types.cpp - src/utilities/deserializers.cpp - src/utilities/object_cache.cpp - src/utilities/serializers.cpp - src/utils.cpp - src/watchers.cpp -) + src/executor.cpp + src/logging.cpp + src/module_registry.cpp + src/module_wrappers/pickle.cpp + src/module_wrappers/shared_memory.cpp + src/node.cpp + src/operators.cpp + src/options.cpp + src/pipeline.cpp + src/plugins.cpp + src/py_segment_module.cpp + src/segment.cpp + src/segment_modules.cpp + src/subscriber.cpp + src/system.cpp + src/types.cpp + src/utilities/deserializers.cpp + src/utilities/object_cache.cpp + src/utilities/serializers.cpp + src/utils.cpp + src/watchers.cpp + ) add_library(${PROJECT_NAME}::pysrf ALIAS pysrf) target_link_libraries(pysrf - PUBLIC - ${PROJECT_NAME}::libsrf - ${Python_LIBRARIES} - prometheus-cpp::core - pybind11::pybind11 - ) + PUBLIC + ${PROJECT_NAME}::libsrf + ${Python_LIBRARIES} + prometheus-cpp::core + pybind11::pybind11 + ) target_include_directories(pysrf - PUBLIC - $ - $ - PRIVATE - ${Python_INCLUDE_DIR} - ) + PUBLIC + $ + $ + PRIVATE + ${Python_INCLUDE_DIR} + ) set_target_properties(pysrf PROPERTIES OUTPUT_NAME ${PROJECT_NAME}_pysrf) set_target_properties(pysrf PROPERTIES CXX_VISIBILITY_PRESET hidden) @@ -64,18 +68,18 @@ set_target_properties(pysrf PROPERTIES CXX_VISIBILITY_PRESET hidden) rapids_cmake_install_lib_dir(lib_dir) install( - TARGETS pysrf - DESTINATION ${lib_dir} - EXPORT ${PROJECT_NAME}-core-exports - COMPONENT Python + TARGETS pysrf + DESTINATION ${lib_dir} + EXPORT ${PROJECT_NAME}-core-exports + COMPONENT Python ) install( - DIRECTORY include/ - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} - COMPONENT Python + DIRECTORY include/ + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT Python ) -if(SRF_BUILD_TESTS) +if (SRF_BUILD_TESTS) add_subdirectory(tests) -endif() +endif () diff --git a/python/srf/_pysrf/include/pysrf/module_registry.hpp b/python/srf/_pysrf/include/pysrf/module_registry.hpp new file mode 100644 index 000000000..98530fa1e --- /dev/null +++ b/python/srf/_pysrf/include/pysrf/module_registry.hpp @@ -0,0 +1,78 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "srf/modules/segment_modules.hpp" // IWYU pragma: keep +#include "srf/segment/forward.hpp" + +#include + +#include +#include +#include +#include + +namespace srf::pysrf { + +// Export everything in the srf::pysrf namespace by default since we compile with -fvisibility=hidden +#pragma GCC visibility push(default) + +class ModuleRegistryProxy +{ + using registry_version_t = std::vector; + + public: + ModuleRegistryProxy() = default; + + static bool contains(const std::string& name, const std::string& registry_namespace); + + static bool contains_namespace(const std::string& registry_namespace); + + static std::map> registered_modules(); + + static bool is_version_compatible(const registry_version_t& release_version); + + static pybind11::cpp_function get_module_constructor(const std::string& name, + const std::string& registry_namespace); + + static void register_module(std::string name, + const registry_version_t& release_version, + std::function fn_py_initializer); + + static void register_module(std::string name, + std::string registry_namespace, + const registry_version_t& release_version, + std::function fn_py_initializer); + + static void unregister_module(const std::string& name, const std::string& registry_namespace, bool optional = true); + + private: + /** + * When we register python modules, we have to capture a python-land initializer function, which is in turn + * stored in the ModuleRegistry -- a global static struct. If the registered modules that capture a python + * function are not unregistered when the python interpreter exits, it will hang, waiting on their ref counts + * to drop to zero. To ensure this doesn't happen, we register an atexit callback here that forces all python + * modules to be unregistered when the interpreter is shut down. + * @param name Name of the module + * @param registry_namespace Namespace of the module + */ + static void register_module_cleanup_fn(const std::string& name, const std::string& registry_namespace); +}; + +#pragma GCC visibility pop +} // namespace srf::pysrf diff --git a/python/srf/_pysrf/include/pysrf/plugins.hpp b/python/srf/_pysrf/include/pysrf/plugins.hpp new file mode 100644 index 000000000..fe170c90a --- /dev/null +++ b/python/srf/_pysrf/include/pysrf/plugins.hpp @@ -0,0 +1,38 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "srf/modules/plugins.hpp" + +#include +#include + +namespace srf::pysrf { + +// Export everything in the srf::pysrf namespace by default since we compile with -fvisibility=hidden +#pragma GCC visibility push(default) + +class PluginProxy +{ + public: + static std::shared_ptr create_or_acquire(const std::string& plugin_library_name); +}; + +#pragma GCC visibility pop + +} // namespace srf::pysrf diff --git a/python/srf/_pysrf/include/pysrf/py_segment_module.hpp b/python/srf/_pysrf/include/pysrf/py_segment_module.hpp new file mode 100644 index 000000000..1caaed299 --- /dev/null +++ b/python/srf/_pysrf/include/pysrf/py_segment_module.hpp @@ -0,0 +1,62 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "srf/modules/segment_modules.hpp" +#include "srf/segment/forward.hpp" + +#include +#include // IWYU pragma: keep +#include // IWYU pragma: keep + +#include +#include + +namespace srf::pysrf { + +class ModuleRegistryProxy; + +// Export everything in the srf::pysrf namespace by default since we compile with -fvisibility=hidden +#pragma GCC visibility push(default) + +/** + * PythonSegmentModule exists to solve one problem: allowing for binding a dynamic initializer for a SegmentModule + * This is accomplished by allowing the builder to set m_py_initialize, and subsequently calling it in the overridden + * `initialize` method. + */ +class PythonSegmentModule : public srf::modules::SegmentModule +{ + using type_t = PythonSegmentModule; + friend ModuleRegistryProxy; + + public: + using py_initializer_t = std::function; + + PythonSegmentModule(std::string module_name); + PythonSegmentModule(std::string module_name, nlohmann::json config); + + protected: + void initialize(segment::Builder& builder) override; + std::string module_type_name() const override; + + private: + py_initializer_t m_py_initialize{}; +}; +} // namespace srf::pysrf + +#pragma GCC visibility pop \ No newline at end of file diff --git a/python/srf/_pysrf/include/pysrf/segment.hpp b/python/srf/_pysrf/include/pysrf/segment.hpp index 1bb66adfc..af2adb77e 100644 --- a/python/srf/_pysrf/include/pysrf/segment.hpp +++ b/python/srf/_pysrf/include/pysrf/segment.hpp @@ -19,6 +19,7 @@ #include "pysrf/types.hpp" +#include "srf/modules/segment_modules.hpp" #include "srf/segment/forward.hpp" #include "srf/segment/object.hpp" @@ -76,7 +77,7 @@ auto wrap_segment_init_callback(void (ClassT::*method)(const std::string&, * method. * * We need to force pybind to pass us a function that expects a srf::segment::Builder* not a srf::segment::Builder&. If - * not it'll try to make a copy and srf::segment::Builder isnt' copy-constructable. Once we have that, we wrap it with + * not it'll try to make a copy and srf::segment::Builder isn't copy-constructable. Once we have that, we wrap it with * our reference based function. * * @tparam ClassT Class where the init method binding is defined. @@ -111,7 +112,7 @@ auto wrap_segment_init_callback( return func; } -class SegmentProxy +class BuilderProxy { public: static std::shared_ptr make_source(srf::segment::Builder& self, @@ -175,8 +176,6 @@ class SegmentProxy const std::string& name, std::function sub_fn); - static void test_fn(srf::segment::Builder& self, pybind11::function py_func); - static void make_py2cxx_edge_adapter(srf::segment::Builder& self, std::shared_ptr source, std::shared_ptr sink, @@ -197,6 +196,22 @@ class SegmentProxy static std::shared_ptr get_egress(srf::segment::Builder& self, const std::string& name); + static std::shared_ptr load_module_from_registry(srf::segment::Builder& self, + const std::string& module_id, + const std::string& registry_namespace, + std::string module_name, + pybind11::dict config = {}); + + static void register_module_input(srf::segment::Builder& self, + std::string input_name, + std::shared_ptr object); + + static void register_module_output(srf::segment::Builder& self, + std::string output_name, + std::shared_ptr object); + + static void init_module(srf::segment::Builder& self, std::shared_ptr module); + static std::shared_ptr make_file_reader(srf::segment::Builder& self, const std::string& name, const std::string& filename); diff --git a/python/srf/_pysrf/include/pysrf/segment_modules.hpp b/python/srf/_pysrf/include/pysrf/segment_modules.hpp new file mode 100644 index 000000000..4da5d1f08 --- /dev/null +++ b/python/srf/_pysrf/include/pysrf/segment_modules.hpp @@ -0,0 +1,64 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "srf/modules/segment_modules.hpp" +#include "srf/segment/object.hpp" + +#include // IWYU pragma: keep +#include +#include // IWYU pragma: keep + +#include +#include +#include + +namespace srf::pysrf { + +// Export everything in the srf::pysrf namespace by default since we compile with -fvisibility=hidden +#pragma GCC visibility push(default) + +class SegmentModuleProxy +{ + public: + static std::string component_prefix(srf::modules::SegmentModule& self); + + static pybind11::dict config(srf::modules::SegmentModule& self); + + static const std::string& name(srf::modules::SegmentModule& self); + + static std::string module_type_name(srf::modules::SegmentModule& self); + + static std::vector input_ids(srf::modules::SegmentModule& self); + + static std::vector output_ids(srf::modules::SegmentModule& self); + + static std::shared_ptr input_port(srf::modules::SegmentModule& self, + const std::string& input_id); + + static const srf::modules::SegmentModule::segment_module_port_map_t& input_ports(srf::modules::SegmentModule& self); + + static std::shared_ptr output_port(srf::modules::SegmentModule& self, + const std::string& output_id); + + static const srf::modules::SegmentModule::segment_module_port_map_t& output_ports( + srf::modules::SegmentModule& self); +}; + +#pragma GCC visibility pop +} // namespace srf::pysrf diff --git a/python/srf/_pysrf/src/executor.cpp b/python/srf/_pysrf/src/executor.cpp index b09de9cc8..04883ab96 100644 --- a/python/srf/_pysrf/src/executor.cpp +++ b/python/srf/_pysrf/src/executor.cpp @@ -191,9 +191,15 @@ void Executor::stop() void Executor::join() { - // Release the GIL before blocking - py::gil_scoped_release nogil; + { + // Release the GIL before blocking + py::gil_scoped_release nogil; + // Wait without the GIL + m_join_future.wait(); + } + + // Call get() with the GIL to rethrow any exceptions m_join_future.get(); } @@ -207,6 +213,9 @@ std::shared_ptr Executor::join_async() // Grab the GIL to return a py::object py::gil_scoped_acquire gil; + // Once we have the GIL, call get() to propagate any exceptions + this->m_join_future.get(); + return py::none(); }); diff --git a/python/srf/_pysrf/src/module_registry.cpp b/python/srf/_pysrf/src/module_registry.cpp new file mode 100644 index 000000000..79d7cdb0b --- /dev/null +++ b/python/srf/_pysrf/src/module_registry.cpp @@ -0,0 +1,113 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pysrf/module_registry.hpp" + +#include "pysrf/py_segment_module.hpp" +#include "pysrf/utils.hpp" + +#include "srf/modules/module_registry.hpp" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace srf::pysrf { + +bool ModuleRegistryProxy::contains(const std::string& name, const std::string& registry_namespace) +{ + return srf::modules::ModuleRegistry::contains(name, registry_namespace); +} + +bool ModuleRegistryProxy::contains_namespace(const std::string& registry_namespace) +{ + return srf::modules::ModuleRegistry::contains_namespace(registry_namespace); +} + +std::map> ModuleRegistryProxy::registered_modules() +{ + return modules::ModuleRegistry::registered_modules(); +} + +bool ModuleRegistryProxy::is_version_compatible(const registry_version_t& release_version) +{ + return modules::ModuleRegistry::is_version_compatible(release_version); +} + +pybind11::cpp_function ModuleRegistryProxy::get_module_constructor(const std::string& name, + const std::string& registry_namespace) +{ + auto fn_constructor = modules::ModuleRegistry::get_module_constructor(name, registry_namespace); + auto py_module_wrapper = [fn_constructor](std::string module_name, pybind11::dict config) { + auto json_config = cast_from_pyobject(config); + return fn_constructor(std::move(module_name), std::move(json_config)); + }; + + return py_module_wrapper; +} + +void ModuleRegistryProxy::register_module(std::string name, + const registry_version_t& release_version, + std::function fn_py_initializer) +{ + register_module(name, "default", release_version, fn_py_initializer); +} + +void ModuleRegistryProxy::register_module(std::string name, + std::string registry_namespace, + const registry_version_t& release_version, + std::function fn_py_initializer) +{ + VLOG(2) << "Registering python module: " << registry_namespace << "::" << name; + auto fn_constructor = [fn_py_initializer](std::string name, nlohmann::json config) { + auto module = std::make_shared(std::move(name), std::move(config)); + module->m_py_initialize = fn_py_initializer; + + return module; + }; + + srf::modules::ModuleRegistry::register_module(name, registry_namespace, release_version, fn_constructor); + + register_module_cleanup_fn(name, registry_namespace); +} + +void ModuleRegistryProxy::unregister_module(const std::string& name, + const std::string& registry_namespace, + bool optional) +{ + return srf::modules::ModuleRegistry::unregister_module(name, registry_namespace, optional); +} + +void ModuleRegistryProxy::register_module_cleanup_fn(const std::string& name, const std::string& registry_namespace) +{ + auto at_exit = pybind11::module_::import("atexit"); + at_exit.attr("register")(pybind11::cpp_function([name, registry_namespace]() { + VLOG(2) << "(atexit) Unregistering " << registry_namespace << "::" << name; + + // Try unregister -- ignore if already unregistered + srf::modules::ModuleRegistry::unregister_module(name, registry_namespace, true); + })); +} + +} // namespace srf::pysrf diff --git a/python/srf/_pysrf/src/plugins.cpp b/python/srf/_pysrf/src/plugins.cpp new file mode 100644 index 000000000..711026d23 --- /dev/null +++ b/python/srf/_pysrf/src/plugins.cpp @@ -0,0 +1,29 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pysrf/plugins.hpp" + +#include + +namespace srf::pysrf { + +std::shared_ptr PluginProxy::create_or_acquire(const std::string& plugin_library_name) +{ + return modules::PluginModule::create_or_acquire(plugin_library_name); +} + +} // namespace srf::pysrf diff --git a/python/srf/_pysrf/src/py_segment_module.cpp b/python/srf/_pysrf/src/py_segment_module.cpp new file mode 100644 index 000000000..4bdd89338 --- /dev/null +++ b/python/srf/_pysrf/src/py_segment_module.cpp @@ -0,0 +1,48 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pysrf/py_segment_module.hpp" + +#include "srf/core/utils.hpp" + +#include + +#include +#include +#include +#include + +namespace srf::pysrf { + +PythonSegmentModule::PythonSegmentModule(std::string module_name) : SegmentModule(std::move(module_name)) {} + +PythonSegmentModule::PythonSegmentModule(std::string module_name, nlohmann::json config) : + SegmentModule(std::move(module_name), std::move(config)) +{} + +void PythonSegmentModule::initialize(segment::Builder& builder) +{ + VLOG(2) << "Calling PythonSegmentModule::initialize"; + m_py_initialize(builder); +} + +std::string PythonSegmentModule::module_type_name() const +{ + return std::string(::srf::type_name()); +} + +} // namespace srf::pysrf \ No newline at end of file diff --git a/python/srf/_pysrf/src/segment.cpp b/python/srf/_pysrf/src/segment.cpp index 203cd04de..153890723 100644 --- a/python/srf/_pysrf/src/segment.cpp +++ b/python/srf/_pysrf/src/segment.cpp @@ -24,6 +24,7 @@ #include "srf/channel/status.hpp" #include "srf/core/utils.hpp" #include "srf/manifold/egress.hpp" +#include "srf/modules/segment_modules.hpp" #include "srf/node/edge_builder.hpp" #include "srf/node/port_registry.hpp" #include "srf/node/sink_properties.hpp" @@ -87,9 +88,6 @@ std::shared_ptr build_source(srf::segment::Build // Get the next value auto next_val = py::cast(*iter); - // Increment it for next loop - ++iter; - { // Release the GIL to call on_next pybind11::gil_scoped_release nogil; @@ -100,6 +98,9 @@ std::shared_ptr build_source(srf::segment::Build subscriber.on_next(std::move(next_val)); } } + + // Increment it for next loop + ++iter; } } catch (const std::exception& e) @@ -122,7 +123,7 @@ std::shared_ptr build_source(srf::segment::Build return self.construct_object>(name, wrapper); } -std::shared_ptr SegmentProxy::make_source(srf::segment::Builder& self, +std::shared_ptr BuilderProxy::make_source(srf::segment::Builder& self, const std::string& name, py::iterator source_iterator) { @@ -142,7 +143,7 @@ std::shared_ptr SegmentProxy::make_source(srf::s }); } -std::shared_ptr SegmentProxy::make_source(srf::segment::Builder& self, +std::shared_ptr BuilderProxy::make_source(srf::segment::Builder& self, const std::string& name, py::iterable source_iterable) { @@ -153,7 +154,7 @@ std::shared_ptr SegmentProxy::make_source(srf::s }); } -std::shared_ptr SegmentProxy::make_source(srf::segment::Builder& self, +std::shared_ptr BuilderProxy::make_source(srf::segment::Builder& self, const std::string& name, py::function gen_factory) { @@ -164,7 +165,7 @@ std::shared_ptr SegmentProxy::make_source(srf::s }); } -std::shared_ptr SegmentProxy::make_sink(srf::segment::Builder& self, +std::shared_ptr BuilderProxy::make_sink(srf::segment::Builder& self, const std::string& name, std::function on_next, std::function on_error, @@ -196,7 +197,7 @@ std::shared_ptr SegmentProxy::make_sink(srf::seg return self.make_sink(name, on_next_w, on_error_w, on_completed_w); } -std::shared_ptr SegmentProxy::get_ingress(srf::segment::Builder& self, +std::shared_ptr BuilderProxy::get_ingress(srf::segment::Builder& self, const std::string& name) { auto it_caster = node::PortRegistry::s_port_to_type_index.find(name); @@ -209,7 +210,7 @@ std::shared_ptr SegmentProxy::get_ingress(srf::s return self.get_ingress(name); } -std::shared_ptr SegmentProxy::get_egress(srf::segment::Builder& self, +std::shared_ptr BuilderProxy::get_egress(srf::segment::Builder& self, const std::string& name) { auto it_caster = node::PortRegistry::s_port_to_type_index.find(name); @@ -223,7 +224,7 @@ std::shared_ptr SegmentProxy::get_egress(srf::se return self.get_egress(name); } -std::shared_ptr SegmentProxy::make_node( +std::shared_ptr BuilderProxy::make_node( srf::segment::Builder& self, const std::string& name, std::function map_f) @@ -251,7 +252,7 @@ std::shared_ptr SegmentProxy::make_node( })); } -std::shared_ptr SegmentProxy::make_node_full( +std::shared_ptr BuilderProxy::make_node_full( srf::segment::Builder& self, const std::string& name, std::function sub_fn) @@ -286,135 +287,43 @@ std::shared_ptr SegmentProxy::make_node_full( return node; } -void SegmentProxy::make_py2cxx_edge_adapter(srf::segment::Builder& self, - std::shared_ptr source, - std::shared_ptr sink, - py::object& sink_t) +std::shared_ptr BuilderProxy::load_module_from_registry( + srf::segment::Builder& self, + const std::string& module_id, + const std::string& registry_namespace, + std::string module_name, + py::dict config) { - using source_type_t = py::object; - - /* + auto json_config = cast_from_pyobject(config); - // https://numpy.org/doc/stable/reference/generated/numpy.dtype.kind.html - pybind11::dtype dtype = pybind11::dtype::from_args(sink_t); - switch (dtype.kind()) - { - case 'b': - self.make_dynamic_edge(source->name(), sink->name()); - break; - case 'i': - if (dtype.itemsize() == 4) - { - self.make_dynamic_edge(source->name(), sink->name()); - break; - } - self.make_dynamic_edge(source->name(), sink->name()); - break; - case 'u': - if (dtype.itemsize() == 4) - { - self.make_dynamic_edge(source->name(), sink->name()); - break; - } - self.make_dynamic_edge(source->name(), sink->name()); - break; - case 'f': - if (dtype.itemsize() == 4) - { - self.make_dynamic_edge(source->name(), sink->name()); - break; - } - self.make_dynamic_edge(source->name(), sink->name()); - break; - case 'c': - throw std::runtime_error("Complex-float datatypes are not currently supported"); - case 'm': - throw std::runtime_error("Timedelta datatypes are not currently supported"); - case 'M': - throw std::runtime_error("Datetime datatypes are not currently supported"); - case 'O': - throw std::runtime_error("Automatic conversion between py::objects is not supported."); - case 'S': - self.make_dynamic_edge(source->name(), sink->name()); - break; - case 'U': - self.make_dynamic_edge(source->name(), sink->name()); - break; - case 'V': - throw std::runtime_error("Void datatypes are not supported"); - default: - throw std::runtime_error("Unknown sink type"); - } + return self.load_module_from_registry( + module_id, registry_namespace, std::move(module_name), std::move(json_config)); +} - */ +void BuilderProxy::init_module(srf::segment::Builder& self, std::shared_ptr module) +{ + self.init_module(module); } -void SegmentProxy::make_cxx2py_edge_adapter(srf::segment::Builder& self, - std::shared_ptr source, - std::shared_ptr sink, - py::object& source_t) +void BuilderProxy::register_module_input(srf::segment::Builder& self, + std::string input_name, + std::shared_ptr object) { - using sink_type_t = pybind11::object; + self.register_module_input(std::move(input_name), object); +} - LOG(FATAL) << "fixme"; - /* - // https://numpy.org/doc/stable/reference/generated/numpy.dtype.kind.html - pybind11::dtype dtype = pybind11::dtype::from_args(source_t); - switch (dtype.kind()) - { - case 'b': - self.make_dynamic_edge(source->name(), sink->name()); - break; - case 'i': - if (dtype.itemsize() == 4) - { - self.make_dynamic_edge(source->name(), sink->name()); - break; - } - self.make_dynamic_edge(source->name(), sink->name()); - break; - case 'u': - if (dtype.itemsize() == 4) - { - self.make_dynamic_edge(source->name(), sink->name()); - break; - } - self.make_dynamic_edge(source->name(), sink->name()); - break; - case 'f': - if (dtype.itemsize() == 4) - { - self.make_dynamic_edge(source->name(), sink->name()); - break; - } - self.make_dynamic_edge(source->name(), sink->name()); - break; - - case 'c': - throw std::runtime_error("Complex-float datatypes are not supported."); - case 'm': - throw std::runtime_error("Timedelta datatypes are not supported."); - case 'M': - throw std::runtime_error("Datetime datatypes are not supported."); - case 'O': - throw std::runtime_error("Automatic conversion to generic py::object is not supported."); - case 'S': - self.make_dynamic_edge(source->name(), sink->name()); - case 'U': - self.make_dynamic_edge(source->name(), sink->name()); - break; - case 'V': - throw std::runtime_error("Void datatypes are not supported"); - default: - throw std::runtime_error("Unknown sink type"); - } - */ +void BuilderProxy::register_module_output(srf::segment::Builder& self, + std::string output_name, + std::shared_ptr object) +{ + self.register_module_output(std::move(output_name), object); } -void SegmentProxy::make_edge(srf::segment::Builder& self, +void BuilderProxy::make_edge(srf::segment::Builder& self, std::shared_ptr source, std::shared_ptr sink) { node::EdgeBuilder::make_edge_typeless(source->source_base(), sink->sink_base()); } + } // namespace srf::pysrf diff --git a/python/srf/_pysrf/src/segment_modules.cpp b/python/srf/_pysrf/src/segment_modules.cpp new file mode 100644 index 000000000..1df97f714 --- /dev/null +++ b/python/srf/_pysrf/src/segment_modules.cpp @@ -0,0 +1,85 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pysrf/segment_modules.hpp" + +#include "pysrf/utils.hpp" + +#include "srf/modules/segment_modules.hpp" +#include "srf/segment/object.hpp" + +#include + +#include +#include + +namespace srf::pysrf { +std::string SegmentModuleProxy::component_prefix(srf::modules::SegmentModule& self) +{ + return self.component_prefix(); +} + +pybind11::dict SegmentModuleProxy::config(srf::modules::SegmentModule& self) +{ + return cast_from_json(self.config()); +} + +const std::string& SegmentModuleProxy::name(srf::modules::SegmentModule& self) +{ + return self.name(); +} + +std::string SegmentModuleProxy::module_type_name(srf::modules::SegmentModule& self) +{ + return self.module_type_name(); +} + +std::vector SegmentModuleProxy::input_ids(srf::modules::SegmentModule& self) +{ + return self.input_ids(); +} + +std::vector SegmentModuleProxy::output_ids(srf::modules::SegmentModule& self) +{ + return self.output_ids(); +} + +std::shared_ptr SegmentModuleProxy::input_port(srf::modules::SegmentModule& self, + const std::string& input_id) +{ + return self.input_port(input_id); +} + +const srf::modules::SegmentModule::segment_module_port_map_t& SegmentModuleProxy::input_ports( + srf::modules::SegmentModule& self) +{ + return self.input_ports(); +} + +std::shared_ptr SegmentModuleProxy::output_port(srf::modules::SegmentModule& self, + const std::string& output_id) +{ + return self.output_port(output_id); +} + +const srf::modules::SegmentModule::segment_module_port_map_t& SegmentModuleProxy::output_ports( + srf::modules::SegmentModule& self) +{ + return self.output_ports(); +} + +} // namespace srf::pysrf \ No newline at end of file diff --git a/python/srf/core/CMakeLists.txt b/python/srf/core/CMakeLists.txt index ceae52b79..545e21c21 100644 --- a/python/srf/core/CMakeLists.txt +++ b/python/srf/core/CMakeLists.txt @@ -15,14 +15,15 @@ list(APPEND CMAKE_MESSAGE_CONTEXT "core") -srf_add_pybind11_module(common SOURCE_FILES common.cpp) -srf_add_pybind11_module(executor SOURCE_FILES executor.cpp) -srf_add_pybind11_module(logging SOURCE_FILES logging.cpp) -srf_add_pybind11_module(node SOURCE_FILES node.cpp) -srf_add_pybind11_module(operators SOURCE_FILES operators.cpp) -srf_add_pybind11_module(options SOURCE_FILES options.cpp) -srf_add_pybind11_module(pipeline SOURCE_FILES pipeline.cpp) -srf_add_pybind11_module(segment SOURCE_FILES segment.cpp) -srf_add_pybind11_module(subscriber SOURCE_FILES subscriber.cpp) +srf_add_pybind11_module(common SOURCE_FILES common.cpp) +srf_add_pybind11_module(executor SOURCE_FILES executor.cpp) +srf_add_pybind11_module(logging SOURCE_FILES logging.cpp) +srf_add_pybind11_module(node SOURCE_FILES node.cpp) +srf_add_pybind11_module(operators SOURCE_FILES operators.cpp) +srf_add_pybind11_module(options SOURCE_FILES options.cpp) +srf_add_pybind11_module(pipeline SOURCE_FILES pipeline.cpp) +srf_add_pybind11_module(plugins SOURCE_FILES plugins.cpp) +srf_add_pybind11_module(segment SOURCE_FILES segment.cpp) +srf_add_pybind11_module(subscriber SOURCE_FILES subscriber.cpp) list(POP_BACK CMAKE_MESSAGE_CONTEXT) diff --git a/python/srf/core/common.cpp b/python/srf/core/common.cpp index 93ba6c24a..7faa80ee9 100644 --- a/python/srf/core/common.cpp +++ b/python/srf/core/common.cpp @@ -24,6 +24,8 @@ #include "srf/manifold/egress.hpp" #include "srf/node/sink_properties.hpp" #include "srf/node/source_properties.hpp" +#include "srf/utils/string_utils.hpp" +#include "srf/version.hpp" #include #include @@ -32,6 +34,7 @@ #include #include +#include #include // IWYU pragma: no_include @@ -45,9 +48,9 @@ namespace srf::pysrf { namespace py = pybind11; using namespace py::literals; -PYBIND11_MODULE(common, m) +PYBIND11_MODULE(common, module) { - m.doc() = R"pbdoc( + module.doc() = R"pbdoc( Python bindings for SRF common functionality / utilities ------------------------------- .. currentmodule:: common @@ -57,10 +60,8 @@ PYBIND11_MODULE(common, m) EdgeAdapterUtil::register_data_adapters(); PortBuilderUtil::register_port_util(); -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif + + module.attr("__version__") = + SRF_CONCAT_STR(srf_VERSION_MAJOR << "." << srf_VERSION_MINOR << "." << srf_VERSION_PATCH); } } // namespace srf::pysrf diff --git a/python/srf/core/executor.cpp b/python/srf/core/executor.cpp index 608e1876b..c4653574f 100644 --- a/python/srf/core/executor.cpp +++ b/python/srf/core/executor.cpp @@ -20,13 +20,17 @@ #include "pysrf/utils.hpp" #include "srf/options/options.hpp" +#include "srf/utils/string_utils.hpp" +#include "srf/version.hpp" #include #include #include // IWYU pragma: keep #include +#include #include // for move + // IWYU thinks we need vectir for py::class_> // IWYU pragma: no_include @@ -34,9 +38,9 @@ namespace srf::pysrf { namespace py = pybind11; -PYBIND11_MODULE(executor, m) +PYBIND11_MODULE(executor, module) { - m.doc() = R"pbdoc( + module.doc() = R"pbdoc( Python bindings for SRF executors ------------------------------- .. currentmodule:: executor @@ -45,17 +49,17 @@ PYBIND11_MODULE(executor, m) )pbdoc"; // Common must be first in every module - pysrf::import(m, "srf.core.common"); - pysrf::import(m, "srf.core.options"); - pysrf::import(m, "srf.core.pipeline"); + pysrf::import(module, "srf.core.common"); + pysrf::import(module, "srf.core.options"); + pysrf::import(module, "srf.core.pipeline"); - py::class_>(m, "Awaitable") + py::class_>(module, "Awaitable") .def(py::init<>()) .def("__iter__", &Awaitable::iter) .def("__await__", &Awaitable::await) .def("__next__", &Awaitable::next); - py::class_>(m, "Executor") + py::class_>(module, "Executor") .def(py::init<>([]() { auto options = std::make_shared(); @@ -74,15 +78,12 @@ PYBIND11_MODULE(executor, m) .def("join_async", &Executor::join_async) .def("register_pipeline", &Executor::register_pipeline); - py::class_(m, "Future") + py::class_(module, "Future") .def(py::init<>([]() { return PyBoostFuture(); })) .def("result", &PyBoostFuture::py_result) .def("set_result", &PyBoostFuture::set_result); -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif + module.attr("__version__") = + SRF_CONCAT_STR(srf_VERSION_MAJOR << "." << srf_VERSION_MINOR << "." << srf_VERSION_PATCH); } } // namespace srf::pysrf diff --git a/python/srf/core/logging.cpp b/python/srf/core/logging.cpp index 474dcdbb6..be2cb5030 100644 --- a/python/srf/core/logging.cpp +++ b/python/srf/core/logging.cpp @@ -18,19 +18,23 @@ #include "pysrf/logging.hpp" #include "srf/core/logging.hpp" +#include "srf/utils/string_utils.hpp" +#include "srf/version.hpp" #include #include #include // IWYU pragma: keep +#include + namespace srf::pysrf { namespace py = pybind11; using namespace std::string_literals; -PYBIND11_MODULE(logging, m) +PYBIND11_MODULE(logging, module) { - m.doc() = R"pbdoc( + module.doc() = R"pbdoc( Python bindings for SRF logging ------------------------------- .. currentmodule:: logging @@ -38,31 +42,29 @@ PYBIND11_MODULE(logging, m) :toctree: _generate )pbdoc"; - m.def("init_logging", - &init_logging, - "Initializes Srf's logger, The return value inidicates if the logger was initialized, which will be `True` " - "on the first call, and `False` for all subsequant calls.", - py::arg("logname"), - py::arg("py_level") = py_log_levels::INFO); + module.def( + "init_logging", + &init_logging, + "Initializes SRF's logger, The return value inidicates if the logger was initialized, which will be `True` " + "on the first call, and `False` for all subsequant calls.", + py::arg("logname"), + py::arg("py_level") = py_log_levels::INFO); - m.def("is_initialized", &srf::is_initialized, "Checks if Srf's logger has been initialized."); + module.def("is_initialized", &srf::is_initialized, "Checks if SRF's logger has been initialized."); - m.def("get_level", &get_level, "Gets the log level for Srf's logger."); + module.def("get_level", &get_level, "Gets the log level for SRF's logger."); - m.def("set_level", &set_level, "Sets the log level for Srf's logger.", py::arg("py_level")); + module.def("set_level", &set_level, "Sets the log level for SRF's logger.", py::arg("py_level")); - m.def("log", - &log, - "Logs a message to Srf's logger.", - py::arg("msg"), - py::arg("py_level") = py_log_levels::INFO, - py::arg("filename") = ""s, - py::arg("line") = 0); + module.def("log", + &log, + "Logs a message to SRF's logger.", + py::arg("msg"), + py::arg("py_level") = py_log_levels::INFO, + py::arg("filename") = ""s, + py::arg("line") = 0); -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif + module.attr("__version__") = + SRF_CONCAT_STR(srf_VERSION_MAJOR << "." << srf_VERSION_MINOR << "." << srf_VERSION_PATCH); } } // namespace srf::pysrf diff --git a/python/srf/core/node.cpp b/python/srf/core/node.cpp index d06e74ea6..6387d4109 100644 --- a/python/srf/core/node.cpp +++ b/python/srf/core/node.cpp @@ -20,19 +20,22 @@ #include "srf/runnable/launch_options.hpp" #include "srf/segment/object.hpp" +#include "srf/utils/string_utils.hpp" +#include "srf/version.hpp" #include // IWYU pragma: keep #include #include +#include #include namespace srf::pysrf { namespace py = pybind11; -PYBIND11_MODULE(node, m) +PYBIND11_MODULE(node, module) { - m.doc() = R"pbdoc( + module.doc() = R"pbdoc( Python bindings for SRF nodes ------------------------------- .. currentmodule:: node @@ -41,23 +44,20 @@ PYBIND11_MODULE(node, m) )pbdoc"; // Common must be first in every module - pysrf::import(m, "srf.core.common"); + pysrf::import(module, "srf.core.common"); - py::class_(m, "LaunchOptions") + py::class_(module, "LaunchOptions") .def_readwrite("pe_count", &srf::runnable::LaunchOptions::pe_count) .def_readwrite("engines_per_pe", &srf::runnable::LaunchOptions::engines_per_pe) .def_readwrite("engine_factory_name", &srf::runnable::LaunchOptions::engine_factory_name); - py::class_>(m, "SegmentObject") + py::class_>(module, "SegmentObject") .def_property_readonly("name", &PyNode::name) .def_property_readonly("launch_options", py::overload_cast<>(&srf::segment::ObjectProperties::launch_options), py::return_value_policy::reference_internal); -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif + module.attr("__version__") = + SRF_CONCAT_STR(srf_VERSION_MAJOR << "." << srf_VERSION_MINOR << "." << srf_VERSION_PATCH); } } // namespace srf::pysrf diff --git a/python/srf/core/operators.cpp b/python/srf/core/operators.cpp index 76ab04aa6..0aa5d1280 100644 --- a/python/srf/core/operators.cpp +++ b/python/srf/core/operators.cpp @@ -19,19 +19,23 @@ #include "pysrf/utils.hpp" +#include "srf/utils/string_utils.hpp" +#include "srf/version.hpp" + #include // IWYU pragma: keep #include #include +#include namespace srf::pysrf { namespace py = pybind11; // Define the pybind11 module m, as 'pipeline'. -PYBIND11_MODULE(operators, m) +PYBIND11_MODULE(operators, module) { - m.doc() = R"pbdoc( + module.doc() = R"pbdoc( Python bindings for SRF operators ------------------------------- .. currentmodule:: operators @@ -40,21 +44,18 @@ PYBIND11_MODULE(operators, m) )pbdoc"; // Common must be first in every module - pysrf::import(m, "srf.core.common"); - - py::class_(m, "Operator").def_property_readonly("name", &OperatorProxy::get_name); - - m.def("filter", &OperatorsProxy::filter); - m.def("flatten", &OperatorsProxy::flatten); - m.def("map", &OperatorsProxy::map); - m.def("on_completed", &OperatorsProxy::on_completed); - m.def("pairwise", &OperatorsProxy::pairwise); - m.def("to_list", &OperatorsProxy::to_list); - -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif + pysrf::import(module, "srf.core.common"); + + py::class_(module, "Operator").def_property_readonly("name", &OperatorProxy::get_name); + + module.def("filter", &OperatorsProxy::filter); + module.def("flatten", &OperatorsProxy::flatten); + module.def("map", &OperatorsProxy::map); + module.def("on_completed", &OperatorsProxy::on_completed); + module.def("pairwise", &OperatorsProxy::pairwise); + module.def("to_list", &OperatorsProxy::to_list); + + module.attr("__version__") = + SRF_CONCAT_STR(srf_VERSION_MAJOR << "." << srf_VERSION_MINOR << "." << srf_VERSION_PATCH); } } // namespace srf::pysrf diff --git a/python/srf/core/options.cpp b/python/srf/core/options.cpp index 8d68576f1..29f020c85 100644 --- a/python/srf/core/options.cpp +++ b/python/srf/core/options.cpp @@ -24,10 +24,13 @@ #include "srf/options/placement.hpp" #include "srf/options/topology.hpp" #include "srf/runnable/types.hpp" +#include "srf/utils/string_utils.hpp" +#include "srf/version.hpp" #include #include +#include namespace srf::pysrf { @@ -37,9 +40,9 @@ class Config {}; // Define the pybind11 module m, as 'pipeline'. -PYBIND11_MODULE(options, m) +PYBIND11_MODULE(options, module) { - m.doc() = R"pbdoc( + module.doc() = R"pbdoc( Python bindings for SRF options ------------------------------- .. currentmodule:: options @@ -48,9 +51,9 @@ PYBIND11_MODULE(options, m) )pbdoc"; // Common must be first in every module - pysrf::import(m, "srf.core.common"); + pysrf::import(module, "srf.core.common"); - py::class_(m, "Config") + py::class_(module, "Config") .def_property_static("default_channel_size", &ConfigProxy::get_default_channel_size, &ConfigProxy::set_default_channel_size, @@ -58,26 +61,26 @@ PYBIND11_MODULE(options, m) Sets the default size of the buffers between edges for all newly created edges. Larger size will reduce backpressure at the cost of memory. )doc"); - py::enum_(m, "PlacementStrategy") + py::enum_(module, "PlacementStrategy") .value("PerMachine", srf::PlacementStrategy::PerMachine) .value("PerNumaNode", srf::PlacementStrategy::PerNumaNode) .export_values(); - py::enum_(m, "EngineType") + py::enum_(module, "EngineType") .value("Fiber", srf::runnable::EngineType::Fiber) .value("Process", srf::runnable::EngineType::Process) .value("Thread", srf::runnable::EngineType::Thread) .export_values(); - py::class_(m, "TopologyOptions") + py::class_(module, "TopologyOptions") .def(py::init<>()) .def_property("user_cpuset", &OptionsProxy::get_user_cpuset, &OptionsProxy::set_user_cpuset); - py::class_(m, "PlacementOptions") + py::class_(module, "PlacementOptions") .def(py::init<>()) .def_property("cpu_strategy", &OptionsProxy::get_cpu_strategy, &OptionsProxy::set_cpu_strategy); - py::class_(m, "EngineFactoryOptions") + py::class_(module, "EngineFactoryOptions") .def(py::init<>()) .def_property("cpu_count", &EngineFactoryOptionsProxy::get_cpu_count, &EngineFactoryOptionsProxy::set_cpu_count) .def_property( @@ -87,7 +90,7 @@ PYBIND11_MODULE(options, m) &EngineFactoryOptionsProxy::get_allow_overlap, &EngineFactoryOptionsProxy::set_allow_overlap); - py::class_(m, "EngineGroups") + py::class_(module, "EngineGroups") .def(py::init<>()) .def_property( "default_engine_type", &srf::EngineGroups::default_engine_type, &srf::EngineGroups::set_default_engine_type) @@ -100,7 +103,7 @@ PYBIND11_MODULE(options, m) &srf::EngineGroups::engine_group_options, py::return_value_policy::reference_internal); - py::class_>(m, "Options") + py::class_>(module, "Options") .def(py::init<>()) .def_property_readonly("placement", &OptionsProxy::get_placement, py::return_value_policy::reference_internal) .def_property_readonly("topology", &OptionsProxy::get_topology, py::return_value_policy::reference_internal) @@ -110,10 +113,8 @@ PYBIND11_MODULE(options, m) // return a const str static_cast(&srf::Options::architect_url), static_cast(&srf::Options::architect_url)); -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif + + module.attr("__version__") = + SRF_CONCAT_STR(srf_VERSION_MAJOR << "." << srf_VERSION_MINOR << "." << srf_VERSION_PATCH); } } // namespace srf::pysrf diff --git a/python/srf/core/pipeline.cpp b/python/srf/core/pipeline.cpp index d6d568cce..14f18036e 100644 --- a/python/srf/core/pipeline.cpp +++ b/python/srf/core/pipeline.cpp @@ -21,11 +21,15 @@ #include "pysrf/utils.hpp" #include "srf/segment/builder.hpp" // IWYU pragma: keep +#include "srf/utils/string_utils.hpp" +#include "srf/version.hpp" #include // IWYU pragma: keep #include #include // IWYU pragma: keep +#include + // IWYU pragma: no_include // IWYU pragma: no_include // IWYU thinks we need array for py::class_ @@ -36,9 +40,9 @@ namespace srf::pysrf { namespace py = pybind11; // Define the pybind11 module m, as 'pipeline'. -PYBIND11_MODULE(pipeline, m) +PYBIND11_MODULE(pipeline, module) { - m.doc() = R"pbdoc( + module.doc() = R"pbdoc( Python bindings for SRF pipelines ------------------------------- .. currentmodule:: pipeline @@ -47,10 +51,10 @@ PYBIND11_MODULE(pipeline, m) )pbdoc"; // Common must be first in every module - pysrf::import(m, "srf.core.common"); - pysrf::import(m, "srf.core.segment"); + pysrf::import(module, "srf.core.common"); + pysrf::import(module, "srf.core.segment"); - py::class_(m, "Pipeline") + py::class_(module, "Pipeline") .def(py::init<>()) .def( "make_segment", @@ -63,10 +67,7 @@ PYBIND11_MODULE(pipeline, m) const std::string&, py::list, py::list, const std::function&)>( &Pipeline::make_segment))); -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif + module.attr("__version__") = + SRF_CONCAT_STR(srf_VERSION_MAJOR << "." << srf_VERSION_MINOR << "." << srf_VERSION_PATCH); } } // namespace srf::pysrf diff --git a/python/srf/core/plugins.cpp b/python/srf/core/plugins.cpp new file mode 100644 index 000000000..45503a3ed --- /dev/null +++ b/python/srf/core/plugins.cpp @@ -0,0 +1,80 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pysrf/plugins.hpp" + +#include "pysrf/utils.hpp" + +#include "srf/modules/plugins.hpp" +#include "srf/utils/string_utils.hpp" +#include "srf/version.hpp" + +#include +#include +#include // IWYU pragma: keep + +#include +#include + +// IWYU thinks the Segment.def calls need array and vector +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include + +const std::vector PybindSegmentModuleVersion{srf_VERSION_MAJOR, srf_VERSION_MINOR, srf_VERSION_PATCH}; + +namespace srf::pysrf { + +namespace py = pybind11; + +PYBIND11_MODULE(plugins, module) +{ + module.doc() = R"pbdoc( + Python bindings for SRF Plugins + ------------------------------- + .. currentmodule:: plugins + .. autosummary:: + :toctree: _generate + )pbdoc"; + + // Common must be first in every module + pysrf::import(module, "srf.core.common"); + pysrf::import_module_object(module, "srf.core.segment", "SegmentModule"); + + auto PluginModule = + py::class_>(module, "PluginModule"); + + /** Module Register Interface Declarations **/ + PluginModule.def("create_or_acquire", &PluginProxy::create_or_acquire, py::return_value_policy::reference_internal); + + PluginModule.def("list_modules", &srf::modules::PluginModule::list_modules); + + PluginModule.def("load", &srf::modules::PluginModule::load, py::arg("throw_on_error") = true); + + PluginModule.def("reload", &srf::modules::PluginModule::reload); + + PluginModule.def("reset_library_directory", &srf::modules::PluginModule::reset_library_directory); + + PluginModule.def("set_library_directory", &srf::modules::PluginModule::set_library_directory, py::arg("path")); + + PluginModule.def("unload", &srf::modules::PluginModule::unload, py::arg("throw_on_error") = true); + + module.attr("__version__") = + SRF_CONCAT_STR(srf_VERSION_MAJOR << "." << srf_VERSION_MINOR << "." << srf_VERSION_PATCH); +} +} // namespace srf::pysrf diff --git a/python/srf/core/segment.cpp b/python/srf/core/segment.cpp index e20db7d98..df349b268 100644 --- a/python/srf/core/segment.cpp +++ b/python/srf/core/segment.cpp @@ -17,21 +17,30 @@ #include "pysrf/segment.hpp" +#include "pysrf/module_registry.hpp" #include "pysrf/node.hpp" // IWYU pragma: keep +#include "pysrf/segment_modules.hpp" #include "pysrf/types.hpp" #include "pysrf/utils.hpp" #include "srf/channel/status.hpp" +#include "srf/modules/segment_modules.hpp" #include "srf/node/edge_connector.hpp" #include "srf/segment/builder.hpp" #include "srf/segment/definition.hpp" #include "srf/segment/object.hpp" // IWYU pragma: keep +#include "srf/utils/string_utils.hpp" +#include "srf/version.hpp" #include #include #include #include +#include +#include +#include +#include // IWYU thinks the Segment.def calls need array and vector // IWYU pragma: no_include @@ -43,9 +52,9 @@ namespace srf::pysrf { namespace py = pybind11; -PYBIND11_MODULE(segment, m) +PYBIND11_MODULE(segment, module) { - m.doc() = R"pbdoc( + module.doc() = R"pbdoc( Python bindings for SRF Segments ------------------------------- .. currentmodule:: segment @@ -54,12 +63,11 @@ PYBIND11_MODULE(segment, m) )pbdoc"; // Common must be first in every module - pysrf::import(m, "srf.core.common"); + pysrf::import(module, "srf.core.common"); + pysrf::import(module, "srf.core.subscriber"); - pysrf::import_module_object(m, "srf.core.node", "SegmentObject"); - pysrf::import(m, "srf.core.subscriber"); + pysrf::import_module_object(module, "srf.core.node", "SegmentObject"); - // Register the converters for make_py2cxx_edge_adapter and make_cxx2py_edge_adapter // Type 'b' node::EdgeConnector::register_converter(); node::EdgeConnector::register_converter(); @@ -89,24 +97,28 @@ PYBIND11_MODULE(segment, m) node::EdgeConnector::register_converter(); node::EdgeConnector::register_converter(); - auto Definition = py::class_(m, "Definition"); - auto Builder = py::class_(m, "Builder"); + auto Builder = py::class_(module, "Builder"); + auto Definition = py::class_(module, "Definition"); + auto SegmentModule = + py::class_>(module, "SegmentModule"); + auto SegmentModuleRegistry = py::class_(module, "ModuleRegistry"); + /** Builder Interface Declarations **/ /* * @brief Make a source node that generates py::object values */ Builder.def("make_source", static_cast (*)( - srf::segment::Builder&, const std::string&, py::iterator)>(&SegmentProxy::make_source)); + srf::segment::Builder&, const std::string&, py::iterator)>(&BuilderProxy::make_source)); Builder.def("make_source", static_cast (*)( - srf::segment::Builder&, const std::string&, py::iterable)>(&SegmentProxy::make_source), + srf::segment::Builder&, const std::string&, py::iterable)>(&BuilderProxy::make_source), py::return_value_policy::reference_internal); Builder.def("make_source", static_cast (*)( - srf::segment::Builder&, const std::string&, py::function)>(&SegmentProxy::make_source)); + srf::segment::Builder&, const std::string&, py::function)>(&BuilderProxy::make_source)); /** * Construct a new py::object sink. @@ -128,7 +140,7 @@ PYBIND11_MODULE(segment, m) * sink = segment.make_sink("test", my_on_next, my_on_error, my_on_completed) * ``` */ - Builder.def("make_sink", &SegmentProxy::make_sink, py::return_value_policy::reference_internal); + Builder.def("make_sink", &BuilderProxy::make_sink, py::return_value_policy::reference_internal); /** * Construct a new 'pure' python::object -> python::object node @@ -138,26 +150,112 @@ PYBIND11_MODULE(segment, m) * (py) @param map_f : a std::function that takes a py::object and returns a py::object. This is your * python-function which will be called on each data element as it flows through the node. */ - Builder.def("make_node", &SegmentProxy::make_node, py::return_value_policy::reference_internal); + Builder.def("make_node", &BuilderProxy::make_node, py::return_value_policy::reference_internal); - Builder.def("make_node_full", &SegmentProxy::make_node_full, py::return_value_policy::reference_internal); + /** + * Find and return an existing egress port -- throws if `name` does not exist + * (py) @param name: Name of the egress port + */ + Builder.def("get_egress", &BuilderProxy::get_egress, py::arg("name")); + + /** + * Find and return an existing ingress port -- throws if `name` does not exist + * (py) @param name: Name of the ingress port + */ + Builder.def("get_ingress", &BuilderProxy::get_ingress, py::arg("name")); + + Builder.def("make_edge", &BuilderProxy::make_edge); + + Builder.def("make_edge", &BuilderProxy::make_edge, py::arg("source"), py::arg("sink")); + + Builder.def("load_module", + &BuilderProxy::load_module_from_registry, + py::arg("module_id"), + py::arg("registry_namespace"), + py::arg("module_name"), + py::arg("module_config"), + py::return_value_policy::reference_internal); + + Builder.def("init_module", &BuilderProxy::init_module, py::arg("module")); + + Builder.def( + "register_module_input", &BuilderProxy::register_module_input, py::arg("input_name"), py::arg("object")); + + Builder.def( + "register_module_output", &BuilderProxy::register_module_output, py::arg("output_name"), py::arg("object")); + + Builder.def("make_node_full", &BuilderProxy::make_node_full, py::return_value_policy::reference_internal); + + /** Segment Module Interface Declarations **/ + SegmentModule.def("config", &SegmentModuleProxy::config); + + SegmentModule.def("component_prefix", &SegmentModuleProxy::component_prefix); + + SegmentModule.def("input_port", &SegmentModuleProxy::input_port, py::arg("input_id")); + + SegmentModule.def("input_ports", &SegmentModuleProxy::input_ports); + + SegmentModule.def("module_type_name", &SegmentModuleProxy::module_type_name); + + SegmentModule.def("name", &SegmentModuleProxy::name); + + SegmentModule.def("output_port", &SegmentModuleProxy::output_port, py::arg("output_id")); + + SegmentModule.def("output_ports", &SegmentModuleProxy::output_ports); + + SegmentModule.def("input_ids", &SegmentModuleProxy::input_ids); + + SegmentModule.def("output_ids", &SegmentModuleProxy::output_ids); + + // TODO(drobison): need to think about if/how we want to expose type_ids to Python... It might allow for some nice + // flexibility SegmentModule.def("input_port_type_id", &SegmentModuleProxy::input_port_type_id, py::arg("input_id")) + // SegmentModule.def("input_port_type_ids", &SegmentModuleProxy::input_port_type_id) + // SegmentModule.def("output_port_type_id", &SegmentModuleProxy::output_port_type_id, py::arg("output_id")) + // SegmentModule.def("output_port_type_ids", &SegmentModuleProxy::output_port_type_id) + + /** Module Register Interface Declarations **/ + SegmentModuleRegistry.def_static( + "contains", &ModuleRegistryProxy::contains, py::arg("name"), py::arg("registry_namespace")); + + SegmentModuleRegistry.def_static( + "contains_namespace", &ModuleRegistryProxy::contains_namespace, py::arg("registry_namespace")); - Builder.def("make_py2cxx_edge_adapter", &SegmentProxy::make_py2cxx_edge_adapter); + SegmentModuleRegistry.def_static("registered_modules", &ModuleRegistryProxy::registered_modules); - Builder.def("make_cxx2py_edge_adapter", &SegmentProxy::make_cxx2py_edge_adapter); + SegmentModuleRegistry.def_static( + "is_version_compatible", &ModuleRegistryProxy::is_version_compatible, py::arg("release_version")); - Builder.def("make_edge", &SegmentProxy::make_edge); + SegmentModuleRegistry.def_static("get_module_constructor", + &ModuleRegistryProxy::get_module_constructor, + py::arg("name"), + py::arg("registry_namespace")); - Builder.def("get_ingress", &SegmentProxy::get_ingress); + SegmentModuleRegistry.def_static( + "register_module", + static_cast&, std::function)>( + &ModuleRegistryProxy::register_module), + py::arg("name"), + py::arg("release_version"), + py::arg("fn_constructor")); - Builder.def("get_egress", &SegmentProxy::get_egress); + SegmentModuleRegistry.def_static( + "register_module", + static_cast&, std::function)>( + &ModuleRegistryProxy::register_module), + py::arg("name"), + py::arg("registry_namespace"), + py::arg("release_version"), + py::arg("fn_constructor")); - Builder.def("make_edge", &SegmentProxy::make_edge, py::arg("source"), py::arg("sink")); + SegmentModuleRegistry.def_static("unregister_module", + &ModuleRegistryProxy::unregister_module, + py::arg("name"), + py::arg("registry_namespace"), + py::arg("optional") = true); -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif + module.attr("__version__") = + SRF_CONCAT_STR(srf_VERSION_MAJOR << "." << srf_VERSION_MINOR << "." << srf_VERSION_PATCH); } } // namespace srf::pysrf diff --git a/python/srf/core/subscriber.cpp b/python/srf/core/subscriber.cpp index 2d6e0788f..7954c5738 100644 --- a/python/srf/core/subscriber.cpp +++ b/python/srf/core/subscriber.cpp @@ -20,6 +20,9 @@ #include "pysrf/types.hpp" // for PyObjectObserver, PyObjectSubscriber, PyObjectObservable, PySubscription #include "pysrf/utils.hpp" +#include "srf/utils/string_utils.hpp" +#include "srf/version.hpp" + #include #include // IWYU pragma: keep #include @@ -27,15 +30,16 @@ #include #include +#include namespace srf::pysrf { namespace py = pybind11; using namespace py::literals; -PYBIND11_MODULE(subscriber, m) +PYBIND11_MODULE(subscriber, module) { - m.doc() = R"pbdoc( + module.doc() = R"pbdoc( Python bindings for SRF subscribers ------------------------------- .. currentmodule:: subscriber @@ -44,11 +48,11 @@ PYBIND11_MODULE(subscriber, m) )pbdoc"; // Common must be first in every module - pysrf::import(m, "srf.core.common"); + pysrf::import(module, "srf.core.common"); - py::class_(m, "Subscription"); + py::class_(module, "Subscription"); - py::class_(m, "Observer") + py::class_(module, "Observer") .def("on_next", &ObserverProxy::on_next, py::call_guard(), @@ -57,13 +61,13 @@ PYBIND11_MODULE(subscriber, m) .def("on_completed", &PyObjectObserver::on_completed, py::call_guard()) .def_static("make_observer", &ObserverProxy::make_observer); - py::class_(m, "Subscriber") + py::class_(module, "Subscriber") .def("on_next", &SubscriberProxy::on_next, py::call_guard()) .def("on_error", &SubscriberProxy::on_error) .def("on_completed", &PyObjectSubscriber::on_completed, py::call_guard()) .def("is_subscribed", &SubscriberProxy::is_subscribed, py::call_guard()); - py::class_(m, "Observable") + py::class_(module, "Observable") .def("subscribe", py::overload_cast(&ObservableProxy::subscribe), py::call_guard()) @@ -72,10 +76,7 @@ PYBIND11_MODULE(subscriber, m) py::call_guard()) .def("pipe", &ObservableProxy::pipe); -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif + module.attr("__version__") = + SRF_CONCAT_STR(srf_VERSION_MAJOR << "." << srf_VERSION_MINOR << "." << srf_VERSION_PATCH); } } // namespace srf::pysrf diff --git a/python/srf/tests/CMakeLists.txt b/python/srf/tests/CMakeLists.txt index 5c6565d6b..439c49fc9 100644 --- a/python/srf/tests/CMakeLists.txt +++ b/python/srf/tests/CMakeLists.txt @@ -15,8 +15,11 @@ list(APPEND CMAKE_MESSAGE_CONTEXT "tests") -srf_add_pybind11_module(test_edges_cpp - SOURCE_FILES test_edges.cpp +srf_add_pybind11_module(test_edges_cpp SOURCE_FILES test_edges.cpp ) +srf_add_pybind11_module(sample_modules SOURCE_FILES sample_modules.cpp) + +srf_add_pybind11_module(utils + SOURCE_FILES utils.cpp ) list(POP_BACK CMAKE_MESSAGE_CONTEXT) diff --git a/python/srf/tests/sample_modules.cpp b/python/srf/tests/sample_modules.cpp new file mode 100644 index 000000000..91621d8d6 --- /dev/null +++ b/python/srf/tests/sample_modules.cpp @@ -0,0 +1,77 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "srf/modules/sample_modules.hpp" + +#include "pysrf/utils.hpp" + +#include "srf/channel/status.hpp" +#include "srf/modules/module_registry_util.hpp" +#include "srf/node/rx_source.hpp" +#include "srf/utils/string_utils.hpp" +#include "srf/version.hpp" + +#include +#include + +#include +#include + +// IWYU thinks the Segment.def calls need array and vector +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include + +const std::vector PybindSegmentModuleVersion{srf_VERSION_MAJOR, srf_VERSION_MINOR, srf_VERSION_PATCH}; + +namespace srf::pysrf { + +namespace py = pybind11; + +PYBIND11_MODULE(sample_modules, module) +{ + module.doc() = R"pbdoc( + Python bindings for SRF Unittest Exports + ------------------------------- + .. currentmodule:: plugins + .. autosummary:: + :toctree: _generate + )pbdoc"; + + pysrf::import(module, "srf.core.common"); + + /** Register test modules -- necessary for python unit tests**/ + modules::ModelRegistryUtil::create_registered_module( + "SimpleModule", "srf_unittest", PybindSegmentModuleVersion); + modules::ModelRegistryUtil::create_registered_module( + "ConfigurableModule", "srf_unittest", PybindSegmentModuleVersion); + modules::ModelRegistryUtil::create_registered_module( + "SourceModule", "srf_unittest", PybindSegmentModuleVersion); + modules::ModelRegistryUtil::create_registered_module( + "SinkModule", "srf_unittest", PybindSegmentModuleVersion); + modules::ModelRegistryUtil::create_registered_module( + "NestedModule", "srf_unittest", PybindSegmentModuleVersion); + modules::ModelRegistryUtil::create_registered_module>( + "TemplateModuleInt", "srf_unittest", PybindSegmentModuleVersion); + modules::ModelRegistryUtil::create_registered_module>( + "TemplateModuleString", "srf_unittest", PybindSegmentModuleVersion); + + module.attr("__version__") = + SRF_CONCAT_STR(srf_VERSION_MAJOR << "." << srf_VERSION_MINOR << "." << srf_VERSION_PATCH); +} +} // namespace srf::pysrf diff --git a/python/srf/tests/test_edges.cpp b/python/srf/tests/test_edges.cpp index 10c2f89c7..986c4f300 100644 --- a/python/srf/tests/test_edges.cpp +++ b/python/srf/tests/test_edges.cpp @@ -29,6 +29,8 @@ #include "srf/node/source_properties.hpp" #include "srf/segment/builder.hpp" #include "srf/segment/object.hpp" +#include "srf/utils/string_utils.hpp" +#include "srf/version.hpp" #include #include @@ -39,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -191,21 +194,21 @@ class SinkBase : public pysrf::PythonSink> SinkBase() : PythonSink(build()) {} }; -PYBIND11_MODULE(test_edges_cpp, m) +PYBIND11_MODULE(test_edges_cpp, module) { - m.doc() = R"pbdoc()pbdoc"; + module.doc() = R"pbdoc()pbdoc"; - pysrf::import(m, "srf"); + pysrf::import(module, "srf"); - py::class_>(m, "Base").def(py::init<>([]() { return std::make_shared(); })); + py::class_>(module, "Base").def(py::init<>([]() { return std::make_shared(); })); srf::pysrf::PortBuilderUtil::register_port_util(); - py::class_>(m, "DerivedA").def(py::init<>([]() { + py::class_>(module, "DerivedA").def(py::init<>([]() { return std::make_shared(); })); srf::pysrf::PortBuilderUtil::register_port_util(); - py::class_>(m, "DerivedB").def(py::init<>([]() { + py::class_>(module, "DerivedB").def(py::init<>([]() { return std::make_shared(); })); srf::pysrf::PortBuilderUtil::register_port_util(); @@ -215,7 +218,7 @@ PYBIND11_MODULE(test_edges_cpp, m) py::class_, srf::segment::ObjectProperties, - std::shared_ptr>>(m, "SourceDerivedB") + std::shared_ptr>>(module, "SourceDerivedB") .def(py::init<>([](srf::segment::Builder& parent, const std::string& name) { auto stage = parent.construct_object(name); @@ -226,7 +229,7 @@ PYBIND11_MODULE(test_edges_cpp, m) py::class_, srf::segment::ObjectProperties, - std::shared_ptr>>(m, "SourcePyHolder") + std::shared_ptr>>(module, "SourcePyHolder") .def(py::init<>([](srf::segment::Builder& parent, const std::string& name) { auto stage = parent.construct_object(name); @@ -236,7 +239,7 @@ PYBIND11_MODULE(test_edges_cpp, m) py::arg("name")); py::class_, srf::segment::ObjectProperties, std::shared_ptr>>( - m, "NodeBase") + module, "NodeBase") .def(py::init<>([](srf::segment::Builder& parent, const std::string& name) { auto stage = parent.construct_object(name); @@ -247,7 +250,7 @@ PYBIND11_MODULE(test_edges_cpp, m) py::class_, srf::segment::ObjectProperties, - std::shared_ptr>>(m, "NodePyHolder") + std::shared_ptr>>(module, "NodePyHolder") .def(py::init<>([](srf::segment::Builder& parent, const std::string& name) { auto stage = parent.construct_object(name); @@ -257,7 +260,7 @@ PYBIND11_MODULE(test_edges_cpp, m) py::arg("name")); py::class_, segment::ObjectProperties, std::shared_ptr>>( - m, "SinkBase") + module, "SinkBase") .def(py::init<>([](segment::Builder& parent, const std::string& name) { auto stage = parent.construct_object(name); @@ -266,10 +269,7 @@ PYBIND11_MODULE(test_edges_cpp, m) py::arg("parent"), py::arg("name")); -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif + module.attr("__version__") = + SRF_CONCAT_STR(srf_VERSION_MAJOR << "." << srf_VERSION_MINOR << "." << srf_VERSION_PATCH); } } // namespace srf::pytests diff --git a/python/srf/tests/utils.cpp b/python/srf/tests/utils.cpp new file mode 100644 index 000000000..9b1ca8fbf --- /dev/null +++ b/python/srf/tests/utils.cpp @@ -0,0 +1,54 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pysrf/utils.hpp" + +#include "srf/utils/string_utils.hpp" +#include "srf/version.hpp" + +#include +#include + +#include +#include + +namespace srf::pytests { + +namespace py = pybind11; + +PYBIND11_MODULE(utils, module) +{ + module.doc() = R"pbdoc()pbdoc"; + + pysrf::import(module, "srf"); + + module.def( + "throw_cpp_error", + [](std::string msg = "") { + if (msg.empty()) + { + msg = "Exception from C++ code"; + } + + throw std::runtime_error(msg); + }, + py::arg("msg") = ""); + + module.attr("__version__") = + SRF_CONCAT_STR(srf_VERSION_MAJOR << "." << srf_VERSION_MINOR << "." << srf_VERSION_PATCH); +} +} // namespace srf::pytests diff --git a/python/tests/test_edges.py b/python/tests/test_edges.py index 93346f48d..771513de5 100644 --- a/python/tests/test_edges.py +++ b/python/tests/test_edges.py @@ -83,13 +83,13 @@ def segment_init(seg: srf.Builder): source = m.SourceDerivedB(seg, "source") def on_next(x: m.Base): - print("Got: {}".format(type(x))) + pass def on_error(e): pass def on_complete(): - print("Complete") + pass sink = seg.make_sink("sink", on_next, on_error, on_complete) seg.make_edge(source, sink) @@ -163,7 +163,6 @@ def create_source(): def on_next(x: int): nonlocal on_next_count - print("Got: {}".format(type(x))) on_next_count += 1 @@ -171,7 +170,7 @@ def on_error(e): pass def on_complete(): - print("Complete") + pass sink = seg.make_sink("sink", on_next, on_error, on_complete) seg.make_edge(node, sink) @@ -232,7 +231,7 @@ def segment_sink(seg: srf.Builder): # This method will get called each time the sink gets a value def sink_on_next(x: MyCustomClass): - print("Sink: Got Obj Name: {}, Value: {}".format(x.name, x.value)) + pass def sink_on_next_untyped(input): pass diff --git a/python/tests/test_executor.py b/python/tests/test_executor.py new file mode 100644 index 000000000..367813f5f --- /dev/null +++ b/python/tests/test_executor.py @@ -0,0 +1,187 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import typing + +import pytest + +import srf +from srf.tests.utils import throw_cpp_error + + +def pairwise(t): + it = iter(t) + return zip(it, it) + + +node_fn_type = typing.Callable[[srf.Builder], srf.SegmentObject] + + +@pytest.fixture +def source_pyexception(): + + def build(builder: srf.Builder): + + def gen_data_and_raise(): + yield 1 + yield 2 + yield 3 + + raise RuntimeError("Raised python error") + + return builder.make_source("source", gen_data_and_raise) + + return build + + +@pytest.fixture +def source_cppexception(): + + def build(builder: srf.Builder): + + def gen_data_and_raise(): + yield 1 + yield 2 + yield 3 + + throw_cpp_error() + + return builder.make_source("source", gen_data_and_raise) + + return build + + +@pytest.fixture +def sink(): + + def build(builder: srf.Builder): + + def sink_on_next(data): + print("Got value: {}".format(data)) + + return builder.make_sink("sink", sink_on_next, None, None) + + return build + + +@pytest.fixture +def build_pipeline(): + + def inner(*node_fns: node_fn_type): + + def init_segment(builder: srf.Builder): + + created_nodes = [] + + # Loop over node creation functions + for n in node_fns: + created_nodes.append(n(builder)) + + # For each pair, call make_edge + for source, sink in pairwise(created_nodes): + builder.make_edge(source, sink) + + pipe = srf.Pipeline() + + pipe.make_segment("TestSegment11", init_segment) + + return pipe + + return inner + + +build_pipeline_type = typing.Callable[[typing.Tuple[node_fn_type, ...]], srf.Pipeline] + + +@pytest.fixture +def build_executor(): + + def inner(pipe: srf.Pipeline): + options = srf.Options() + + executor = srf.Executor(options) + executor.register_pipeline(pipe) + + executor.start() + + return executor + + return inner + + +build_executor_type = typing.Callable[[srf.Pipeline], srf.Executor] + + +def test_pyexception_in_source(source_pyexception: node_fn_type, + sink: node_fn_type, + build_pipeline: build_pipeline_type, + build_executor: build_executor_type): + + pipe = build_pipeline(source_pyexception, sink) + + executor = build_executor(pipe) + + with pytest.raises(RuntimeError): + executor.join() + + +def test_cppexception_in_source(source_cppexception: node_fn_type, + sink: node_fn_type, + build_pipeline: build_pipeline_type, + build_executor: build_executor_type): + + pipe = build_pipeline(source_cppexception, sink) + + executor = build_executor(pipe) + + with pytest.raises(RuntimeError): + executor.join() + + +def test_pyexception_in_source_async(source_pyexception: node_fn_type, + sink: node_fn_type, + build_pipeline: build_pipeline_type, + build_executor: build_executor_type): + + pipe = build_pipeline(source_pyexception, sink) + + async def run_pipeline(): + executor = build_executor(pipe) + + with pytest.raises(RuntimeError): + await executor.join_async() + + asyncio.run(run_pipeline()) + + +def test_cppexception_in_source_async(source_cppexception: node_fn_type, + sink: node_fn_type, + build_pipeline: build_pipeline_type, + build_executor: build_executor_type): + + pipe = build_pipeline(source_cppexception, sink) + + async def run_pipeline(): + executor = build_executor(pipe) + + with pytest.raises(RuntimeError): + await executor.join_async() + + asyncio.run(run_pipeline()) + + +if (__name__ in ("__main__", )): + test_pyexception_in_source() diff --git a/python/tests/test_module_registry.py b/python/tests/test_module_registry.py new file mode 100644 index 000000000..4f0fd1e91 --- /dev/null +++ b/python/tests/test_module_registry.py @@ -0,0 +1,509 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import random + +import pytest + +import srf +import srf.tests.sample_modules + +VERSION = [int(cmpt) for cmpt in srf.tests.sample_modules.__version__.split(".")] + +packet_count = 0 + + +def test_contains_namespace(): + registry = srf.ModuleRegistry + + assert registry.contains_namespace("xyz") is not True + assert registry.contains_namespace("default") + + +def test_contains(): + registry = srf.ModuleRegistry + + assert registry.contains("SimpleModule", "srf_unittest") + assert registry.contains("SourceModule", "srf_unittest") + assert registry.contains("SinkModule", "srf_unittest") + assert registry.contains("SimpleModule", "default") is not True + + +def test_is_version_compatible(): + registry = srf.ModuleRegistry + + release_version = [22, 11, 0] + old_release_version = [22, 10, 0] + no_version_patch = [22, 10] + no_version_minor_and_patch = [22] + + assert registry.is_version_compatible(release_version) + assert registry.is_version_compatible(old_release_version) is not True + assert registry.is_version_compatible(no_version_patch) is not True + assert registry.is_version_compatible(no_version_minor_and_patch) is not True + + +def test_unregister_module(): + registry = srf.ModuleRegistry + + registry_namespace = "srf_unittest2" + simple_mod_name = "SimpleModule" + + registry.unregister_module(simple_mod_name, registry_namespace) + + with pytest.raises(Exception): + registry.unregister_module(simple_mod_name, registry_namespace, False) + + registry.unregister_module(simple_mod_name, registry_namespace, True) + + +def test_registered_modules(): + registry = srf.ModuleRegistry + registered_mod_dict = registry.registered_modules() + + assert "default" in registered_mod_dict + assert "srf_unittest" in registered_mod_dict + assert len(registered_mod_dict) == 2 + + +def module_init_fn(builder: srf.Builder): + pass + + +def module_init_nested_fn(builder: srf.Builder): + pass + + +# Purpose: Test basic dynamic module registration fails when given an incompatible version number +def test_module_registry_register_bad_version(): + registry = srf.ModuleRegistry + + # Bad version should result in a raised exception + with pytest.raises(Exception): + registry.register_module("a_module", "srf_unittests", [99, 99, 99], module_init_fn) + + +# Purpose: Test basic dynamic module registration and un-registration +def test_module_registry_register_good_version(): + registry = srf.ModuleRegistry + + registry.register_module("test_module_registry_register_good_version_module", + "srf_unittests", + VERSION, + module_init_fn) + registry.unregister_module("test_module_registry_register_good_version_module", "srf_unittests") + + +# Purpose: Test basic dynamic module registration, and indirectly test correct shutdown/cleanup behavior +def test_module_registry_register_good_version_no_unregister(): + # Ensure that we don"t throw any errors or hang if we don"t explicitly unregister the python module + registry = srf.ModuleRegistry + + registry.register_module("test_module_registry_register_good_version_no_unregister_module", + "srf_unittests", + VERSION, + module_init_fn) + + +def test_get_module_constructor(): + registry = srf.ModuleRegistry + + # Retrieve the module constructor + fn_constructor = registry.get_module_constructor("SimpleModule", "srf_unittest") + + # Instantiate a version of the module + config = {"config_key_1": True} + module = fn_constructor("ModuleInitializationTest_mod", config) + + assert "config_key_1" in module.config() + + with pytest.raises(Exception): + registry.get_module_constructor("SimpleModule", "default") + + +def test_module_intitialize(): + + module_name = "test_py_source_from_cpp" + config = {"source_count": 42} + registry = srf.ModuleRegistry + + def module_initializer(builder: srf.Builder): + + source_mod = builder.load_module("SourceModule", "srf_unittest", "ModuleSourceTest_mod1", config) + builder.register_module_output("source", source_mod.output_port("source")) + + def init_wrapper(builder: srf.Builder): + + global packet_count + packet_count = 0 + + def on_next(data): + global packet_count + packet_count += 1 + logging.info("Sinking {}".format(data)) + + def on_error(): + pass + + def on_complete(): + pass + + # Retrieve the module constructor + fn_constructor = registry.get_module_constructor(module_name, "srf_unittest") + # Instantiate a version of the module + source_module = fn_constructor("ModuleSourceTest_mod1", config) + + sink = builder.make_sink("sink", on_next, on_error, on_complete) + + builder.init_module(source_module) + builder.make_edge(source_module.output_port('source'), sink) + + # Register the module + registry.register_module(module_name, "srf_unittest", VERSION, module_initializer) + + pipeline = srf.Pipeline() + pipeline.make_segment("ModuleAsSource_Segment", init_wrapper) + + options = srf.Options() + options.topology.user_cpuset = "0-1" + + executor = srf.Executor(options) + executor.register_pipeline(pipeline) + executor.start() + executor.join() + + assert packet_count == 42 + + +# Purpose: Create a self-contained (no input/output ports), nested, dynamic module, and instantiate two copies in our +# init wrapper +def test_py_registered_nested_modules(): + global packet_count + + # Stand-alone module, no input or output ports + # 1. We register a python module definition as being built by "init_registered" + # 2. We then create a segment with a separate init function "init_caller" that loads our python module from + # the registry and initializes it, running + def module_initializer(builder: srf.Builder): + global packet_count + + def on_next(data): + global packet_count + packet_count += 1 + logging.info("Sinking {}".format(data)) + + def on_error(): + pass + + def on_complete(): + pass + + config = {"source_count": 42} + + source_mod = builder.load_module("SourceModule", "srf_unittest", "ModuleSourceTest_mod1", config) + sink = builder.make_sink("sink", on_next, on_error, on_complete) + builder.make_edge(source_mod.output_port("source"), sink) + + def init_caller(builder: srf.Builder): + global packet_count + packet_count = 0 + builder.load_module("test_py_registered_nested_module", "srf_unittests", "my_loaded_module!", {}) + + registry = srf.ModuleRegistry + registry.register_module("test_py_registered_nested_module", "srf_unittests", VERSION, module_initializer) + + pipeline = srf.Pipeline() + pipeline.make_segment("ModuleAsSource_Segment", init_caller) + + options = srf.Options() + options.topology.user_cpuset = "0-1" + + executor = srf.Executor(options) + executor.register_pipeline(pipeline) + executor.start() + executor.join() + + # We loaded two copies of the module, and each one captures the same packet_count, so we should see 2*42 packets + assert packet_count == 42 + + +# Purpose: Create a self-contained (no input/output ports), nested, dynamic module, and instantiate two copies in our +# init wrapper -- since both versions capture our global 'packet_count', we should see double the packets. +def test_py_registered_nested_copied_modules(): + global packet_count + + def module_initializer(builder: srf.Builder): + global packet_count + + def on_next(data): + global packet_count + packet_count += 1 + logging.info("Sinking {}".format(data)) + + def on_error(): + pass + + def on_complete(): + pass + + config = {"source_count": 42} + + source_mod = builder.load_module("SourceModule", "srf_unittest", "ModuleSourceTest_mod1", config) + sink = builder.make_sink("sink", on_next, on_error, on_complete) + builder.make_edge(source_mod.output_port("source"), sink) + + registry = srf.ModuleRegistry + registry.register_module("test_py_registered_nested_copied_module", "srf_unittests", VERSION, module_initializer) + + def init_wrapper(builder: srf.Builder): + global packet_count + packet_count = 0 + builder.load_module("test_py_registered_nested_copied_module", "srf_unittests", "my_loaded_module!", {}) + builder.load_module("test_py_registered_nested_copied_module", "srf_unittests", "my_loaded_module_copy!", {}) + + pipeline = srf.Pipeline() + pipeline.make_segment("ModuleAsSource_Segment", init_wrapper) + + options = srf.Options() + options.topology.user_cpuset = "0-1" + + executor = srf.Executor(options) + executor.register_pipeline(pipeline) + executor.start() + executor.join() + + # We loaded two copies of the module, and each one captures the same packet_count, so we should see 2*42 packets + assert packet_count == 84 + + +# Test if we can create a [source_module] -> [sink] configuration, where the source module is a python source created +# via builder.make_source. +# +# Purpose: This is intended to check dynamic module creation, registration, and retrieval +def test_py_dynamic_module_source(): + global packet_count + module_name = "test_py_dyn_source" + + def module_initializer(builder: srf.Builder): + + def gen_data(): + for x in range(42): + yield random.choice([True, False]) + + source1 = builder.make_source("dynamic_module_source", gen_data) + builder.register_module_output("source", source1) + + registry = srf.ModuleRegistry + registry.register_module(module_name, "srf_unittests", VERSION, module_initializer) + + def init_wrapper(builder: srf.Builder): + global packet_count + packet_count = 0 + + def on_next(data): + global packet_count + packet_count += 1 + logging.info("Sinking {}".format(data)) + + def on_error(): + pass + + def on_complete(): + pass + + # Load our registered module + source_mod = builder.load_module(module_name, "srf_unittests", "my_loaded_module!", {}) + sink = builder.make_sink("sink", on_next, on_error, on_complete) + + builder.make_edge(source_mod.output_port("source"), sink) + + pipeline = srf.Pipeline() + pipeline.make_segment("ModuleAsSource_Segment", init_wrapper) + + options = srf.Options() + options.topology.user_cpuset = "0-1" + + executor = srf.Executor(options) + executor.register_pipeline(pipeline) + executor.start() + executor.join() + + assert packet_count == 42 + + +# Test if we can create a [source_module] -> [sink] configuration, where the source module loads a c++ defined module +# from the registry and exposes its source as the source for our dynamic module. +# +# Purpose: This is intended to check dynamic module creation, registration, and retrieval, in conjunction with module +# nesting. +def test_py_dynamic_module_from_cpp_source(): + global packet_count + module_name = "test_py_dyn_source_from_cpp" + + def module_initializer(builder: srf.Builder): + config = {"source_count": 42} + + source_mod = builder.load_module("SourceModule", "srf_unittest", "ModuleSourceTest_mod1", config) + builder.register_module_output("source", source_mod.output_port("source")) + + registry = srf.ModuleRegistry + registry.register_module(module_name, "srf_unittests", VERSION, module_initializer) + + def init_wrapper(builder: srf.Builder): + global packet_count + packet_count = 0 + + def on_next(data): + global packet_count + packet_count += 1 + logging.info("Sinking {}".format(data)) + + def on_error(): + pass + + def on_complete(): + pass + + # Load our registered module + source_mod = builder.load_module(module_name, "srf_unittests", "my_loaded_module!", {}) + sink = builder.make_sink("sink", on_next, on_error, on_complete) + + builder.make_edge(source_mod.output_port("source"), sink) + + pipeline = srf.Pipeline() + pipeline.make_segment("ModuleAsSource_Segment", init_wrapper) + + options = srf.Options() + options.topology.user_cpuset = "0-1" + + executor = srf.Executor(options) + executor.register_pipeline(pipeline) + executor.start() + executor.join() + + assert packet_count == 42 + + +# Purpose: Test creation of a dynamic module that acts as a sink [source] -> [sink_module] +def test_py_dynamic_module_sink(): + global packet_count + module_name = "test_py_dyn_sink" + + def module_initializer(builder: srf.Builder): + global packet_count + packet_count = 0 + + def on_next(data): + global packet_count + packet_count += 1 + logging.info("Sinking {}".format(data)) + + def on_error(): + pass + + def on_complete(): + pass + + sink = builder.make_sink("sink", on_next, on_error, on_complete) + builder.register_module_input("sink", sink) + + registry = srf.ModuleRegistry + registry.register_module(module_name, "srf_unittests", VERSION, module_initializer) + + def init_wrapper(builder: srf.Builder): + global packet_count + packet_count = 0 + + def gen_data(): + for x in range(42): + yield random.choice([True, False]) + + source = builder.make_source("source", gen_data) + sink_mod = builder.load_module(module_name, "srf_unittests", "loaded_sink_module", {}) + + builder.make_edge(source, sink_mod.input_port("sink")) + + pipeline = srf.Pipeline() + pipeline.make_segment("ModuleAsSource_Segment", init_wrapper) + + options = srf.Options() + options.topology.user_cpuset = "0-1" + + executor = srf.Executor(options) + executor.register_pipeline(pipeline) + executor.start() + executor.join() + + assert packet_count == 42 + + +# Purpose: Test creation of a dynamic module that acts as a sink [source] -> [sink_module], where sink module loads a +# c++ defined module from the registry and exposes its sink as the sink for the dynamic module. +def test_py_dynamic_module_from_cpp_sink(): + global packet_count + module_name = "test_py_dyn_sink_from_cpp" + + def module_initializer(builder: srf.Builder): + config = {"source_count": 42} + + sink_mod = builder.load_module("SinkModule", "srf_unittest", "ModuleSinkTest_Mod1", config) + builder.register_module_input("sink", sink_mod.input_port("sink")) + + registry = srf.ModuleRegistry + registry.register_module(module_name, "srf_unittests", VERSION, module_initializer) + + def gen_data(): + for x in range(42): + yield random.choice([True, False]) + + def init_wrapper(builder: srf.Builder): + global packet_count + packet_count = 0 + + source = builder.make_source("source", gen_data) + sink_mod = builder.load_module(module_name, "srf_unittests", "loaded_sink_module", {}) + + builder.make_edge(source, sink_mod.input_port("sink")) + + pipeline = srf.Pipeline() + pipeline.make_segment("ModuleAsSource_Segment", init_wrapper) + + options = srf.Options() + options.topology.user_cpuset = "0-1" + + executor = srf.Executor(options) + executor.register_pipeline(pipeline) + executor.start() + executor.join() + + +if (__name__ in ("__main__", )): + test_module_intitialize() + test_contains_namespace() + test_contains() + test_get_module_constructor() + test_is_version_compatible() + test_unregister_module() + test_registered_modules() + test_module_registry_register_bad_version() + test_module_registry_register_good_version() + test_module_registry_register_good_version_no_unregister() + test_py_registered_nested_modules() + test_py_registered_nested_copied_modules() + test_py_dynamic_module_source() + test_py_dynamic_module_from_cpp_source() + test_py_dynamic_module_sink() + test_py_dynamic_module_from_cpp_sink() diff --git a/python/tests/test_node.py b/python/tests/test_node.py index a59897a74..696bdb6bd 100644 --- a/python/tests/test_node.py +++ b/python/tests/test_node.py @@ -22,7 +22,6 @@ @pytest.mark.parametrize("pe_count", [1, 3]) @pytest.mark.parametrize("source_type", ["iterator", "iterable", "function"]) def test_launch_options_source(source_type: str, pe_count: int, engines_per_pe: int): - hit_count = 0 source = None @@ -61,8 +60,6 @@ def node_fn(x: int): hit_count += 1 - print("Hit with value: {}".format(x)) - hit_counter = seg.make_node("hit_counter", node_fn) seg.make_edge(src_node, hit_counter) @@ -92,14 +89,12 @@ def node_fn(x: int): def test_launch_options_iterable(): - pe_count = 2 engines_per_pe = 4 hit_count = 0 def segment_init(seg: srf.Builder): - src_node = seg.make_source("my_src", [1, 2, 3]) src_node.launch_options.pe_count = pe_count @@ -110,8 +105,6 @@ def node_fn(x: int): hit_count += 1 - print("Hit with value: {}".format(x)) - hit_counter = seg.make_node("hit_counter", node_fn) seg.make_edge(src_node, hit_counter) diff --git a/python/tests/test_operators.py b/python/tests/test_operators.py index f420cd52b..e73f86836 100644 --- a/python/tests/test_operators.py +++ b/python/tests/test_operators.py @@ -21,6 +21,7 @@ @pytest.fixture def ex_runner(): + def run_exec(segment_init): pipeline = srf.Pipeline() @@ -44,6 +45,7 @@ def run_exec(segment_init): @pytest.fixture def run_segment(ex_runner): + def run(input_data, node_fn): actual = [] @@ -149,6 +151,7 @@ def test_on_complete_none(run_segment): on_completed_hit = False def node_fn(input: srf.Observable, output: srf.Subscriber): + def on_completed_fn(): nonlocal on_completed_hit on_completed_hit = True diff --git a/python/tests/test_pipeline.py b/python/tests/test_pipeline.py index 820fb733d..d04905aa9 100644 --- a/python/tests/test_pipeline.py +++ b/python/tests/test_pipeline.py @@ -326,7 +326,7 @@ def gen_data(): def init1(builder: srf.Builder): source = builder.make_source("source", gen_data) - egress = builder.get_egress("b") + egress = builder.get_egress("a") builder.make_edge(source, egress) @@ -341,7 +341,7 @@ def on_error(): def on_complete(): pass - ingress = builder.get_ingress("b") + ingress = builder.get_ingress("a") sink = builder.make_sink("sink", on_next, on_error, on_complete) builder.make_edge(ingress, sink) diff --git a/python/tests/test_plugin_modules.py b/python/tests/test_plugin_modules.py new file mode 100644 index 000000000..031bc5f98 --- /dev/null +++ b/python/tests/test_plugin_modules.py @@ -0,0 +1,185 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import pathlib + +import pytest + +import srf + +whereami = pathlib.Path(__file__).parent.resolve() + +FOUND_DYN_LIB = False +DYN_LIB_DIR = None + +dyn_lib_candidates = list(glob.iglob(f"{whereami}/../../**/libdynamic_test_module.so", recursive=True)) +if (len(dyn_lib_candidates) >= 1): + FOUND_DYN_LIB = True + DYN_LIB_DIR = pathlib.Path(dyn_lib_candidates[0]).parent.resolve() + + +def test_plugin_module_create_or_acquire(): + mod = srf.PluginModule.create_or_acquire("doesnt_exist.so") + + assert mod is not None + + +@pytest.mark.skipif(not FOUND_DYN_LIB, reason="Missing: libdynamic_test_module.so") +def test_dynamic_module_plugin_interface(): + plugin_module = srf.PluginModule.create_or_acquire("libdynamic_test_module.so") + plugin_module.set_library_directory(f"{DYN_LIB_DIR}") + plugin_module.load() + plugin_module.reload() + + +@pytest.mark.skipif(not FOUND_DYN_LIB, reason="Missing: libdynamic_test_module.so") +def test_dynamic_module_registration(): + plugin_module = srf.PluginModule.create_or_acquire("libdynamic_test_module.so") + plugin_module.set_library_directory(f"{DYN_LIB_DIR}") + plugin_module.load() + + module_namespace = "srf_unittest_cpp_dynamic" + module_name = "DynamicSourceModule" + + registry = srf.ModuleRegistry + + assert registry.contains_namespace(module_namespace) + assert registry.contains(module_name, module_namespace) + + def init_wrapper(builder: srf.Builder): + global packet_count + packet_count = 0 + + def on_next(input): + global packet_count + packet_count += 1 + + def on_error(): + pass + + def on_complete(): + pass + + config = {"source_count": 42} + + dynamic_source_mod = builder.load_module("DynamicSourceModule", + "srf_unittest_cpp_dynamic", + "DynamicModuleSourceTest_mod1", + config) + sink = builder.make_sink("sink", on_next, on_error, on_complete) + + builder.make_edge(dynamic_source_mod.output_port("source"), sink) + + pipeline = srf.Pipeline() + pipeline.make_segment("DynamicSourceModule_Segment", init_wrapper) + + options = srf.Options() + options.topology.user_cpuset = "0-1" + + executor = srf.Executor(options) + executor.register_pipeline(pipeline) + executor.start() + executor.join() + + assert packet_count == 42 + assert plugin_module.unload() + + +@pytest.mark.skipif(not FOUND_DYN_LIB, reason="Missing: libdynamic_test_module.so") +def test_dynamic_module_plugin_registration(): + plugin_module = srf.PluginModule.create_or_acquire("libdynamic_test_module.so") + plugin_module.set_library_directory(f"{DYN_LIB_DIR}") + plugin_module.load() + + module_namespace = "srf_unittest_cpp_dynamic" + + module_name = "DynamicSourceModule" + + registry = srf.ModuleRegistry + + assert registry.contains_namespace(module_namespace) + assert registry.contains(module_name, module_namespace) + + registered_modules = registry.registered_modules() + + assert "srf_unittest_cpp_dynamic" in registered_modules + + ns_1 = registered_modules["srf_unittest_cpp_dynamic"] + assert len(ns_1) == 1 + assert ns_1[0] == "DynamicSourceModule" + + assert "srf_unittest_cpp_dynamic_2" in registered_modules + + ns_2 = registered_modules["srf_unittest_cpp_dynamic_2"] + assert len(ns_2) == 1 + assert ns_2[0] == "DynamicSourceModule" + + assert "srf_unittest_cpp_dynamic_3" in registered_modules + + ns_3 = registered_modules["srf_unittest_cpp_dynamic_3"] + assert len(ns_3) == 1 + assert ns_3[0] == "DynamicSourceModule" + + actual_modules = plugin_module.list_modules() + assert len(actual_modules) == 3 + + expected_modules = [ + "srf_unittest_cpp_dynamic::DynamicSourceModule", + "srf_unittest_cpp_dynamic_2::DynamicSourceModule", + "srf_unittest_cpp_dynamic_3::DynamicSourceModule" + ] + + assert actual_modules == expected_modules + + plugin_module.unload() + + registered_modules = registry.registered_modules() + + assert "srf_unittest_cpp_dynamic" not in registered_modules + assert "srf_unittest_cpp_dynamic_2" not in registered_modules + assert "srf_unittest_cpp_dynamic_3" not in registered_modules + + +@pytest.mark.skipif(not FOUND_DYN_LIB, reason="Failed to find libdynamic_test_module.so") +def test_dynamic_module_bad_version_test(): + + BAD_VERSION = [13, 14, 15] + module_name = "DynamicSourceModule_BAD" + module_namespace = "srf_unittest_cpp_dynamic_BAD" + + def module_initializer(builder: srf.Builder): + config = {"source_count": 42} + + builder.load_module("DynamicSourceModule_BAD", + "srf_unittest_cpp_dynamic_BAD", + "DynamicSourceModule_BAD_Test", + config) + + registry = srf.ModuleRegistry + + with pytest.raises(Exception): + registry.register_module(module_name, module_namespace, BAD_VERSION, module_initializer) + + assert registry.contains_namespace(module_namespace) is not True + assert registry.contains(module_namespace, module_namespace) is not True + + +if (__name__ in ("__main__", )): + test_plugin_module_create_or_acquire() + test_dynamic_module_plugin_interface() + test_dynamic_module_plugin_registration() + test_dynamic_module_bad_version_test() + test_dynamic_module_registration() diff --git a/python/tests/test_segment_modules.py b/python/tests/test_segment_modules.py new file mode 100644 index 000000000..30b83bb8c --- /dev/null +++ b/python/tests/test_segment_modules.py @@ -0,0 +1,362 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import pytest + +import srf +# Required to register sample modules with the ModuleRegistry +import srf.tests.sample_modules + +packets_1 = 0 +packets_2 = 0 +packets_3 = 0 + + +def test_py_end_to_end(): + + def gen_data_1(): + yield True + yield False + yield True + yield True + + def gen_data_2(): + yield True + yield False + yield False + yield False + yield True + yield False + + # Visualization of What's Going On + # SimpleModule + # __________________________________ + # source1 -> emitted boolean -> --- | input1 -- _internal1_ -- output1 | --- emitted string --- sink1 + # | | + # source2 -> emitted boolean -> --- | input2 -- _internal2_ -- output2 | --- emitted string --- sink2 + # |__________________________________| + # + # ConfigurableModule + # ________________________________________________________________ + # source3 -> emitted boolean -> --- | configurable_input_a -- _internal1_ -- configurable_output_x | --- ... sink3 + # |_______________________________________________________________ + # + + def init_wrapper(builder: srf.Builder): + global packets_1, packets_2, packets_3 + packets_1, packets_2, packets_3 = 0, 0, 0 + + def on_next_sink_1(input): + global packets_1 + packets_1 += 1 + + def on_next_sink_2(input): + global packets_2 + packets_2 += 1 + + def on_next_sink_3(input): + global packets_3 + packets_3 += 1 + + def on_error(): + pass + + def on_complete(): + pass + + simple_mod = builder.load_module("SimpleModule", "srf_unittest", "ModuleEndToEndTest_mod1", {}) + configurable_mod = builder.load_module("ConfigurableModule", "srf_unittest", "ModuleEndToEndTest_mod2", {}) + + source1 = builder.make_source("src1", gen_data_1) + builder.make_edge(source1, simple_mod.input_port("input1")) + + source2 = builder.make_source("src2", gen_data_2) + builder.make_edge(source2, simple_mod.input_port("input2")) + + sink1 = builder.make_sink("sink1", on_next_sink_1, on_error, on_complete) + builder.make_edge(simple_mod.output_port("output1"), sink1) + + sink2 = builder.make_sink("sink2", on_next_sink_2, on_error, on_complete) + builder.make_edge(simple_mod.output_port("output2"), sink2) + + source3 = builder.make_source("src3", gen_data_1) + builder.make_edge(source3, configurable_mod.input_port("configurable_input_a")) + + sink3 = builder.make_sink("sink3", on_next_sink_3, on_error, on_complete) + builder.make_edge(configurable_mod.output_port("configurable_output_x"), sink3) + + pipe = srf.Pipeline() + + pipe.make_segment("EndToEnd_Segment", [], [], init_wrapper) + + options = srf.Options() + + executor = srf.Executor(options) + executor.register_pipeline(pipe) + + executor.start() + executor.join() + + assert (packets_1 == 4) + assert (packets_2 == 6) + assert (packets_3 == 4) + + +def test_py_constructor(): + + config = {"config_key_1": True} + + registry = srf.ModuleRegistry + + # Retrieve the module constructor + fn_constructor = registry.get_module_constructor("SimpleModule", "srf_unittest") + + # Instantiate a version of the module + config = {"config_key_1": True} + module = fn_constructor("ModuleInitializationTest_mod", config) + + assert "config_key_1" in module.config() + + with pytest.raises(Exception): + registry.get_module_constructor("SimpleModule", "default") + + +def test_py_module_initialization(): + + def gen_data(): + yield True + yield False + yield True + yield True + + def init_wrapper(builder: srf.Builder): + + def on_next(input): + pass + + def on_error(): + pass + + def on_complete(): + pass + + config = {"config_key_1": True} + + registry = srf.ModuleRegistry + + source = builder.make_source("source", gen_data) + source2 = builder.make_source("source2", gen_data) + fn_constructor = registry.get_module_constructor("SimpleModule", "srf_unittest") + simple_mod = fn_constructor("ModuleInitializationTest_mod2", config) + sink = builder.make_sink("sink", on_next, on_error, on_complete) + + builder.init_module(simple_mod) + + assert len(simple_mod.input_ids()) == 2 + assert len(simple_mod.output_ids()) == 2 + assert len(simple_mod.input_ports()) == 2 + assert len(simple_mod.output_ports()) == 2 + + assert ("input1" in simple_mod.input_ports()) + assert ("input2" in simple_mod.input_ports()) + assert ("output1" in simple_mod.output_ports()) + assert ("output2" in simple_mod.output_ports()) + + with pytest.raises(Exception): + simple_mod.input_port("DOES_NOT_EXIST") + with pytest.raises(Exception): + simple_mod.output_port("DOES_NOT_EXIST") + with pytest.raises(Exception): + simple_mod.input_port_type_id("DOES_NOT_EXIST") + with pytest.raises(Exception): + simple_mod.output_port_type_id("DOES_NOT_EXIST") + + builder.make_edge(source, simple_mod.input_port("input1")) + builder.make_edge(source2, simple_mod.input_port("input2")) + builder.make_edge(simple_mod.output_port("output1"), sink) + builder.make_edge(simple_mod.output_port("output2"), sink) + + pipeline = srf.Pipeline() + pipeline.make_segment("Initialization_Segment", init_wrapper) + + options = srf.Options() + options.topology.user_cpuset = "0-1" + + executor = srf.Executor(options) + executor.register_pipeline(pipeline) + executor.start() + executor.join() + + +def test_py_module_as_source(): + + def init_wrapper(builder: srf.Builder): + global packet_count + packet_count = 0 + + def on_next(input): + global packet_count + packet_count += 1 + logging.info("Sinking {}".format(input)) + + def on_error(): + pass + + def on_complete(): + pass + + config = {} + config["source_count"] = 42 + + source_mod = builder.load_module("SourceModule", "srf_unittest", "ModuleSourceTest_mod1", config) + sink = builder.make_sink("sink", on_next, on_error, on_complete) + builder.make_edge(source_mod.output_port("source"), sink) + + pipeline = srf.Pipeline() + pipeline.make_segment("ModuleAsSource_Segment", init_wrapper) + + options = srf.Options() + options.topology.user_cpuset = "0-1" + + executor = srf.Executor(options) + executor.register_pipeline(pipeline) + executor.start() + executor.join() + assert packet_count == 42 + + +def test_py_module_as_sink(): + + def gen_data(): + for i in range(0, 43): + yield True + global packet_count + packet_count += 1 + + def init_wrapper(builder: srf.Builder): + global packet_count + packet_count = 0 + + source = builder.make_source("source", gen_data()) + sink_mod = builder.load_module("SinkModule", "srf_unittest", "ModuleSinkTest_mod1", {}) + + builder.make_edge(source, sink_mod.input_port("sink")) + + pipeline = srf.Pipeline() + pipeline.make_segment("ModuleAsSink_Segment", init_wrapper) + + options = srf.Options() + options.topology.user_cpuset = "0-1" + + executor = srf.Executor(options) + executor.register_pipeline(pipeline) + executor.start() + executor.join() + + assert packet_count == 43 + + +def test_py_module_chaining(): + + def init_wrapper(builder: srf.Builder): + global packet_count + packet_count = 0 + + def on_next(input): + global packet_count + packet_count += 1 + logging.info("Sinking {}".format(input)) + + def on_error(): + pass + + def on_complete(): + pass + + config = {"source_count": 42} + + source_mod = builder.load_module("SourceModule", "srf_unittest", "ModuleChainingTest_mod1", config) + configurable_mod = builder.load_module("ConfigurableModule", "srf_unittest", "ModuleEndToEndTest_mod2", {}) + sink = builder.make_sink("sink", on_next, on_error, on_complete) + + builder.make_edge(source_mod.output_port("source"), configurable_mod.input_port("configurable_input_a")) + builder.make_edge(configurable_mod.output_port("configurable_output_x"), sink) + + pipeline = srf.Pipeline() + pipeline.make_segment("ModuleChaining_Segment", init_wrapper) + + options = srf.Options() + options.topology.user_cpuset = "0-1" + + executor = srf.Executor(options) + executor.register_pipeline(pipeline) + executor.start() + executor.join() + + assert packet_count == 42 + + +def test_py_module_nesting(): + + def gen_data(): + for i in range(0, 43): + yield True + global packet_count + packet_count += 1 + + def init_wrapper(builder: srf.Builder): + global packet_count + packet_count = 0 + + def on_next(input): + global packet_count + packet_count += 1 + logging.info("Sinking {}".format(input)) + + def on_error(): + pass + + def on_complete(): + pass + + nested_mod = builder.load_module("NestedModule", "srf_unittest", "ModuleNestingTest_mod1", {}) + nested_sink = builder.make_sink("nested_sink", on_next, on_error, on_complete) + + builder.make_edge(nested_mod.output_port("nested_module_output"), nested_sink) + + pipeline = srf.Pipeline() + pipeline.make_segment("ModuleNesting_Segment", init_wrapper) + + options = srf.Options() + options.topology.user_cpuset = "0-1" + + executor = srf.Executor(options) + executor.register_pipeline(pipeline) + executor.start() + executor.join() + + assert packet_count == 4 + + +if (__name__ in ("__main__", )): + test_py_end_to_end() + test_py_module_as_source() + test_py_module_as_sink() + test_py_module_chaining() + test_py_module_nesting() + test_py_constructor() + test_py_module_initialization() diff --git a/python/tests/test_stat_gather.py b/python/tests/test_stat_gather.py index 051e1f598..9a6d653d5 100644 --- a/python/tests/test_stat_gather.py +++ b/python/tests/test_stat_gather.py @@ -65,7 +65,7 @@ def double_float_type2(x): def init_double_segment(builder: srf.Builder): - ## CXX double source with heterogesrfus segment node composition + # CXX double source with heterogesrfus segment node composition python_source_double = builder.make_source("python_source_double", double_source) python_node_2x_1 = builder.make_node("python_node_2x_1", double_float_type1) builder.make_edge(python_source_double, python_node_2x_1) @@ -158,8 +158,6 @@ def test_stat_gather_full(): framework_stats_info = srf.benchmarking.get_tracing_stats() component_metrics = framework_stats_info["aggregations"]["components"]["metrics"] - import json - print(json.dumps(component_metrics, indent=2)) for key, _type in required_components: component = component_metrics[key] assert (len(component.keys()) > 0) @@ -241,7 +239,7 @@ def test_stat_gather_full_noreset_start_stop(): srf.benchmarking.reset_tracing_stats() -if (__name__ in ("__main__",)): +if (__name__ in ("__main__", )): test_stat_gather_operators() test_stat_gather_channels() test_stat_gather_full() diff --git a/src/internal/system/device_info.cpp b/src/internal/system/device_info.cpp index 3d33ff599..e8d57f365 100644 --- a/src/internal/system/device_info.cpp +++ b/src/internal/system/device_info.cpp @@ -32,25 +32,26 @@ #define TEST_BIT(_n, _p) (_n & (1UL << _p)) +#define SRF_CHECK_NVML(expression) \ + { \ + auto status = (expression); \ + if (status != NVML_SUCCESS) \ + { \ + LOG(FATAL) << "NVML failed: " << nvmlErrorString(status); \ + } \ + } + namespace { struct NvmlState { NvmlState() { - switch (nvmlInit_v2()) + auto nvml_status = nvmlInit_v2(); + if (nvml_status != NVML_SUCCESS) { - case NVML_SUCCESS: - break; - case NVML_ERROR_DRIVER_NOT_LOADED: - LOG(WARNING) - << "NVML: No NVIDIA GPU driver was detected; setting DeviceCount to 0, CUDA will not be initialized"; - return; - case NVML_ERROR_NO_PERMISSION: - LOG(WARNING) << "NVML: Access to the NVIDIA GPU driver failed; setting DeviceCount to 0, CUDA will not be " - "initialized"; + LOG(WARNING) << "NVML: Error initializing due to '" << nvmlErrorString(nvml_status) + << "'. Setting DeviceCount to 0, CUDA will not be initialized"; return; - default: - LOG(FATAL) << "NVML_ERROR_UNKNOWN: is a hard fail"; } unsigned int visible_devices = 0; @@ -60,6 +61,9 @@ struct NvmlState for (decltype(visible_devices) i = 0; i < visible_devices; i++) { nvmlDevice_t device_handle; + unsigned int current_mig_mode; + unsigned int pending_mig_mode; + auto device_status = nvmlDeviceGetHandleByIndex_v2(i, &device_handle); if (device_status != NVML_SUCCESS) { @@ -67,6 +71,33 @@ struct NvmlState << " will be ignored"; continue; } + + auto mig_status = nvmlDeviceGetMigMode(device_handle, ¤t_mig_mode, &pending_mig_mode); + if (mig_status == NVML_SUCCESS && + (current_mig_mode == NVML_DEVICE_MIG_ENABLE || pending_mig_mode == NVML_DEVICE_MIG_ENABLE)) + { + // let's treat pending as current - MIG mode cannot swap while a cuda process is active, but we may not + // have initailized CUDA yet, so to avoid any race conditions, we'll error on the side of caution + if (visible_devices == 1) + { + // if have 1 visible device and it's in MIG mode, then we expect to see only one visible mig + // instance + m_using_mig = true; + + // if(number_of_visible_mig_instances == 1) { + // if all conditions for running on MIG are met, then we early return from this scope + // return; + // } + + LOG(FATAL) << "SRF Issue #205: mig instance queries and enumeration is current not supported"; + } + else + { + LOG(WARNING) << "NVML visible device #" << i + << " has MIG mode enabled with multiple GPUs visible; this device will be ignored"; + continue; + } + } m_accessible_indexes.insert(i); } } @@ -80,6 +111,11 @@ struct NvmlState return m_accessible_indexes; } + bool using_mig() const + { + return m_using_mig; + } + private: // this object can also hold the list of device handles that we have access to. // - nvmlDeviceGetCount_v2 - will tell us the total number of devices we have access to, i.e. the range of [0, N) @@ -87,6 +123,8 @@ struct NvmlState // which we have permission to access, the call to nvmlDeviceGetHandleByIndex_v2 may return // NVML_ERROR_NO_PERMISSION. std::set m_accessible_indexes; + + bool m_using_mig{false}; }; auto nvmlInstatnce = std::make_unique(); @@ -175,7 +213,7 @@ std::set DeviceInfo::AccessibleDeviceIndexes() nvmlMemory_t DeviceInfo::MemoryInfo(int device_id) { nvmlMemory_t info; - CHECK_EQ(nvmlDeviceGetMemoryInfo(DeviceInfo::GetHandleById(device_id), &info), NVML_SUCCESS); + SRF_CHECK_NVML(nvmlDeviceGetMemoryInfo(DeviceInfo::GetHandleById(device_id), &info)); return info; } diff --git a/src/public/modules/module_registry.cpp b/src/public/modules/module_registry.cpp new file mode 100644 index 000000000..ada93be32 --- /dev/null +++ b/src/public/modules/module_registry.cpp @@ -0,0 +1,205 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "srf/modules/module_registry.hpp" + +#include "srf/version.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { +std::string version_to_string(const srf::modules::ModuleRegistry::registry_version_t& release_version) +{ + if (release_version.empty()) + { + return {""}; + } + + std::stringstream sstream; + sstream << release_version[0]; + std::for_each(release_version.begin() + 1, release_version.end(), [&sstream](unsigned int element) { + sstream << "." << element; + }); + + return sstream.str(); +} +} // namespace + +namespace srf::modules { + +const unsigned int ModuleRegistry::VersionElements{3}; + +const ModuleRegistry::registry_version_t ModuleRegistry::Version{ + srf_VERSION_MAJOR, srf_VERSION_MINOR, srf_VERSION_PATCH}; + +ModuleRegistry::module_namespace_map_t ModuleRegistry::s_module_namespace_registry{ + {"default", ModuleRegistry::module_registry_map_t{}}}; + +ModuleRegistry::module_name_map_t ModuleRegistry::s_module_name_map{{"default", std::vector{}}}; + +std::recursive_mutex ModuleRegistry::s_mutex{}; + +bool ModuleRegistry::contains(const std::string& name, const std::string& registry_namespace) +{ + std::lock_guard lock(s_mutex); + + if (!contains_namespace(registry_namespace)) + { + return false; + } + + auto& module_registry = s_module_namespace_registry[registry_namespace]; + auto iter_reg = module_registry.find(name); + + return iter_reg != module_registry.end(); +} + +bool ModuleRegistry::contains_namespace(const std::string& registry_namespace) +{ + std::lock_guard lock(s_mutex); + + return s_module_namespace_registry.find(registry_namespace) != s_module_namespace_registry.end(); +} + +ModuleRegistry::module_constructor_t ModuleRegistry::get_module_constructor(const std::string& name, + const std::string& registry_namespace) +{ + std::lock_guard lock(s_mutex); + + if (contains(name, registry_namespace)) + { + return s_module_namespace_registry[registry_namespace][name]; + } + + std::stringstream sstream; + + sstream << "Module does not exist -> " << registry_namespace << "::" << name; + LOG(ERROR) << sstream.str(); + throw std::invalid_argument(sstream.str()); +} + +const ModuleRegistry::module_name_map_t& ModuleRegistry::registered_modules() +{ + return ModuleRegistry::s_module_name_map; +} + +void ModuleRegistry::register_module(std::string name, + const registry_version_t& release_version, + srf::modules::ModuleRegistry::module_constructor_t fn_constructor) +{ + register_module(std::move(name), "default", release_version, fn_constructor); +} + +void ModuleRegistry::register_module(std::string name, + std::string registry_namespace, + const registry_version_t& release_version, + srf::modules::ModuleRegistry::module_constructor_t fn_constructor) +{ + std::lock_guard lock(s_mutex); + VLOG(2) << "Registering module: " << registry_namespace << "::" << name; + if (!is_version_compatible(release_version)) + { + std::stringstream sstream; + sstream << "Failed to register module -> module version is: '" << ::version_to_string(release_version) + << "' and registry requires: '" << ::version_to_string(Version); + + throw std::runtime_error(sstream.str()); + } + + if (!contains_namespace(registry_namespace)) + { + s_module_namespace_registry[registry_namespace] = ModuleRegistry::module_registry_map_t{}; + s_module_name_map[registry_namespace] = std::vector(); + + VLOG(2) << "Creating namespace because it does not exist: " << registry_namespace; + } + + if (!contains(name, registry_namespace)) + { + auto& module_registry = s_module_namespace_registry[registry_namespace]; + module_registry[name] = fn_constructor; + + auto& module_name_map = s_module_name_map[registry_namespace]; + module_name_map.push_back(name); + + std::sort(module_name_map.begin(), module_name_map.end()); + + VLOG(2) << "Registered module: " << registry_namespace << "::" << name << std::endl; + return; + } + + std::stringstream sstream; + + sstream << "Attempt to register duplicate module -> " << registry_namespace << ":" << name; + VLOG(2) << sstream.str(); + throw std::invalid_argument(sstream.str()); +} + +void ModuleRegistry::unregister_module(const std::string& name, const std::string& registry_namespace, bool optional) +{ + std::lock_guard lock(s_mutex); + + VLOG(2) << "Unregistering module " << registry_namespace << "::" << name; + + if (contains(name, registry_namespace)) + { + s_module_namespace_registry[registry_namespace].erase(name); + + auto& name_map = s_module_name_map[registry_namespace]; + auto iter_erase = std::find(name_map.begin(), name_map.end(), name); + + name_map.erase(iter_erase); + + if (s_module_namespace_registry[registry_namespace].empty()) + { + VLOG(2) << "Namespace " << registry_namespace << " is empty, removing."; + s_module_namespace_registry.erase(registry_namespace); + s_module_name_map.erase(registry_namespace); + } + + return; + } + + if (optional) + { + return; + } + + std::stringstream sstream; + + sstream << "Failed to unregister module -> " << registry_namespace << "::" << name << " does not exist."; + VLOG(5) << sstream.str(); + throw std::invalid_argument(sstream.str()); +} + +bool ModuleRegistry::is_version_compatible(const registry_version_t& release_version) +{ + // TODO(devin) improve criteria for module compatibility + return std::equal(ModuleRegistry::Version.begin(), + ModuleRegistry::Version.begin() + ModuleRegistry::VersionElements, + release_version.begin()); +} + +} // namespace srf::modules diff --git a/src/public/modules/plugins.cpp b/src/public/modules/plugins.cpp new file mode 100644 index 000000000..956ebced9 --- /dev/null +++ b/src/public/modules/plugins.cpp @@ -0,0 +1,313 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "srf/modules/plugins.hpp" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace fs = boost::filesystem; + +namespace srf::modules { + +std::map> PluginModule::s_plugin_map{}; +std::recursive_mutex PluginModule::s_mutex{}; + +const std::string PluginModule::PluginEntrypointList{"SRF_MODULE_entrypoint_list"}; +const std::string PluginModule::PluginEntrypointLoad{"SRF_MODULE_entrypoint_load"}; +const std::string PluginModule::PluginEntrypointUnload{"SRF_MODULE_entrypoint_unload"}; + +std::shared_ptr PluginModule::create_or_acquire(const std::string& plugin_library_name) +{ + std::lock_guard lock(s_mutex); + auto iter_lock = s_plugin_map.find(plugin_library_name); + if (iter_lock != s_plugin_map.end()) + { + return iter_lock->second; + } + + auto plugin_ptr = std::shared_ptr(new PluginModule(plugin_library_name)); + s_plugin_map[plugin_library_name] = plugin_ptr; + + return plugin_ptr; +} + +PluginModule::PluginModule(std::string plugin_library_name) : m_plugin_library_name(std::move(plugin_library_name)) {} + +void PluginModule::set_library_directory(std::string directory_path) +{ + fs::path lib_dir(directory_path); + + if (!fs::is_directory(lib_dir)) + { + std::stringstream sstream; + + sstream << "Failed to set library directory -> '" << directory_path << "' is not a directory"; + throw std::invalid_argument(sstream.str()); + } + + m_plugin_library_dir = std::move(directory_path); +} + +void PluginModule::reset_library_directory() +{ + m_plugin_library_dir = ""; +} + +void PluginModule::get_entrypoint(const std::string& entrypoint_name, void** entrypoint) +{ + *entrypoint = nullptr; + + dlerror(); + void* _fn = dlsym(m_plugin_handle, entrypoint_name.c_str()); + + const char* dlsym_error = dlerror(); + if (dlsym_error != nullptr) + { + std::stringstream sstream; + + sstream << "Failed to find entrypoint -> '" << entrypoint_name << "' in '" << m_plugin_library_name << " : " + << dlsym_error; + + VLOG(2) << sstream.str(); + throw std::invalid_argument(sstream.str()); + } + + *entrypoint = _fn; +} + +std::vector PluginModule::list_modules() +{ + const char** module_list; + unsigned int module_count = m_plugin_list(&module_list); + + std::vector ret{}; + for (int i = 0; i < module_count; i++) + { + ret.emplace_back(module_list[i]); + } + + return ret; +} + +bool PluginModule::load(bool throw_on_error) +{ + std::lock_guard lock(s_mutex); + + if (m_loaded) + { + return true; + } + + if (!try_open_library_handle(throw_on_error)) + { + return false; + } + + if (!try_build_plugin_interface(throw_on_error)) + { + return false; + } + + return try_load_plugin(throw_on_error); +} + +bool PluginModule::unload(bool throw_on_error) +{ + std::lock_guard lock(s_mutex); + + if (!m_loaded) + { + return true; + } + + // TODO(Devin): If we want to close the library handle here, then we need to implement a tracking mechanism to be + // sure that there are no existing SegmentModules which have been created from a Plugin library. + + return try_unload_plugin(throw_on_error); +} + +void PluginModule::reload() +{ + unload(); + try_close_library_handle(); + load(); +} + +bool PluginModule::try_open_library_handle(bool throw_on_error) +{ + if (m_plugin_handle != nullptr) + { + return true; + } + + std::string library_path = + m_plugin_library_dir.empty() ? m_plugin_library_name : m_plugin_library_dir + "/" + m_plugin_library_name; + + m_plugin_handle = dlopen(library_path.c_str(), RTLD_NOW | RTLD_LOCAL); + if (m_plugin_handle == nullptr) + { + std::stringstream sstream; + + sstream << "Failed to open plugin module -> " << dlerror(); + if (throw_on_error) + { + LOG(ERROR) << sstream.str(); + throw std::runtime_error(sstream.str()); + } + + LOG(WARNING) << sstream.str(); + return false; + } + + return true; +} + +bool PluginModule::try_close_library_handle(bool throw_on_error) +{ + if (m_plugin_handle == nullptr) + { + return true; + } + + if (dlclose(m_plugin_handle) != 0) + { + std::stringstream sstream; + + sstream << "Failed to close plugin module -> " << dlerror(); + if (throw_on_error) + { + LOG(ERROR) << sstream.str(); + throw std::runtime_error(sstream.str()); + } + + LOG(WARNING) << sstream.str(); + return false; + } + + m_plugin_handle = nullptr; + + return true; +} + +bool PluginModule::try_load_plugin(bool throw_on_error) +{ + try + { + m_plugin_load(); + } catch (std::exception& error) + { + if (throw_on_error) + { + LOG(ERROR) << "Plugin entrypoint load failed: " << error.what(); + throw; + } + + LOG(WARNING) << "Plugin entrypoint load failed: " << error.what(); + return false; + } catch (...) + { + if (throw_on_error) + { + LOG(ERROR) << "Plugin entrypoint load failed: [Unknown Exception]"; + throw; + } + + LOG(WARNING) << "Plugin entrypoint load failed: [Unknown Exception]"; + return false; + } + + m_loaded = true; + + return true; +} + +bool PluginModule::try_unload_plugin(bool throw_on_error) +{ + try + { + m_plugin_unload(); + clear_plugin_interface(); + + m_loaded = false; + } catch (std::exception& error) + { + if (throw_on_error) + { + LOG(ERROR) << "Plugin entrypoint unload failed: " << error.what(); + throw; + } + + LOG(WARNING) << "Plugin entrypoint unload failed: " << error.what(); + return false; + } catch (...) + { + if (throw_on_error) + { + LOG(ERROR) << "Plugin entrypoint unload failed: [Unknown Exception]"; + throw; + } + + LOG(WARNING) << "Plugin entrypoint unload failed: [Unknown Exception]"; + return false; + } + + return true; +} + +bool PluginModule::try_build_plugin_interface(bool throw_on_error) +{ + try + { + get_entrypoint(PluginEntrypointList, reinterpret_cast(&m_plugin_list)); + get_entrypoint(PluginEntrypointLoad, reinterpret_cast(&m_plugin_load)); + get_entrypoint(PluginEntrypointUnload, reinterpret_cast(&m_plugin_unload)); + } catch (std::invalid_argument& error) + { + clear_plugin_interface(); + if (throw_on_error) + { + LOG(ERROR) << "Failed to load SRF plugin-> " << error.what(); + throw error; + } + + LOG(WARNING) << "Failed to load SRF plugin-> " << error.what(); + return false; + } + + return true; +} + +void PluginModule::clear_plugin_interface() +{ + m_plugin_list = nullptr; + m_plugin_load = nullptr; + m_plugin_unload = nullptr; +} + +} // namespace srf::modules diff --git a/src/public/modules/sample_modules.cpp b/src/public/modules/sample_modules.cpp new file mode 100644 index 000000000..36e1e2438 --- /dev/null +++ b/src/public/modules/sample_modules.cpp @@ -0,0 +1,231 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "srf/modules/sample_modules.hpp" + +#include "rxcpp/operators/rx-map.hpp" +#include "rxcpp/sources/rx-iterate.hpp" + +#include "srf/channel/status.hpp" +#include "srf/core/utils.hpp" +#include "srf/modules/segment_modules.hpp" +#include "srf/node/rx_node.hpp" +#include "srf/node/rx_sink.hpp" +#include "srf/node/rx_source.hpp" +#include "srf/segment/object.hpp" + +#include +#include + +#include +#include +#include +#include + +namespace srf::modules { +SimpleModule::SimpleModule(std::string module_name) : SegmentModule(std::move(module_name)) {} + +SimpleModule::SimpleModule(std::string module_name, nlohmann::json config) : + SegmentModule(std::move(module_name), std::move(config)) +{} + +void SimpleModule::initialize(segment::Builder& builder) +{ + if (config().contains("simple_key_1")) + { + m_was_configured = true; + } + + /** First linear path **/ + auto input1 = builder.make_node("input1", rxcpp::operators::map([](bool input) { + unsigned int output = 42; + return output; + })); + + auto internal1 = + builder.make_node("_internal1_", rxcpp::operators::map([](unsigned int input) { + auto output = std::to_string(input); + VLOG(10) << "Created output1 << " << output << std::endl; + return output; + })); + + builder.make_edge(input1, internal1); + + auto output1 = builder.make_node( + "output1", rxcpp::operators::map([](std::string input) { return input; })); + + builder.make_edge(internal1, output1); + + /** Second linear path **/ + auto input2 = builder.make_node("input2", rxcpp::operators::map([](bool input) { + unsigned int output = 42; + return output; + })); + + auto internal2 = + builder.make_node("_internal2_", rxcpp::operators::map([](unsigned int input) { + auto output = std::to_string(input); + VLOG(10) << "Created output2: " << output << std::endl; + return output; + })); + + builder.make_edge(input2, internal2); + + auto output2 = builder.make_node( + "output2", rxcpp::operators::map([](std::string input) { return input; })); + + builder.make_edge(internal2, output2); + + register_input_port("input1", input1); + register_output_port("output1", output1); + + register_input_port("input2", input2); + register_output_port("output2", output2); + + m_initialized = true; +} + +std::string SimpleModule::module_type_name() const +{ + return std::string(::srf::type_name()); +} + +ConfigurableModule::ConfigurableModule(std::string module_name) : SegmentModule(std::move(module_name)) {} +ConfigurableModule::ConfigurableModule(std::string module_name, nlohmann::json config) : + SegmentModule(std::move(module_name), std::move(config)) +{} + +void ConfigurableModule::initialize(segment::Builder& builder) +{ + if (config().contains("config_key_1")) + { + m_was_configured = true; + } + + auto input1 = builder.make_node("configurable_input_a", rxcpp::operators::map([](bool input) { + unsigned int output = 42; + return output; + })); + + auto internal1 = + builder.make_node("_internal1_", rxcpp::operators::map([](unsigned int input) { + auto output = std::to_string(input); + VLOG(10) << "Created output1: " << output << std::endl; + return output; + })); + + builder.make_edge(input1, internal1); + + auto output1 = builder.make_node( + "configurable_output_x", rxcpp::operators::map([](std::string input) { return input; })); + + builder.make_edge(internal1, output1); + + register_input_port("configurable_input_a", input1); + register_output_port("configurable_output_x", output1); + + m_initialized = true; +} + +std::string ConfigurableModule::module_type_name() const +{ + return std::string(::srf::type_name()); +} + +SourceModule::SourceModule(std::string module_name) : SegmentModule(std::move(module_name)) {} +SourceModule::SourceModule(std::string module_name, nlohmann::json config) : + SegmentModule(std::move(module_name), std::move(config)) +{} + +void SourceModule::initialize(segment::Builder& builder) +{ + unsigned int count{1}; + + if (config().contains("source_count")) + { + count = config()["source_count"]; + } + + auto source = builder.make_source("source", [count](rxcpp::subscriber& sub) { + if (sub.is_subscribed()) + { + for (unsigned int i = 0; i < count; ++i) + { + sub.on_next(true); + } + } + + sub.on_completed(); + }); + + // Register the submodules output as one of this module's outputs + register_output_port("source", source); +} + +std::string SourceModule::module_type_name() const +{ + return std::string(::srf::type_name()); +} + +SinkModule::SinkModule(std::string module_name) : SegmentModule(std::move(module_name)) {} +SinkModule::SinkModule(std::string module_name, nlohmann::json config) : + SegmentModule(std::move(module_name), std::move(config)) +{} + +void SinkModule::initialize(segment::Builder& builder) +{ + auto sink = builder.make_sink("sink", [this](bool input) { + m_packet_count++; + VLOG(10) << "Sinking " << input << std::endl; + }); + + // Register the submodules output as one of this module's outputs + register_input_port("sink", sink); +} + +std::string SinkModule::module_type_name() const +{ + return std::string(::srf::type_name()); +} + +NestedModule::NestedModule(std::string module_name) : SegmentModule(std::move(module_name)) {} +NestedModule::NestedModule(std::string module_name, nlohmann::json config) : + SegmentModule(std::move(module_name), std::move(config)) +{} + +void NestedModule::initialize(segment::Builder& builder) +{ + auto configurable_mod = builder.make_module("NestedModule_submod2"); + + auto config = nlohmann::json(); + config["source_count"] = 4; + + // Create a data source and attach it to our submodule + auto source1 = builder.make_module("source", config); + + builder.make_dynamic_edge(source1->output_port("source"), + configurable_mod->input_port("configurable_input_a")); + + // Register the submodules output as one of this module's outputs + register_output_port("nested_module_output", configurable_mod->output_port("configurable_output_x")); +} + +std::string NestedModule::module_type_name() const +{ + return std::string(::srf::type_name()); +} +} // namespace srf::modules \ No newline at end of file diff --git a/src/public/modules/segment_modules.cpp b/src/public/modules/segment_modules.cpp new file mode 100644 index 000000000..fc16b8427 --- /dev/null +++ b/src/public/modules/segment_modules.cpp @@ -0,0 +1,182 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "srf/modules/segment_modules.hpp" + +#include "srf/node/sink_properties.hpp" +#include "srf/node/source_properties.hpp" + +#include +#include +#include +#include + +namespace srf::modules { + +SegmentModule::SegmentModule(std::string module_name) : m_module_instance_name(std::move(module_name)) +{ + if (m_module_instance_name.find_first_of("/") != std::string::npos) + { + throw std::invalid_argument("Module name cannot contain '/' characters"); + } +} + +SegmentModule::SegmentModule(std::string module_name, nlohmann::json config) : + m_module_instance_name(std::move(module_name)), + m_config(std::move(config)) +{ + if (m_module_instance_name.find_first_of("/") != std::string::npos) + { + throw std::invalid_argument("Module name cannot contain '/' characters"); + } +} + +std::string SegmentModule::component_prefix() const +{ + return module_type_name() + "::" + name(); +} + +const nlohmann::json& SegmentModule::config() const +{ + return m_config; +} + +const std::vector& SegmentModule::input_ids() const +{ + return m_input_port_ids; +} + +const SegmentModule::segment_module_port_map_t& SegmentModule::input_ports() const +{ + return m_input_ports; +} + +SegmentModule::segment_module_port_t SegmentModule::input_port(const std::string& input_name) +{ + if (m_input_ports.find(input_name) != m_input_ports.end()) + { + return m_input_ports[input_name]; + } + + std::stringstream sstream; + + sstream << "Invalid input port: " << input_name; + throw std::invalid_argument(sstream.str()); +} + +const SegmentModule::segment_module_typeindex_map_t& SegmentModule::input_port_type_ids() const +{ + return m_input_port_type_indices; +} + +std::type_index SegmentModule::input_port_type_id(const std::string& input_name) +{ + auto ipt_iter = m_input_port_type_indices.find(input_name); + if (ipt_iter != m_input_port_type_indices.end()) + { + return ipt_iter->second; + } + + std::stringstream sstream; + + sstream << "Invalid input port: " << input_name; + throw std::invalid_argument(sstream.str()); +} + +const SegmentModule::segment_module_port_map_t& SegmentModule::output_ports() const +{ + return m_output_ports; +} + +SegmentModule::segment_module_port_t SegmentModule::output_port(const std::string& output_name) +{ + if (m_output_ports.find(output_name) != m_output_ports.end()) + { + return m_output_ports[output_name]; + } + + std::stringstream sstream; + + sstream << "Invalid output port: " << output_name; + throw std::invalid_argument(sstream.str()); +} + +const SegmentModule::segment_module_typeindex_map_t& SegmentModule::output_port_type_ids() const +{ + return m_output_port_type_indices; +} + +std::type_index SegmentModule::output_port_type_id(const std::string& output_name) +{ + auto opt_iter = m_output_port_type_indices.find(output_name); + if (opt_iter != m_output_port_type_indices.end()) + { + return opt_iter->second; + } + + std::stringstream sstream; + + sstream << "Invalid output port: " << output_name; + throw std::invalid_argument(sstream.str()); +} + +const std::vector& SegmentModule::output_ids() const +{ + return m_output_port_ids; +} + +const std::string& SegmentModule::name() const +{ + return m_module_instance_name; +} + +void SegmentModule::operator()(segment::Builder& builder) +{ + this->initialize(builder); +} + +void SegmentModule::register_input_port(std::string input_name, std::shared_ptr object) +{ + if (m_input_ports.find(input_name) != m_input_ports.end()) + { + std::stringstream sstream; + + sstream << "Attempt to register duplicate module input port: " + std::move(input_name); + throw std::invalid_argument(sstream.str()); + } + + m_input_port_ids.push_back(input_name); + m_input_ports[input_name] = object; + m_input_port_type_indices.try_emplace(input_name, object->sink_base().sink_type()); +} + +void SegmentModule::register_output_port(std::string output_name, std::shared_ptr object) +{ + if (m_output_ports.find(output_name) != m_output_ports.end()) + { + std::stringstream sstream; + + sstream << "Attempt to register duplicate module output port: " + std::move(output_name); + throw std::invalid_argument(sstream.str()); + } + + m_output_port_ids.push_back(output_name); + m_output_ports[output_name] = object; + m_output_port_type_indices.try_emplace(output_name, object->source_base().source_type()); +} + +} // namespace srf::modules diff --git a/src/public/node/edge_builder.cpp b/src/public/node/edge_builder.cpp index e255c801a..1cea17366 100644 --- a/src/public/node/edge_builder.cpp +++ b/src/public/node/edge_builder.cpp @@ -27,6 +27,7 @@ #include #include +#include #include #include @@ -43,6 +44,7 @@ std::shared_ptr EdgeBuilder::ingress_adapter_for_sink( { VLOG(2) << "Looking for edge adapter: (" << source.source_type_name() << ", " << sink.sink_type_name() << ")"; VLOG(2) << "- (" << source.source_type_hash() << ", " << sink.sink_type_hash() << ")"; + if (EdgeAdapterRegistry::has_source_adapter(source.source_type())) { auto adapter = EdgeAdapterRegistry::find_source_adapter(source.source_type()); @@ -56,8 +58,20 @@ std::shared_ptr EdgeBuilder::ingress_adapter_for_sink( } // Fallback -- probably fail - auto fn_converter = srf::node::EdgeRegistry::find_converter(source.source_type(), sink.sink_type()); - return fn_converter(ingress_handle); + try + { + auto fn_converter = srf::node::EdgeRegistry::find_converter(source.source_type(), sink.sink_type()); + return fn_converter(ingress_handle); + } catch (std::runtime_error e) + { + // Last attempt, check if types are the same and return ingress handle. + if (source.source_type() == sink.sink_type()) + { + return ingress_handle; + } + + throw e; + } } std::shared_ptr EdgeBuilder::ingress_for_source_type( diff --git a/src/public/options/fiber_pool.cpp b/src/public/options/fiber_pool.cpp index ed0897de6..a05846297 100644 --- a/src/public/options/fiber_pool.cpp +++ b/src/public/options/fiber_pool.cpp @@ -40,7 +40,7 @@ bool FiberPoolOptions::enable_memory_binding() const } bool FiberPoolOptions::enable_thread_binding() const { - return m_enable_memory_binding; + return m_enable_thread_binding; } bool FiberPoolOptions::enable_tracing_scheduler() const { diff --git a/src/public/segment/builder.cpp b/src/public/segment/builder.cpp index 1ddb9ed1c..5e8e3ac9f 100644 --- a/src/public/segment/builder.cpp +++ b/src/public/segment/builder.cpp @@ -17,8 +17,29 @@ #include "srf/segment/builder.hpp" +#include "srf/modules/module_registry.hpp" #include "srf/node/port_registry.hpp" +#include + +#include +#include +#include + +namespace { + +std::string accum_merge(std::string lhs, std::string rhs) +{ + if (lhs.empty()) + { + return std::move(rhs); + } + + return std::move(lhs) + "/" + std::move(rhs); +} + +} // namespace + namespace srf::segment { std::shared_ptr Builder::get_ingress(std::string name, std::type_index type_index) { @@ -56,4 +77,78 @@ std::shared_ptr Builder::get_egress(std::string name, std::typ return port; } + +void Builder::init_module(sp_segment_module_t module) +{ + ns_push(module); + VLOG(2) << "Initializing module: " << m_namespace_prefix; + module->m_module_instance_registered_namespace = m_namespace_prefix; + module->initialize(*this); + ns_pop(); +} + +std::shared_ptr Builder::load_module_from_registry(const std::string& module_id, + const std::string& registry_namespace, + std::string module_name, + nlohmann::json config) +{ + auto fn_module_constructor = srf::modules::ModuleRegistry::get_module_constructor(module_id, registry_namespace); + auto module = fn_module_constructor(std::move(module_name), std::move(config)); + + init_module(module); + + return module; +} + +/** private implementations **/ + +void Builder::ns_push(sp_segment_module_t module) +{ + m_module_stack.push_back(module); + m_namespace_stack.push_back(module->component_prefix()); + m_namespace_prefix = + std::accumulate(m_namespace_stack.begin(), m_namespace_stack.end(), std::string(""), ::accum_merge); +} + +void Builder::ns_pop() +{ + m_module_stack.pop_back(); + m_namespace_stack.pop_back(); + m_namespace_prefix = + std::accumulate(m_namespace_stack.begin(), m_namespace_stack.end(), std::string(""), ::accum_merge); +} + +void Builder::register_module_input(std::string input_name, sp_obj_prop_t object) +{ + if (m_module_stack.empty()) + { + std::stringstream sstream; + + sstream << "Failed to register module input '" << input_name << "' -> no module context exists"; + VLOG(2) << sstream.str(); + + throw std::invalid_argument(sstream.str()); + } + + auto current_module = m_module_stack.back(); + current_module->register_input_port(std::move(input_name), object); +} + +void Builder::register_module_output(std::string output_name, sp_obj_prop_t object) +{ + if (m_module_stack.empty()) + { + std::stringstream sstream; + + sstream << "Failed to register module output'" << output_name << "' -> no module context exists"; + VLOG(2) << sstream.str(); + + throw std::invalid_argument(sstream.str()); + } + + auto current_module = m_module_stack.back(); + + current_module->register_output_port(std::move(output_name), object); +} + } // namespace srf::segment diff --git a/src/tests/test_ucx.cpp b/src/tests/test_ucx.cpp index 21642953b..f289b1a4e 100644 --- a/src/tests/test_ucx.cpp +++ b/src/tests/test_ucx.cpp @@ -204,6 +204,11 @@ TEST_F(TestUCX, Get) worker_get_src->progress(); } future.get(); + + // unregister memory + ucp_rkey_buffer_release(src_rbuff); + context->unregister_memory(src_lkey); + context->unregister_memory(dst_lkey); } // Recv diff --git a/srf.code-workspace b/srf.code-workspace index ff3e4526e..df7ea65d6 100644 --- a/srf.code-workspace +++ b/srf.code-workspace @@ -272,7 +272,9 @@ "--rcfile=${workspaceFolder}/python/.pylintrc" ], "python.linting.pylintEnabled": true, - "python.sortImports.args": ["--settings-file=${workspaceFolder}/python/setup.cfg"], + "python.sortImports.args": [ + "--settings-file=${workspaceFolder}/python/setup.cfg" + ], "python.testing.cwd": "${workspaceFolder}/python", "python.testing.pytestArgs": [ "-s" @@ -335,6 +337,7 @@ "matepek.vscode-catch2-test-adapter", "ms-vscode.cmake-tools", "stkb.rewrap", + "formulahendry.terminal", ], // List of extensions recommended by VS Code that should not be recommended for users of this workspace. "unwantedRecommendations": [ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index df220c4d0..d40eae15a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -15,6 +15,9 @@ # Keep all source files sorted!!! add_executable(test_srf + modules/test_module_registry.cpp + modules/test_module_util.cpp + modules/test_segment_modules.cpp test_channel.cpp test_codable.cpp test_executor.cpp @@ -40,3 +43,4 @@ add_test( add_subdirectory(benchmarking) add_subdirectory(logging) +add_subdirectory(modules) diff --git a/tests/modules/CMakeLists.txt b/tests/modules/CMakeLists.txt new file mode 100644 index 000000000..81deb2c59 --- /dev/null +++ b/tests/modules/CMakeLists.txt @@ -0,0 +1,34 @@ +# ============================================================================= +# SPDX-FileCopyrightText: Copyright (c) 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +list(APPEND CMAKE_MESSAGE_CONTEXT "tests") + +add_library(dynamic_test_module MODULE + dynamic_module.cpp +) + +include_directories(dynamic_test_module + PUBLIC + ${CMAKE_SOURCE_DIR}/include +) + +target_link_libraries(dynamic_test_module + PUBLIC + rxcpp::rxcpp + PRIVATE + libsrf +) + +list(POP_BACK CMAKE_MESSAGE_CONTEXT) diff --git a/tests/modules/dynamic_module.cpp b/tests/modules/dynamic_module.cpp new file mode 100644 index 000000000..b68cc37fb --- /dev/null +++ b/tests/modules/dynamic_module.cpp @@ -0,0 +1,168 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "srf/core/utils.hpp" +#include "srf/modules/module_registry_util.hpp" +#include "srf/modules/segment_modules.hpp" +#include "srf/segment/builder.hpp" +#include "srf/version.hpp" + +#include +#include + +namespace srf::modules { +class DynamicSourceModule : public SegmentModule +{ + using type_t = DynamicSourceModule; + + public: + DynamicSourceModule(std::string module_name); + DynamicSourceModule(std::string module_name, nlohmann::json config); + + protected: + void initialize(segment::Builder& builder) override; + std::string module_type_name() const override; +}; + +DynamicSourceModule::DynamicSourceModule(std::string module_name) : SegmentModule(std::move(module_name)) {} +DynamicSourceModule::DynamicSourceModule(std::string module_name, nlohmann::json config) : + SegmentModule(std::move(module_name), std::move(config)) +{} + +void DynamicSourceModule::initialize(segment::Builder& builder) +{ + unsigned int count{1}; + + if (config().contains("source_count")) + { + count = config()["source_count"]; + } + + auto source = builder.make_source("source", [count](rxcpp::subscriber& sub) { + if (sub.is_subscribed()) + { + for (unsigned int i = 0; i < count; ++i) + { + sub.on_next(true); + } + } + + sub.on_completed(); + }); + + // Register the submodules output as one of this module's outputs + register_output_port("source", source); +} + +std::string DynamicSourceModule::module_type_name() const +{ + return std::string(::srf::type_name()); +} + +} // namespace srf::modules + +extern "C" { + +const std::vector DynamicTestModuleVersion{srf_VERSION_MAJOR, srf_VERSION_MINOR, srf_VERSION_PATCH}; + +const char* MODULES[] = {"srf_unittest_cpp_dynamic::DynamicSourceModule", + "srf_unittest_cpp_dynamic_2::DynamicSourceModule", + "srf_unittest_cpp_dynamic_3::DynamicSourceModule"}; + +[[maybe_unused]] bool SRF_MODULE_dummy_entrypoint() // NOLINT +{ + return true; +} + +[[maybe_unused]] bool SRF_MODULE_entrypoint_load() // NOLINT +{ + using namespace srf::modules; + + try + { + ModuleRegistry::register_module("DynamicSourceModule", + "srf_unittest_cpp_dynamic", + DynamicTestModuleVersion, + [](std::string module_name, nlohmann::json config) { + return std::make_shared( + std::move(module_name), std::move(config)); + }); + + ModuleRegistry::register_module("DynamicSourceModule", + "srf_unittest_cpp_dynamic_2", + DynamicTestModuleVersion, + [](std::string module_name, nlohmann::json config) { + return std::make_shared( + std::move(module_name), std::move(config)); + }); + + ModuleRegistry::register_module("DynamicSourceModule", + "srf_unittest_cpp_dynamic_3", + DynamicTestModuleVersion, + [](std::string module_name, nlohmann::json config) { + return std::make_shared( + std::move(module_name), std::move(config)); + }); + } catch (...) + { + return false; + } + + return true; +} + +[[maybe_unused]] bool SRF_MODULE_entrypoint_unload() // NOLINT +{ + using namespace srf::modules; + + try + { + ModuleRegistry::unregister_module("DynamicSourceModule", "srf_unittest_cpp_dynamic"); + ModuleRegistry::unregister_module("DynamicSourceModule", "srf_unittest_cpp_dynamic_2"); + ModuleRegistry::unregister_module("DynamicSourceModule", "srf_unittest_cpp_dynamic_3"); + } catch (...) + { + return false; + } + + return true; +} + +[[maybe_unused]] unsigned int SRF_MODULE_entrypoint_list(const char** result) // NOLINT +{ + *result = (const char*)(&MODULES); + + return 3; +} + +[[maybe_unused]] bool SRF_MODULE_bad_version_entrypoint() // NOLINT +{ + using namespace srf::modules; + + auto BadVersion = std::vector{13, 14, 15}; + + ModuleRegistry::register_module("DynamicSourceModule_BAD", + "srf_unittest_cpp_dynamic_BAD", + BadVersion, + [](std::string module_name, nlohmann::json config) { + return std::make_shared( + std::move(module_name), std::move(config)); + }); + + return true; +} +} diff --git a/tests/modules/test_module_registry.cpp b/tests/modules/test_module_registry.cpp new file mode 100644 index 000000000..32444f296 --- /dev/null +++ b/tests/modules/test_module_registry.cpp @@ -0,0 +1,371 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "test_modules.hpp" + +#include "srf/modules/module_registry.hpp" +#include "srf/modules/plugins.hpp" +#include "srf/modules/sample_modules.hpp" +#include "srf/version.hpp" + +#include +#include + +TEST_F(TestModuleRegistry, RegistryModuleTest) +{ + using namespace modules; + + const auto* registry_namespace = "module_registry_unittest"; + const auto* simple_mod_name = "SimpleModule"; + const auto* configurable_mod_name = "ConfigurableModule"; + + auto config = nlohmann::json(); + config["source_count"] = 42; + + const std::vector release_version = {srf_VERSION_MAJOR, srf_VERSION_MINOR, srf_VERSION_PATCH}; + + auto simple_mod_func = [](std::string module_name, nlohmann::json config) { + return std::make_shared(std::move(module_name), std::move(config)); + }; + + auto configurable_mod_func = [](std::string module_name, nlohmann::json config) { + return std::make_shared(std::move(module_name), std::move(config)); + }; + ModuleRegistry::register_module(simple_mod_name, release_version, simple_mod_func); + ModuleRegistry::register_module(configurable_mod_name, release_version, configurable_mod_func); + ModuleRegistry::register_module(simple_mod_name, registry_namespace, release_version, simple_mod_func); + ModuleRegistry::register_module(configurable_mod_name, registry_namespace, release_version, configurable_mod_func); + + // Registering duplicate module throws an exception. + EXPECT_THROW(ModuleRegistry::register_module(simple_mod_name, release_version, simple_mod_func), + std::invalid_argument); + EXPECT_THROW(ModuleRegistry::register_module(configurable_mod_name, release_version, simple_mod_func), + std::invalid_argument); + EXPECT_THROW(ModuleRegistry::register_module(simple_mod_name, registry_namespace, release_version, simple_mod_func), + std::invalid_argument); + EXPECT_THROW( + ModuleRegistry::register_module(configurable_mod_name, registry_namespace, release_version, simple_mod_func), + std::invalid_argument); +} + +TEST_F(TestModuleRegistry, ContainsNamespaceTest) +{ + using namespace modules; + + const auto* registry_namespace = "module_registry_unittest"; + const auto* registry_namespace_2 = "module_registry_unittest2"; + const auto* module_name = "SimpleModule"; + + auto hasNamespace = ModuleRegistry::contains_namespace(registry_namespace); + auto hasNamespace_2 = ModuleRegistry::contains_namespace(registry_namespace_2); + + EXPECT_EQ(hasNamespace, true); + EXPECT_EQ(hasNamespace_2, false); +} + +TEST_F(TestModuleRegistry, ContainsModuleTest) +{ + using namespace modules; + + const auto* registry_namespace = "module_registry_unittest"; + const auto* registry_namespace_2 = "module_registry_unittest2"; + const auto* module_name = "SimpleModule"; + + auto hasModule = ModuleRegistry::contains(module_name, registry_namespace); + auto hasModule_2 = ModuleRegistry::contains(module_name, registry_namespace_2); + + EXPECT_EQ(hasModule, true); + EXPECT_EQ(hasModule_2, false); +} + +TEST_F(TestModuleRegistry, FindModuleTest) +{ + using namespace modules; + + const auto* registry_namespace = "module_registry_unittest"; + const auto* module_name = "SimpleModule"; + const auto* module_name_3 = "SimpleModuleTest"; + + auto fn_module_constructor = ModuleRegistry::get_module_constructor(module_name, registry_namespace); + + // Finding a module that does not exists in the registry throws an exception. + EXPECT_THROW(ModuleRegistry::get_module_constructor(module_name_3), std::invalid_argument); + EXPECT_THROW(ModuleRegistry::get_module_constructor(module_name_3, registry_namespace), std::invalid_argument); +} + +TEST_F(TestModuleRegistry, UnRegistrerModuleTest) +{ + using namespace modules; + + std::string period = "."; + std::string release_version_str = std::to_string(srf_VERSION_MAJOR) + period + std::to_string(srf_VERSION_MINOR) + + period + std::to_string(srf_VERSION_PATCH); + + std::string registry_namespace = "module_registry_unittest"; + std::string simple_mod_name = "SimpleModule"; + + ModuleRegistry::unregister_module(simple_mod_name, release_version_str); + + ModuleRegistry::unregister_module(simple_mod_name, release_version_str, true); + + // Throws an exception when there is no registered module. + EXPECT_THROW(ModuleRegistry::unregister_module(simple_mod_name, release_version_str, false), std::invalid_argument); +} + +TEST_F(TestModuleRegistry, VersionCompatibleTest) +{ + using namespace modules; + + const std::vector release_version = {srf_VERSION_MAJOR, srf_VERSION_MINOR, srf_VERSION_PATCH}; + const std::vector old_release_version = {22, 10, 0}; + const std::vector no_version_patch = {22, 10}; + const std::vector no_version_minor_and_patch = {22}; + + EXPECT_EQ(ModuleRegistry::is_version_compatible(release_version), true); + EXPECT_EQ(ModuleRegistry::is_version_compatible(old_release_version), false); + EXPECT_EQ(ModuleRegistry::is_version_compatible(no_version_patch), false); + EXPECT_EQ(ModuleRegistry::is_version_compatible(no_version_minor_and_patch), false); +} + +TEST_F(TestModuleRegistry, RegisteredModulesTest) +{ + using namespace modules; + + auto rigestered_mods_map = ModuleRegistry::registered_modules(); + + EXPECT_EQ(rigestered_mods_map.size(), 2); + EXPECT_EQ(rigestered_mods_map.find("default") != rigestered_mods_map.end(), true); + EXPECT_EQ(rigestered_mods_map.find("module_registry_unittest") != rigestered_mods_map.end(), true); +} + +std::string get_modules_path() +{ + int pid = getpid(); + std::stringstream sstream; + sstream << "/proc/" << pid << "/exe"; + + std::string link_id = sstream.str(); + unsigned int sz_path_buffer = 8102; + std::vector path_buffer(sz_path_buffer + 1); + readlink(link_id.c_str(), path_buffer.data(), sz_path_buffer); + + boost::filesystem::path whereami(path_buffer.data()); + + std::string modules_path = whereami.parent_path().string() + "/modules/"; + + return modules_path; +} + +TEST_F(TestModuleRegistry, DynamicModuleLoadTest) +{ + void* module_handle; + bool (*dummy_entrypoint)(); + + std::string module_path = get_modules_path() + "libdynamic_test_module.so"; + + module_handle = dlopen(module_path.c_str(), RTLD_NOW | RTLD_LOCAL); + if (module_handle == nullptr) + { + std::cerr << "Error: " << dlerror() << std::endl; + } + EXPECT_TRUE(module_handle); + + dummy_entrypoint = (bool (*)())dlsym(module_handle, "SRF_MODULE_dummy_entrypoint"); + const char* dlsym_error = dlerror(); + if (dlsym_error != nullptr) + { + std::cerr << "Error: " << dlsym_error << std::endl; + } + EXPECT_TRUE(dlsym_error == nullptr); + EXPECT_TRUE(dummy_entrypoint()); +} + +TEST_F(TestModuleRegistry, DynamicModuleRegistrationTest) +{ + using namespace srf::modules; + void* module_handle; + bool (*entrypoint_load)(); + bool (*entrypoint_unload)(); + + std::string module_path = get_modules_path() + "libdynamic_test_module.so"; + + module_handle = dlopen(module_path.c_str(), RTLD_NOW | RTLD_LOCAL); + if (module_handle == nullptr) + { + std::cerr << "Error: " << dlerror() << std::endl; + } + EXPECT_TRUE(module_handle); + + entrypoint_load = (bool (*)())dlsym(module_handle, "SRF_MODULE_entrypoint_load"); + const char* dlsym_error = dlerror(); + if (dlsym_error != nullptr) + { + std::cerr << "Error: " << dlsym_error << std::endl; + } + EXPECT_TRUE(dlsym_error == nullptr); + EXPECT_TRUE(entrypoint_load()); + + std::string module_namespace{"srf_unittest_cpp_dynamic"}; + std::string module_name{"DynamicSourceModule"}; + + EXPECT_TRUE(ModuleRegistry::contains_namespace(module_namespace)); + EXPECT_TRUE(ModuleRegistry::contains(module_name, module_namespace)); + + entrypoint_unload = (bool (*)())dlsym(module_handle, "SRF_MODULE_entrypoint_unload"); + const char* dlsym_unload_error = dlerror(); + if (dlsym_unload_error != nullptr) + { + std::cerr << "Error: " << dlsym_unload_error << std::endl; + } + EXPECT_TRUE(dlsym_unload_error == nullptr); + + unsigned int packet_count{0}; + + auto init_wrapper = [&packet_count](segment::Builder& builder) { + auto config = nlohmann::json(); + unsigned int source_count{42}; + config["source_count"] = source_count; + + auto dynamic_source_mod = builder.load_module_from_registry( + "DynamicSourceModule", "srf_unittest_cpp_dynamic", "DynamicModuleSourceTest_mod1", config); + + auto sink = builder.make_sink("sink", [&packet_count](bool input) { + packet_count++; + VLOG(10) << "Sinking " << input << std::endl; + }); + + builder.make_edge(dynamic_source_mod->output_port("source"), sink); + }; + + m_pipeline->make_segment("DynamicSourceModule_Segment", init_wrapper); + + auto options = std::make_shared(); + options->topology().user_cpuset("0-1"); + options->topology().restrict_gpus(true); + + Executor executor(options); + executor.register_pipeline(std::move(m_pipeline)); + executor.start(); + executor.join(); + + EXPECT_EQ(packet_count, 42); + EXPECT_TRUE(entrypoint_unload()); + dlclose(module_handle); +} + +TEST_F(TestModuleRegistry, DynamicModulePluginInterfaceTest) +{ + using namespace srf::modules; + + auto plugin = PluginModule::create_or_acquire("libdynamic_test_module.so"); + plugin->set_library_directory(get_modules_path()); + plugin->load(); + plugin->reload(); + + auto plugin_copy = PluginModule::create_or_acquire("libdynamic_test_module.so"); + // Should be 1 in the global plugin tracker, 1 for plugin, and 1 for plugin_copy + EXPECT_TRUE(plugin_copy.use_count() == 3); + plugin_copy.reset(); + + EXPECT_TRUE(plugin.use_count() == 2); +} + +TEST_F(TestModuleRegistry, DynamicModulePluginRegistrationTest) +{ + using namespace srf::modules; + + // auto plugin = std::unique_ptr{}; + auto plugin = PluginModule::create_or_acquire("libdynamic_test_module.so"); + plugin->set_library_directory(get_modules_path()); + plugin->load(); + plugin->reload(); + + std::string module_namespace{"srf_unittest_cpp_dynamic"}; + std::string module_name{"DynamicSourceModule"}; + + EXPECT_TRUE(ModuleRegistry::contains_namespace(module_namespace)); + EXPECT_TRUE(ModuleRegistry::contains(module_name, module_namespace)); + + /* + * The dynamic_test_module registers DynamicSourceModule in three test namespaces: + * srf_unittest_cpp_dynamic[1|2|3]. Double check this here. + */ + auto registered_modules = ModuleRegistry::registered_modules(); + + EXPECT_TRUE(registered_modules.find("srf_unittest_cpp_dynamic") != registered_modules.end()); + auto& ns_1 = registered_modules["srf_unittest_cpp_dynamic"]; + EXPECT_EQ(ns_1.size(), 1); + EXPECT_TRUE(ns_1[0] == "DynamicSourceModule"); + + EXPECT_TRUE(registered_modules.find("srf_unittest_cpp_dynamic_2") != registered_modules.end()); + auto& ns_2 = registered_modules["srf_unittest_cpp_dynamic_2"]; + EXPECT_EQ(ns_2.size(), 1); + EXPECT_TRUE(ns_2[0] == "DynamicSourceModule"); + + EXPECT_TRUE(registered_modules.find("srf_unittest_cpp_dynamic_3") != registered_modules.end()); + auto& ns_3 = registered_modules["srf_unittest_cpp_dynamic_3"]; + EXPECT_EQ(ns_3.size(), 1); + EXPECT_TRUE(ns_3[0] == "DynamicSourceModule"); + + std::vector expected_modules{ + "srf_unittest_cpp_dynamic::DynamicSourceModule", + "srf_unittest_cpp_dynamic_2::DynamicSourceModule", + "srf_unittest_cpp_dynamic_3::DynamicSourceModule", + }; + + auto actual_modules = plugin->list_modules(); + EXPECT_EQ(actual_modules.size(), 3); + EXPECT_TRUE(std::equal(expected_modules.begin(), expected_modules.begin() + 3, actual_modules.begin())); + + plugin->unload(); + registered_modules = ModuleRegistry::registered_modules(); + + EXPECT_TRUE(registered_modules.find("srf_unittest_cpp_dynamic") == registered_modules.end()); + EXPECT_TRUE(registered_modules.find("srf_unittest_cpp_dynamic_2") == registered_modules.end()); + EXPECT_TRUE(registered_modules.find("srf_unittest_cpp_dynamic_3") == registered_modules.end()); +} + +TEST_F(TestModuleRegistry, DynamicModuleBadVersionTest) +{ + using namespace srf::modules; + void* module_handle; + bool (*entrypoint)(); + + std::string module_path = get_modules_path() + "libdynamic_test_module.so"; + + module_handle = dlopen(module_path.c_str(), RTLD_NOW | RTLD_LOCAL); + if (module_handle == nullptr) + { + std::cerr << "Error: " << dlerror() << std::endl; + } + EXPECT_TRUE(module_handle); + + entrypoint = (bool (*)())dlsym(module_handle, "SRF_MODULE_bad_version_entrypoint"); + const char* dlsym_error = dlerror(); + if (dlsym_error != nullptr) + { + std::cerr << "Error: " << dlsym_error << std::endl; + } + EXPECT_TRUE(dlsym_error == nullptr); + EXPECT_THROW(entrypoint(), std::runtime_error); + + std::string module_namespace{"srf_unittest_cpp_dynamic_BAD"}; + std::string module_name{"DynamicSourceModule_BAD"}; + + EXPECT_FALSE(ModuleRegistry::contains_namespace(module_namespace)); + EXPECT_FALSE(ModuleRegistry::contains(module_name, module_namespace)); +} diff --git a/tests/modules/test_module_util.cpp b/tests/modules/test_module_util.cpp new file mode 100644 index 000000000..e2458ac23 --- /dev/null +++ b/tests/modules/test_module_util.cpp @@ -0,0 +1,46 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "test_modules.hpp" + +#include "srf/modules/module_registry_util.hpp" +#include "srf/modules/sample_modules.hpp" +#include "srf/version.hpp" + +TEST_F(TestModuleUtil, ModuleRegistryUtilTest) +{ + using namespace modules; + + const auto* registry_namespace = "srf_unittest"; + + const std::vector release_version = {srf_VERSION_MAJOR, srf_VERSION_MINOR, srf_VERSION_PATCH}; + + ModelRegistryUtil::create_registered_module("SimpleModule", registry_namespace, release_version); + ModelRegistryUtil::create_registered_module("SourceModule", registry_namespace, release_version); + ModelRegistryUtil::create_registered_module("SinkModule", registry_namespace, release_version); + ModelRegistryUtil::create_registered_module("NestedModule", registry_namespace, release_version); + ModelRegistryUtil::create_registered_module( + "ConfigurableModule", registry_namespace, release_version); + ModelRegistryUtil::create_registered_module>( + "TemplateModuleInt", registry_namespace, release_version); + ModelRegistryUtil::create_registered_module>( + "TemplateModuleString", registry_namespace, release_version); + + EXPECT_THROW( + ModelRegistryUtil::create_registered_module("SimpleModule", registry_namespace, release_version), + std::invalid_argument); +} diff --git a/tests/modules/test_modules.hpp b/tests/modules/test_modules.hpp new file mode 100644 index 000000000..93a385245 --- /dev/null +++ b/tests/modules/test_modules.hpp @@ -0,0 +1,72 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../test_srf.hpp" // IWYU pragma: keep + +#include "srf/core/executor.hpp" // IWYU pragma: keep +#include "srf/options/options.hpp" +#include "srf/options/topology.hpp" +#include "srf/pipeline/pipeline.hpp" +#include "srf/segment/builder.hpp" // IWYU pragma: keep +#include "srf/segment/egress_ports.hpp" +#include "srf/segment/ingress_ports.hpp" +#include "srf/segment/segment.hpp" // IWYU pragma: keep + +#include +#include +#include +#include + +// IWYU pragma: no_include "gtest/gtest_pred_impl.h" + +namespace srf::segment { +struct ObjectProperties; +} + +class TestSegmentResources +{ + public: + TestSegmentResources() = default; + + static std::unique_ptr make_options() + { + auto options = std::make_unique(); + options->topology().user_cpuset("0"); + return options; + } +}; + +class TestModules : public ::testing::Test +{ + protected: + void SetUp() override + { + m_pipeline = pipeline::make_pipeline(); + m_resources = std::make_shared(); + } + + void TearDown() override {} + + std::unique_ptr m_pipeline; + std::shared_ptr m_resources; +}; + +using TestModuleRegistry = TestModules; // NOLINT +using TestModuleUtil = TestModules; // NOLINT +using TestSegmentModules = TestModules; // NOLINT diff --git a/tests/modules/test_segment_modules.cpp b/tests/modules/test_segment_modules.cpp new file mode 100644 index 000000000..384d388b4 --- /dev/null +++ b/tests/modules/test_segment_modules.cpp @@ -0,0 +1,465 @@ +/** + * SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "test_modules.hpp" + +#include "srf/core/executor.hpp" +#include "srf/engine/pipeline/ipipeline.hpp" +#include "srf/modules/module_registry.hpp" +#include "srf/modules/plugins.hpp" +#include "srf/modules/sample_modules.hpp" +#include "srf/options/options.hpp" +#include "srf/segment/builder.hpp" + +#include +#include +#include + +#include +#include +#include +#include + +TEST_F(TestSegmentModules, ModuleConstructorTest) +{ + using namespace modules; + + auto config_1 = nlohmann::json(); + auto config_2 = nlohmann::json(); + config_2["config_key_1"] = true; + + auto mod1 = SimpleModule("InitModuleTest_mod1"); + auto mod2 = ConfigurableModule("InitModuleTest_2"); + auto mod3 = ConfigurableModule("InitModuleTest_3", config_1); + auto mod4 = ConfigurableModule("InitModuleTest_4", config_2); + + ASSERT_EQ(mod4.config().contains("config_key_1"), true); + + ASSERT_THROW(SimpleModule("bad/module/name"), std::invalid_argument); +} + +TEST_F(TestSegmentModules, ModuleInitializationTest) +{ + using namespace modules; + + auto init_wrapper = [](segment::Builder& builder) { + auto config_1 = nlohmann::json(); + auto config_2 = nlohmann::json(); + config_2["config_key_1"] = true; + + auto simple_mod = builder.make_module("ModuleInitializationTest_mod1"); + auto configurable_1_mod = builder.make_module("ModuleInitializationTest_mod2", config_1); + auto configurable_2_mod = builder.make_module("ModuleInitializationTest_mod3", config_2); + auto configurable_mod_3 = ConfigurableModule("ModuleInitializationTest_mod4", config_2); + + configurable_mod_3(builder); + + EXPECT_EQ(simple_mod->input_ids().size(), 2); + EXPECT_EQ(simple_mod->output_ids().size(), 2); + EXPECT_EQ(simple_mod->input_ports().size(), 2); + EXPECT_EQ(simple_mod->output_ports().size(), 2); + EXPECT_EQ(simple_mod->input_ports().find("input1") != simple_mod->input_ports().end(), true); + EXPECT_EQ(simple_mod->input_ports().find("input2") != simple_mod->input_ports().end(), true); + EXPECT_EQ(simple_mod->input_port_type_id("input1"), typeid(bool)); + EXPECT_EQ(simple_mod->input_port_type_id("input2"), typeid(bool)); + EXPECT_EQ(simple_mod->input_port_type_ids().find("input1")->second, typeid(bool)); + EXPECT_EQ(simple_mod->input_port_type_ids().find("input2")->second, typeid(bool)); + EXPECT_EQ(simple_mod->output_ports().find("output1") != simple_mod->input_ports().end(), true); + EXPECT_EQ(simple_mod->output_ports().find("output2") != simple_mod->input_ports().end(), true); + EXPECT_EQ(simple_mod->output_port_type_id("output1"), typeid(std::string)); + EXPECT_EQ(simple_mod->output_port_type_id("output2"), typeid(std::string)); + EXPECT_EQ(simple_mod->output_port_type_ids().find("output1")->second, typeid(std::string)); + EXPECT_EQ(simple_mod->output_port_type_ids().find("output2")->second, typeid(std::string)); + + EXPECT_THROW(simple_mod->input_port("DOES_NOT_EXIST"), std::invalid_argument); + EXPECT_THROW(simple_mod->output_port("DOES_NOT_EXIST"), std::invalid_argument); + EXPECT_THROW(simple_mod->input_port_type_id("DOES_NOT_EXIST"), std::invalid_argument); + EXPECT_THROW(simple_mod->output_port_type_id("DOES_NOT_EXIST"), std::invalid_argument); + + EXPECT_EQ(configurable_1_mod->input_ports().size(), 1); + EXPECT_EQ(configurable_1_mod->output_ports().size(), 1); + EXPECT_EQ(configurable_1_mod->m_was_configured, false); + + EXPECT_EQ(configurable_2_mod->input_ports().size(), 1); + EXPECT_EQ(configurable_2_mod->output_ports().size(), 1); + EXPECT_EQ(configurable_2_mod->m_was_configured, true); + }; + + m_pipeline->make_segment("Initialization_Segment", init_wrapper); + + auto options = std::make_shared(); + options->topology().user_cpuset("0-1"); + options->topology().restrict_gpus(true); + + Executor executor(options); + executor.register_pipeline(std::move(m_pipeline)); + executor.stop(); + executor.join(); +} + +TEST_F(TestSegmentModules, ModuleEndToEndTest) +{ + using namespace modules; + unsigned int packets_1{0}; + unsigned int packets_2{0}; + unsigned int packets_3{0}; + + auto init_wrapper = [&packets_1, &packets_2, &packets_3](segment::Builder& builder) { + auto simple_mod = builder.make_module("ModuleEndToEndTest_mod1"); + auto configurable_mod = builder.make_module("ModuleEndToEndTest_mod2"); + + auto source1 = builder.make_source("src1", [](rxcpp::subscriber& sub) { + if (sub.is_subscribed()) + { + sub.on_next(true); + sub.on_next(false); + sub.on_next(true); + sub.on_next(true); + } + + sub.on_completed(); + }); + + // Ex1. Partially dynamic edge construction + builder.make_edge(source1, simple_mod->input_port("input1")); + + auto source2 = builder.make_source("src2", [](rxcpp::subscriber& sub) { + if (sub.is_subscribed()) + { + sub.on_next(true); + sub.on_next(false); + sub.on_next(false); + sub.on_next(false); + sub.on_next(true); + sub.on_next(false); + } + + sub.on_completed(); + }); + + // Ex2. Dynamic edge construction -- requires type specification + builder.make_dynamic_edge(source2, simple_mod->input_port("input2")); + + auto sink1 = builder.make_sink("sink1", [&packets_1](std::string input) { + packets_1++; + VLOG(10) << "Sinking " << input << std::endl; + }); + + builder.make_edge(simple_mod->output_port("output1"), sink1); + + auto sink2 = builder.make_sink("sink2", [&packets_2](std::string input) { + packets_2++; + VLOG(10) << "Sinking " << input << std::endl; + }); + + builder.make_edge(simple_mod->output_port("output2"), sink2); + + auto source3 = builder.make_source("src3", [](rxcpp::subscriber& sub) { + if (sub.is_subscribed()) + { + sub.on_next(true); + sub.on_next(false); + sub.on_next(true); + sub.on_next(true); + } + + sub.on_completed(); + }); + + builder.make_edge(source3, configurable_mod->input_port("configurable_input_a")); + + auto sink3 = builder.make_sink("sink3", [&packets_3](std::string input) { + packets_3++; + VLOG(10) << "Sinking " << input << std::endl; + }); + + builder.make_edge(configurable_mod->output_port("configurable_output_x"), sink3); + }; + + m_pipeline->make_segment("EndToEnd_Segment", init_wrapper); + + auto options = std::make_shared(); + options->topology().user_cpuset("0-1"); + options->topology().restrict_gpus(true); + + Executor executor(options); + executor.register_pipeline(std::move(m_pipeline)); + executor.start(); + executor.join(); + + EXPECT_EQ(packets_1, 4); + EXPECT_EQ(packets_2, 6); + EXPECT_EQ(packets_3, 4); +} + +TEST_F(TestSegmentModules, ModuleAsSourceTest) +{ + using namespace modules; + + unsigned int packet_count{0}; + + auto init_wrapper = [&packet_count](segment::Builder& builder) { + auto config = nlohmann::json(); + unsigned int source_count{42}; + config["source_count"] = source_count; + + auto source_mod = builder.make_module("ModuleSourceTest_mod1", config); + + auto sink = builder.make_sink("sink", [&packet_count](bool input) { + packet_count++; + VLOG(10) << "Sinking " << input << std::endl; + }); + + builder.make_edge(source_mod->output_port("source"), sink); + }; + + m_pipeline->make_segment("SimpleModule_Segment", init_wrapper); + + auto options = std::make_shared(); + options->topology().user_cpuset("0-1"); + options->topology().restrict_gpus(true); + + Executor executor(options); + executor.register_pipeline(std::move(m_pipeline)); + executor.start(); + executor.join(); + + EXPECT_EQ(packet_count, 42); +} + +TEST_F(TestSegmentModules, ModuleAsSinkTest) +{ + using namespace modules; + + unsigned int packet_count{0}; + + auto init_wrapper = [&packet_count](segment::Builder& builder) { + auto source = builder.make_source("source", [&packet_count](rxcpp::subscriber& sub) { + if (sub.is_subscribed()) + { + for (unsigned int i = 0; i < 43; ++i) + { + sub.on_next(true); + packet_count++; + } + } + + sub.on_completed(); + }); + + auto sink_mod = builder.make_module("ModuleSinkTest_mod1"); + + builder.make_edge(source, sink_mod->input_port("sink")); + }; + + m_pipeline->make_segment("SimpleModule_Segment", init_wrapper); + + auto options = std::make_shared(); + options->topology().user_cpuset("0-1"); + options->topology().restrict_gpus(true); + + Executor executor(options); + executor.register_pipeline(std::move(m_pipeline)); + executor.start(); + executor.join(); + + EXPECT_EQ(packet_count, 43); +} + +TEST_F(TestSegmentModules, ModuleChainingTest) +{ + using namespace modules; + + auto sink_mod = std::make_shared("ModuleChainingTest_mod2"); + auto init_wrapper = [&sink_mod](segment::Builder& builder) { + auto config = nlohmann::json(); + unsigned int source_count{42}; + config["source_count"] = source_count; + + auto source_mod = builder.make_module("ModuleChainingTest_mod1", config); + builder.init_module(sink_mod); + + builder.make_dynamic_edge(source_mod->output_port("source"), sink_mod->input_port("sink")); + }; + + m_pipeline->make_segment("SimpleModule_Segment", init_wrapper); + + auto options = std::make_shared(); + options->topology().user_cpuset("0-1"); + options->topology().restrict_gpus(true); + + Executor executor(options); + executor.register_pipeline(std::move(m_pipeline)); + executor.start(); + executor.join(); + + EXPECT_EQ(sink_mod->m_packet_count, 42); +} + +TEST_F(TestSegmentModules, ModuleNestingTest) +{ + using namespace modules; + + unsigned int packet_count{0}; + + auto init_wrapper = [&packet_count](segment::Builder& builder) { + auto nested_mod = builder.make_module("ModuleNestingTest_mod1"); + + auto nested_sink = builder.make_sink("nested_sink", [&packet_count](std::string input) { + packet_count++; + VLOG(10) << "Sinking " << input << std::endl; + }); + + builder.make_edge(nested_mod->output_port("nested_module_output"), nested_sink); + }; + + m_pipeline->make_segment("SimpleModule_Segment", init_wrapper); + + auto options = std::make_shared(); + options->topology().user_cpuset("0-1"); + options->topology().restrict_gpus(true); + + Executor executor(options); + executor.register_pipeline(std::move(m_pipeline)); + executor.start(); + executor.join(); + + EXPECT_EQ(packet_count, 4); +} + +TEST_F(TestSegmentModules, ModuleTemplateTest) +{ + using namespace modules; + + unsigned int packet_count_1{0}; + unsigned int packet_count_2{0}; + + auto init_wrapper = [&packet_count_1, &packet_count_2](segment::Builder& builder) { + using data_type_1_t = int; + using data_type_2_t = std::string; + + auto config_1 = nlohmann::json(); + auto config_2 = nlohmann::json(); + + unsigned int source_count_1{42}; + unsigned int source_count_2{24}; + + config_1["source_count"] = source_count_1; + config_2["source_count"] = source_count_2; + + auto source_1_mod = builder.make_module>("ModuleTemplateTest_mod1", config_1); + + auto sink_1 = builder.make_sink("sink_1", [&packet_count_1](data_type_1_t input) { + packet_count_1++; + VLOG(10) << "Sinking " << input << std::endl; + }); + + builder.make_edge(source_1_mod->output_port("source"), sink_1); + + auto source_2_mod = builder.make_module>("ModuleTemplateTest_mod2", config_2); + + auto sink_2 = builder.make_sink("sink_2", [&packet_count_2](data_type_2_t input) { + packet_count_2++; + VLOG(10) << "Sinking " << input << std::endl; + }); + + builder.make_edge(source_2_mod->output_port("source"), sink_2); + }; + + m_pipeline->make_segment("SimpleModule_Segment", init_wrapper); + + auto options = std::make_shared(); + options->topology().user_cpuset("0-1"); + options->topology().restrict_gpus(true); + + Executor executor(options); + executor.register_pipeline(std::move(m_pipeline)); + executor.start(); + executor.join(); + + EXPECT_EQ(packet_count_1, 42); + EXPECT_EQ(packet_count_2, 24); +} + +#if !defined(__clang__) && defined(__GNUC__) +// Work around for GCC : https://gcc.gnu.org/bugzilla/show_bug.cgi?id=83258 +auto F_1 = []() -> int { return 15; }; +auto F_2 = []() -> std::string { return "test string"; }; +#endif + +TEST_F(TestSegmentModules, ModuleTemplateWithInitTest) +{ + using namespace modules; + + unsigned int packet_count_1{0}; + unsigned int packet_count_2{0}; + + auto init_wrapper = [&packet_count_1, &packet_count_2](segment::Builder& builder) { + using data_type_1_t = int; + using data_type_2_t = std::string; + + auto config_1 = nlohmann::json(); + auto config_2 = nlohmann::json(); + + unsigned int source_count_1{42}; + unsigned int source_count_2{24}; + + config_1["source_count"] = source_count_1; + config_2["source_count"] = source_count_2; + +#if defined(__clang__) + auto F_1 = []() -> int { return 15; }; + auto F_2 = []() -> std::string { return "test string"; }; +#endif + + auto source_1_mod = builder.make_module>( + "ModuleTemplateWithInitTest_mod1", config_1); + + auto sink_1 = builder.make_sink("sink_1", [&packet_count_1](data_type_1_t input) { + assert(input == 15); + packet_count_1++; + VLOG(10) << "Sinking " << input << std::endl; + }); + + builder.make_edge(source_1_mod->output_port("source"), sink_1); + + auto source_2_mod = builder.make_module>( + "ModuleTemplateWithInitTest_mod2", config_2); + + auto sink_2 = builder.make_sink("sink_2", [&packet_count_2](data_type_2_t input) { + assert(input == "test string"); + packet_count_2++; + VLOG(10) << "Sinking " << input << std::endl; + }); + + builder.make_edge(source_2_mod->output_port("source"), sink_2); + }; + + m_pipeline->make_segment("SimpleModule_Segment", init_wrapper); + + auto options = std::make_shared(); + options->topology().user_cpuset("0-1"); + options->topology().restrict_gpus(true); + + Executor executor(options); + executor.register_pipeline(std::move(m_pipeline)); + executor.start(); + executor.join(); + + EXPECT_EQ(packet_count_1, 42); + EXPECT_EQ(packet_count_2, 24); +} diff --git a/tests/test_segment.cpp b/tests/test_segment.cpp index fcfbfcaa5..e6c8fe3cf 100644 --- a/tests/test_segment.cpp +++ b/tests/test_segment.cpp @@ -84,12 +84,12 @@ struct SrfRuntimeError; using namespace std::literals::string_literals; -TEST_F(SegmentTests, CreateSegmentDefinition) +TEST_F(TestSegment, CreateSegmentDefinition) { auto segdef = segment::Definition::create("segment_test", m_initializer); } -TEST_F(SegmentTests, InitializeSegmentFromDefinition) +TEST_F(TestSegment, InitializeSegmentFromDefinition) { auto segdef = segment::Definition::create("segment_test", m_initializer); // // auto builder = std::make_unique(segdef, 42); @@ -97,12 +97,12 @@ TEST_F(SegmentTests, InitializeSegmentFromDefinition) // --- // -TEST_F(SegmentTests, CreateSegmentDefinitionIngressOnly) +TEST_F(TestSegment, CreateSegmentDefinitionIngressOnly) { auto segdef = segment::Definition::create("segment_test", m_ingress_multi_port, m_initializer); } -TEST_F(SegmentTests, InitializeSegmentIngressOnlyFromDefinition) +TEST_F(TestSegment, InitializeSegmentIngressOnlyFromDefinition) { auto segdef = segment::Definition::create("segment_test", m_ingress_multi_port, m_initializer); // // auto builder = std::make_unique(segdef, 42); @@ -110,12 +110,12 @@ TEST_F(SegmentTests, InitializeSegmentIngressOnlyFromDefinition) // --- // -TEST_F(SegmentTests, CreateSegmentDefinitionEgressOnly) +TEST_F(TestSegment, CreateSegmentDefinitionEgressOnly) { auto segdef = segment::Definition::create("segment_test", m_egress_multi_port, m_initializer); } -TEST_F(SegmentTests, InitializeSegmentEgressOnlyFromDefinition) +TEST_F(TestSegment, InitializeSegmentEgressOnlyFromDefinition) { auto segdef = segment::Definition::create("segment_test", m_egress_multi_port, m_initializer); // // auto builder = std::make_unique(segdef, 42); @@ -123,12 +123,12 @@ TEST_F(SegmentTests, InitializeSegmentEgressOnlyFromDefinition) // --- // -TEST_F(SegmentTests, CreateSegmentDefinitionIngressEgress) +TEST_F(TestSegment, CreateSegmentDefinitionIngressEgress) { auto segdef = segment::Definition::create("segment_test", m_ingress_multi_port, m_egress_multi_port, m_initializer); } -TEST_F(SegmentTests, InitializeSegmentIngressEgressFromDefinition) +TEST_F(TestSegment, InitializeSegmentIngressEgressFromDefinition) { auto segdef = segment::Definition::create("segment_test", m_ingress_multi_port, m_egress_multi_port, m_initializer); // // auto builder = std::make_unique(segdef, 42); @@ -142,7 +142,7 @@ TEST_F(SegmentTests, InitializeSegmentIngressEgressFromDefinition) */ } -TEST_F(SegmentTests, PortsConstructorBadNameBuilderSizeMismatch) +TEST_F(TestSegment, PortsConstructorBadNameBuilderSizeMismatch) { using port_type_t = segment::Ports; @@ -152,7 +152,7 @@ TEST_F(SegmentTests, PortsConstructorBadNameBuilderSizeMismatch) EXPECT_THROW(port_type_t BadPorts(port_names, port_builder_fns), exceptions::SrfRuntimeError); } -TEST_F(SegmentTests, PortsConstructorBadDuplicateName) +TEST_F(TestSegment, PortsConstructorBadDuplicateName) { using port_type_t = segment::Ports; @@ -167,7 +167,7 @@ TEST_F(SegmentTests, PortsConstructorBadDuplicateName) EXPECT_THROW(port_type_t BadPorts(port_names, port_builder_fns), exceptions::SrfRuntimeError); } -TEST_F(SegmentTests, UserLambdaIsCalled) +TEST_F(TestSegment, UserLambdaIsCalled) { GTEST_SKIP() << "Skipping until issue #59 is resolved"; @@ -179,7 +179,7 @@ TEST_F(SegmentTests, UserLambdaIsCalled) EXPECT_EQ(m_initializer_called, true); } -TEST_F(SegmentTests, SegmentRxSinkCreation) +TEST_F(TestSegment, SegmentRxSinkCreation) { auto init = [](segment::Builder& segment) { auto x = segment.make_sink( @@ -192,7 +192,7 @@ TEST_F(SegmentTests, SegmentRxSinkCreation) // // auto builder = std::make_unique(segdef, 42); } -TEST_F(SegmentTests, SegmentRxSourceCreation) +TEST_F(TestSegment, SegmentRxSourceCreation) { auto init = [](segment::Builder& segment) { auto x = segment.make_source("x_src", [&](rxcpp::subscriber s) { @@ -207,7 +207,7 @@ TEST_F(SegmentTests, SegmentRxSourceCreation) // // auto builder = std::make_unique(segdef, 42); } -TEST_F(SegmentTests, SegmentRxNodeCreation) +TEST_F(TestSegment, SegmentRxNodeCreation) { auto init = [](segment::Builder& segment) { auto x = segment.make_node("x"); @@ -226,7 +226,7 @@ TEST_F(SegmentTests, SegmentRxNodeCreation) // auto builder = std::make_unique(segdef, 42); } -TEST_F(SegmentTests, SegmentRxNodeStaticEdges) +TEST_F(TestSegment, SegmentRxNodeStaticEdges) { auto init = [this](segment::Builder& segment) { auto x = segment.make_node("x"); @@ -251,7 +251,7 @@ TEST_F(SegmentTests, SegmentRxNodeStaticEdges) // auto builder = std::make_unique(segdef, 42); } -TEST_F(SegmentTests, SegmentRxNodeValidTypeConversionWorks) +TEST_F(TestSegment, SegmentRxNodeValidTypeConversionWorks) { auto init = [this](segment::Builder& segment) { auto x = segment.make_node("x"); @@ -310,7 +310,7 @@ TEST_F(SegmentTests, SegmentRxNodeDynamicEdges) */ -TEST_F(SegmentTests, SegmentEndToEndTest) +TEST_F(TestSegment, SegmentEndToEndTest) { auto init = [](segment::Builder& segment) { auto src = segment.make_source("src", [&](rxcpp::subscriber& s) { @@ -343,7 +343,7 @@ TEST_F(SegmentTests, SegmentEndToEndTest) // // auto builder = std::make_unique(segdef, 42); } -TEST_F(SegmentTests, CompileTimeConversionValuesWorkAsExpected) +TEST_F(TestSegment, CompileTimeConversionValuesWorkAsExpected) { auto init = [](segment::Builder& segment) { auto src = segment.make_source("src", [&](rxcpp::subscriber& s) { @@ -397,7 +397,7 @@ TEST_F(SegmentTests, CompileTimeConversionValuesWorkAsExpected) // auto builder = std::make_unique(segdef, 42); } -TEST_F(SegmentTests, RuntimeConversionValuesWorkAsExpected) +TEST_F(TestSegment, RuntimeConversionValuesWorkAsExpected) { auto init = [](segment::Builder& segment) { auto src = segment.make_source("src", [&](rxcpp::subscriber& s) { @@ -453,7 +453,7 @@ TEST_F(SegmentTests, RuntimeConversionValuesWorkAsExpected) // auto builder = std::make_unique(segdef, 42); } -TEST_F(SegmentTests, SegmentEndToEndTestRx) +TEST_F(TestSegment, SegmentEndToEndTestRx) { auto init = [](segment::Builder& segment) { auto src = segment.make_source("src", [&](rxcpp::subscriber s) { @@ -491,7 +491,7 @@ void execute_pipeline(std::unique_ptr pipeline) exec.join(); } -TEST_F(SegmentTests, ChannelClose) +TEST_F(TestSegment, ChannelClose) { auto p = pipeline::make_pipeline(); @@ -539,7 +539,7 @@ TEST_F(SegmentTests, ChannelClose) EXPECT_EQ(complete_count, 1); } -TEST_F(SegmentTests, SegmentEndToEndTestSinkOutput) +TEST_F(TestSegment, SegmentEndToEndTestSinkOutput) { unsigned int iterations{10}; std::atomic sink_results{0}; @@ -571,7 +571,7 @@ TEST_F(SegmentTests, SegmentEndToEndTestSinkOutput) // auto builder = std::make_unique(segdef, 42); } -TEST_F(SegmentTests, SegmentSingleSourceTwoNodesException) +TEST_F(TestSegment, SegmentSingleSourceTwoNodesException) { unsigned int iterations{3}; std::atomic sink1_results{0}; @@ -624,7 +624,7 @@ TEST_F(SegmentTests, SegmentSingleSourceTwoNodesException) // auto builder = std::make_unique(segdef, 42); } -TEST_F(SegmentTests, SegmentSingleSourceTwoNodes) +TEST_F(TestSegment, SegmentSingleSourceTwoNodes) { unsigned int iterations{3}; std::atomic sink1_results{0}; @@ -687,7 +687,7 @@ TEST_F(SegmentTests, SegmentSingleSourceTwoNodes) EXPECT_EQ(sink2_results, 5.5F * iterations); } -TEST_F(SegmentTests, SegmentSingleSourceMultiNodes) +TEST_F(TestSegment, SegmentSingleSourceMultiNodes) { constexpr unsigned int NumChildren{10}; unsigned int iterations{3}; @@ -746,7 +746,7 @@ TEST_F(SegmentTests, SegmentSingleSourceMultiNodes) } } -TEST_F(SegmentTests, EnsureMove) +TEST_F(TestSegment, EnsureMove) { auto init = [&](segment::Builder& segment) { auto src = segment.make_source("src", [](rxcpp::subscriber& s) { @@ -778,7 +778,7 @@ TEST_F(SegmentTests, EnsureMove) execute_pipeline(std::move(pipeline)); } -TEST_F(SegmentTests, EnsureMoveMultiChildren) +TEST_F(TestSegment, EnsureMoveMultiChildren) { constexpr unsigned int NumChildren{10}; auto init = [&](segment::Builder& segment) { @@ -828,7 +828,7 @@ TEST_F(SegmentTests, EnsureMoveMultiChildren) execute_pipeline(std::move(pipeline)); } -TEST_F(SegmentTests, EnsureMoveConstructor) +TEST_F(TestSegment, EnsureMoveConstructor) { // First ensure CopyMoveCounter is working as expected { @@ -944,7 +944,7 @@ TEST_F(SegmentTests, EnsureMoveConstructor) } } -TEST_F(SegmentTests, SegmentTestRxcppHigherLevelNodes) +TEST_F(TestSegment, SegmentTestRxcppHigherLevelNodes) { std::size_t iterations = 5; using srf::benchmarking::TraceStatistics; @@ -1011,7 +1011,7 @@ TEST_F(SegmentTests, SegmentTestRxcppHigherLevelNodes) nlohmann::json j = TraceStatistics::aggregate(); auto _j = j["aggregations"]["components"]["metrics"]; - std::cerr << j.dump(2); + // std::cerr << j.dump(2); EXPECT_EQ(_j.contains("src"), true); auto src_json = j["src"]; // stat_check_helper(src_json, 0, 0, iterations, iterations); @@ -1030,7 +1030,7 @@ TEST_F(SegmentTests, SegmentTestRxcppHigherLevelNodes) TraceStatistics::reset(); } -TEST_F(SegmentTests, SegmentGetEgressError) +TEST_F(TestSegment, SegmentGetEgressError) { auto init = [](segment::Builder& segment) { segment.get_egress("test"); }; auto segdef = segment::Definition::create("segment_test", init); @@ -1050,7 +1050,7 @@ TEST_F(SegmentTests, SegmentGetEgressError) */ } -TEST_F(SegmentTests, SegmentGetEgressNotEgressError) +TEST_F(TestSegment, SegmentGetEgressNotEgressError) { auto init = [](segment::Builder& segment) { auto src = diff --git a/tests/test_segment.hpp b/tests/test_segment.hpp index 6f90d3f05..67dc0079b 100644 --- a/tests/test_segment.hpp +++ b/tests/test_segment.hpp @@ -52,7 +52,7 @@ class TestSegmentResources } }; -class SegmentTests : public ::testing::Test +class TestSegment : public ::testing::Test { protected: void SetUp() override