diff --git a/applications/dds/CMakeLists.txt b/applications/dds/CMakeLists.txt index 70ddad6070..7319f093b5 100644 --- a/applications/dds/CMakeLists.txt +++ b/applications/dds/CMakeLists.txt @@ -17,6 +17,14 @@ cmake_minimum_required(VERSION 3.20) project(distributed_applications LANGUAGES NONE) +add_holohub_application(dds_h264 DEPENDS + OPERATORS dds_video_publisher + dds_video_subscriber + video_encoder + tensor_to_video_buffer + append_timestamp) + + add_holohub_application(dds_video DEPENDS OPERATORS dds_shapes_subscriber dds_video_publisher diff --git a/applications/dds/dds_h264/CMakeLists.txt b/applications/dds/dds_h264/CMakeLists.txt new file mode 100644 index 0000000000..a64cc52a4e --- /dev/null +++ b/applications/dds/dds_h264/CMakeLists.txt @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.24) +project(dds_h264) + +find_package(holoscan 2.0 REQUIRED CONFIG + PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install") + +add_executable(dds_h264 dds_h264.cpp) +target_link_libraries(dds_h264 + holoscan::core + holoscan::ops::v4l2 + holoscan::ops::gxf_codelet + holoscan::ops::holoviz + holoscan::ops::video_encoder + holoscan::ops::dds_video_publisher + holoscan::ops::dds_video_subscriber + holoscan::ops::append_timestamp + holoscan::ops::format_converter + holoscan::ops::tensor_to_video_buffer + holoscan::ops::video_stream_replayer +) + +# Copy qos_profiles.xml to the binary directory +add_custom_target(dds_video_qos_profiles_xml + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/qos_profiles.xml" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/qos_profiles.xml" +) +add_dependencies(dds_h264 dds_video_qos_profiles_xml) + +# Copy config file +add_custom_target(dds_h264_yaml + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${CMAKE_CURRENT_SOURCE_DIR}/dds_h264.yaml" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "dds_h264.yaml" + BYPRODUCTS "dds_h264.yaml" +) +add_dependencies(dds_h264 dds_h264_yaml) diff --git a/applications/dds/dds_h264/Dockerfile b/applications/dds/dds_h264/Dockerfile new file mode 100644 index 0000000000..be4c73e2ff --- /dev/null +++ b/applications/dds/dds_h264/Dockerfile @@ -0,0 +1,113 @@ +# syntax=docker/dockerfile:1 + +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BASE_IMAGE +ARG GPU_TYPE + +############################################################ +# Base image +############################################################ + +ARG BASE_IMAGE +ARG GPU_TYPE + +FROM ${BASE_IMAGE} AS base + +ARG DEBIAN_FRONTEND=noninteractive + +# -------------------------------------------------------------------------- +# +# Holohub run setup +# + +RUN mkdir -p /tmp/scripts +COPY run /tmp/scripts/ +RUN mkdir -p /tmp/scripts/utilities +COPY utilities/holohub_autocomplete /tmp/scripts/utilities/ +RUN chmod +x /tmp/scripts/run +RUN /tmp/scripts/run setup + +# Enable autocomplete +RUN echo ". /etc/bash_completion.d/holohub_autocomplete" >> /etc/bash.bashrc + +# - This variable is consumed by all dependencies below as an environment variable (CMake 3.22+) +# - We use ARG to only set it at docker build time, so it does not affect cmake builds +# performed at docker run time in case users want to use a different BUILD_TYPE +ARG CMAKE_BUILD_TYPE=Release + +# Qcap dependency +RUN apt update \ + && apt install --no-install-recommends -y \ + libgstreamer1.0-0 \ + libgstreamer-plugins-base1.0-0 \ + libgles2 \ + libopengl0 + +# For benchmarking +RUN apt update \ + && apt install --no-install-recommends -y \ + libcairo2-dev \ + libgirepository1.0-dev \ + gobject-introspection \ + libgtk-3-dev \ + libcanberra-gtk-module \ + graphviz\ + ninja-build + +RUN pip install meson + +RUN if ! grep -q "VERSION_ID=\"22.04\"" /etc/os-release; then \ + pip install setuptools; \ + fi +COPY benchmarks/holoscan_flow_benchmarking/requirements.txt /tmp/benchmarking_requirements.txt +RUN pip install -r /tmp/benchmarking_requirements.txt + +# For RTI Connext DDS +RUN apt update \ + && apt install --no-install-recommends -y \ + openjdk-21-jre +RUN echo 'export JREHOME=$(readlink /etc/alternatives/java | sed -e "s/\/bin\/java//")' >> /etc/bash.bashrc + +# Set default Holohub data directory +ENV HOLOSCAN_INPUT_PATH=/workspace/holohub/data + +ENV NDDSHOME=/opt/rti.com/rti_connext_dds-7.3.0 +ENV RTI_CONNEXT_DDS_DIR=$NDDSHOME + +# - Install libv4l-dev required for nvv4l2 +# - Install kmod as a workaround to fix iGPU support. +# GXF ENC / DEC needs lsmod to check whether it's dGPU or iGPU. +RUN apt update && apt install -y libv4l-dev kmod + +# Below workarounds are required to get H.264 Encode and Decode working inside +# the docker container. +RUN if [ ! -e "/usr/lib/$(arch)-linux-gnu/libnvidia-encode.so" ]; then \ + ln -s /usr/lib/$(arch)-linux-gnu/libnvidia-encode.so.1 /usr/lib/$(arch)-linux-gnu/libnvidia-encode.so; \ + fi +RUN mkdir /usr/lib/$(arch)-linux-gnu/libv4l/plugins/nv && \ + ln -s /usr/lib/$(arch)-linux-gnu/tegra/libv4l2_nvcuvidvideocodec.so /usr/lib/$(arch)-linux-gnu/libv4l/plugins/nv/libv4l2_nvcuvidvideocodec.so + +COPY applications/h264/install_dependencies.sh / + +WORKDIR / + +# Uncomment the following line for aarch64 support +#ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/aarch64-linux-gnu/tegra/ + +RUN /install_dependencies.sh + +CMD ["/bin/bash", "-c", "source $NDDSHOME/resource/scripts/rtisetenv_x64Linux4gcc7.3.0.bash && exec /bin/bash"] \ No newline at end of file diff --git a/applications/dds/dds_h264/README.md b/applications/dds/dds_h264/README.md new file mode 100644 index 0000000000..2b9f325f2d --- /dev/null +++ b/applications/dds/dds_h264/README.md @@ -0,0 +1,165 @@ +# DDS Video: Real-time Video Streaming with RTI Connext & H.264 + +This application demonstrates how to encode video frames with H.264 using the multimedia +extension over DDS. + +The application can be run as either a publisher or as a subscriber. In either case, +it will use the [VideoFrame](../../../operators/dds/video/VideoFrame.idl) data topic +registered by the `DDSVideoPublisherOp` or `DDSVideoSubscriberOp` operators in order +to write or read the video frame data to/from the DDS databus, respectively. + +When run as a publisher, the source for the input video frames can come from either an +attached V4L2-compatible camera via the `V4L2VideoCaptureOp` operator or a video file via the +`VideoStreamReplayerOp`. This can be configured in the `source` field inside the +[dds_h264.yaml](./dds_h264.yaml) configuration file. + +When run as a subscriber, the application will use Holoviz to render the received +video frames to the display. + +## Prerequisites + +- This application requires [RTI Connext](https://content.rti.com/l/983311/2024-04-30/pz1wms) +be installed and configured with a valid RTI Connext license prior to use. +- V4L2 capable device + +> [!NOTE] +> Instructions below are based on the `.run' installer from RTI Connext. Refer to the +> [Linux installation](https://community.rti.com/static/documentation/developers/get-started/full-install.html) +> for details. + + +## Quick Start + +```bash +# Start the publisher +./dev_container build_and_run dds_h264 --container_args "-v $HOME/rti_connext_dds-7.3.0:/opt/rti.com/rti_connext_dds-7.3.0/" --run_args "-p" + +# Start the subscriber +./dev_container build_and_run dds_h264 --container_args "-v $HOME/rti_connext_dds-7.3.0:/opt/rti.com/rti_connext_dds-7.3.0/" --run_args "-s" +``` + + +## Building the Application + +To build on an IGX devkit (using the `armv8` architecture), follow the +[instructions to build Connext DDS applications for embedded Arm targets](https://community.rti.com/kb/how-do-i-create-connext-dds-application-rti-code-generator-and-build-it-my-embedded-target-arm) +up to, and including, step 5 (Installing Java and setting JREHOME). + +To build the application, the `RTI_CONNEXT_DDS_DIR` CMake variable must point to +the installation path for RTI Connext. This can be done automatically by setting +the `NDDSHOME` environment variable to the RTI Connext installation directory +(such as when using the RTI `setenv` scripts), or manually at build time, e.g.: + +```sh +$ ./run build dds_h264 --configure-args -DRTI_CONNEXT_DDS_DIR=~/rti/rti_connext_dds-7.3.0 +``` + +### Building with a Container + +Due to the license requirements of RTI Connext it is not currently supported to +install RTI Connext into a development container. Instead, Connext should be +installed onto the host as above and then the development container can be +launched with the RTI Connext folder mounted at runtime. To do so, ensure that +the `NDDSHOME` and `CONNEXTDDS_ARCH` environment variables are set (which can be +done using the RTI `setenv` script) and use the following: + +```sh +# 1. Build the container +./dev_container build --docker_file applications/dds/dds_h264/Dockerfile +# 2. Launch the container +./dev_container launch --docker_opts "-v $HOME/rti_connext_dds-7.3.0:/opt/rti.com/rti_connext_dds-7.3.0/" +# 3. Build the application +./run build dds_h264 +# Continue to the next section to run the application with the publisher. +# Open a new terminal to repeat step #2 and launch a new container for the subscriber. +``` + + + +## Running the Application + +Both a publisher and subscriber process must be launched to see the result of +writing to and reading the video stream from DDS, respectively. + +To run the publisher process, use the `-p` option: + +```sh +$ ./run launch dds_h264 --extra_args "-p" +``` + +To run the subscriber process, use the `-s` option: + +```sh +$ ./run launch dds_h264 --extra_args "-s" +``` + +If running the application generates an error about `RTI Connext DDS No Source +for License information`, ensure that the RTI Connext license has either been +installed system-wide or the `NDDSHOME` environment variable has been set to +point to your user's RTI Connext installation path. + +Note that these processes can be run on the same or different systems, so long as they +are both discoverable by the other via RTI Connext. If the processes are run on +different systems then they will communicate using UDPv4, for which optimizations have +been defined in the default `qos_profiles.xml` file. These optimizations include +increasing the buffer size used by RTI Connext for network sockets, and so the systems +running the application must also be configured to increase their maximum send and +receive socket buffer sizes. This can be done by running the `set_socket_buffer_sizes.sh` +script within this directory: + +```sh +$ ./set_socket_buffer_sizes.sh +``` + +For more details, see the [RTI Connext Guide to Improve DDS Network Performance on Linux Systems](https://community.rti.com/howto/improve-rti-connext-dds-network-performance-linux-systems) + +The QoS profiles used by the application can also be modified by editing the +`qos_profiles.xml` file in the application directory. For more information about modifying +the QoS profiles, see the [RTI Connext Basic QoS](https://community.rti.com/static/documentation/connext-dds/7.3.0/doc/manuals/connext_dds_professional/getting_started_guide/cpp11/intro_qos.html) +tutorial or the [RTI Connext QoS Reference Guide](https://community.rti.com/static/documentation/connext-dds/7.3.0/doc/manuals/connext_dds_professional/qos_reference/index.htm). + +## Benchmarks + +We collected latency benchmark results from the log output of the subscriber. The benchmark is conducted on x86_64 with NVIDIA ADA6000 GPU. + +### Single System Setup + +**Source**: Video Stream Replayer +**Resolution**: 854x480 + +| Configuration | FPS | AVg. Transfer Time | Jitter | Input Size | Avg. Encoded Size | +|-------------------|---------|--------------------|---------|------------|-------------------| +| `realtime: false` | 685.068 | 1.576ms | 0.840ms | 1,229,760 | 25,053 | +| `realtime: true` | 30.049 | 0.150ms | 0.059ms | 1,229,760 | 26,800 | + +**Source** : V4L2 Camera +**Frame Rate**: 30 + +| Resolution | FPS | AVg. Transfer Time | Jitter | Input Size | Avg. Encoded Size | +|------------|--------|--------------------|---------|------------|-------------------| +| 640x480 | 30.169 | 0.098ms | 0.030ms | 921,600 | 16,176 | +| 1920x1080 | 30.281 | 0.104ms | 0.040ms | 6,220,800 | 86,222 | + +### Multiple System Setup + +The two systems are connected via VPNin this scenario. + +**Average Ping Latency**: 22.529ms + + +**Source**: Video Stream Replayer +**Resolution**: 854x480 + +| Configuration | FPS | AVg. Transfer Time | Jitter | Input Size | Avg. Encoded Size | +|-------------------|---------|--------------------|---------|------------|-------------------| +| `realtime: false` | 607.581 | 12.278ms | 3.595ms | 1,229,760 | 22,679 | +| `realtime: true` | 30.050 | 12.937ms | 3.856ms | 1,229,760 | 26,741 | + + +**Source** : V4L2 Camera +**Frame Rate**: 30 + +| Resolution | FPS | AVg. Transfer Time | Jitter | Input Size | Avg. Encoded Size | +|------------|--------|--------------------|---------|------------|-------------------| +| 640x480 | 30.047 | 10.771ms | 4.621ms | 921,600 | 11,571 | +| 1920x1080 | 28.877 | 14.322ms | 3.420ms | 6,220,800 | 52,273 | diff --git a/applications/dds/dds_h264/dds_h264.cpp b/applications/dds/dds_h264/dds_h264.cpp new file mode 100644 index 0000000000..ecda42940d --- /dev/null +++ b/applications/dds/dds_h264/dds_h264.cpp @@ -0,0 +1,384 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "holoscan/core/resources/gxf/gxf_component_resource.hpp" +#include "holoscan/operators/gxf_codelet/gxf_codelet.hpp" + +#include "dds_video_publisher.hpp" +#include "dds_video_subscriber.hpp" + +#include "append_timestamp.hpp" +#include "tensor_to_video_buffer.hpp" +#include "video_encoder.hpp" + +#include +#include + +// Import h.264 GXF codelets and components as Holoscan operators and resources +// Starting with Holoscan SDK v2.1.0, importing GXF codelets/components as Holoscan operators/ +// resources can be done using the HOLOSCAN_WRAP_GXF_CODELET_AS_OPERATOR and +// HOLOSCAN_WRAP_GXF_COMPONENT_AS_RESOURCE macros. This new feature allows using GXF codelets +// and components in Holoscan applications without writing custom class wrappers (for C++) and +// Python wrappers (for Python) for each GXF codelet and component. +// For the VideoEncoderRequestOp class, since it needs to override the setup() to provide custom +// parameters and override the initialize() to register custom converters, it requires a custom +// class that extends the holoscan::ops::GXFCodeletOp class. + +// The VideoEncoderResponseOp implements nvidia::gxf::VideoEncoderResponse and handles the output +// of the encoded YUV frames. +// Parameters: +// - pool (std::shared_ptr): Memory pool for allocating output data. +// - videoencoder_context (std::shared_ptr): Encoder context +// handle. +// - outbuf_storage_type (uint32_t): Output Buffer Storage(memory) type used by this allocator. +// Can be 0: kHost, 1: kDevice. Default: 1. +HOLOSCAN_WRAP_GXF_CODELET_AS_OPERATOR(VideoEncoderResponseOp, "nvidia::gxf::VideoEncoderResponse") + +// The VideoEncoderContext implements nvidia::gxf::VideoEncoderContext and holds common variables +// and underlying context. +// Parameters: +// - async_scheduling_term (std::shared_ptr): Asynchronous +// scheduling condition required to get/set event state. +HOLOSCAN_WRAP_GXF_COMPONENT_AS_RESOURCE(VideoEncoderContext, "nvidia::gxf::VideoEncoderContext") + +// The VideoDecoderResponseOp implements nvidia::gxf::VideoDecoderResponse and handles the output +// of the decoded H264 bit stream. +// Parameters: +// - pool (std::shared_ptr): Memory pool for allocating output data. +// - outbuf_storage_type (uint32_t): Output Buffer Storage(memory) type used by this allocator. +// Can be 0: kHost, 1: kDevice. +// - videodecoder_context (std::shared_ptr): Decoder context +// Handle. +HOLOSCAN_WRAP_GXF_CODELET_AS_OPERATOR(VideoDecoderResponseOp, "nvidia::gxf::VideoDecoderResponse") + +// The VideoDecoderRequestOp implements nvidia::gxf::VideoDecoderRequest and handles the input +// for the H264 bit stream decode. +// Parameters: +// - inbuf_storage_type (uint32_t): Input Buffer storage type, 0:kHost, 1:kDevice. +// - async_scheduling_term (std::shared_ptr): Asynchronous +// scheduling condition. +// - videodecoder_context (std::shared_ptr): Decoder +// context Handle. +// - codec (uint32_t): Video codec to use, 0:H264, only H264 supported. Default:0. +// - disableDPB (uint32_t): Enable low latency decode, works only for IPPP case. +// - output_format (std::string): VidOutput frame video format, nv12pl and yuv420planar are +// supported. +HOLOSCAN_WRAP_GXF_CODELET_AS_OPERATOR(VideoDecoderRequestOp, "nvidia::gxf::VideoDecoderRequest") + +// The VideoDecoderContext implements nvidia::gxf::VideoDecoderContext and holds common variables +// and underlying context. +// Parameters: +// - async_scheduling_term (std::shared_ptr): Asynchronous +// scheduling condition required to get/set event state. +HOLOSCAN_WRAP_GXF_COMPONENT_AS_RESOURCE(VideoDecoderContext, "nvidia::gxf::VideoDecoderContext") + +/** + * @brief Application to publish a V4L2 video stream to DDS. + */ +class StreamingServer : public holoscan::Application { + public: + explicit StreamingServer(std::string video_path) : video_path_(video_path) {} + + void configure_extension() { + auto extension_manager = executor().extension_manager(); + extension_manager->load_extension("libgxf_videoencoder.so"); + extension_manager->load_extension("libgxf_videoencoderio.so"); + } + + void compose() override { + using namespace holoscan; + + configure_extension(); + + uint32_t width = 854; + uint32_t height = 480; + uint32_t source_block_size = width * height * 3 * 4; + uint32_t source_num_blocks = 2; + + auto source = from_config("source").as(); + + std::shared_ptr source_operator; + std::shared_ptr format_converter_rgba8888; + std::shared_ptr format_converter_rgb888; + + if (source == "replayer") { + HOLOSCAN_LOG_INFO("Using video path: {}", video_path_); + source_operator = make_operator( + "replayer", + Arg("allocator", make_resource("video_replayer_allocator")), + from_config("replayer"), + Arg("directory", video_path_)); + + HOLOSCAN_LOG_INFO("Using format converter"); + format_converter_rgb888 = make_operator( + "format_converter", + from_config("format_converter_rgb888"), + Arg("pool") = + make_resource("pool", 1, source_block_size, source_num_blocks)); + } else if (source == "v4l2") { + HOLOSCAN_LOG_INFO("Using v4l2"); + width = from_config("v4l2.width").as(); + height = from_config("v4l2.height").as(); + source_block_size = width * height * 3 * 4; + source_num_blocks = 2; + source_operator = make_operator( + "v4l2", + from_config("v4l2"), + Arg("allocator") = make_resource("pool")); + + format_converter_rgba8888 = make_operator( + "format_converter_rgba8888", + from_config("format_converter_rgba8888"), + Arg("pool") = + make_resource("pool", 1, source_block_size, source_num_blocks)); + + format_converter_rgb888 = make_operator( + "format_converter_rgb888", + from_config("format_converter_rgb888"), + Arg("pool") = + make_resource("pool", 1, source_block_size, source_num_blocks)); + } else { + HOLOSCAN_LOG_ERROR("Invalid source: {}", source); + throw std::runtime_error("Invalid source configuration: " + source); + } + HOLOSCAN_LOG_INFO("Video width: {}, height: {}", width, height); + + auto video_replayer = make_operator( + "replayer", + Arg("allocator", make_resource("video_replayer_allocator")), + from_config("replayer"), + Arg("directory", video_path_)); + + auto tensor_to_video_buffer = make_operator( + "tensor_to_video_buffer", from_config("tensor_to_video_buffer")); + + auto encoder_async_condition = make_condition("encoder_async_condition"); + auto video_encoder_context = + make_resource(Arg("scheduling_term") = encoder_async_condition); + + auto video_encoder_request = make_operator( + "video_encoder_request", + Arg("input_width", width), + Arg("input_height", height), + from_config("video_encoder_request"), + Arg("videoencoder_context") = video_encoder_context); + + auto video_encoder_response = + make_operator("video_encoder_response", + from_config("video_encoder_response"), + Arg("pool") = make_resource( + "pool", 1, source_block_size, source_num_blocks), + Arg("videoencoder_context") = video_encoder_context); + auto video_publisher = make_operator( + "video_publisher", + Arg("width", width), + Arg("height", height), + from_config("video_publisher")); + + auto holoviz = make_operator("holoviz", + Arg("window_title") = "DDS Publisher", + Arg("width", width), + Arg("height", height), + from_config("holoviz")); + + if (source == "replayer") { + add_flow(source_operator, format_converter_rgb888, {{"output", "source_video"}}); + add_flow(source_operator, holoviz, {{"output", "receivers"}}); + } else if (source == "v4l2") { + add_flow(source_operator, format_converter_rgba8888, {{"signal", "source_video"}}); + add_flow(format_converter_rgba8888, format_converter_rgb888, {{"tensor", "source_video"}}); + add_flow(format_converter_rgba8888, holoviz, {{"tensor", "receivers"}}); + } + add_flow(format_converter_rgb888, tensor_to_video_buffer, {{"tensor", "in_tensor"}}); + add_flow(tensor_to_video_buffer, video_encoder_request, {{"out_video_buffer", "input_frame"}}); + add_flow(video_encoder_response, video_publisher, {{"output_transmitter", "input"}}); + } + + private: + uint32_t domain_id_; + uint32_t stream_id_; + std::string video_path_; +}; + +/** + * @brief Application to render a DDS video stream (published by the DDSVideoPublisher) + * and shapes (published by the RTI Connext Shapes Demo) to Holoviz. + */ +class StreamingClient : public holoscan::Application { + public: + explicit StreamingClient() {} + + void configure_extension() { + auto extension_manager = executor().extension_manager(); + extension_manager->load_extension("libgxf_videodecoder.so"); + extension_manager->load_extension("libgxf_videodecoderio.so"); + } + + void compose() override { + using namespace holoscan; + + configure_extension(); + + uint32_t width = 854; + uint32_t height = 480; + uint32_t source_block_size = width * height * 3 * 4; + uint32_t source_num_blocks = 2; + + if (from_config("source").as() == "v4l2") { + width = from_config("v4l2.width").as(); + height = from_config("v4l2.height").as(); + } + + HOLOSCAN_LOG_INFO("Video width: {}, height: {}", width, height); + + std::shared_ptr allocator = make_resource("pool"); + + // DDS Video Subscriber + auto video_subscriber = make_operator( + "video_subscriber", + Arg("allocator", allocator), + from_config("video_subscriber")); + + auto append_timestamp = make_operator("append_timestamp"); + + auto response_condition = make_condition("response_condition"); + auto video_decoder_context = + make_resource(Arg("async_scheduling_term") = response_condition); + + auto request_condition = make_condition("request_condition"); + auto video_decoder_request = + make_operator("video_decoder_request", + from_config("video_decoder_request"), + Arg("async_scheduling_term") = request_condition, + Arg("videodecoder_context") = video_decoder_context); + + auto video_decoder_response = + make_operator("video_decoder_response", + from_config("video_decoder_response"), + Arg("pool") = make_resource( + "pool", 1, source_block_size, source_num_blocks), + Arg("videodecoder_context") = video_decoder_context); + + // Holoviz (initialize with the default input spec for the video stream) + auto holoviz = make_operator("holoviz", + Arg("window_title") = "DDS Subscriber", + Arg("width", width), + Arg("height", height), + from_config("holoviz")); + + add_flow(video_subscriber, append_timestamp, {{"output", "in_tensor"}}); + add_flow(append_timestamp, video_decoder_request, {{"out_tensor", "input_frame"}}); + add_flow(video_decoder_response, holoviz, {{"output_transmitter", "receivers"}}); + } +}; + +void usage() { + std::cout << "Usage: dds_video {-p | -s} [options]" << std::endl + << std::endl + << "Options" << std::endl + << " -p, --publisher Run as a publisher" << std::endl + << " -s, --subscriber Run as a subscriber" << std::endl + << " -v VIDEO_PATH, --video=VIDEO_PATH Use the specified video path" + << std::endl + << " -c CONFIG_PATH, --config=CONFIG_PATH Use the specified config path" + << std::endl; +} + +int main(int argc, char** argv) { + bool publisher = false; + bool subscriber = false; + std::string video_path = ""; + std::string config_path = ""; + struct option long_options[] = {{"help", no_argument, 0, 'h'}, + {"publisher", no_argument, 0, 'p'}, + {"subscriber", no_argument, 0, 's'}, + {"video", required_argument, 0, 'v'}, + {"config", optional_argument, 0, 'c'}, + {0, 0, 0, 0}}; + + while (true) { + int option_index = 0; + + const int c = getopt_long(argc, argv, "hpsi:d:v:c::", long_options, &option_index); + if (c == -1) { + break; + } + + const std::string argument(optarg ? optarg : ""); + switch (c) { + case 'h': + usage(); + return 0; + case 'p': + publisher = true; + break; + case 's': + subscriber = true; + break; + case 'v': + video_path = argument; + break; + case 'c': + config_path = argument; + break; + default: + HOLOSCAN_LOG_ERROR("Unhandled option '{}'", static_cast(c)); + } + } + + if (!publisher && !subscriber) { + HOLOSCAN_LOG_ERROR("Must provide either -p or -s for publisher or subscriber, respectively"); + usage(); + return -1; + } + + if (publisher && video_path.empty()) { + HOLOSCAN_LOG_ERROR("Video path is required when running as publisher"); + usage(); + return -1; + } + + if (config_path.empty()) { + auto exe_path = std::filesystem::canonical(argv[0]).parent_path(); + config_path = exe_path / "dds_h264.yaml"; + HOLOSCAN_LOG_INFO("No config path provided, using default config: {}", config_path); + } else { + auto canonical_path = std::filesystem::canonical(config_path); + config_path = canonical_path.string(); + HOLOSCAN_LOG_INFO("Using config path: {}", config_path); + } + + HOLOSCAN_LOG_INFO("Starting {}...", publisher ? "publisher" : "subscriber"); + + if (publisher) { + auto app = holoscan::make_application(video_path); + app->config(config_path); + app->run(); + } else if (subscriber) { + auto app = holoscan::make_application(); + app->config(config_path); + app->run(); + } + + return 0; +} diff --git a/applications/dds/dds_h264/dds_h264.yaml b/applications/dds/dds_h264/dds_h264.yaml new file mode 100644 index 0000000000..12b0ba05ff --- /dev/null +++ b/applications/dds/dds_h264/dds_h264.yaml @@ -0,0 +1,90 @@ +%YAML 1.2 +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +source: "replayer" # default: "replayer". "v4l2" or "replayer" + +replayer: + basename: "surgical_video" + frame_rate: 0 # as specified in timestamps + repeat: true # default: false + realtime: true # default: false + count: 0 # default: 0 (no frame count restriction) + +v4l2: + device: "/dev/video0" + pixel_format: "auto" + width: 640 + height: 480 + frame_rate: 30 + +format_converter_rgba8888: + in_dtype: "rgba8888" + out_dtype: "rgb888" + +format_converter_rgb888: + in_dtype: "rgb888" + out_dtype: "yuv420" + +tensor_to_video_buffer: + video_format: "yuv420" + +video_encoder_request: + inbuf_storage_type: 1 + codec: 0 + input_format: yuv420planar + profile: 2 + bitrate: 20000000 + framerate: 30 + config: pframe_cqp + rate_control_mode: 0 + qp: 20 + iframe_interval: 1 + +video_encoder_response: + outbuf_storage_type: 1 + +video_decoder_request: + inbuf_storage_type: 1 + output_format: "nv12pl" + +video_decoder_response: + outbuf_storage_type: 1 + +decoder_output_format_converter: + in_dtype: "nv12" + out_dtype: "rgb888" + +holoviz: + tensors: + - name: "" + type: color + opacity: 1.0 + priority: 0 + +video_subscriber: + fps_report_interval: 1.0 + log_missing_frames: true + reader_qos: "HoloscanDDSDataFlow::Video" + participant_qos: "HoloscanDDSTransport::SHMEM+LAN" + stream_id: 1 + domain_id: 1 + log_frame_warning_threshold: 0 + +video_publisher: + stream_id: 1 + domain_id: 1 + participant_qos: "HoloscanDDSTransport::SHMEM+LAN" + writer_qos: "HoloscanDDSDataFlow::Video" \ No newline at end of file diff --git a/applications/dds/dds_h264/metadata.json b/applications/dds/dds_h264/metadata.json new file mode 100644 index 0000000000..1e0ef7e005 --- /dev/null +++ b/applications/dds/dds_h264/metadata.json @@ -0,0 +1,44 @@ +{ + "application": { + "name": "DDS Video: Real-time Video Streaming with RTI Connext & H.264", + "authors": [ + { + "name": "Victor Chang", + "affiliation": "NVIDIA" + } + ], + "language": "C++", + "version": "1.0", + "changelog": { + "1.0": "Initial Release" + }, + "dockerfile": "applications/dds/dds_h264/Dockerfile", + "holoscan_sdk": { + "minimum_required_version": "3.0.0", + "tested_versions": [ + "3.0.0" + ] + }, + "platforms": [ + "x86_64", + "aarch64" + ], + "tags": ["Networking and Distributed Computing", "DDS", "RTI Connext", "Video", "Visualization"], + "ranking": 2, + "dependencies": { + "packages": [ + { + "name": "RTI Connext", + "author": "Real-Time Innovations", + "license": "Closed", + "version": "7.3.0", + "url": "https://www.rti.com/products" + } + ] + }, + "run": { + "command": "./dds_h264 --video /endoscopy", + "workdir": "holohub_app_bin" + } + } +} \ No newline at end of file diff --git a/applications/dds/dds_h264/qos_profiles.xml b/applications/dds/dds_h264/qos_profiles.xml new file mode 100644 index 0000000000..923a858dc8 --- /dev/null +++ b/applications/dds/dds_h264/qos_profiles.xml @@ -0,0 +1,94 @@ + + + + + + + + + + + Holoscan DDS Video + + + + SHMEM | UDPv4 + + 4194304 + 10 + 41943040 + + + + 4194304 + 4194304 + + + + + + + + + + + BuiltinQosSnippetLib::Optimization.Discovery.Common + BuiltinQosSnippetLib::Optimization.Discovery.Endpoint.Fast + BuiltinQosSnippetLib::Optimization.ReliabilityProtocol.Common + + + + + + + + + + + + + VideoFrameWriter + + + + + VideoFrameReader + + + + + + diff --git a/applications/h264/CMakeLists.txt b/applications/h264/CMakeLists.txt index e6b3271a9a..5f36a73f81 100644 --- a/applications/h264/CMakeLists.txt +++ b/applications/h264/CMakeLists.txt @@ -17,8 +17,10 @@ add_holohub_application(h264_endoscopy_tool_tracking DEPENDS OPERATORS video_encoder tensor_to_video_buffer lstm_tensor_rt_inference - tool_tracking_postprocessor - ) + tool_tracking_postprocessor) + +add_holohub_application(h264_video_encode_decode DEPENDS + OPERATORS video_decoder tensor_to_video_buffer video_read_bitstream append_timestamp) add_holohub_application(h264_video_decode DEPENDS OPERATORS video_decoder video_read_bitstream) diff --git a/applications/h264/README.md b/applications/h264/README.md index 40f7b3e095..6d93790031 100644 --- a/applications/h264/README.md +++ b/applications/h264/README.md @@ -21,3 +21,12 @@ operator for reading H.264 elementary stream input and uses Holoviz operator for rendering decoded data to the native window. [Building and Running the H.264 Video Decode Application](./h264_video_decode//README.md) + + +## H.264 Video Encode Decode Application + +Here's a simple example showing how to chain the H.264 video encode and decode operators together. +In a streaming setup, one app might use the encoder, while another app receives the encoded frames +and decodes each one. + +[Building and Running the H.264 Video Decode Application](./h264_video_encode_decode/README.md) diff --git a/applications/h264/h264_video_encode_decode/CMakeLists.txt b/applications/h264/h264_video_encode_decode/CMakeLists.txt new file mode 100644 index 0000000000..728c803cc3 --- /dev/null +++ b/applications/h264/h264_video_encode_decode/CMakeLists.txt @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.20) +project(h264_video_encode_decode_apps LANGUAGES NONE) + +add_subdirectory(python) diff --git a/applications/h264/h264_video_encode_decode/README.md b/applications/h264/h264_video_encode_decode/README.md new file mode 100644 index 0000000000..3fafd48c64 --- /dev/null +++ b/applications/h264/h264_video_encode_decode/README.md @@ -0,0 +1,69 @@ +# H.264 Video Encode Decode + +This is a minimal reference application demonstrating usage of H.264 video +decode operators. This application makes use of H.264 elementary stream reader +operator for reading H.264 elementary stream input and uses Holoviz operator +for rendering decoded data to the native window. + +_The H.264 video decode operators do not adjust framerate as it reads the +elementary stream input. As a result the video stream can be displayed as +quickly as the decoding can be performed. This application uses +`PeriodicCondition` to play video at the same speed as the source video._ + +## Requirements + +This application is configured to use H.264 elementary stream from endoscopy +sample data as input. To use any other stream, the filename / path for the +input file can be specified in the 'h264_video_decode.yaml' file. + +### Data + +[📦️ (NGC) Sample App Data for AI-based Endoscopy Tool Tracking](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/clara-holoscan/resources/holoscan_endoscopy_sample_data) + +The data is automatically downloaded when building the application. + +## Building and Running H.264 Endoscopy Tool Tracking Application + +* Building and running the application from the top level Holohub directory: + +```bash +# C++ version +./dev_container build_and_run h264_video_decode --docker_file applications/h264/Dockerfile --language cpp + +# Python version +./dev_container build_and_run h264_video_decode --docker_file applications/h264/Dockerfile --language python + +``` + +Important: on aarch64, applications also need tegra folder mounted inside the container and +the `LD_LIBRARY_PATH` environment variable should be updated to include +tegra folder path. + +Open and edit the [Dockerfile](../Dockerfile) and uncomment line 66: + +```bash +# Uncomment the following line for aarch64 support +ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/aarch64-linux-gnu/tegra/ +``` + + +## Dev Container + +To start the the Dev Container, run the following command from the root directory of Holohub: + +```bash +./dev_container vscode h264 +``` + +### VS Code Launch Profiles + +#### C++ + +Use the **(gdb) h264_video_decode/cpp** launch profile to run and debug the C++ application. + +#### Python + +There are a couple of launch profiles configured for this application: + +1. **(debugpy) h264_video_decode/python**: Launch the h.264 Video Decode application with the ability to debug Python code. +2. **(pythoncpp) h264_video_decode/python**: Launch the h.264 Video Decode application with the ability to debug both Python and C++ code. diff --git a/applications/h264/h264_video_encode_decode/python/CMakeLists.txt b/applications/h264/h264_video_encode_decode/python/CMakeLists.txt new file mode 100644 index 0000000000..f83ef3b78e --- /dev/null +++ b/applications/h264/h264_video_encode_decode/python/CMakeLists.txt @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +cmake_minimum_required(VERSION 3.20) + +find_package(holoscan 2.1.0 REQUIRED CONFIG + PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install") + +# Add testing +if(BUILD_TESTING) + add_test(NAME h264_video_decode_python_test + COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/h264_video_decode.py + --config ${CMAKE_CURRENT_SOURCE_DIR}/h264_video_decode.yaml + --data ${HOLOHUB_DATA_DIR}/endoscopy + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + + set_property(TEST h264_video_decode_python_test PROPERTY ENVIRONMENT + "PYTHONPATH=${GXF_LIB_DIR}/../python/lib:${CMAKE_BINARY_DIR}/python/lib") + + set_tests_properties(h264_video_decode_python_test PROPERTIES + PASS_REGULAR_EXPRESSION "Deactivating Graph" + FAIL_REGULAR_EXPRESSION "[^a-z]Error;ERROR;Failed") + + # For aarch64 LD_LIBRARY_PATH needs to be set + if(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL arm64) + set_property(TEST h264_video_decode_python_test APPEND PROPERTY ENVIRONMENT + "LD_LIBRARY_PATH=/usr/lib/aarch64-linux-gnu/tegra/") + endif() +endif() diff --git a/applications/h264/h264_video_encode_decode/python/h264_video_encode_decode.py b/applications/h264/h264_video_encode_decode/python/h264_video_encode_decode.py new file mode 100644 index 0000000000..8730c7a00d --- /dev/null +++ b/applications/h264/h264_video_encode_decode/python/h264_video_encode_decode.py @@ -0,0 +1,316 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from argparse import ArgumentParser + +try: + from holoscan.conditions import AsynchronousCondition +except ImportError as e: + raise ImportError( + "This example requires Holoscan SDK >= 2.1.0 so AsynchronousCondition is available." + ) from e +from holoscan.core import Application, Tracker +from holoscan.gxf import load_extensions +from holoscan.operators import FormatConverterOp, GXFCodeletOp, HolovizOp, VideoStreamReplayerOp +from holoscan.resources import ( + BlockMemoryPool, + GXFComponentResource, + MemoryStorageType, + RMMAllocator, +) + +from holohub.append_timestamp import AppendTimestampOp +from holohub.tensor_to_video_buffer import TensorToVideoBufferOp + +# Import h.264 GXF codelets and components as Holoscan operators and resources +# Starting with Holoscan SDK v2.1.0, importing GXF codelets/components as Holoscan operators/ +# resources can be done by extending the GXFCodeletOp class and the GXFComponentResource class. +# This new feature allows GXF codelets and components in Holoscan applications without writing +# custom class wrappers in C++ and Python wrappers for each GXF codelet and component. + + +# The VideoDecoderResponseOp implements nvidia::gxf::VideoDecoderResponse and handles the output +# of the decoded H264 bit stream. +# Parameters: +# - pool (Allocator): Memory pool for allocating output data. +# - outbuf_storage_type (int): Output Buffer Storage(memory) type used by this allocator. +# Can be 0: kHost, 1: kDevice. +# - videodecoder_context (VideoDecoderContext): Decoder context +# Handle. +class VideoDecoderResponseOp(GXFCodeletOp): + def __init__(self, fragment, *args, **kwargs): + super().__init__(fragment, "nvidia::gxf::VideoDecoderResponse", *args, **kwargs) + + +# The VideoDecoderRequestOp implements nvidia::gxf::VideoDecoderRequest and handles the input +# for the H264 bit stream decode. +# Parameters: +# - inbuf_storage_type (int): Input Buffer storage type, 0:kHost, 1:kDevice. +# - async_scheduling_term (AsynchronousCondition): Asynchronous scheduling condition. +# - videodecoder_context (VideoDecoderContext): Decoder context Handle. +# - codec (int): Video codec to use, 0:H264, only H264 supported. Default:0. +# - disableDPB (int): Enable low latency decode, works only for IPPP case. +# - output_format (str): VidOutput frame video format, nv12pl and yuv420planar are supported. +class VideoDecoderRequestOp(GXFCodeletOp): + def __init__(self, fragment, *args, **kwargs): + super().__init__(fragment, "nvidia::gxf::VideoDecoderRequest", *args, **kwargs) + + +# The VideoDecoderContext implements nvidia::gxf::VideoDecoderContext and holds common variables +# and underlying context. +# Parameters: +# - async_scheduling_term (AsynchronousCondition): Asynchronous scheduling condition required to get/set event state. +class VideoDecoderContext(GXFComponentResource): + def __init__(self, fragment, *args, **kwargs): + super().__init__(fragment, "nvidia::gxf::VideoDecoderContext", *args, **kwargs) + + +# The VideoReadBitstreamOp implements nvidia::gxf::VideoReadBitStream and reads h.264 video files +# from the disk at the specified input file path. +# Parameters: +# - input_file_path (str): Path to image file +# - pool (Allocator): Memory pool for allocating output data +# - outbuf_storage_type (int): Output Buffer storage type, 0:kHost, 1:kDevice +class VideoReadBitstreamOp(GXFCodeletOp): + def __init__(self, fragment, *args, **kwargs): + super().__init__(fragment, "nvidia::gxf::VideoReadBitStream", *args, **kwargs) + + +# The VideoWriteBitstreamOp implements nvidia::gxf::VideoWriteBitstream and writes bit stream to +# the disk at specified output path. +# Parameters: +# - output_video_path (str): The file path of the output video +# - frame_width (int): The width of the output video +# - frame_height (int): The height of the output video +# - inbuf_storage_type (int): Input Buffer storage type, 0:kHost, 1:kDevice +class VideoWriteBitstreamOp(GXFCodeletOp): + def __init__(self, fragment, *args, **kwargs): + super().__init__(fragment, "nvidia::gxf::VideoWriteBitstream", *args, **kwargs) + + +# The VideoEncoderResponseOp implements nvidia::gxf::VideoEncoderResponse and handles the output +# of the encoded YUV frames. +# Parameters: +# - pool (Allocator): Memory pool for allocating output data. +# - videoencoder_context (VideoEncoderContext): Encoder context handle. +# - outbuf_storage_type (int): Output Buffer Storage(memory) type used by this allocator. +# Can be 0: kHost, 1: kDevice. Default: 1. +class VideoEncoderResponseOp(GXFCodeletOp): + def __init__(self, fragment, *args, **kwargs): + super().__init__(fragment, "nvidia::gxf::VideoEncoderResponse", *args, **kwargs) + + +# The VideoEncoderContext implements nvidia::gxf::VideoEncoderContext and holds common variables +# and underlying context. +# Parameters: +# - async_scheduling_term (AsynchronousCondition): Asynchronous scheduling condition required to get/set event state. +class VideoEncoderContext(GXFComponentResource): + def __init__(self, fragment, *args, **kwargs): + super().__init__(fragment, "nvidia::gxf::VideoEncoderContext", *args, **kwargs) + + +# The VideoEncoderRequestOp implements nvidia::gxf::VideoEncoderRequest and handles the input for +# encoding YUV frames to H264 bit stream. +# Refer to operators/video_encoder/video_encoder_request/README.md for details +class VideoEncoderRequestOp(GXFCodeletOp): + def __init__(self, fragment, *args, **kwargs): + super().__init__(fragment, "nvidia::gxf::VideoEncoderRequest", *args, **kwargs) + + +class H264VideoEncodeDecodeApp(Application): + def __init__(self, data): + """Initialize the H264 video decode application""" + super().__init__() + + # set name + self.name = "H264 video encode decode App" + + if (data is None) or (data == "none"): + data = os.environ.get("HOLOHUB_DATA_PATH", "../data") + + self.sample_data_path = data + + def compose(self): + width = 854 + height = 480 + source_block_size = width * height * 3 * 4 + source_num_blocks = 2 + + video_dir = self.sample_data_path + if not os.path.exists(video_dir): + raise ValueError(f"Could not find video data: {video_dir=}") + + source = VideoStreamReplayerOp( + self, + name="replayer", + directory=video_dir, + allocator=RMMAllocator(self, name="video_replayer_allocator"), + **self.kwargs("replayer"), + ) + + format_converter = FormatConverterOp( + self, + name="format_converter", + pool=BlockMemoryPool( + self, + name="pool", + storage_type=MemoryStorageType.DEVICE, + block_size=source_block_size, + num_blocks=source_num_blocks, + ), + **self.kwargs("format_converter"), + ) + + tensor_to_video_buffer = TensorToVideoBufferOp( + self, name="tensor_to_video_buffer", **self.kwargs("tensor_to_video_buffer") + ) + encoder_async_condition = AsynchronousCondition(self, "encoder_async_condition") + video_encoder_context = VideoEncoderContext(self, scheduling_term=encoder_async_condition) + + video_encoder_request = VideoEncoderRequestOp( + self, + name="video_encoder_request", + videoencoder_context=video_encoder_context, + **self.kwargs("video_encoder_request"), + ) + video_encoder_response = VideoEncoderResponseOp( + self, + name="video_encoder_response", + pool=BlockMemoryPool( + self, + name="pool", + storage_type=MemoryStorageType.DEVICE, + block_size=source_block_size, + num_blocks=source_num_blocks, + ), + videoencoder_context=video_encoder_context, + **self.kwargs("video_encoder_response"), + ) + + append_timestamp = AppendTimestampOp(self, name="append_timestamp") + + response_condition = AsynchronousCondition(self, "response_condition") + video_decoder_context = VideoDecoderContext(self, async_scheduling_term=response_condition) + + request_condition = AsynchronousCondition(self, "request_condition") + video_decoder_request = VideoDecoderRequestOp( + self, + name="video_decoder_request", + async_scheduling_term=request_condition, + videodecoder_context=video_decoder_context, + **self.kwargs("video_decoder_request"), + ) + + video_decoder_response = VideoDecoderResponseOp( + self, + name="video_decoder_response", + pool=BlockMemoryPool( + self, + name="pool", + storage_type=MemoryStorageType.DEVICE, + block_size=source_block_size, + num_blocks=source_num_blocks, + ), + videodecoder_context=video_decoder_context, + **self.kwargs("video_decoder_response"), + ) + + decoder_output_format_converter = FormatConverterOp( + self, + name="decoder_output_format_converter", + pool=BlockMemoryPool( + self, + name="pool", + storage_type=MemoryStorageType.DEVICE, + block_size=source_block_size, + num_blocks=source_num_blocks, + ), + **self.kwargs("decoder_output_format_converter"), + ) + + visualizer_allocator = BlockMemoryPool( + self, + name="allocator", + storage_type=MemoryStorageType.DEVICE, + block_size=source_block_size, + num_blocks=source_num_blocks, + ) + visualizer = HolovizOp( + self, + name="holoviz", + width=width, + height=height, + enable_render_buffer_input=False, + enable_render_buffer_output=False, + allocator=visualizer_allocator, + **self.kwargs("holoviz"), + ) + + self.add_flow(source, format_converter, {("output", "source_video")}) + self.add_flow(format_converter, tensor_to_video_buffer, {("tensor", "in_tensor")}) + self.add_flow( + tensor_to_video_buffer, video_encoder_request, {("out_video_buffer", "input_frame")} + ) + self.add_flow( + video_encoder_response, append_timestamp, {("output_transmitter", "in_tensor")} + ) + self.add_flow(append_timestamp, video_decoder_request, {("out_tensor", "input_frame")}) + self.add_flow( + video_decoder_response, + decoder_output_format_converter, + {("output_transmitter", "source_video")}, + ) + self.add_flow(decoder_output_format_converter, visualizer, {("tensor", "receivers")}) + + +if __name__ == "__main__": + # Parse args + parser = ArgumentParser(description="H264 video decode demo application.") + + parser.add_argument( + "-c", + "--config", + default="none", + help=("Set config path to override the default config file location"), + ) + parser.add_argument( + "-d", + "--data", + default="none", + help=("Set the data path"), + ) + args = parser.parse_args() + + if args.config == "none": + config_file = os.path.join(os.path.dirname(__file__), "h264_video_encode_decode.yaml") + else: + config_file = args.config + + app = H264VideoEncodeDecodeApp(data=args.data) + + context = app.executor.context_uint64 + exts = [ + "libgxf_videodecoder.so", + "libgxf_videodecoderio.so", + "libgxf_videoencoder.so", + "libgxf_videoencoderio.so", + ] + load_extensions(context, exts) + + app.config(config_file) + with Tracker(app) as trackers: + app.run() + trackers.print() diff --git a/applications/h264/h264_video_encode_decode/python/h264_video_encode_decode.yaml b/applications/h264/h264_video_encode_decode/python/h264_video_encode_decode.yaml new file mode 100644 index 0000000000..fd4a906de0 --- /dev/null +++ b/applications/h264/h264_video_encode_decode/python/h264_video_encode_decode.yaml @@ -0,0 +1,64 @@ +%YAML 1.2 +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +replayer: + basename: "surgical_video" + frame_rate: 0 # as specified in timestamps + repeat: false # default: false + realtime: false # default: false + count: 0 # default: 0 (no frame count restriction) + +format_converter: + in_dtype: "rgb888" + out_dtype: "yuv420" + +tensor_to_video_buffer: + video_format: "yuv420" + +video_encoder_request: + inbuf_storage_type: 1 + codec: 0 + input_width: 854 + input_height: 480 + input_format: yuv420planar + profile: 2 + bitrate: 20000000 + framerate: 30 + config: pframe_cqp + rate_control_mode: 0 + qp: 20 + iframe_interval: 1 + +video_encoder_response: + outbuf_storage_type: 1 + +video_decoder_request: + inbuf_storage_type: 1 + output_format: "nv12pl" + +video_decoder_response: + outbuf_storage_type: 1 + +decoder_output_format_converter: + in_dtype: "nv12" + out_dtype: "rgb888" + +holoviz: + tensors: + - name: "" + type: color + opacity: 1.0 + priority: 0 \ No newline at end of file diff --git a/applications/h264/h264_video_encode_decode/python/metadata.json b/applications/h264/h264_video_encode_decode/python/metadata.json new file mode 100644 index 0000000000..a08a70ded2 --- /dev/null +++ b/applications/h264/h264_video_encode_decode/python/metadata.json @@ -0,0 +1,40 @@ +{ + "application": { + "name": "H.264 Video Encode Decode", + "authors": [ + { + "name": "Holoscan Team", + "affiliation": "NVIDIA" + } + ], + "language": "Python", + "version": "1.0", + "changelog": { + "1.0": "Initial Release" + }, + "dockerfile": "applications/h264/Dockerfile", + "holoscan_sdk": { + "minimum_required_version": "2.6.0", + "tested_versions": ["2.6.0"] + }, + "platforms": ["x86_64", "aarch64"], + "tags": ["Healthcare AI", "Video", "Hardware Accelerated Encode Decode", "Endoscopy"], + "ranking": 1, + "dependencies": { + "operators": [ + { + "name": "videodecoder", + "version": "1.2.0" + }, + { + "name": "videodecoderio", + "version": "1.2.0" + } + ] + }, + "run": { + "command": "python3 /h264_video_encode_decode.py --data /endoscopy", + "workdir": "holohub_bin" + } + } +} diff --git a/operators/CMakeLists.txt b/operators/CMakeLists.txt index accbf7b575..1d140891b9 100644 --- a/operators/CMakeLists.txt +++ b/operators/CMakeLists.txt @@ -45,7 +45,7 @@ add_holohub_operator(vtk_renderer) add_holohub_operator(yuan_qcap DEPENDS EXTENSIONS yuan_qcap) add_holohub_operator(ehr_query_llm) add_holohub_operator(xr) - +add_holohub_operator(append_timestamp) # install install( DIRECTORY "${CMAKE_BINARY_DIR}/python/lib/holohub" diff --git a/operators/append_timestamp/CMakeLists.txt b/operators/append_timestamp/CMakeLists.txt new file mode 100644 index 0000000000..395920eb2f --- /dev/null +++ b/operators/append_timestamp/CMakeLists.txt @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +cmake_minimum_required(VERSION 3.20) +project(append_timestamp) + +find_package(holoscan 0.5 REQUIRED CONFIG + PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install") + +add_library(append_timestamp SHARED + append_timestamp.hpp + append_timestamp.cpp + ) +add_library(holoscan::ops::append_timestamp ALIAS append_timestamp) + +target_include_directories(append_timestamp INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) + +target_link_libraries( + append_timestamp + holoscan::core + GXF::std +) + +if(HOLOHUB_BUILD_PYTHON) + add_subdirectory(python) +endif() diff --git a/operators/append_timestamp/README.md b/operators/append_timestamp/README.md new file mode 100644 index 0000000000..5c7c192c7e --- /dev/null +++ b/operators/append_timestamp/README.md @@ -0,0 +1,3 @@ +### Append Timestamp Operator + +The `append_timestamp` operators add a `nvidia::gxf::Timestamp` to the incoming `gxf::Entity` objectS. diff --git a/operators/append_timestamp/append_timestamp.cpp b/operators/append_timestamp/append_timestamp.cpp new file mode 100644 index 0000000000..d40b78686d --- /dev/null +++ b/operators/append_timestamp/append_timestamp.cpp @@ -0,0 +1,63 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights + * reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +// If GXF has gxf/std/dlpack_utils.hpp it has DLPack support +#if __has_include("gxf/std/dlpack_utils.hpp") +#define GXF_HAS_DLPACK_SUPPORT 1 +#include "gxf/std/tensor.hpp" +#else +#define GXF_HAS_DLPACK_SUPPORT 0 +#include "holoscan/core/gxf/gxf_tensor.hpp" +#endif + +#include +#include "gxf/std/timestamp.hpp" + +#include "holoscan/core/execution_context.hpp" +#include "holoscan/core/gxf/entity.hpp" +#include "holoscan/core/operator_spec.hpp" + +#include "append_timestamp.hpp" + +namespace holoscan::ops { + +void AppendTimestampOp::setup(OperatorSpec& spec) { + auto& input = spec.input("in_tensor"); + auto& output = spec.output("out_tensor"); +} + +void AppendTimestampOp::compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) { + // Process input message + // The type of `in_message` is 'holoscan::gxf::Entity'. + auto in_message = op_input.receive("in_tensor").value(); + + // Add timestamp to the tensor + auto timestamp = + static_cast(in_message).add("timestamp"); + if (timestamp) { + (*timestamp)->pubtime = std::chrono::system_clock::now().time_since_epoch().count(); + (*timestamp)->acqtime = std::chrono::system_clock::now().time_since_epoch().count(); + } + + // Transmit the gxf video buffer to target + op_output.emit(in_message, "out_tensor"); +} + +} // namespace holoscan::ops diff --git a/operators/append_timestamp/append_timestamp.hpp b/operators/append_timestamp/append_timestamp.hpp new file mode 100644 index 0000000000..e47a37f894 --- /dev/null +++ b/operators/append_timestamp/append_timestamp.hpp @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HOLOSCAN_OPERATORS_APPEND_TIMESTAMP_HPP +#define HOLOSCAN_OPERATORS_APPEND_TIMESTAMP_HPP + +#include "holoscan/core/operator.hpp" + +namespace holoscan::ops { + +/** + * @brief Operator class to append timestamp to a tensor. + * + * This operator takes a tensor as input and appends a timestamp to it, + * then outputs the tensor with the timestamp. + */ +class AppendTimestampOp : public Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(AppendTimestampOp) + + AppendTimestampOp() = default; + + void setup(OperatorSpec& spec) override; + void compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) override; +}; + +} // namespace holoscan::ops + +#endif // HOLOSCAN_OPERATORS_APPEND_TIMESTAMP_HPP diff --git a/operators/append_timestamp/metadata.json b/operators/append_timestamp/metadata.json new file mode 100644 index 0000000000..9d17645a67 --- /dev/null +++ b/operators/append_timestamp/metadata.json @@ -0,0 +1,29 @@ +{ + "operator": { + "name": "append_timestamp", + "authors": [ + { + "name": "Holoscan Team", + "affiliation": "NVIDIA" + } + ], + "version": "1.0", + "changelog": { + "1.0": "Initial Release" + }, + "language": ["C++", "Python"], + "holoscan_sdk": { + "minimum_required_version": "2.0.0", + "tested_versions": [ + "3.0.0" + ] + }, + "platforms": [ + "x86_64", + "aarch64" + ], + "tags": ["Healthcare AI", "Video", "Timestamp"], + "ranking": 1, + "dependencies": {} + } +} \ No newline at end of file diff --git a/operators/append_timestamp/python/CMakeLists.txt b/operators/append_timestamp/python/CMakeLists.txt new file mode 100644 index 0000000000..b0b93e337d --- /dev/null +++ b/operators/append_timestamp/python/CMakeLists.txt @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include(pybind11_add_holohub_module) +pybind11_add_holohub_module( + CPP_CMAKE_TARGET append_timestamp + CLASS_NAME "AppendTimestampOp" + SOURCES append_timestamp.cpp +) \ No newline at end of file diff --git a/operators/append_timestamp/python/append_timestamp.cpp b/operators/append_timestamp/python/append_timestamp.cpp new file mode 100644 index 0000000000..feb101bc97 --- /dev/null +++ b/operators/append_timestamp/python/append_timestamp.cpp @@ -0,0 +1,90 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights + * reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../append_timestamp.hpp" +#include "./append_timestamp_pydoc.hpp" +#include "../../operator_util.hpp" + +#include +#include // for unordered_map -> dict, etc. + +#include +#include +#include + +#include +#include +#include +#include +#include "holoscan/core/resources/gxf/cuda_stream_pool.hpp" + +using std::string_literals::operator""s; +using pybind11::literals::operator""_a; + +#define STRINGIFY(x) #x +#define MACRO_STRINGIFY(x) STRINGIFY(x) + +namespace py = pybind11; + +namespace holoscan::ops { + +/* Trampoline classes for handling Python kwargs + * + * These add a constructor that takes a Fragment for which to initialize the operator. + * The explicit parameter list and default arguments take care of providing a Pythonic + * kwarg-based interface with appropriate default values matching the operator's + * default parameters in the C++ API `setup` method. + * + * The sequence of events in this constructor is based on Fragment::make_operator + */ + +class PyAppendTimestampOp : public AppendTimestampOp { + public: + /* Inherit the constructors */ + using AppendTimestampOp::AppendTimestampOp; + + // Define a constructor that fully initializes the object. + PyAppendTimestampOp(Fragment* fragment, const py::args& args, + const std::string& name = "append_timestamp") + : AppendTimestampOp() { + add_positional_condition_and_resource_args(this, args); + name_ = name; + fragment_ = fragment; + spec_ = std::make_shared(fragment); + setup(*spec_.get()); + } +}; + +PYBIND11_MODULE(_append_timestamp, m) { + m.doc() = R"pbdoc( + AppendTimestampOp Python Bindings + ------------------------------------- + .. currentmodule:: _tensor_to_video_buffer + )pbdoc"; + py::class_>( + m, "AppendTimestampOp", doc::AppendTimestampOp::doc_AppendTimestampOp) + .def(py::init(), + "fragment"_a, + "name"_a = "append_timestamp"s, + doc::AppendTimestampOp::doc_AppendTimestampOp); +} // PYBIND11_MODULE NOLINT +} // namespace holoscan::ops diff --git a/operators/append_timestamp/python/append_timestamp_pydoc.hpp b/operators/append_timestamp/python/append_timestamp_pydoc.hpp new file mode 100644 index 0000000000..7ef75d85a2 --- /dev/null +++ b/operators/append_timestamp/python/append_timestamp_pydoc.hpp @@ -0,0 +1,54 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PYHOLOHUB_OPERATORS_APPEND_TIMESTAMP_PYDOC_HPP +#define PYHOLOHUB_OPERATORS_APPEND_TIMESTAMP_PYDOC_HPP + +#include + +#include "macros.hpp" + +namespace holoscan::doc { + +namespace AppendTimestampOp { + +PYDOC(AppendTimestampOp, R"doc( +Operator class to convert Tensor to VideoBuffer. + + +**==Named Inputs==** + + in_tensor : gxf::Entity + +**==Named Outputs==** + + out_tensor : gxf::Entity + +Parameters +---------- +fragment : Fragment + The fragment that the operator belongs to. +name : str, optional + The name of the operator. + +)doc") + +} // namespace AppendTimestampOp + +} // namespace holoscan::doc + +#endif // PYHOLOHUB_OPERATORS_APPEND_TIMESTAMP_PYDOC_HPP diff --git a/operators/dds/video/VideoFrame.idl b/operators/dds/video/VideoFrame.idl index 8e94888cf1..847c184c71 100644 --- a/operators/dds/video/VideoFrame.idl +++ b/operators/dds/video/VideoFrame.idl @@ -15,10 +15,17 @@ const string VIDEO_FRAME_TOPIC = "VideoFrame"; +enum Codec +{ + None, + H264 +}; struct VideoFrame { @key unsigned long stream_id; unsigned long frame_num; unsigned long width; unsigned long height; sequence data; + Codec codec; + int64 transfer_start_time; }; diff --git a/operators/dds/video/dds_video_publisher/dds_video_publisher.cpp b/operators/dds/video/dds_video_publisher/dds_video_publisher.cpp index ad5839e817..af1e8170b9 100644 --- a/operators/dds/video/dds_video_publisher/dds_video_publisher.cpp +++ b/operators/dds/video/dds_video_publisher/dds_video_publisher.cpp @@ -28,6 +28,18 @@ void DDSVideoPublisherOp::setup(OperatorSpec& spec) { spec.param(writer_qos_, "writer_qos", "Writer QoS", "Data Writer QoS Profile", std::string()); spec.param(stream_id_, "stream_id", "Stream ID", "Stream ID for the DDS Video Stream", 0u); + spec.param(width_, + "width", + "Width", + "Width of the video stream", + 0u, + holoscan::ParameterFlag::kOptional); + spec.param(height_, + "height", + "Height", + "Height of the video stream", + 0u, + holoscan::ParameterFlag::kOptional); } void DDSVideoPublisherOp::initialize() { @@ -43,37 +55,62 @@ void DDSVideoPublisherOp::initialize() { } // Create the writer for the VideoFrame - writer_ = dds::pub::DataWriter(publisher, topic, - qos_provider_.datawriter_qos(writer_qos_.get())); + writer_ = dds::pub::DataWriter( + publisher, topic, qos_provider_.datawriter_qos(writer_qos_.get())); } -void DDSVideoPublisherOp::compute(InputContext& op_input, - OutputContext& op_output, +void DDSVideoPublisherOp::compute(InputContext& op_input, OutputContext& op_output, ExecutionContext& context) { auto input = op_input.receive("input").value(); if (!input) { throw std::runtime_error("No input available"); } - const auto& buffer = static_cast(input).get(); - if (!buffer) { - throw std::runtime_error("No video buffer attached to input"); + const auto& maybe_buffer = + static_cast(input).get(); + const auto& maybe_tensor = static_cast(input).get(); + if (!maybe_buffer && !maybe_tensor) { + throw std::runtime_error("No video buffer or tensor attached to input"); } - const auto& info = buffer.value()->video_frame_info(); - if (info.color_format != nvidia::gxf::VideoFormat::GXF_VIDEO_FORMAT_RGBA) { - throw std::runtime_error("Invalid buffer format; Only RGBA is supported"); - } - - // Create the VideoFrame sample from the input buffer - std::vector data(buffer.value()->size()); - if (buffer.value()->storage_type() == nvidia::gxf::MemoryStorageType::kHost) { - memcpy(data.data(), buffer.value()->pointer(), data.size()); + VideoFrame frame; + frame.frame_num(frame_num_++); + frame.stream_id(stream_id_.get()); + if (maybe_buffer) { + auto buffer = maybe_buffer.value(); + const auto& info = buffer->video_frame_info(); + if (info.color_format != nvidia::gxf::VideoFormat::GXF_VIDEO_FORMAT_RGBA) { + throw std::runtime_error("Invalid buffer format; Only RGBA is supported"); + } + + // Create the VideoFrame sample from the input buffer + std::vector data(buffer->size()); + if (buffer->storage_type() == nvidia::gxf::MemoryStorageType::kHost) { + memcpy(data.data(), buffer->pointer(), data.size()); + } else { + cudaMemcpy(data.data(), buffer->pointer(), data.size(), cudaMemcpyDeviceToHost); + } + frame.width(info.width); + frame.height(info.height); + frame.data(data); + frame.codec(Codec::None); } else { - cudaMemcpy(data.data(), buffer.value()->pointer(), data.size(), cudaMemcpyDeviceToHost); + auto tensor = maybe_tensor.value(); + std::vector data(tensor->size()); + const auto& info = tensor->shape(); + if (tensor->storage_type() == nvidia::gxf::MemoryStorageType::kHost) { + memcpy(data.data(), tensor->pointer(), data.size()); + } else { + cudaMemcpy(data.data(), tensor->pointer(), data.size(), cudaMemcpyDeviceToHost); + } + frame.width(width_.get()); + frame.height(height_.get()); + frame.data(data); + frame.codec(Codec::H264); } - VideoFrame frame(stream_id_.get(), frame_num_++, info.width, info.height, data); - + frame.transfer_start_time(std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()); // Write the VideoFrame to the writer writer_.write(frame); } diff --git a/operators/dds/video/dds_video_publisher/dds_video_publisher.hpp b/operators/dds/video/dds_video_publisher/dds_video_publisher.hpp index 22112a9509..1344d753b9 100644 --- a/operators/dds/video/dds_video_publisher/dds_video_publisher.hpp +++ b/operators/dds/video/dds_video_publisher/dds_video_publisher.hpp @@ -41,6 +41,8 @@ class DDSVideoPublisherOp : public DDSOperatorBase { private: Parameter writer_qos_; Parameter stream_id_; + Parameter width_; + Parameter height_; dds::pub::DataWriter writer_ = dds::core::null; diff --git a/operators/dds/video/dds_video_subscriber/dds_video_subscriber.cpp b/operators/dds/video/dds_video_subscriber/dds_video_subscriber.cpp index 8b0420a29b..f50c4a5fb2 100644 --- a/operators/dds/video/dds_video_subscriber/dds_video_subscriber.cpp +++ b/operators/dds/video/dds_video_subscriber/dds_video_subscriber.cpp @@ -29,6 +29,21 @@ void DDSVideoSubscriberOp::setup(OperatorSpec& spec) { spec.param(allocator_, "allocator", "Allocator", "Allocator for output buffers."); spec.param(reader_qos_, "reader_qos", "Reader QoS", "Data Reader QoS Profile", std::string()); spec.param(stream_id_, "stream_id", "Stream ID for the video stream"); + spec.param(fps_report_interval_, + "fps_report_interval", + "FPS Report Interval", + "Interval in seconds to report FPS statistics", + 1.0); + spec.param(log_frame_warning_threshold_, + "log_frame_warning_threshold", + "Log Frame Warning Threshold", + "Log warning message when time to transfer frame is greater than this threshold in ms", + 10u); + spec.param(log_missing_frames_, + "log_missing_frames", + "Log Missing Frames", + "Log warning message when frame is missing", + false); } void DDSVideoSubscriberOp::initialize() { @@ -44,13 +59,14 @@ void DDSVideoSubscriberOp::initialize() { } // Create the filtered topic for the requested stream id. - dds::topic::ContentFilteredTopic filtered_topic(topic, + dds::topic::ContentFilteredTopic filtered_topic( + topic, "FilteredVideoFrame", dds::topic::Filter("stream_id = %0", {std::to_string(stream_id_.get())})); // Create the reader for the VideoFrame - reader_ = dds::sub::DataReader(subscriber, filtered_topic, - qos_provider_.datareader_qos(reader_qos_.get())); + reader_ = dds::sub::DataReader( + subscriber, filtered_topic, qos_provider_.datareader_qos(reader_qos_.get())); // Obtain the reader's status condition status_condition_ = dds::core::cond::StatusCondition(reader_); @@ -60,43 +76,166 @@ void DDSVideoSubscriberOp::initialize() { // Attach the status condition to the waitset waitset_ += status_condition_; + + // Initialize FPS tracking + frame_count_ = 0; + timing_initialized_ = false; } -void DDSVideoSubscriberOp::compute(InputContext& op_input, - OutputContext& op_output, +void DDSVideoSubscriberOp::compute(InputContext& op_input, OutputContext& op_output, ExecutionContext& context) { - auto allocator = nvidia::gxf::Handle::Create( - context.context(), allocator_->gxf_cid()); + auto allocator = + nvidia::gxf::Handle::Create(context.context(), allocator_->gxf_cid()); auto output = nvidia::gxf::Entity::New(context.context()); if (!output) { throw std::runtime_error("Failed to allocate message for output"); } - auto video_buffer = output.value().add(); - if (!video_buffer) { - throw std::runtime_error("Failed to allocate video buffer"); - } - bool output_written = false; while (!output_written) { // Wait for a new frame dds::core::cond::WaitSet::ConditionSeq active_conditions = waitset_.wait(dds::core::Duration::from_secs(1)); + for (const auto& cond : active_conditions) { if (cond == status_condition_) { // Take the available frame dds::sub::LoanedSamples frames = reader_.take(); + auto current_time = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); for (const auto& frame : frames) { if (frame.info().valid()) { - // Copy the frame to the output buffer - video_buffer.value()->resize( - frame.data().width(), frame.data().height(), - nvidia::gxf::SurfaceLayout::GXF_SURFACE_LAYOUT_PITCH_LINEAR, - nvidia::gxf::MemoryStorageType::kHost, allocator.value()); - memcpy(video_buffer.value()->pointer(), frame.data().data().data(), - frame.data().data().size()); - output_written = true; + auto selected_frame = frame.data(); + // Initialize timing on first frame + auto transfer_time = current_time - selected_frame.transfer_start_time(); + if (!timing_initialized_) { + start_time_ = std::chrono::steady_clock::now(); + last_fps_report_time_ = start_time_; + timing_initialized_ = true; + } else { + transfer_times_.push_back(transfer_time); + // Keep only the last 1000 samples + if (transfer_times_.size() > 1000) { + transfer_times_.erase(transfer_times_.begin()); + } + + frame_sizes.push_back(selected_frame.data().size()); + if (frame_sizes.size() > 1000) { + frame_sizes.erase(frame_sizes.begin()); + } + + // Print transfer time if it's greater than 10ms + if (log_frame_warning_threshold_.get() > 0 && + transfer_time > log_frame_warning_threshold_.get() * 1000000) { + HOLOSCAN_LOG_WARN("Transfer time: {}ns/{}ms, Frame Number: {}", + transfer_time, + transfer_time / 1000000.0, + selected_frame.frame_num()); + } + } + + // Increment frame count + frame_count_++; + if (selected_frame.frame_num() != last_frame_num_ + 1) { + if (log_missing_frames_.get()) { + HOLOSCAN_LOG_WARN("Expected frame number: {} != {}", last_frame_num_ + 1, + selected_frame.frame_num()); + } + last_frame_num_ = selected_frame.frame_num(); + } + + // Calculate and report FPS at specified intervals + auto current_time = std::chrono::steady_clock::now(); + auto duration_since_last_report = + std::chrono::duration_cast>(current_time - + last_fps_report_time_) + .count(); + + if (duration_since_last_report >= fps_report_interval_.get()) { + auto total_duration = std::chrono::duration_cast>( + current_time - start_time_) + .count(); + + double average_fps = frame_count_ / total_duration; + + if (fps_report_interval_.get() > 0.0) { + double average_transfer_time = 0.0; + double jitter_time = 0.0; + if (transfer_times_.size() > 0) { + // Calculate average + auto sum = std::accumulate(transfer_times_.begin(), transfer_times_.end(), 0LL); + double avg_ns = + static_cast(sum) / static_cast(transfer_times_.size()); + average_transfer_time = avg_ns / 1000000.0; + + // Calculate jitter (standard deviation) + if (transfer_times_.size() > 1) { + double variance_sum = 0.0; + for (const auto& time : transfer_times_) { + double diff = static_cast(time) - avg_ns; + variance_sum += diff * diff; + } + double variance = + variance_sum / static_cast(transfer_times_.size() - 1); + jitter_time = std::sqrt(variance) / 1000000.0; // Convert to ms + } + } + + auto sum_frame_sizes = std::accumulate(frame_sizes.begin(), frame_sizes.end(), 0LL); + double avg_frame_size = + static_cast(sum_frame_sizes) / static_cast(frame_sizes.size()); + + HOLOSCAN_LOG_INFO( + "DDS Video Subscriber - Stream ID: {} | Total Frames: {} | " + "Average FPS: {:.3f} | Total Time: {:.3f}s | Width: {} | Height: {} | Avg " + "Size: {} " + "| Codec: {} | Avg Transfer Time: {:.3f}ms | Jitter: {:.3f}ms", + stream_id_.get(), + frame_count_, + average_fps, + total_duration, + selected_frame.width(), + selected_frame.height(), + avg_frame_size, + static_cast(selected_frame.codec()), + average_transfer_time, + jitter_time); + } + + last_fps_report_time_ = current_time; + } + + if (selected_frame.codec() == Codec::H264) { + auto tensor = output.value().add(); + if (!tensor) { + throw std::runtime_error("Failed to allocate tensor"); + } + tensor.value()->reshape(nvidia::gxf::Shape({selected_frame.data().size()}), + nvidia::gxf::MemoryStorageType::kHost, + allocator.value()); + memcpy(tensor.value()->pointer(), + selected_frame.data().data(), + selected_frame.data().size()); + output_written = true; + } else { + auto video_buffer = output.value().add(); + if (!video_buffer) { + throw std::runtime_error("Failed to allocate video buffer"); + } + // Copy the frame to the output buffer + video_buffer.value()->resize( + selected_frame.width(), + selected_frame.height(), + nvidia::gxf::SurfaceLayout::GXF_SURFACE_LAYOUT_PITCH_LINEAR, + nvidia::gxf::MemoryStorageType::kHost, + allocator.value()); + memcpy(video_buffer.value()->pointer(), + selected_frame.data().data(), + selected_frame.data().size()); + output_written = true; + } } } } diff --git a/operators/dds/video/dds_video_subscriber/dds_video_subscriber.hpp b/operators/dds/video/dds_video_subscriber/dds_video_subscriber.hpp index 5d82489660..7775456b80 100644 --- a/operators/dds/video/dds_video_subscriber/dds_video_subscriber.hpp +++ b/operators/dds/video/dds_video_subscriber/dds_video_subscriber.hpp @@ -18,6 +18,7 @@ #pragma once #include +#include #include "dds_operator_base.hpp" #include "VideoFrame.hpp" @@ -42,10 +43,24 @@ class DDSVideoSubscriberOp : public DDSOperatorBase { Parameter> allocator_; Parameter reader_qos_; Parameter stream_id_; + Parameter fps_report_interval_; + Parameter log_frame_warning_threshold_; + Parameter log_missing_frames_; dds::sub::DataReader reader_ = dds::core::null; dds::core::cond::StatusCondition status_condition_ = dds::core::null; dds::core::cond::WaitSet waitset_; + + // FPS calculation variables + uint64_t frame_count_ = 0; + std::chrono::steady_clock::time_point start_time_; + std::chrono::steady_clock::time_point last_fps_report_time_; + bool timing_initialized_ = false; + + uint64_t last_frame_num_ = 0; + + std::vector transfer_times_; + std::vector frame_sizes; }; } // namespace holoscan::ops