[NEW APP] BCI Visualization with Kernel Flow2 and Real-Time Volume Rendering

applications/CMakeLists.txt

@@ -25,6 +25,9 @@ add_holohub_application(async_buffer_deadline)
 add_holohub_application(basic_networking_ping DEPENDS
                         OPERATORS basic_network)
 
+add_holohub_application(bci_visualization DEPENDS
+                        OPERATORS volume_renderer)
+
 add_holohub_application(body_pose_estimation DEPENDS
                         OPERATORS OPTIONAL dds_video_subscriber dds_video_publisher)
 

applications/bci_visualization/CMakeLists.txt

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cmake_minimum_required(VERSION 3.20)
+project(bci_visualization LANGUAGES NONE)
+
+find_package(holoscan 2.0 REQUIRED CONFIG
+             PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install")
+
+add_subdirectory(operators)

applications/bci_visualization/Dockerfile

@@ -0,0 +1,32 @@
+# syntax=docker/dockerfile:1
+
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE} AS base
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Install curl for downloading data files
+RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/*
+
+ENV HOLOSCAN_INPUT_PATH=/workspace/holohub/data/bci_visualization
+
+# Install Python dependencies
+COPY applications/bci_visualization/requirements.txt /tmp/requirements.txt
+RUN pip install -r /tmp/requirements.txt --no-cache-dir
+

applications/bci_visualization/README.md

@@ -0,0 +1,54 @@
+# BCI Visualization
+
+## Running
+1. Download data
+   * Please download data from [here](https://drive.google.com/drive/folders/1RpQ6UzjIZAr90FdW9VIbtTFYR6-up7w2) and put everything under `data/bci_visualization`.
+   * Includes an activation volume and a segmentation volume.
+2. Run application
+   ```bash
+   ./holohub run bci_visualization
+   ```
+   This command will build the docker and run the application.
+
+## Expected Results
+
+![Example output for BCI Visualization](docs/brain.png)
+### Components
+
+1. **VoxelStreamToVolumeOp** (`operators/voxel_stream_to_volume/voxel_stream_to_volume.py`): Bridge operator that converts sparse voxel data to dense volume
+   - Inputs: `affine_4x4`, `hb_voxel_data`
+   - Outputs: `volume`, `spacing`, `permute_axis`, `flip_axes`, `mask_volume`, `mask_spacing`, `mask_permute_axis`, `mask_flip_axes`
+
+2. **VolumeRendererOp**: ClaraViz-based volume renderer
+   - Renders the 3D volume with transfer functions
+   - Supports interactive camera control
+
+3. **HolovizOp**: Interactive visualization
+   - Displays the rendered volume
+   - Provides camera pose feedback
+
+
+## Data Flow
+
+```
+Reconstruction → VoxelStreamToVolume → VolumeRenderer → Holoviz
+                                             ↑               ↓
+                                             └─── camera ────┘
+```
+
+## Volume Renderer Configuration
+Here are some of the important config:
+1. `timeSlot`: Rendering time in ms. The longer the better the quality.
+2. `TransferFunction`
+
+   a. `activeRegions`:  (0: SKIN, 1: SKULL, 2: CSF, 3: GRAY MATER, 4: WHITE MATTER, 5: AIR). Here, we select [3, 4] for our ROI. Set everything else as opacity=0 (default).
+
+   b. `blendingProfile`: If there're overlapping components configured, how to blend.
+
+   c. `range`: Volume's value within this range to be configured. 
+
+   d. `opacityProfile`: How opacity is being applied within this range. Select `Square` for constant.
+
+   e. `diffuseStart/End`: The component's color. Linear interpolation between start/end.
+
+   Note: there are three components configured. First is for overall ROI, set color as white. Second is for positive activation, set as red. Third is for negative activation, set as blue.

applications/bci_visualization/bci_visualization.py

@@ -0,0 +1,255 @@
+"""
+SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+SPDX-License-Identifier: Apache-2.0
+
+BCI Visualization Application - streams synthetic voxel data and renders as 3D volume.
+"""
+
+import os
+import argparse
+from pathlib import Path
+
+
+from holoscan.core import Application
+from holoscan.operators import HolovizOp
+from holoscan.resources import CudaStreamPool, UnboundedAllocator
+from holoscan.schedulers import EventBasedScheduler, MultiThreadScheduler
+from holoscan.core import ConditionType
+
+# Import local operators
+from operators.voxel_stream_to_volume import VoxelStreamToVolumeOp
+from operators.reconstruction import (
+    BuildRHSOperator,
+    ConvertToVoxelsOperator,
+    NormalizeOperator,
+    RegularizedSolverOperator,
+)
+from operators.stream import StreamOperator
+
+# Import processing utilities
+from processing.reconstruction import REG_DEFAULT, get_assets
+from streams.base_nirs import BaseNirsStream
+from streams.snirf import SNIRFStream
+
+from holohub.color_buffer_passthrough import ColorBufferPassthroughOp
+from holohub.volume_renderer import VolumeRendererOp
+
+
+class BciVisualizationApp(Application):
+    """BCI Visualization Application with ClaraViz.
+
+    This application integrates the full pipeline from NIRS data streaming through
+    reconstruction to 3D volume rendering and visualization.
+    """
+
+    def __init__(self, 
+        argv=None,
+        *args,
+        render_config_file,
+        stream: BaseNirsStream,
+        jacobian_path: Path | str,
+        channel_mapping_path: Path | str,
+        voxel_info_dir: Path | str,
+        coefficients_path: Path | str,
+        mask_path=None,
+        reg: float = REG_DEFAULT,
+        tol: float = 1e-4,
+        use_gpu: bool = False,
+        **kwargs,
+    ):
+        self._rendering_config = render_config_file
+        self._mask_path = mask_path
+
+        # Reconstruction pipeline parameters
+        self._stream = stream
+        self._reg = reg
+        self._jacobian_path = Path(jacobian_path)
+        self._channel_mapping_path = Path(channel_mapping_path)
+        self._coefficients_path = Path(coefficients_path)
+        self._voxel_info_dir = Path(voxel_info_dir)
+        self._tol = tol
+        self._use_gpu = use_gpu
+
+        super().__init__(argv, *args, **kwargs)
+
+    def compose(self):
+        # Resources
+        volume_allocator = UnboundedAllocator(self, name="allocator")
+        cuda_stream_pool = CudaStreamPool(
+            self,
+            name="cuda_stream",
+            dev_id=0,
+            stream_flags=0,
+            stream_priority=0,
+            reserved_size=1,
+            max_size=5,
+        )
+
+        # Get reconstruction pipeline assets
+        pipeline_assets = get_assets(
+            jacobian_path=self._jacobian_path,
+            channel_mapping_path=self._channel_mapping_path,
+            voxel_info_dir=self._voxel_info_dir,
+            coefficients_path=self._coefficients_path,
+        )
+
+        # ========== Reconstruction Pipeline Operators ==========
+        stream_operator = StreamOperator(stream=self._stream, fragment=self)
+
+        build_rhs_operator = BuildRHSOperator(
+            assets=pipeline_assets,
+            fragment=self,
+        )
+
+        normalize_operator = NormalizeOperator(
+            fragment=self,
+            use_gpu=self._use_gpu,
+        )
+
+        regularized_solver_operator = RegularizedSolverOperator(
+            reg=self._reg,
+            use_gpu=self._use_gpu,
+            fragment=self,
+        )
+
+        convert_to_voxels_operator = ConvertToVoxelsOperator(
+            fragment=self,
+            coefficients=pipeline_assets.extinction_coefficients,
+            ijk=pipeline_assets.ijk,
+            xyz=pipeline_assets.xyz,
+            use_gpu=self._use_gpu,
+        )
+
+        # ========== Visualization Pipeline Operators ==========
+        voxel_to_volume = VoxelStreamToVolumeOp(
+            self,
+            name="voxel_to_volume",
+            pool=volume_allocator,
+            mask_nifti_path=self._mask_path,
+        )
+
+        volume_renderer = VolumeRendererOp(
+            self,
+            name="volume_renderer",
+            config_file=self._rendering_config,
+            allocator=volume_allocator,
+            cuda_stream_pool=cuda_stream_pool,
+            **self.kwargs("volume_renderer"),
+        )
+
+        # IMPORTANT changes to avoid deadlocks of volume_renderer and holoviz 
+        # when running in multi-threading mode
+        # 1. Set the output port condition to NONE to remove backpressure
+        volume_renderer.spec.outputs["color_buffer_out"].condition(ConditionType.NONE)
+        # 2. Use a passthrough operator to configure queue policy as POP to use latest frame
+        color_buffer_passthrough = ColorBufferPassthroughOp(
+            self,
+            name="color_buffer_passthrough",
+        )
+
+        holoviz = HolovizOp(
+            self,
+            name="holoviz",
+            window_title="BCI Visualization with ClaraViz",
+            enable_camera_pose_output=True,
+            cuda_stream_pool=cuda_stream_pool,
+        )
+
+        # ========== Connect Reconstruction Pipeline ==========
+        self.add_flow(stream_operator, build_rhs_operator, {
+            ("samples", "moments"),
+        })
+        self.add_flow(build_rhs_operator, normalize_operator, {
+            ("batch", "batch"),
+        })
+        self.add_flow(normalize_operator, regularized_solver_operator, {
+            ("normalized", "batch"),
+        })
+        self.add_flow(regularized_solver_operator, convert_to_voxels_operator, {
+            ("result", "result"),
+        })
+        self.add_flow(convert_to_voxels_operator, voxel_to_volume, {
+            ("affine_4x4", "affine_4x4"),
+            ("hb_voxel_data", "hb_voxel_data"),
+        })
+
+        # ========== Connect Visualization Pipeline ==========
+        # voxel_to_volume → volume_renderer
+        self.add_flow(voxel_to_volume, volume_renderer, {
+            ("volume", "density_volume"),
+            ("spacing", "density_spacing"),
+            ("permute_axis", "density_permute_axis"),
+            ("flip_axes", "density_flip_axes"),
+        })
+        # Add mask connections to VolumeRendererOp
+        self.add_flow(voxel_to_volume, volume_renderer, {
+            ("mask_volume", "mask_volume"),
+            ("mask_spacing", "mask_spacing"),
+            ("mask_permute_axis", "mask_permute_axis"),
+            ("mask_flip_axes", "mask_flip_axes"),
+        })
+
+        # volume_renderer ↔ holoviz
+        self.add_flow(volume_renderer, color_buffer_passthrough,
+                      {("color_buffer_out", "color_buffer_in")})
+        self.add_flow(color_buffer_passthrough, holoviz, {("color_buffer_out", "receivers")})
+        self.add_flow(holoviz, volume_renderer, {("camera_pose_output", "camera_pose")})
+
+
+def main():
+    parser = argparse.ArgumentParser(description="BCI Visualization Application", add_help=False)
+
+    parser.add_argument(
+        "-c",
+        "--renderer_config",
+        action="store",
+        dest="renderer_config",
+        help="Path to the renderer JSON configuration file to load",
+    )
+
+    parser.add_argument(
+        "-m",
+        "--mask_path",
+        action="store",
+        type=str,
+        dest="mask_path",
+        help="Path to the mask NIfTI file containing 3D integer labels (I, J, K). Optional.",
+    )
+
+    parser.add_argument(
+        "-h", "--help", action="help", default=argparse.SUPPRESS, help="Help message"
+    )
+
+    args = parser.parse_args()
+
+    # Setup data paths
+    default_data_path = os.path.join(os.getcwd(), "data/bci_visualization")
+    kernel_data = Path(os.environ.get("HOLOSCAN_INPUT_PATH", default_data_path))
+
+    stream = SNIRFStream(kernel_data / "data.snirf")
+    app = BciVisualizationApp(
+        render_config_file=args.renderer_config,
+        stream=stream,
+        jacobian_path=kernel_data / "flow_mega_jacobian.npy",
+        channel_mapping_path=kernel_data / "flow_channel_map.json",
+        voxel_info_dir=kernel_data / "voxel_info",
+        coefficients_path=kernel_data / "extinction_coefficients_mua.csv",
+        mask_path=args.mask_path,
+        reg=REG_DEFAULT,
+        use_gpu=True,
+    )
+
+    # Load YAML configuration
+    config_file = os.path.join(os.path.dirname(__file__), "bci_visualization.yaml")
+
+    app.config(config_file)
+    app.scheduler(EventBasedScheduler(app, worker_thread_number=5, stop_on_deadlock=True))
+
+    app.run()
+
+    print("BCI Visualization Application has finished running.")
+
+
+if __name__ == "__main__":
+    main()
+

applications/bci_visualization/bci_visualization.yaml

@@ -0,0 +1,9 @@
+%YAML 1.2
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+# SPDX-License-Identifier: Apache-2.0
+---
+volume_renderer:
+  alloc_width: 1024
+  alloc_height: 768
+  density_min: -100
+  density_max: 100