Add support for GPU inference on windows using D3d11

.github/workflows/dls-coverity.yaml

@@ -83,7 +83,7 @@ jobs:
           libssh2-1-dev cmake git valgrind numactl libvpx-dev libopus-dev libsrtp2-dev libxv-dev \
           linux-libc-dev libpmix2t64 libhwloc15 libhwloc-plugins libxcb1-dev libx11-xcb-dev \
           ffmpeg librdkafka-dev libpaho-mqtt-dev libopencv-dev libpostproc-dev libavfilter-dev libavdevice-dev \
-          libswscale-dev libswresample-dev libavutil-dev libavformat-dev libavcodec-dev libtbb12
+          libswscale-dev libswresample-dev libavutil-dev libavformat-dev libavcodec-dev libtbb12 ocl-icd-opencl-dev
 
           echo "Setting up a Python environment"
           python3 -m venv python3venv

docker/ubuntu/ubuntu22.Dockerfile

@@ -95,7 +95,8 @@ RUN \
     libssh2-1-dev=\* cmake=\* git=\* valgrind=\* numactl=\* libvpx-dev=\* libopus-dev=\* libsrtp2-dev=\* libxv-dev=\* \
     linux-libc-dev=\* libpmix2=\* libhwloc15=\* libhwloc-plugins=\* libxcb1-dev=\* libx11-xcb-dev=\* \
     ffmpeg=\* libpaho-mqtt-dev=\* libpostproc-dev=\* libavfilter-dev=\* libavdevice-dev=\* \
-    libswscale-dev=\* libswresample-dev=\* libavutil-dev=\* libavformat-dev=\* libavcodec-dev=\* libxml2-dev=\* libsoup-3.0-0=\* && \
+    libswscale-dev=\* libswresample-dev=\* libavutil-dev=\* libavformat-dev=\* libavcodec-dev=\* libxml2-dev=\* libsoup-3.0-0=\* \
+    ocl-icd-opencl-dev=\* && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 

docker/ubuntu/ubuntu24.Dockerfile

@@ -97,7 +97,8 @@ RUN \
     libssh2-1-dev=\* cmake=\* git=\* valgrind=\* numactl=\* libvpx-dev=\* libopus-dev=\* libsrtp2-dev=\* libxv-dev=\* \
     linux-libc-dev=\* libpmix2t64=\* libhwloc15=\* libhwloc-plugins=\* libxcb1-dev=\* libx11-xcb-dev=\* \
     ffmpeg=\* libpaho-mqtt-dev=\* libpostproc-dev=\* libavfilter-dev=\* libavdevice-dev=\* \
-    libswscale-dev=\* libswresample-dev=\* libavutil-dev=\* libavformat-dev=\* libavcodec-dev=\* libxml2-dev=\* libsoup-3.0-0=\* &&  \
+    libswscale-dev=\* libswresample-dev=\* libavutil-dev=\* libavformat-dev=\* libavcodec-dev=\* libxml2-dev=\* libsoup-3.0-0=\* \
+    ocl-icd-opencl-dev=\* &&  \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 

docs/source/dev_guide/advanced_install/advanced_install_guide_compilation.md

@@ -30,7 +30,8 @@ Follow the instructions in
       libssh2-1-dev cmake git valgrind numactl libvpx-dev libopus-dev libsrtp2-dev libxv-dev \
       linux-libc-dev libpmix2t64 libhwloc15 libhwloc-plugins libxcb1-dev libx11-xcb-dev \
       ffmpeg libpaho-mqtt-dev libopencv-dev libpostproc-dev libavfilter-dev libavdevice-dev \
-      libswscale-dev libswresample-dev libavutil-dev libavformat-dev libavcodec-dev libtbb12 libxml2-dev libopencv-dev
+      libswscale-dev libswresample-dev libavutil-dev libavformat-dev libavcodec-dev libtbb12 libxml2-dev libopencv-dev \
+      ocl-icd-opencl-dev
   ```
 
 <!--hide_directive:::
@@ -47,7 +48,8 @@ Follow the instructions in
       libssh2-1-dev cmake git valgrind numactl libvpx-dev libopus-dev libsrtp2-dev libxv-dev \
       linux-libc-dev libpmix2 libhwloc15 libhwloc-plugins libxcb1-dev libx11-xcb-dev \
       ffmpeg libpaho-mqtt-dev libpostproc-dev libavfilter-dev libavdevice-dev \
-      libswscale-dev libswresample-dev libavutil-dev libavformat-dev libavcodec-dev libxml2-dev
+      libswscale-dev libswresample-dev libavutil-dev libavformat-dev libavcodec-dev libxml2-dev ocl-icd-opencl-dev \
+      opencl-headers
   ```
 
 <!--hide_directive:::

include/dlstreamer/openvino/context.h

@@ -9,9 +9,12 @@
 #include "dlstreamer/base/context.h"
 #include "dlstreamer/tensor.h"
 
-#include <openvino/openvino.hpp>
-#include <openvino/runtime/intel_gpu/properties.hpp>
-#include <openvino/runtime/intel_gpu/remote_properties.hpp>
+#ifdef _WIN32
+#include <gst/d3d11/gstd3d11device.h>
+#include <openvino/runtime/intel_gpu/ocl/dx.hpp>
+#else
+#include <openvino/runtime/intel_gpu/ocl/va.hpp>
+#endif
 
 namespace dlstreamer {
 
@@ -33,17 +36,23 @@ class OpenVINOContext : public BaseContext {
     OpenVINOContext(ov::Core core, const std::string &device, ContextPtr context) : BaseContext(MemoryType::OpenVINO) {
         ov::AnyMap context_params;
         if (device.find("GPU") != std::string::npos && context) {
+#ifdef _WIN32
+            auto d3d_device = context->handle(BaseContext::key::d3d_device);
+            if (d3d_device) {
+                auto gst_device = static_cast<GstD3D11Device *>(d3d_device);
+                _remote_context = ov::intel_gpu::ocl::D3DContext(core, gst_d3d11_device_get_device_handle(gst_device));
+            }
+#else
             auto va_display = context->handle(BaseContext::key::va_display);
-            //_remote_context = ov::intel_gpu::ocl::VAContext(core, va_display);
             if (va_display) {
                 int tile_id =
                     static_cast<int>(reinterpret_cast<intptr_t>(context->handle(BaseContext::key::va_tile_id)));
-                context_params = {{ov::intel_gpu::context_type.name(), "VA_SHARED"},
-                                  {ov::intel_gpu::va_device.name(), static_cast<void *>(va_display)},
-                                  {ov::intel_gpu::tile_id.name(), tile_id}};
+                _remote_context = ov::intel_gpu::ocl::VAContext(core, static_cast<void *>(va_display), tile_id);
             }
+#endif
         }
-        _remote_context = core.create_context(device, context_params);
+        if (!_remote_context)
+            _remote_context = core.get_default_context(device);
     }
 
     OpenVINOContext(ov::CompiledModel compiled_model) : BaseContext(MemoryType::OpenVINO) {

src/base/openvino_inference_backend/CMakeLists.txt

@@ -11,11 +11,12 @@ pkg_search_module(VA va libva REQUIRED)
 if(WIN32)
     pkg_check_modules(GSTD3D11 gstreamer-d3d11-1.0>=1.16 REQUIRED)
 endif()
+find_package(OpenCL REQUIRED)
 
 add_library(${TARGET_NAME} OBJECT openvino.cpp openvino.hpp)
 set_compile_flags(${TARGET_NAME})
 
-target_include_directories(${TARGET_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${VA_INCLUDE_DIRS})
+target_include_directories(${TARGET_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${VA_INCLUDE_DIRS} ${OpenCL_INCLUDE_DIRS})
 
 if(WIN32)
     target_include_directories(${TARGET_NAME} PUBLIC ${GSTD3D11_INCLUDE_DIRS})
@@ -27,6 +28,7 @@ PUBLIC
 PRIVATE
     openvino::runtime
     dlstreamer_logger
+    OpenCL::OpenCL
 )
 
 if(WIN32)

src/monolithic/gst/inference_elements/base/gva_base_inference.cpp

@@ -179,7 +179,8 @@ static GstCaps *gva_base_inference_transform_caps(GstBaseTransform *trans, GstPa
             GstCapsFeatures *features = gst_caps_get_features(filtered, i);
             if ((gst_caps_features_contains(features, "memory:VASurface")) ||
                 (gst_caps_features_contains(features, "memory:VAMemory")) ||
-                (gst_caps_features_contains(features, "memory:DMABuf"))) {
+                (gst_caps_features_contains(features, "memory:DMABuf")) ||
+                (gst_caps_features_contains(features, "memory:D3D11Memory"))) {
                 gst_caps_remove_structure(filtered, i);
                 GST_WARNING("Filtered out structure %d from caps, it contains unsupported memory type", i);
             }
@@ -862,7 +863,8 @@ gboolean gva_base_inference_set_caps(GstBaseTransform *trans, GstCaps *incaps, G
     if ((base_inference->device && g_strcmp0(base_inference->device, "CPU") == 0 &&
          ((gst_caps_features_contains(gst_caps_get_features(incaps, 0), "memory:VASurface")) ||
           (gst_caps_features_contains(gst_caps_get_features(incaps, 0), "memory:VAMemory")) ||
-          (gst_caps_features_contains(gst_caps_get_features(incaps, 0), "memory:DMABuf"))))) {
+          (gst_caps_features_contains(gst_caps_get_features(incaps, 0), "memory:DMABuf")) ||
+          (gst_caps_features_contains(gst_caps_get_features(incaps, 0), "memory:D3D11Memory"))))) {
         GST_ELEMENT_WARNING(base_inference, RESOURCE, SETTINGS,
                             ("Refusing caps other than SYSTEM_MEMORY_CAPS because device is set to CPU"),
                             ("Set device property to a hardware accelerator (e.g., GPU) to enable VA memory types."));

src/monolithic/inference_backend/image_inference/openvino/CMakeLists.txt

@@ -9,6 +9,7 @@ set (TARGET_NAME "image_inference_openvino")
 # FIXME: update macro below
 find_package(PkgConfig REQUIRED)
 find_package(OpenVINO REQUIRED Runtime)
+find_package(OpenCL REQUIRED)
 
 set(OpenVINO_MIN_VERSION "2024.0.0")
 if(${OpenVINO_VERSION} VERSION_LESS ${OpenVINO_MIN_VERSION})
@@ -44,6 +45,7 @@ PUBLIC
         utils
         logger
         dlstreamer_logger
+        OpenCL::OpenCL
 )
 
 if(WIN32)

src/monolithic/inference_backend/image_inference/openvino/openvino_image_inference.cpp

@@ -9,7 +9,9 @@
 #include <openvino/runtime/properties.hpp>
 
 #include <dlstreamer/openvino/context.h>
-#include <dlstreamer/vaapi/context.h>
+#ifdef _WIN32
+#include <dlstreamer/d3d11/context.h>
+#endif
 // For logger_name
 #include <dlstreamer/element.h>
 
@@ -1417,20 +1419,39 @@ class OpenVinoNewApiImpl {
 
     dlstreamer::ContextPtr create_remote_context() {
         // FIXME: invert to reduce nesting
-        if (is_device_gpu() && !is_device_multi() &&
-            (_memory_type == MemoryType::VAAPI || _memory_type == MemoryType::SYSTEM)) {
-            if (_app_context) {
-                try {
-                    dlstreamer::VAAPIContextPtr vaapi_ctx = dlstreamer::VAAPIContext::create(_app_context);
-                    _openvino_context = std::make_shared<dlstreamer::OpenVINOContext>(core(), _device, vaapi_ctx);
-                } catch (std::exception &e) {
-                    GVA_ERROR("Exception occurred when creating OpenVINO™ toolkit remote context: %s", e.what());
-                    std::throw_with_nested(std::runtime_error("couldn't create OV remote context"));
+        if (is_device_gpu() && !is_device_multi()) {
+#ifdef _WIN32
+            if (_memory_type == MemoryType::D3D11 || _memory_type == MemoryType::SYSTEM) {
+                if (_app_context) {
+                    try {
+                        dlstreamer::D3D11ContextPtr d3d11_ctx = dlstreamer::D3D11Context::create(_app_context);
+                        _openvino_context = std::make_shared<dlstreamer::OpenVINOContext>(core(), _device, d3d11_ctx);
+                    } catch (std::exception &e) {
+                        GVA_ERROR("Exception occurred when creating OpenVINO™ toolkit remote context: %s", e.what());
+                        std::throw_with_nested(std::runtime_error("couldn't create OV remote context"));
+                    }
+
+                } else if (_memory_type == MemoryType::D3D11) {
+                    throw std::runtime_error("Display must be provided for GPU device with d3d11 backend");
                 }
+            }
+#else
+            if (_memory_type == MemoryType::VAAPI || _memory_type == MemoryType::SYSTEM) {
+                if (_app_context) {
+                    try {
+                        dlstreamer::VAAPIContextPtr vaapi_ctx = dlstreamer::VAAPIContext::create(_app_context);
+                        _openvino_context = std::make_shared<dlstreamer::OpenVINOContext>(core(), _device, vaapi_ctx);
+                    } catch (std::exception &e) {
+                        GVA_ERROR("Exception occurred when creating OpenVINO™ toolkit remote context: %s", e.what());
+                        std::throw_with_nested(std::runtime_error("couldn't create OV remote context"));
+                    }
 
-            } else if (_memory_type == MemoryType::VAAPI) {
-                throw std::runtime_error("Display must be provided for GPU device with vaapi-surface-sharing backend");
+                } else if (_memory_type == MemoryType::VAAPI) {
+                    throw std::runtime_error(
+                        "Display must be provided for GPU device with vaapi-surface-sharing backend");
+                }
             }
+#endif
         }
         return _openvino_context;
     }

src/openvino/openvino_inference/CMakeLists.txt

@@ -11,6 +11,7 @@ if(WIN32)
     pkg_check_modules(GSTD3D11 gstreamer-d3d11-1.0>=1.16 REQUIRED)
 endif()
 pkg_search_module(VA va libva REQUIRED)
+find_package(OpenCL REQUIRED)
 
 add_library(${TARGET_NAME} OBJECT openvino_inference.cpp)
 set_compile_flags(${TARGET_NAME})
@@ -20,6 +21,7 @@ PRIVATE
     openvino::runtime
     dlstreamer_api
     dlstreamer_logger
+    OpenCL::OpenCL
 )
 
 if(WIN32)