From b8ecfb2ace25e7c33c9fd05c075d0941d736bfca Mon Sep 17 00:00:00 2001 From: Xiaoxia Liang Date: Tue, 7 Feb 2023 22:50:17 +0800 Subject: [PATCH] enable blur rectangle feature and add the watermark flag interface to support set which parts will be rendered. Signed-off-by: Pengfei Qu Signed-off-by: Xiaoxia Liang --- .../gvawatermark/gstgvawatermarkimpl.cpp | 98 ++++++++++++++++--- .../gvawatermark/gstgvawatermarkimpl.h | 9 ++ gst/elements/gvawatermark/gvawatermark.cpp | 52 +++++++++- gst/elements/gvawatermark/gvawatermark.h | 4 + .../renderer/cpu/renderer_cpu.cpp | 47 +++++++++ .../gvawatermark/renderer/cpu/renderer_cpu.h | 4 + .../gvawatermark/renderer/render_prim.h | 12 ++- 7 files changed, 208 insertions(+), 18 deletions(-) diff --git a/gst/elements/gvawatermark/gstgvawatermarkimpl.cpp b/gst/elements/gvawatermark/gstgvawatermarkimpl.cpp index 52599379..cd45f479 100644 --- a/gst/elements/gvawatermark/gstgvawatermarkimpl.cpp +++ b/gst/elements/gvawatermark/gstgvawatermarkimpl.cpp @@ -44,6 +44,8 @@ GST_DEBUG_CATEGORY_STATIC(gst_gva_watermark_impl_debug_category); #define DEFAULT_DEVICE nullptr +#define DEFAULT_FILTER "none" + typedef enum { DEVICE_CPU, DEVICE_GPU, DEVICE_GPU_AUTOSELECTED } DEVICE_SELECTOR; namespace { @@ -115,12 +117,20 @@ dlstreamer::MemoryMapperPtr createMapperToDMA(InferenceBackend::MemoryType in_me struct Impl { Impl(GstVideoInfo *info, DEVICE_SELECTOR device, InferenceBackend::MemoryType mem_type, - dlstreamer::ContextPtr context); + dlstreamer::ContextPtr context, WATERMARK_FLAG flag, std::string filter); bool render(GstBuffer *buffer); const std::string &getBackendType() const { return _backend_type; } + const int &getFlag() const { + return flag; + } + + const std::string &getFilter() const { + return filter; + } + private: void preparePrimsForRoi(GVA::RegionOfInterest &roi, std::vector &prims) const; void preparePrimsForTensor(const GVA::Tensor &tensor, GVA::Rect rect, @@ -154,9 +164,15 @@ struct Impl { const int type = cv::FONT_HERSHEY_TRIPLEX; const double scale = 1.0; } _font; + int flag; + std::string filter; }; -enum { PROP_0, PROP_DEVICE }; +enum { PROP_0, + PROP_DEVICE, + PROP_WATERMARK_FLAG, + PROP_WATERMARK_FILTER, +}; G_DEFINE_TYPE_WITH_CODE(GstGvaWatermarkImpl, gst_gva_watermark_impl, GST_TYPE_BASE_TRANSFORM, GST_DEBUG_CATEGORY_INIT(gst_gva_watermark_impl_debug_category, "gvawatermarkimpl", 0, @@ -176,6 +192,12 @@ void gst_gva_watermark_impl_set_property(GObject *object, guint prop_id, const G g_free(gvawatermark->device); gvawatermark->device = g_value_dup_string(value); break; + case PROP_WATERMARK_FLAG: + gvawatermark->flag = g_value_get_uint(value); + break; + case PROP_WATERMARK_FILTER: + gvawatermark->filter = g_value_dup_string(value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); break; @@ -195,6 +217,20 @@ void gst_gva_watermark_impl_get_property(GObject *object, guint prop_id, GValue g_value_set_string(value, self->device); } break; + case PROP_WATERMARK_FLAG: + if (self->impl) { + g_value_set_uint(value, self->impl->getFlag()); + } else { + g_value_set_uint(value, self->flag); + } + break; + case PROP_WATERMARK_FILTER: + if (self->impl) { + g_value_set_string(value, self->impl->getFilter().c_str()); + } else { + g_value_set_string(value, self->filter); + } + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); break; @@ -222,6 +258,9 @@ void gst_gva_watermark_impl_finalize(GObject *object) { g_free(gvawatermark->device); gvawatermark->device = nullptr; + g_free(gvawatermark->filter); + gvawatermark->filter=nullptr; + G_OBJECT_CLASS(gst_gva_watermark_impl_parent_class)->finalize(object); } @@ -305,7 +344,7 @@ static gboolean gst_gva_watermark_impl_set_caps(GstBaseTransform *trans, GstCaps } try { - gvawatermark->impl = new Impl(&gvawatermark->info, device, mem_type, va_dpy); + gvawatermark->impl = new Impl(&gvawatermark->info, device, mem_type, va_dpy, (WATERMARK_FLAG)gvawatermark->flag, gvawatermark->filter); } catch (const std::exception &e) { GST_ELEMENT_ERROR(gvawatermark, CORE, FAILED, ("Could not initialize"), ("Cannot create watermark instance. %s", Utils::createNestedErrorMsg(e).c_str())); @@ -392,11 +431,26 @@ static void gst_gva_watermark_impl_class_init(GstGvaWatermarkImplClass *klass) { "device", "Target device", "Supported devices are CPU and GPU. Default is CPU on system memory and GPU on video memory", DEFAULT_DEVICE, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property( + gobject_class, PROP_WATERMARK_FLAG, + g_param_spec_uint("flag", "flag", + "watermark flag used to trigger the blur/draw rectangle and text. " + "(0x01)draw rectangle, (0x02)draw text,(0x04)draw landmark,(0x08)blur rectangle" + "please see user guide for more details", + 1, 16, WATERMARK_DRAW_RECTANGLE, static_cast(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property( + gobject_class, PROP_WATERMARK_FILTER, + g_param_spec_string("filter", "filter", + "watermark filter the specified object label list. seperated by comma " + "please see user guide for more details", + DEFAULT_FILTER, static_cast(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); } Impl::Impl(GstVideoInfo *info, DEVICE_SELECTOR device, InferenceBackend::MemoryType mem_type, - dlstreamer::ContextPtr context) - : _vinfo(info) { + dlstreamer::ContextPtr context, WATERMARK_FLAG flag, std::string filter) + : _vinfo(info), flag(flag), filter(filter) { assert(_vinfo); if (GST_VIDEO_INFO_COLORIMETRY(_vinfo).matrix == GstVideoColorMatrix::GST_VIDEO_COLOR_MATRIX_UNKNOWN) throw std::runtime_error("GST_VIDEO_COLOR_MATRIX_UNKNOWN"); @@ -446,11 +500,12 @@ bool Impl::render(GstBuffer *buffer) { for (auto &tensor : video_frame.tensors()) { if (tensor.is_detection()) continue; - preparePrimsForTensor(tensor, ff_rect, prims); + if (flag & WATERMARK_DRAW_CIRCLE) + preparePrimsForTensor(tensor, ff_rect, prims); appendStr(ff_text, tensor.label()); } - if (ff_text.tellp() != 0) + if (ff_text.tellp() != 0 && flag & WATERMARK_DRAW_TEXT) prims.emplace_back(render::Text(ff_text.str(), _ff_text_position, _font.type, _font.scale, _default_color)); // Skip render if there are no primitives to draw @@ -489,23 +544,34 @@ void Impl::preparePrimsForRoi(GVA::RegionOfInterest &roi, std::vector(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | G_PARAM_DEPRECATED))); + + g_object_class_install_property( + gobject_class, PROP_WATERMARK_FLAG, + g_param_spec_uint("flag", "flag", + "watermark flag used to trigger the blur/draw rectangle and text. " + "(0x01)draw rectangle, (0x02)draw text,(0x04)draw landmark,(0x08)blur rectangle" + "please see user guide for more details", + 1, 16, WATERMARK_DRAW_RECTANGLE, static_cast(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property( + gobject_class, PROP_WATERMARK_FILTER, + g_param_spec_string("filter", "filter", + "watermark filter the specified object label list. seperated by comma " + "please see user guide for more details", + DEFAULT_FILTER, static_cast(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); } static void gst_gva_watermark_init(GstGvaWatermark *self) { + GST_DEBUG_OBJECT(self, "gst_gva_watermark_init"); + + if (self == NULL) + return; + + self->flag = WATERMARK_DRAW_RECTANGLE; + self->filter = g_strdup(DEFAULT_FILTER); + GstPadTemplate *pad_tmpl = gst_static_pad_template_get(&sinktemplate); self->sinkpad = gst_ghost_pad_new_no_target_from_template("sink", pad_tmpl); gst_object_unref(pad_tmpl); @@ -146,6 +181,12 @@ void gst_gva_watermark_set_property(GObject *object, guint property_id, const GV gvawatermark->device = g_value_dup_string(value); g_object_set(gvawatermark->watermarkimpl, "device", gvawatermark->device, nullptr); break; + case PROP_WATERMARK_FLAG: + gvawatermark->flag = g_value_get_uint(value); + break; + case PROP_WATERMARK_FILTER: + gvawatermark->filter = g_value_dup_string(value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); break; @@ -161,6 +202,12 @@ void gst_gva_watermark_get_property(GObject *object, guint property_id, GValue * case PROP_DEVICE: g_value_set_string(value, gvawatermark->device); break; + case PROP_WATERMARK_FLAG: + g_value_set_uint(value, gvawatermark->flag); + break; + case PROP_WATERMARK_FILTER: + g_value_set_string(value, gvawatermark->filter); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); break; @@ -179,6 +226,9 @@ void gst_gva_watermark_finalize(GObject *object) { g_free(gvawatermark->device); gvawatermark->device = nullptr; + + g_free(gvawatermark->filter); + gvawatermark->filter=nullptr; G_OBJECT_CLASS(gst_gva_watermark_parent_class)->finalize(object); } diff --git a/gst/elements/gvawatermark/gvawatermark.h b/gst/elements/gvawatermark/gvawatermark.h index 6533cb86..fb289c64 100644 --- a/gst/elements/gvawatermark/gvawatermark.h +++ b/gst/elements/gvawatermark/gvawatermark.h @@ -48,6 +48,9 @@ struct _GstGvaWatermark { GstElement *preproc; GstElement *capsfilter; GstElement *convert; + + guint flag; + gchar *filter; }; struct _GstGvaWatermarkClass { @@ -56,6 +59,7 @@ struct _GstGvaWatermarkClass { GType gst_gva_watermark_get_type(void); + G_END_DECLS #endif diff --git a/gst/elements/gvawatermark/renderer/cpu/renderer_cpu.cpp b/gst/elements/gvawatermark/renderer/cpu/renderer_cpu.cpp index 3734fcee..f3e85c8b 100644 --- a/gst/elements/gvawatermark/renderer/cpu/renderer_cpu.cpp +++ b/gst/elements/gvawatermark/renderer/cpu/renderer_cpu.cpp @@ -51,6 +51,8 @@ void RendererYUV::draw_backend(std::vector &image_planes, std::vector(p)); } else if (std::holds_alternative(p)) { draw_text(image_planes, std::get(p)); + } else if (std::holds_alternative(p)) { + blur_rectangle(image_planes, std::get(p)); } } } @@ -133,6 +135,26 @@ void RendererI420::draw_line(std::vector &mats, render::Line line) { cv::line(v, pos1_u_v, pos2_u_v, line.color[2], thick); } +void RendererI420::blur_rectangle(std::vector &mats, render::Blur blur) { + check_planes<3>(mats); + cv::Mat &y = mats[0]; + cv::Mat &u = mats[1]; + cv::Mat &v = mats[2]; + + cv::Rect r =blur.rect; + + cv::Mat roi_u(u, cv::Rect(r.x/2, r.y/2, r.width/2, r.height/2)); + cv::blur(roi_u, roi_u, cv::Size(11, 11)); + cv::GaussianBlur(roi_u, roi_u, cv::Size(11, 11), 0, 0); + + cv::Mat roi_v(v, cv::Rect(r.x/2, r.y/2, r.width/2, r.height/2)); + cv::blur(roi_v, roi_v, cv::Size(11, 11)); + cv::GaussianBlur(roi_v, roi_v, cv::Size(11, 11), 0, 0); + + cv::Mat roi_y(y, cv::Rect(r.x, r.y, r.width, r.height)); + cv::blur(roi_y, roi_y, cv::Size(11, 11)); + cv::GaussianBlur(roi_y, roi_y, cv::Size(11, 11), 0, 0); +} void RendererNV12::draw_rectangle(std::vector &mats, render::Rect rect) { check_planes<2>(mats); cv::Mat &y = mats[0]; @@ -180,6 +202,22 @@ void RendererNV12::draw_line(std::vector &mats, render::Line line) { cv::line(u_v, pos1_u_v, pos2_u_v, {line.color[1], line.color[2]}, calc_thick_for_u_v_planes(line.thick)); } +void RendererNV12::blur_rectangle(std::vector &mats, render::Blur blur) { + check_planes<2>(mats); + cv::Mat &y = mats[0]; + cv::Mat &u_v = mats[1]; + + cv::Rect r =blur.rect; + + cv::Mat roi_uv(u_v, cv::Rect(r.x/2, r.y/2, r.width/2, r.height/2)); + cv::blur(roi_uv, roi_uv, cv::Size(11, 11)); + cv::GaussianBlur(roi_uv, roi_uv, cv::Size(11, 11), 0, 0); + + cv::Mat roi(y, cv::Rect(r.x, r.y, r.width, r.height)); + cv::blur(roi, roi, cv::Size(11, 11)); + cv::GaussianBlur(roi, roi, cv::Size(11, 11), 0, 0); +} + void RendererBGR::draw_rectangle(std::vector &mats, render::Rect rect) { cv::rectangle(mats[0], rect.rect.tl(), rect.rect.br(), rect.color, rect.thick); } @@ -195,3 +233,12 @@ void RendererBGR::draw_text(std::vector &mats, render::Text text) { void RendererBGR::draw_line(std::vector &mats, render::Line line) { cv::line(mats[0], line.pt1, line.pt2, line.color, line.thick); } + +void RendererBGR::blur_rectangle(std::vector &mats, render::Blur blur) { + cv::Mat &mat = mats[0]; + cv::Rect r = blur.rect; + cv::Mat roi(mat, cv::Rect(r.x, r.y, r.width, r.height)); + cv::blur(roi, roi, cv::Size(11, 11)); + cv::GaussianBlur(roi, roi, cv::Size(11, 11), 0, 0); +} + diff --git a/gst/elements/gvawatermark/renderer/cpu/renderer_cpu.h b/gst/elements/gvawatermark/renderer/cpu/renderer_cpu.h index 4a2c3dc0..42ee1cdc 100644 --- a/gst/elements/gvawatermark/renderer/cpu/renderer_cpu.h +++ b/gst/elements/gvawatermark/renderer/cpu/renderer_cpu.h @@ -35,6 +35,7 @@ class RendererYUV : public RendererCPU { virtual void draw_circle(std::vector &mats, render::Circle circle) = 0; virtual void draw_text(std::vector &mats, render::Text text) = 0; virtual void draw_line(std::vector &mats, render::Line line) = 0; + virtual void blur_rectangle(std::vector &mats, render::Blur blur) = 0; void draw_rect_y_plane(cv::Mat &y, cv::Point2i pt1, cv::Point2i pt2, double color, int thick); }; @@ -50,6 +51,7 @@ class RendererI420 : public RendererYUV { void draw_circle(std::vector &mats, render::Circle circle) override; void draw_text(std::vector &mats, render::Text text) override; void draw_line(std::vector &mats, render::Line line) override; + void blur_rectangle(std::vector &mats, render::Blur blur) override; }; class RendererNV12 : public RendererYUV { @@ -63,6 +65,7 @@ class RendererNV12 : public RendererYUV { void draw_circle(std::vector &mats, render::Circle circle) override; void draw_text(std::vector &mats, render::Text text) override; void draw_line(std::vector &mats, render::Line line) override; + void blur_rectangle(std::vector &mats, render::Blur blur) override; }; class RendererBGR : public RendererYUV { @@ -76,4 +79,5 @@ class RendererBGR : public RendererYUV { void draw_circle(std::vector &mats, render::Circle circle) override; void draw_text(std::vector &mats, render::Text text) override; void draw_line(std::vector &mats, render::Line line) override; + void blur_rectangle(std::vector &mats, render::Blur blur) override; }; diff --git a/gst/elements/gvawatermark/renderer/render_prim.h b/gst/elements/gvawatermark/renderer/render_prim.h index fa0fdf95..17676bf7 100644 --- a/gst/elements/gvawatermark/renderer/render_prim.h +++ b/gst/elements/gvawatermark/renderer/render_prim.h @@ -66,6 +66,16 @@ struct Line { } }; -using Prim = std::variant; +struct Blur { + cv::Rect rect; + int thick; + + Blur() = default; + + Blur(const cv::Rect &rect) : rect(rect) { + } +}; + +using Prim = std::variant; } // namespace render