Skip to content

enable blur rectangle feature and add the watermark flag interface to support set which parts will be rendered. #346

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
xiaoxial wants to merge 1 commit into from
Closed

enable blur rectangle feature and add the watermark flag interface to support set which parts will be rendered. #346

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 82 additions & 16 deletions gst/elements/gvawatermark/gstgvawatermarkimpl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ GST_DEBUG_CATEGORY_STATIC(gst_gva_watermark_impl_debug_category);

#define DEFAULT_DEVICE nullptr

#define DEFAULT_FILTER "none"

typedef enum { DEVICE_CPU, DEVICE_GPU, DEVICE_GPU_AUTOSELECTED } DEVICE_SELECTOR;

namespace {
Expand Down Expand Up @@ -115,12 +117,20 @@ dlstreamer::MemoryMapperPtr createMapperToDMA(InferenceBackend::MemoryType in_me

struct Impl {
Impl(GstVideoInfo *info, DEVICE_SELECTOR device, InferenceBackend::MemoryType mem_type,
dlstreamer::ContextPtr context);
dlstreamer::ContextPtr context, WATERMARK_FLAG flag, std::string filter);
bool render(GstBuffer *buffer);
const std::string &getBackendType() const {
return _backend_type;
}

const int &getFlag() const {
return flag;
}

const std::string &getFilter() const {
return filter;
}

private:
void preparePrimsForRoi(GVA::RegionOfInterest &roi, std::vector<render::Prim> &prims) const;
void preparePrimsForTensor(const GVA::Tensor &tensor, GVA::Rect<double> rect,
Expand Down Expand Up @@ -154,9 +164,15 @@ struct Impl {
const int type = cv::FONT_HERSHEY_TRIPLEX;
const double scale = 1.0;
} _font;
int flag;
std::string filter;
};

enum { PROP_0, PROP_DEVICE };
enum { PROP_0,
PROP_DEVICE,
PROP_WATERMARK_FLAG,
PROP_WATERMARK_FILTER,
};

G_DEFINE_TYPE_WITH_CODE(GstGvaWatermarkImpl, gst_gva_watermark_impl, GST_TYPE_BASE_TRANSFORM,
GST_DEBUG_CATEGORY_INIT(gst_gva_watermark_impl_debug_category, "gvawatermarkimpl", 0,
Expand All @@ -176,6 +192,12 @@ void gst_gva_watermark_impl_set_property(GObject *object, guint prop_id, const G
g_free(gvawatermark->device);
gvawatermark->device = g_value_dup_string(value);
break;
case PROP_WATERMARK_FLAG:
gvawatermark->flag = g_value_get_uint(value);
break;
case PROP_WATERMARK_FILTER:
gvawatermark->filter = g_value_dup_string(value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
break;
Expand All @@ -195,6 +217,20 @@ void gst_gva_watermark_impl_get_property(GObject *object, guint prop_id, GValue
g_value_set_string(value, self->device);
}
break;
case PROP_WATERMARK_FLAG:
if (self->impl) {
g_value_set_uint(value, self->impl->getFlag());
} else {
g_value_set_uint(value, self->flag);
}
break;
case PROP_WATERMARK_FILTER:
if (self->impl) {
g_value_set_string(value, self->impl->getFilter().c_str());
} else {
g_value_set_string(value, self->filter);
}
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
break;
Expand Down Expand Up @@ -222,6 +258,9 @@ void gst_gva_watermark_impl_finalize(GObject *object) {
g_free(gvawatermark->device);
gvawatermark->device = nullptr;

g_free(gvawatermark->filter);
gvawatermark->filter=nullptr;

G_OBJECT_CLASS(gst_gva_watermark_impl_parent_class)->finalize(object);
}

Expand Down Expand Up @@ -305,7 +344,7 @@ static gboolean gst_gva_watermark_impl_set_caps(GstBaseTransform *trans, GstCaps
}

try {
gvawatermark->impl = new Impl(&gvawatermark->info, device, mem_type, va_dpy);
gvawatermark->impl = new Impl(&gvawatermark->info, device, mem_type, va_dpy, (WATERMARK_FLAG)gvawatermark->flag, gvawatermark->filter);
} catch (const std::exception &e) {
GST_ELEMENT_ERROR(gvawatermark, CORE, FAILED, ("Could not initialize"),
("Cannot create watermark instance. %s", Utils::createNestedErrorMsg(e).c_str()));
Expand Down Expand Up @@ -392,11 +431,26 @@ static void gst_gva_watermark_impl_class_init(GstGvaWatermarkImplClass *klass) {
"device", "Target device",
"Supported devices are CPU and GPU. Default is CPU on system memory and GPU on video memory",
DEFAULT_DEVICE, (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));

g_object_class_install_property(
gobject_class, PROP_WATERMARK_FLAG,
g_param_spec_uint("flag", "flag",
"watermark flag used to trigger the blur/draw rectangle and text. "
"(0x01)draw rectangle, (0x02)draw text,(0x04)draw landmark,(0x08)blur rectangle"
"please see user guide for more details",
1, 16, WATERMARK_DRAW_RECTANGLE, static_cast<GParamFlags>(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));

g_object_class_install_property(
gobject_class, PROP_WATERMARK_FILTER,
g_param_spec_string("filter", "filter",
"watermark filter the specified object label list. seperated by comma "
"please see user guide for more details",
DEFAULT_FILTER, static_cast<GParamFlags>(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
}

Impl::Impl(GstVideoInfo *info, DEVICE_SELECTOR device, InferenceBackend::MemoryType mem_type,
dlstreamer::ContextPtr context)
: _vinfo(info) {
dlstreamer::ContextPtr context, WATERMARK_FLAG flag, std::string filter)
: _vinfo(info), flag(flag), filter(filter) {
assert(_vinfo);
if (GST_VIDEO_INFO_COLORIMETRY(_vinfo).matrix == GstVideoColorMatrix::GST_VIDEO_COLOR_MATRIX_UNKNOWN)
throw std::runtime_error("GST_VIDEO_COLOR_MATRIX_UNKNOWN");
Expand Down Expand Up @@ -446,11 +500,12 @@ bool Impl::render(GstBuffer *buffer) {
for (auto &tensor : video_frame.tensors()) {
if (tensor.is_detection())
continue;
preparePrimsForTensor(tensor, ff_rect, prims);
if (flag & WATERMARK_DRAW_CIRCLE)
preparePrimsForTensor(tensor, ff_rect, prims);
appendStr(ff_text, tensor.label());
}

if (ff_text.tellp() != 0)
if (ff_text.tellp() != 0 && flag & WATERMARK_DRAW_TEXT)
prims.emplace_back(render::Text(ff_text.str(), _ff_text_position, _font.type, _font.scale, _default_color));

// Skip render if there are no primitives to draw
Expand Down Expand Up @@ -489,23 +544,34 @@ void Impl::preparePrimsForRoi(GVA::RegionOfInterest &roi, std::vector<render::Pr

// Prepare primitives for tensors
for (auto &tensor : roi.tensors()) {
preparePrimsForTensor(tensor, rect, prims);
if (flag & WATERMARK_DRAW_CIRCLE)
preparePrimsForTensor(tensor, rect, prims);
if (!tensor.is_detection()) {
appendStr(text, tensor.label());
}
}

// put rectangle
Color color = indexToColor(color_index);
cv::Rect bbox_rect(rect.x, rect.y, rect.w, rect.h);
prims.emplace_back(render::Rect(bbox_rect, color, _thickness));
if (flag & WATERMARK_DRAW_RECTANGLE) {
Color color = indexToColor(color_index);
cv::Rect bbox_rect(rect.x, rect.y, rect.w, rect.h);
prims.emplace_back(render::Rect(bbox_rect, color, _thickness));
}
// draw blur rectangle
if (flag & WATERMARK_BLUR_RECTANGLE) {
cv::Rect blur_rect(rect.x, rect.y, rect.w, rect.h);
prims.emplace_back(render::Blur(blur_rect));
}

// put text
if (text.str().size() != 0) {
cv::Point2f pos(rect.x, rect.y - 5.f);
if (pos.y < 0)
pos.y = rect.y + 30.f;
prims.emplace_back(render::Text(text.str(), pos, _font.type, _font.scale, color));
if (flag & WATERMARK_DRAW_TEXT) {
Color color = indexToColor(color_index);
if (text.str().size() != 0) {
cv::Point2f pos(rect.x, rect.y - 5.f);
if (pos.y < 0)
pos.y = rect.y + 30.f;
prims.emplace_back(render::Text(text.str(), pos, _font.type, _font.scale, color));
}
}
}

Expand Down
9 changes: 9 additions & 0 deletions gst/elements/gvawatermark/gstgvawatermarkimpl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ struct _GstGvaWatermarkImpl {
GstVideoInfo info;
gchar *device;
struct Impl *impl;
guint flag;
gchar *filter;
};

struct _GstGvaWatermarkImplClass {
Expand All @@ -36,6 +38,13 @@ struct _GstGvaWatermarkImplClass {

GType gst_gva_watermark_impl_get_type(void);

typedef enum {
WATERMARK_DRAW_RECTANGLE=0x0001,
WATERMARK_DRAW_TEXT=0x0002,
WATERMARK_DRAW_CIRCLE=0x0004,
WATERMARK_BLUR_RECTANGLE=0x0008
} WATERMARK_FLAG;

G_END_DECLS

#endif
52 changes: 51 additions & 1 deletion gst/elements/gvawatermark/gvawatermark.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,20 @@ GST_DEBUG_CATEGORY_STATIC(gst_gva_watermark_debug_category);
#define GST_CAT_DEFAULT gst_gva_watermark_debug_category

#define DEFAULT_DEVICE "CPU"
#define DEFAULT_FILTER "none"

enum { PROP_0, PROP_DEVICE };
enum { PROP_0,
PROP_DEVICE,
PROP_WATERMARK_FLAG,
PROP_WATERMARK_FILTER,
};

enum {
WATERMARK_DRAW_RECTANGLE=0x0001,
WATERMARK_DRAW_TEXT=0x0002,
WATERMARK_DRAW_CIRCLE=0x0004,
WATERMARK_BLUR_RECTANGLE=0x0008
};

G_DEFINE_TYPE_WITH_CODE(GstGvaWatermark, gst_gva_watermark, GST_TYPE_BIN,
GST_DEBUG_CATEGORY_INIT(gst_gva_watermark_debug_category, "gvawatermark", 0,
Expand Down Expand Up @@ -89,9 +101,32 @@ static void gst_gva_watermark_class_init(GstGvaWatermarkClass *klass) {
gobject_class, PROP_DEVICE,
g_param_spec_string("device", "Target device", "Deprecated, don't use", DEFAULT_DEVICE,
static_cast<GParamFlags>(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | G_PARAM_DEPRECATED)));

g_object_class_install_property(
gobject_class, PROP_WATERMARK_FLAG,
g_param_spec_uint("flag", "flag",
"watermark flag used to trigger the blur/draw rectangle and text. "
"(0x01)draw rectangle, (0x02)draw text,(0x04)draw landmark,(0x08)blur rectangle"
"please see user guide for more details",
1, 16, WATERMARK_DRAW_RECTANGLE, static_cast<GParamFlags>(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));

g_object_class_install_property(
gobject_class, PROP_WATERMARK_FILTER,
g_param_spec_string("filter", "filter",
"watermark filter the specified object label list. seperated by comma "
"please see user guide for more details",
DEFAULT_FILTER, static_cast<GParamFlags>(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
}

static void gst_gva_watermark_init(GstGvaWatermark *self) {
GST_DEBUG_OBJECT(self, "gst_gva_watermark_init");

if (self == NULL)
return;

self->flag = WATERMARK_DRAW_RECTANGLE;
self->filter = g_strdup(DEFAULT_FILTER);

GstPadTemplate *pad_tmpl = gst_static_pad_template_get(&sinktemplate);
self->sinkpad = gst_ghost_pad_new_no_target_from_template("sink", pad_tmpl);
gst_object_unref(pad_tmpl);
Expand Down Expand Up @@ -146,6 +181,12 @@ void gst_gva_watermark_set_property(GObject *object, guint property_id, const GV
gvawatermark->device = g_value_dup_string(value);
g_object_set(gvawatermark->watermarkimpl, "device", gvawatermark->device, nullptr);
break;
case PROP_WATERMARK_FLAG:
gvawatermark->flag = g_value_get_uint(value);
break;
case PROP_WATERMARK_FILTER:
gvawatermark->filter = g_value_dup_string(value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec);
break;
Expand All @@ -161,6 +202,12 @@ void gst_gva_watermark_get_property(GObject *object, guint property_id, GValue *
case PROP_DEVICE:
g_value_set_string(value, gvawatermark->device);
break;
case PROP_WATERMARK_FLAG:
g_value_set_uint(value, gvawatermark->flag);
break;
case PROP_WATERMARK_FILTER:
g_value_set_string(value, gvawatermark->filter);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec);
break;
Expand All @@ -179,6 +226,9 @@ void gst_gva_watermark_finalize(GObject *object) {

g_free(gvawatermark->device);
gvawatermark->device = nullptr;

g_free(gvawatermark->filter);
gvawatermark->filter=nullptr;

G_OBJECT_CLASS(gst_gva_watermark_parent_class)->finalize(object);
}
Expand Down
4 changes: 4 additions & 0 deletions gst/elements/gvawatermark/gvawatermark.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ struct _GstGvaWatermark {
GstElement *preproc;
GstElement *capsfilter;
GstElement *convert;

guint flag;
gchar *filter;
};

struct _GstGvaWatermarkClass {
Expand All @@ -56,6 +59,7 @@ struct _GstGvaWatermarkClass {

GType gst_gva_watermark_get_type(void);


G_END_DECLS

#endif
47 changes: 47 additions & 0 deletions gst/elements/gvawatermark/renderer/cpu/renderer_cpu.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ void RendererYUV::draw_backend(std::vector<cv::Mat> &image_planes, std::vector<r
draw_circle(image_planes, std::get<render::Circle>(p));
} else if (std::holds_alternative<render::Text>(p)) {
draw_text(image_planes, std::get<render::Text>(p));
} else if (std::holds_alternative<render::Blur>(p)) {
blur_rectangle(image_planes, std::get<render::Blur>(p));
}
}
}
Expand Down Expand Up @@ -133,6 +135,26 @@ void RendererI420::draw_line(std::vector<cv::Mat> &mats, render::Line line) {
cv::line(v, pos1_u_v, pos2_u_v, line.color[2], thick);
}

void RendererI420::blur_rectangle(std::vector<cv::Mat> &mats, render::Blur blur) {
check_planes<3>(mats);
cv::Mat &y = mats[0];
cv::Mat &u = mats[1];
cv::Mat &v = mats[2];

cv::Rect r =blur.rect;

cv::Mat roi_u(u, cv::Rect(r.x/2, r.y/2, r.width/2, r.height/2));
cv::blur(roi_u, roi_u, cv::Size(11, 11));
cv::GaussianBlur(roi_u, roi_u, cv::Size(11, 11), 0, 0);

cv::Mat roi_v(v, cv::Rect(r.x/2, r.y/2, r.width/2, r.height/2));
cv::blur(roi_v, roi_v, cv::Size(11, 11));
cv::GaussianBlur(roi_v, roi_v, cv::Size(11, 11), 0, 0);

cv::Mat roi_y(y, cv::Rect(r.x, r.y, r.width, r.height));
cv::blur(roi_y, roi_y, cv::Size(11, 11));
cv::GaussianBlur(roi_y, roi_y, cv::Size(11, 11), 0, 0);
}
void RendererNV12::draw_rectangle(std::vector<cv::Mat> &mats, render::Rect rect) {
check_planes<2>(mats);
cv::Mat &y = mats[0];
Expand Down Expand Up @@ -180,6 +202,22 @@ void RendererNV12::draw_line(std::vector<cv::Mat> &mats, render::Line line) {
cv::line(u_v, pos1_u_v, pos2_u_v, {line.color[1], line.color[2]}, calc_thick_for_u_v_planes(line.thick));
}

void RendererNV12::blur_rectangle(std::vector<cv::Mat> &mats, render::Blur blur) {
check_planes<2>(mats);
cv::Mat &y = mats[0];
cv::Mat &u_v = mats[1];

cv::Rect r =blur.rect;

cv::Mat roi_uv(u_v, cv::Rect(r.x/2, r.y/2, r.width/2, r.height/2));
cv::blur(roi_uv, roi_uv, cv::Size(11, 11));
cv::GaussianBlur(roi_uv, roi_uv, cv::Size(11, 11), 0, 0);

cv::Mat roi(y, cv::Rect(r.x, r.y, r.width, r.height));
cv::blur(roi, roi, cv::Size(11, 11));
cv::GaussianBlur(roi, roi, cv::Size(11, 11), 0, 0);
}

void RendererBGR::draw_rectangle(std::vector<cv::Mat> &mats, render::Rect rect) {
cv::rectangle(mats[0], rect.rect.tl(), rect.rect.br(), rect.color, rect.thick);
}
Expand All @@ -195,3 +233,12 @@ void RendererBGR::draw_text(std::vector<cv::Mat> &mats, render::Text text) {
void RendererBGR::draw_line(std::vector<cv::Mat> &mats, render::Line line) {
cv::line(mats[0], line.pt1, line.pt2, line.color, line.thick);
}

void RendererBGR::blur_rectangle(std::vector<cv::Mat> &mats, render::Blur blur) {
cv::Mat &mat = mats[0];
cv::Rect r = blur.rect;
cv::Mat roi(mat, cv::Rect(r.x, r.y, r.width, r.height));
cv::blur(roi, roi, cv::Size(11, 11));
cv::GaussianBlur(roi, roi, cv::Size(11, 11), 0, 0);
}

Loading